diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2018-08-17 12:44:48 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2018-08-17 12:44:48 -0700 |
| commit | 9bd553929f68921be0f2014dd06561e0c8249a0d (patch) | |
| tree | 720e556374e3500af9a0210178fabfc6bd0f754c /include | |
| parent | 022ff62c3d8c3758d15ccc6b58615fd8f257ba85 (diff) | |
| parent | 0a3173a5f09bc58a3638ecfd0a80bdbae55e123c (diff) | |
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma
Pull rdma updates from Jason Gunthorpe:
"This has been a large cycle for RDMA, with several major patch series
reworking parts of the core code.
- Rework the so-called 'gid cache' and internal APIs to use a kref'd
pointer to a struct instead of copying, push this upwards into the
callers and add more stuff to the struct. The new design avoids
some ugly races the old one suffered with. This is part of the
namespace enablement work as the new struct is learning to be
namespace aware.
- Various uapi cleanups, moving more stuff to include/uapi and fixing
some long standing bugs that have recently been discovered.
- Driver updates for mlx5, mlx4 i40iw, rxe, cxgb4, hfi1, usnic,
pvrdma, and hns
- Provide max_send_sge and max_recv_sge attributes to better support
HW where these values are asymmetric.
- mlx5 user API 'devx' allows sending commands directly to the device
FW, instead of trying to cram every wild and niche feature into the
common API. Sort of like what GPU does.
- Major write() and ioctl() API rework to cleanly support PCI device
hot unplug and advance the ioctl conversion work
- Sparse and compile warning cleanups
- Add 'const' to the ib_poll_cq() signature, and permit a NULL
'bad_wr', which is the common use case
- Various patches to avoid high order allocations across the stack
- SRQ support for cxgb4, hns and qedr
- Changes to IPoIB to better follow the netdev model for working with
struct net_device liftime"
* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (312 commits)
Revert "net/smc: Replace ib_query_gid with rdma_get_gid_attr"
RDMA/hns: Fix usage of bitmap allocation functions return values
IB/core: Change filter function return type from int to bool
IB/core: Update GID entries for netdevice whose mac address changes
IB/core: Add default GIDs of the bond master netdev
IB/core: Consider adding default GIDs of bond device
IB/core: Delete lower netdevice default GID entries in bonding scenario
IB/core: Avoid confusing del_netdev_default_ips
IB/core: Add comment for change upper netevent handling
qedr: Add user space support for SRQ
qedr: Add support for kernel mode SRQ's
qedr: Add wrapping generic structure for qpidr and adjust idr routines.
IB/mlx5: Fix leaking stack memory to userspace
Update the e-mail address of Bart Van Assche
IB/ucm: Fix compiling ucm.c
IB/uverbs: Do not check for device disassociation during ioctl
IB/uverbs: Remove struct uverbs_root_spec and all supporting code
IB/uverbs: Use uverbs_api to unmarshal ioctl commands
IB/uverbs: Use uverbs_alloc for allocations
IB/uverbs: Add a simple allocator to uverbs_attr_bundle
...
Diffstat (limited to 'include')
28 files changed, 1136 insertions, 618 deletions
diff --git a/include/linux/idr.h b/include/linux/idr.h index e856f4e0ab35..3e8215b2c371 100644 --- a/include/linux/idr.h +++ b/include/linux/idr.h @@ -98,6 +98,17 @@ static inline void idr_set_cursor(struct idr *idr, unsigned int val) * period). */ +#define idr_lock(idr) xa_lock(&(idr)->idr_rt) +#define idr_unlock(idr) xa_unlock(&(idr)->idr_rt) +#define idr_lock_bh(idr) xa_lock_bh(&(idr)->idr_rt) +#define idr_unlock_bh(idr) xa_unlock_bh(&(idr)->idr_rt) +#define idr_lock_irq(idr) xa_lock_irq(&(idr)->idr_rt) +#define idr_unlock_irq(idr) xa_unlock_irq(&(idr)->idr_rt) +#define idr_lock_irqsave(idr, flags) \ + xa_lock_irqsave(&(idr)->idr_rt, flags) +#define idr_unlock_irqrestore(idr, flags) \ + xa_unlock_irqrestore(&(idr)->idr_rt, flags) + void idr_preload(gfp_t gfp_mask); int idr_alloc(struct idr *, void *ptr, int start, int end, gfp_t); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 54f385cc8811..7a452716de4b 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1235,14 +1235,11 @@ struct net_device *mlx5_rdma_netdev_alloc(struct mlx5_core_dev *mdev, { return ERR_PTR(-EOPNOTSUPP); } - -static inline void mlx5_rdma_netdev_free(struct net_device *netdev) {} #else struct net_device *mlx5_rdma_netdev_alloc(struct mlx5_core_dev *mdev, struct ib_device *ibdev, const char *name, void (*setup)(struct net_device *)); -void mlx5_rdma_netdev_free(struct net_device *netdev); #endif /* CONFIG_MLX5_CORE_IPOIB */ struct mlx5_profile { diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index 71fb503b2b52..804516e4f483 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -89,6 +89,7 @@ struct mlx5_flow_destination { enum mlx5_flow_destination_type type; union { u32 tir_num; + u32 ft_num; struct mlx5_flow_table *ft; struct mlx5_fc *counter; struct { diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 6ead9c1a5396..f043d65b9bac 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -82,6 +82,7 @@ enum { enum { MLX5_OBJ_TYPE_UCTX = 0x0004, + MLX5_OBJ_TYPE_UMEM = 0x0005, }; enum { @@ -246,12 +247,15 @@ enum { MLX5_CMD_OP_DEALLOC_ENCAP_HEADER = 0x93e, MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT = 0x940, MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT = 0x941, + MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT = 0x942, MLX5_CMD_OP_FPGA_CREATE_QP = 0x960, MLX5_CMD_OP_FPGA_MODIFY_QP = 0x961, MLX5_CMD_OP_FPGA_QUERY_QP = 0x962, MLX5_CMD_OP_FPGA_DESTROY_QP = 0x963, MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS = 0x964, MLX5_CMD_OP_CREATE_GENERAL_OBJECT = 0xa00, + MLX5_CMD_OP_MODIFY_GENERAL_OBJECT = 0xa01, + MLX5_CMD_OP_QUERY_GENERAL_OBJECT = 0xa02, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT = 0xa03, MLX5_CMD_OP_MAX }; @@ -1185,6 +1189,7 @@ enum mlx5_flow_destination_type { MLX5_FLOW_DESTINATION_TYPE_PORT = 0x99, MLX5_FLOW_DESTINATION_TYPE_COUNTER = 0x100, + MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM = 0x101, }; struct mlx5_ifc_dest_format_struct_bits { diff --git a/include/linux/overflow.h b/include/linux/overflow.h index 8712ff70995f..40b48e2133cb 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -202,6 +202,37 @@ #endif /* COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW */ +/** check_shl_overflow() - Calculate a left-shifted value and check overflow + * + * @a: Value to be shifted + * @s: How many bits left to shift + * @d: Pointer to where to store the result + * + * Computes *@d = (@a << @s) + * + * Returns true if '*d' cannot hold the result or when 'a << s' doesn't + * make sense. Example conditions: + * - 'a << s' causes bits to be lost when stored in *d. + * - 's' is garbage (e.g. negative) or so large that the result of + * 'a << s' is guaranteed to be 0. + * - 'a' is negative. + * - 'a << s' sets the sign bit, if any, in '*d'. + * + * '*d' will hold the results of the attempted shift, but is not + * considered "safe for use" if false is returned. + */ +#define check_shl_overflow(a, s, d) ({ \ + typeof(a) _a = a; \ + typeof(s) _s = s; \ + typeof(d) _d = d; \ + u64 _a_full = _a; \ + unsigned int _to_shift = \ + _s >= 0 && _s < 8 * sizeof(*d) ? _s : 0; \ + *_d = (_a_full << _to_shift); \ + (_to_shift != _s || *_d < 0 || _a < 0 || \ + (*_d >> _to_shift) != _a); \ +}) + /** * array_size() - Calculate size of 2-dimensional array. * diff --git a/include/rdma/ib.h b/include/rdma/ib.h index 66dbed0c146d..4f385ec54f80 100644 --- a/include/rdma/ib.h +++ b/include/rdma/ib.h @@ -53,12 +53,12 @@ struct ib_addr { #define sib_interface_id ib_u.uib_addr64[1] }; -static inline int ib_addr_any(const struct ib_addr *a) +static inline bool ib_addr_any(const struct ib_addr *a) { return ((a->sib_addr64[0] | a->sib_addr64[1]) == 0); } -static inline int ib_addr_loopback(const struct ib_addr *a) +static inline bool ib_addr_loopback(const struct ib_addr *a) { return ((a->sib_addr32[0] | a->sib_addr32[1] | a->sib_addr32[2] | (a->sib_addr32[3] ^ htonl(1))) == 0); diff --git a/include/rdma/ib_addr.h b/include/rdma/ib_addr.h index c2c8b1fdeead..77c7908b7d73 100644 --- a/include/rdma/ib_addr.h +++ b/include/rdma/ib_addr.h @@ -58,6 +58,7 @@ * @bound_dev_if: An optional device interface index. * @transport: The transport type used. * @net: Network namespace containing the bound_dev_if net_dev. + * @sgid_attr: GID attribute to use for identified SGID */ struct rdma_dev_addr { unsigned char src_dev_addr[MAX_ADDR_LEN]; @@ -67,6 +68,7 @@ struct rdma_dev_addr { int bound_dev_if; enum rdma_transport_type transport; struct net *net; + const struct ib_gid_attr *sgid_attr; enum rdma_network_type network; int hoplimit; }; @@ -95,7 +97,7 @@ int rdma_translate_ip(const struct sockaddr *addr, * or been canceled. A status of 0 indicates success. * @context: User-specified context associated with the call. */ -int rdma_resolve_ip(struct sockaddr *src_addr, struct sockaddr *dst_addr, +int rdma_resolve_ip(struct sockaddr *src_addr, const struct sockaddr *dst_addr, struct rdma_dev_addr *addr, int timeout_ms, void (*callback)(int status, struct sockaddr *src_addr, struct rdma_dev_addr *addr, void *context), @@ -107,7 +109,7 @@ void rdma_copy_addr(struct rdma_dev_addr *dev_addr, const struct net_device *dev, const unsigned char *dst_dev_addr); -int rdma_addr_size(struct sockaddr *addr); +int rdma_addr_size(const struct sockaddr *addr); int rdma_addr_size_in6(struct sockaddr_in6 *addr); int rdma_addr_size_kss(struct __kernel_sockaddr_storage *addr); diff --git a/include/rdma/ib_cache.h b/include/rdma/ib_cache.h index a5f249828115..3e11e7cc60b7 100644 --- a/include/rdma/ib_cache.h +++ b/include/rdma/ib_cache.h @@ -37,45 +37,23 @@ #include <rdma/ib_verbs.h> -/** - * ib_get_cached_gid - Returns a cached GID table entry - * @device: The device to query. - * @port_num: The port number of the device to query. - * @index: The index into the cached GID table to query. - * @gid: The GID value found at the specified index. - * @attr: The GID attribute found at the specified index (only in RoCE). - * NULL means ignore (output parameter). - * - * ib_get_cached_gid() fetches the specified GID table entry stored in - * the local software cache. - */ -int ib_get_cached_gid(struct ib_device *device, - u8 port_num, - int index, - union ib_gid *gid, - struct ib_gid_attr *attr); - -int ib_find_cached_gid(struct ib_device *device, - const union ib_gid *gid, - enum ib_gid_type gid_type, - struct net_device *ndev, - u8 *port_num, - u16 *index); - -int ib_find_cached_gid_by_port(struct ib_device *device, - const union ib_gid *gid, - enum ib_gid_type gid_type, - u8 port_num, - struct net_device *ndev, - u16 *index); +int rdma_query_gid(struct ib_device *device, u8 port_num, int index, + union ib_gid *gid); +const struct ib_gid_attr *rdma_find_gid(struct ib_device *device, + const union ib_gid *gid, + enum ib_gid_type gid_type, + struct net_device *ndev); +const struct ib_gid_attr *rdma_find_gid_by_port(struct ib_device *ib_dev, + const union ib_gid *gid, + enum ib_gid_type gid_type, + u8 port, + struct net_device *ndev); +const struct ib_gid_attr *rdma_find_gid_by_filter( + struct ib_device *device, const union ib_gid *gid, u8 port_num, + bool (*filter)(const union ib_gid *gid, const struct ib_gid_attr *, + void *), + void *context); -int ib_find_gid_by_filter(struct ib_device *device, - const union ib_gid *gid, - u8 port_num, - bool (*filter)(const union ib_gid *gid, - const struct ib_gid_attr *, - void *), - void *context, u16 *index); /** * ib_get_cached_pkey - Returns a cached PKey table entry * @device: The device to query. @@ -150,4 +128,33 @@ int ib_get_cached_port_state(struct ib_device *device, enum ib_port_state *port_active); bool rdma_is_zero_gid(const union ib_gid *gid); +const struct ib_gid_attr *rdma_get_gid_attr(struct ib_device *device, + u8 port_num, int index); +void rdma_put_gid_attr(const struct ib_gid_attr *attr); +void rdma_hold_gid_attr(const struct ib_gid_attr *attr); + +/* + * This is to be removed. It only exists to make merging rdma and smc simpler. + */ +static inline __deprecated int ib_query_gid(struct ib_device *device, + u8 port_num, int index, + union ib_gid *gid, + struct ib_gid_attr *attr_out) +{ + const struct ib_gid_attr *attr; + + memset(attr_out, 0, sizeof(*attr_out)); + attr = rdma_get_gid_attr(device, port_num, index); + if (IS_ERR(attr)) + return PTR_ERR(attr); + + if (attr->ndev) + dev_hold(attr->ndev); + *attr_out = *attr; + + rdma_put_gid_attr(attr); + + return 0; +} + #endif /* _IB_CACHE_H */ diff --git a/include/rdma/ib_cm.h b/include/rdma/ib_cm.h index 7979cb04f529..c10f4b5ea8ab 100644 --- a/include/rdma/ib_cm.h +++ b/include/rdma/ib_cm.h @@ -120,6 +120,13 @@ struct ib_cm_req_event_param { struct sa_path_rec *primary_path; struct sa_path_rec *alternate_path; + /* + * SGID attribute of the primary path. Currently only + * useful for RoCE. Alternate path GID attributes + * are not yet supported. + */ + const struct ib_gid_attr *ppath_sgid_attr; + __be64 remote_ca_guid; u32 remote_qkey; u32 remote_qpn; @@ -226,6 +233,12 @@ struct ib_cm_apr_event_param { struct ib_cm_sidr_req_event_param { struct ib_cm_id *listen_id; __be64 service_id; + + /* + * SGID attribute of the request. Currently only + * useful for RoCE. + */ + const struct ib_gid_attr *sgid_attr; /* P_Key that was used by the GMP's BTH header */ u16 bth_pkey; u8 port; @@ -246,6 +259,7 @@ struct ib_cm_sidr_rep_event_param { u32 qkey; u32 qpn; void *info; + const struct ib_gid_attr *sgid_attr; u8 info_len; }; @@ -297,7 +311,7 @@ struct ib_cm_event { * destroy the @cm_id after the callback completes. */ typedef int (*ib_cm_handler)(struct ib_cm_id *cm_id, - struct ib_cm_event *event); + const struct ib_cm_event *event); struct ib_cm_id { ib_cm_handler cm_handler; @@ -365,6 +379,7 @@ struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device, struct ib_cm_req_param { struct sa_path_rec *primary_path; struct sa_path_rec *alternate_path; + const struct ib_gid_attr *ppath_sgid_attr; __be64 service_id; u32 qp_num; enum ib_qp_type qp_type; @@ -566,6 +581,7 @@ int ib_send_cm_apr(struct ib_cm_id *cm_id, struct ib_cm_sidr_req_param { struct sa_path_rec *path; + const struct ib_gid_attr *sgid_attr; __be64 service_id; int timeout_ms; const void *private_data; diff --git a/include/rdma/ib_mad.h b/include/rdma/ib_mad.h index 2f4f1768ded4..f6ba366051c7 100644 --- a/include/rdma/ib_mad.h +++ b/include/rdma/ib_mad.h @@ -262,6 +262,39 @@ struct ib_class_port_info { __be32 trap_qkey; }; +/* PortInfo CapabilityMask */ +enum ib_port_capability_mask_bits { + IB_PORT_SM = 1 << 1, + IB_PORT_NOTICE_SUP = 1 << 2, + IB_PORT_TRAP_SUP = 1 << 3, + IB_PORT_OPT_IPD_SUP = 1 << 4, + IB_PORT_AUTO_MIGR_SUP = 1 << 5, + IB_PORT_SL_MAP_SUP = 1 << 6, + IB_PORT_MKEY_NVRAM = 1 << 7, + IB_PORT_PKEY_NVRAM = 1 << 8, + IB_PORT_LED_INFO_SUP = 1 << 9, + IB_PORT_SM_DISABLED = 1 << 10, + IB_PORT_SYS_IMAGE_GUID_SUP = 1 << 11, + IB_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12, + IB_PORT_EXTENDED_SPEEDS_SUP = 1 << 14, + IB_PORT_CM_SUP = 1 << 16, + IB_PORT_SNMP_TUNNEL_SUP = 1 << 17, + IB_PORT_REINIT_SUP = 1 << 18, + IB_PORT_DEVICE_MGMT_SUP = 1 << 19, + IB_PORT_VENDOR_CLASS_SUP = 1 << 20, + IB_PORT_DR_NOTICE_SUP = 1 << 21, + IB_PORT_CAP_MASK_NOTICE_SUP = 1 << 22, + IB_PORT_BOOT_MGMT_SUP = 1 << 23, + IB_PORT_LINK_LATENCY_SUP = 1 << 24, + IB_PORT_CLIENT_REG_SUP = 1 << 25, + IB_PORT_OTHER_LOCAL_CHANGES_SUP = 1 << 26, + IB_PORT_LINK_SPEED_WIDTH_TABLE_SUP = 1 << 27, + IB_PORT_VENDOR_SPECIFIC_MADS_TABLE_SUP = 1 << 28, + IB_PORT_MCAST_PKEY_TRAP_SUPPRESSION_SUP = 1 << 29, + IB_PORT_MCAST_FDB_TOP_SUP = 1 << 30, + IB_PORT_HIERARCHY_INFO_SUP = 1ULL << 31, +}; + #define OPA_CLASS_PORT_INFO_PR_SUPPORT BIT(26) struct opa_class_port_info { diff --git a/include/rdma/ib_sa.h b/include/rdma/ib_sa.h index bacb144f7780..b6ddf2a1b9d8 100644 --- a/include/rdma/ib_sa.h +++ b/include/rdma/ib_sa.h @@ -172,12 +172,7 @@ struct sa_path_rec_ib { */ struct sa_path_rec_roce { bool route_resolved; - u8 dmac[ETH_ALEN]; - /* ignored in IB */ - int ifindex; - /* ignored in IB */ - struct net *net; - + u8 dmac[ETH_ALEN]; }; struct sa_path_rec_opa { @@ -556,13 +551,10 @@ int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num, enum ib_gid_type gid_type, struct rdma_ah_attr *ah_attr); -/** - * ib_init_ah_attr_from_path - Initialize address handle attributes based on - * an SA path record. - */ int ib_init_ah_attr_from_path(struct ib_device *device, u8 port_num, struct sa_path_rec *rec, - struct rdma_ah_attr *ah_attr); + struct rdma_ah_attr *ah_attr, + const struct ib_gid_attr *sgid_attr); /** * ib_sa_pack_path - Conert a path record from struct ib_sa_path_rec @@ -667,45 +659,10 @@ static inline void sa_path_set_dmac_zero(struct sa_path_rec *rec) eth_zero_addr(rec->roce.dmac); } -static inline void sa_path_set_ifindex(struct sa_path_rec *rec, int ifindex) -{ - if (sa_path_is_roce(rec)) - rec->roce.ifindex = ifindex; -} - -static inline void sa_path_set_ndev(struct sa_path_rec *rec, struct net *net) -{ - if (sa_path_is_roce(rec)) - rec->roce.net = net; -} - static inline u8 *sa_path_get_dmac(struct sa_path_rec *rec) { if (sa_path_is_roce(rec)) return rec->roce.dmac; return NULL; } - -static inline int sa_path_get_ifindex(struct sa_path_rec *rec) -{ - if (sa_path_is_roce(rec)) - return rec->roce.ifindex; - return 0; -} - -static inline struct net *sa_path_get_ndev(struct sa_path_rec *rec) -{ - if (sa_path_is_roce(rec)) - return rec->roce.net; - return NULL; -} - -static inline struct net_device *ib_get_ndev_from_path(struct sa_path_rec *rec) -{ - return sa_path_get_ndev(rec) ? - dev_get_by_index(sa_path_get_ndev(rec), - sa_path_get_ifindex(rec)) - : NULL; -} - #endif /* IB_SA_H */ diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 6c003995347a..e950c2a68f06 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -94,6 +94,7 @@ enum ib_gid_type { struct ib_gid_attr { struct net_device *ndev; struct ib_device *device; + union ib_gid gid; enum ib_gid_type gid_type; u16 index; u8 port_num; @@ -148,13 +149,13 @@ static inline enum ib_gid_type ib_network_to_gid_type(enum rdma_network_type net return IB_GID_TYPE_IB; } -static inline enum rdma_network_type ib_gid_to_network_type(enum ib_gid_type gid_type, - union ib_gid *gid) +static inline enum rdma_network_type +rdma_gid_attr_network_type(const struct ib_gid_attr *attr) { - if (gid_type == IB_GID_TYPE_IB) + if (attr->gid_type == IB_GID_TYPE_IB) return RDMA_NETWORK_IB; - if (ipv6_addr_v4mapped((struct in6_addr *)gid)) + if (ipv6_addr_v4mapped((struct in6_addr *)&attr->gid)) return RDMA_NETWORK_IPV4; else return RDMA_NETWORK_IPV6; @@ -344,7 +345,8 @@ struct ib_device_attr { int max_qp; int max_qp_wr; u64 device_cap_flags; - int max_sge; + int max_send_sge; + int max_recv_sge; int max_sge_rd; int max_cq; int max_cqe; @@ -430,33 +432,6 @@ enum ib_port_state { IB_PORT_ACTIVE_DEFER = 5 }; -enum ib_port_cap_flags { - IB_PORT_SM = 1 << 1, - IB_PORT_NOTICE_SUP = 1 << 2, - IB_PORT_TRAP_SUP = 1 << 3, - IB_PORT_OPT_IPD_SUP = 1 << 4, - IB_PORT_AUTO_MIGR_SUP = 1 << 5, - IB_PORT_SL_MAP_SUP = 1 << 6, - IB_PORT_MKEY_NVRAM = 1 << 7, - IB_PORT_PKEY_NVRAM = 1 << 8, - IB_PORT_LED_INFO_SUP = 1 << 9, - IB_PORT_SM_DISABLED = 1 << 10, - IB_PORT_SYS_IMAGE_GUID_SUP = 1 << 11, - IB_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12, - IB_PORT_EXTENDED_SPEEDS_SUP = 1 << 14, - IB_PORT_CM_SUP = 1 << 16, - IB_PORT_SNMP_TUNNEL_SUP = 1 << 17, - IB_PORT_REINIT_SUP = 1 << 18, - IB_PORT_DEVICE_MGMT_SUP = 1 << 19, - IB_PORT_VENDOR_CLASS_SUP = 1 << 20, - IB_PORT_DR_NOTICE_SUP = 1 << 21, - IB_PORT_CAP_MASK_NOTICE_SUP = 1 << 22, - IB_PORT_BOOT_MGMT_SUP = 1 << 23, - IB_PORT_LINK_LATENCY_SUP = 1 << 24, - IB_PORT_CLIENT_REG_SUP = 1 << 25, - IB_PORT_IP_BASED_GIDS = 1 << 26, -}; - enum ib_port_width { IB_WIDTH_1X = 1, IB_WIDTH_4X = 2, @@ -554,6 +529,7 @@ static inline struct rdma_hw_stats *rdma_alloc_hw_stats_struct( #define RDMA_CORE_CAP_AF_IB 0x00001000 #define RDMA_CORE_CAP_ETH_AH 0x00002000 #define RDMA_CORE_CAP_OPA_AH 0x00004000 +#define RDMA_CORE_CAP_IB_GRH_REQUIRED 0x00008000 /* Protocol 0xFFF00000 */ #define RDMA_CORE_CAP_PROT_IB 0x00100000 @@ -563,6 +539,10 @@ static inline struct rdma_hw_stats *rdma_alloc_hw_stats_struct( #define RDMA_CORE_CAP_PROT_RAW_PACKET 0x01000000 #define RDMA_CORE_CAP_PROT_USNIC 0x02000000 +#define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \ + | RDMA_CORE_CAP_PROT_ROCE \ + | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP) + #define RDMA_CORE_PORT_IBA_IB (RDMA_CORE_CAP_PROT_IB \ | RDMA_CORE_CAP_IB_MAD \ | RDMA_CORE_CAP_IB_SMI \ @@ -595,6 +575,8 @@ struct ib_port_attr { enum ib_mtu max_mtu; enum ib_mtu active_mtu; int gid_tbl_len; + unsigned int ip_gids:1; + /* This is the value from PortInfo CapabilityMask, defined by IBA */ u32 port_cap_flags; u32 max_msg_sz; u32 bad_pkey_cntr; @@ -610,7 +592,6 @@ struct ib_port_attr { u8 active_width; u8 active_speed; u8 phys_state; - bool grh_required; }; enum ib_device_modify_flags { @@ -689,6 +670,7 @@ struct ib_event_handler { } while (0) struct ib_global_route { + const struct ib_gid_attr *sgid_attr; union ib_gid dgid; u32 flow_label; u8 sgid_index; @@ -1370,7 +1352,7 @@ struct ib_rdma_wr { u32 rkey; }; -static inline struct ib_rdma_wr *rdma_wr(struct ib_send_wr *wr) +static inline const struct ib_rdma_wr *rdma_wr(const struct ib_send_wr *wr) { return container_of(wr, struct ib_rdma_wr, wr); } @@ -1385,7 +1367,7 @@ struct ib_atomic_wr { u32 rkey; }; -static inline struct ib_atomic_wr *atomic_wr(struct ib_send_wr *wr) +static inline const struct ib_atomic_wr *atomic_wr(const struct ib_send_wr *wr) { return container_of(wr, struct ib_atomic_wr, wr); } @@ -1402,7 +1384,7 @@ struct ib_ud_wr { u8 port_num; /* valid for DR SMPs on switch only */ }; -static inline struct ib_ud_wr *ud_wr(struct ib_send_wr *wr) +static inline const struct ib_ud_wr *ud_wr(const struct ib_send_wr *wr) { return container_of(wr, struct ib_ud_wr, wr); } @@ -1414,7 +1396,7 @@ struct ib_reg_wr { int access; }; -static inline struct ib_reg_wr *reg_wr(struct ib_send_wr *wr) +static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr) { return container_of(wr, struct ib_reg_wr, wr); } @@ -1427,7 +1409,8 @@ struct ib_sig_handover_wr { struct ib_sge *prot; }; -static inline struct ib_sig_handover_wr *sig_handover_wr(struct ib_send_wr *wr) +static inline const struct ib_sig_handover_wr * +sig_handover_wr(const struct ib_send_wr *wr) { return container_of(wr, struct ib_sig_handover_wr, wr); } @@ -1443,14 +1426,16 @@ struct ib_recv_wr { }; enum ib_access_flags { - IB_ACCESS_LOCAL_WRITE = 1, - IB_ACCESS_REMOTE_WRITE = (1<<1), - IB_ACCESS_REMOTE_READ = (1<<2), - IB_ACCESS_REMOTE_ATOMIC = (1<<3), - IB_ACCESS_MW_BIND = (1<<4), - IB_ZERO_BASED = (1<<5), - IB_ACCESS_ON_DEMAND = (1<<6), - IB_ACCESS_HUGETLB = (1<<7), + IB_ACCESS_LOCAL_WRITE = IB_UVERBS_ACCESS_LOCAL_WRITE, + IB_ACCESS_REMOTE_WRITE = IB_UVERBS_ACCESS_REMOTE_WRITE, + IB_ACCESS_REMOTE_READ = IB_UVERBS_ACCESS_REMOTE_READ, + IB_ACCESS_REMOTE_ATOMIC = IB_UVERBS_ACCESS_REMOTE_ATOMIC, + IB_ACCESS_MW_BIND = IB_UVERBS_ACCESS_MW_BIND, + IB_ZERO_BASED = IB_UVERBS_ACCESS_ZERO_BASED, + IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND, + IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB, + + IB_ACCESS_SUPPORTED = ((IB_ACCESS_HUGETLB << 1) - 1) }; /* @@ -1473,14 +1458,17 @@ struct ib_fmr_attr { struct ib_umem; enum rdma_remove_reason { - /* Userspace requested uobject deletion. Call could fail */ + /* + * Userspace requested uobject deletion or initial try + * to remove uobject via cleanup. Call could fail + */ RDMA_REMOVE_DESTROY, /* Context deletion. This call should delete the actual object itself */ RDMA_REMOVE_CLOSE, /* Driver is being hot-unplugged. This call should delete the actual object itself */ RDMA_REMOVE_DRIVER_REMOVE, - /* Context is being cleaned-up, but commit was just completed */ - RDMA_REMOVE_DURING_CLEANUP, + /* uobj is being cleaned-up before being committed */ + RDMA_REMOVE_ABORT, }; struct ib_rdmacg_object { @@ -1492,14 +1480,14 @@ struct ib_rdmacg_object { struct ib_ucontext { struct ib_device *device; struct ib_uverbs_file *ufile; + /* + * 'closing' can be read by the driver only during a destroy callback, + * it is set when we are closing the file descriptor and indicates + * that mm_sem may be locked. + */ int closing; - /* locking the uobjects_list */ - struct mutex uobjects_lock; - struct list_head uobjects; - /* protects cleanup process from other actions */ - struct rw_semaphore cleanup_rwsem; - enum rdma_remove_reason cleanup_reason; + bool cleanup_retryable; struct pid *tgid; #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING @@ -1524,6 +1512,9 @@ struct ib_ucontext { struct ib_uobject { u64 user_handle; /* handle given to us by userspace */ + /* ufile & ucontext owning this object */ + struct ib_uverbs_file *ufile; + /* FIXME, save memory: ufile->context == context */ struct ib_ucontext *context; /* associated user context */ void *object; /* containing object */ struct list_head list; /* link to context's list */ @@ -1533,13 +1524,7 @@ struct ib_uobject { atomic_t usecnt; /* protects exclusive access */ struct rcu_head rcu; /* kfree_rcu() overhead */ - const struct uverbs_obj_type *type; -}; - -struct ib_uobject_file { - struct ib_uobject uobj; - /* ufile contains the lock between context release and file close */ - struct ib_uverbs_file *ufile; + const struct uverbs_api_object *uapi_object; }; struct ib_udata { @@ -1578,6 +1563,7 @@ struct ib_ah { struct ib_device *device; struct ib_pd *pd; struct ib_uobject *uobject; + const struct ib_gid_attr *sgid_attr; enum rdma_ah_attr_type type; }; @@ -1776,6 +1762,9 @@ struct ib_qp { struct ib_uobject *uobject; void (*event_handler)(struct ib_event *, void *); void *qp_context; + /* sgid_attrs associated with the AV's */ + const struct ib_gid_attr *av_sgid_attr; + const struct ib_gid_attr *alt_path_sgid_attr; u32 qp_num; u32 max_write_sge; u32 max_read_sge; @@ -2098,6 +2087,7 @@ struct ib_flow_attr { struct ib_flow { struct ib_qp *qp; + struct ib_device *device; struct ib_uobject *uobject; }; @@ -2213,7 +2203,11 @@ struct rdma_netdev { struct ib_device *hca; u8 port_num; - /* cleanup function must be specified */ + /* + * cleanup function must be specified. + * FIXME: This is only used for OPA_VNIC and that usage should be + * removed too. + */ void (*free_rdma_netdev)(struct net_device *netdev); /* control functions */ @@ -2242,11 +2236,6 @@ struct ib_counters { atomic_t usecnt; }; -enum ib_read_counters_flags { - /* prefer read values from driver cache */ - IB_READ_COUNTERS_ATTR_PREFER_CACHED = 1 << 0, -}; - struct ib_counters_read_attr { u64 *counters_buff; u32 ncounters; @@ -2341,8 +2330,7 @@ struct ib_device { * concurrently for different ports. This function is only called when * roce_gid_table is used. */ - int (*add_gid)(const union ib_gid *gid, - const struct ib_gid_attr *attr, + int (*add_gid)(const struct ib_gid_attr *attr, void **context); /* When calling del_gid, the HW vendor's driver should delete the * gid of device @device at gid index gid_index of port port_num @@ -2389,8 +2377,8 @@ struct ib_device { struct ib_srq_attr *srq_attr); int (*destroy_srq)(struct ib_srq *srq); int (*post_srq_recv)(struct ib_srq *srq, - struct ib_recv_wr *recv_wr, - struct ib_recv_wr **bad_recv_wr); + const struct ib_recv_wr *recv_wr, + const struct ib_recv_wr **bad_recv_wr); struct ib_qp * (*create_qp)(struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata); @@ -2404,11 +2392,11 @@ struct ib_device { struct ib_qp_init_attr *qp_init_attr); int (*destroy_qp)(struct ib_qp *qp); int (*post_send)(struct ib_qp *qp, - struct ib_send_wr *send_wr, - struct ib_send_wr **bad_send_wr); + const struct ib_send_wr *send_wr, + const struct ib_send_wr **bad_send_wr); int (*post_recv)(struct ib_qp *qp, - struct ib_recv_wr *recv_wr, - struct ib_recv_wr **bad_recv_wr); + const struct ib_recv_wr *recv_wr, + const struct ib_recv_wr **bad_recv_wr); struct ib_cq * (*create_cq)(struct ib_device *device, const struct ib_cq_init_attr *attr, struct ib_ucontext *context, @@ -2592,7 +2580,7 @@ struct ib_device { const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev, int comp_vector); - struct uverbs_root_spec *specs_root; + const struct uverbs_object_tree_def *const *driver_specs; enum rdma_driver_id driver_id; }; @@ -2679,6 +2667,46 @@ static inline bool ib_is_udata_cleared(struct ib_udata *udata, } /** + * ib_is_destroy_retryable - Check whether the uobject destruction + * is retryable. + * @ret: The initial destruction return code + * @why: remove reason + * @uobj: The uobject that is destroyed + * + * This function is a helper function that IB layer and low-level drivers + * can use to consider whether the destruction of the given uobject is + * retry-able. + * It checks the original return code, if it wasn't success the destruction + * is retryable according to the ucontext state (i.e. cleanup_retryable) and + * the remove reason. (i.e. why). + * Must be called with the object locked for destroy. + */ +static inline bool ib_is_destroy_retryable(int ret, enum rdma_remove_reason why, + struct ib_uobject *uobj) +{ + return ret && (why == RDMA_REMOVE_DESTROY || + uobj->context->cleanup_retryable); +} + +/** + * ib_destroy_usecnt - Called during destruction to check the usecnt + * @usecnt: The usecnt atomic + * @why: remove reason + * @uobj: The uobject that is destroyed + * + * Non-zero usecnts will block destruction unless destruction was triggered by + * a ucontext cleanup. + */ +static inline int ib_destroy_usecnt(atomic_t *usecnt, + enum rdma_remove_reason why, + struct ib_uobject *uobj) +{ + if (atomic_read(usecnt) && ib_is_destroy_retryable(-EBUSY, why, uobj)) + return -EBUSY; + return 0; +} + +/** * ib_modify_qp_is_ok - Check that the supplied attribute mask * contains all required attributes and no attributes not allowed for * the given QP state transition. @@ -2755,6 +2783,13 @@ static inline int rdma_is_port_valid(const struct ib_device *device, port <= rdma_end_port(device)); } +static inline bool rdma_is_grh_required(const struct ib_device *device, + u8 port_num) +{ + return device->port_immutable[port_num].core_cap_flags & + RDMA_CORE_PORT_IB_GRH_REQUIRED; +} + static inline bool rdma_protocol_ib(const struct ib_device *device, u8 port_num) { return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_PROT_IB; @@ -3046,10 +3081,6 @@ static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num) return rdma_protocol_iwarp(dev, port_num); } -int ib_query_gid(struct ib_device *device, - u8 port_num, int index, union ib_gid *gid, - struct ib_gid_attr *attr); - int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port, int state); int ib_get_vf_config(struct ib_device *device, int vf, u8 port, @@ -3148,6 +3179,13 @@ int ib_get_rdma_header_version(const union rdma_network_hdr *hdr); * ignored unless the work completion indicates that the GRH is valid. * @ah_attr: Returned attributes that can be used when creating an address * handle for replying to the message. + * When ib_init_ah_attr_from_wc() returns success, + * (a) for IB link layer it optionally contains a reference to SGID attribute + * when GRH is present for IB link layer. + * (b) for RoCE link layer it contains a reference to SGID attribute. + * User must invoke rdma_cleanup_ah_attr_gid_attr() to release reference to SGID + * attributes which are initialized using ib_init_ah_attr_from_wc(). + * */ int ib_init_ah_attr_from_wc(struct ib_device *device, u8 port_num, const struct ib_wc *wc, const struct ib_grh *grh, @@ -3247,10 +3285,12 @@ int ib_destroy_srq(struct ib_srq *srq); * the work request that failed to be posted on the QP. */ static inline int ib_post_srq_recv(struct ib_srq *srq, - struct ib_recv_wr *recv_wr, - struct ib_recv_wr **bad_recv_wr) + const struct ib_recv_wr *recv_wr, + const struct ib_recv_wr **bad_recv_wr) { - return srq->device->post_srq_recv(srq, recv_wr, bad_recv_wr); + const struct ib_recv_wr *dummy; + + return srq->device->post_srq_recv(srq, recv_wr, bad_recv_wr ? : &dummy); } /** @@ -3348,10 +3388,12 @@ int ib_close_qp(struct ib_qp *qp); * earlier work requests in the list. */ static inline int ib_post_send(struct ib_qp *qp, - struct ib_send_wr *send_wr, - struct ib_send_wr **bad_send_wr) + const struct ib_send_wr *send_wr, + const struct ib_send_wr **bad_send_wr) { - return qp->device->post_send(qp, send_wr, bad_send_wr); + const struct ib_send_wr *dummy; + + return qp->device->post_send(qp, send_wr, bad_send_wr ? : &dummy); } /** @@ -3363,10 +3405,12 @@ static inline int ib_post_send(struct ib_qp *qp, * the work request that failed to be posted on the QP. */ static inline int ib_post_recv(struct ib_qp *qp, - struct ib_recv_wr *recv_wr, - struct ib_recv_wr **bad_recv_wr) + const struct ib_recv_wr *recv_wr, + const struct ib_recv_wr **bad_recv_wr) { - return qp->device->post_recv(qp, recv_wr, bad_recv_wr); + const struct ib_recv_wr *dummy; + + return qp->device->post_recv(qp, recv_wr, bad_recv_wr ? : &dummy); } struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, @@ -3801,10 +3845,6 @@ struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller); */ int ib_dealloc_xrcd(struct ib_xrcd *xrcd); -struct ib_flow *ib_create_flow(struct ib_qp *qp, - struct ib_flow_attr *flow_attr, int domain); -int ib_destroy_flow(struct ib_flow *flow_id); - static inline int ib_check_mr_access(int flags) { /* @@ -4033,8 +4073,19 @@ static inline void rdma_ah_set_grh(struct rdma_ah_attr *attr, grh->sgid_index = sgid_index; grh->hop_limit = hop_limit; grh->traffic_class = traffic_class; + grh->sgid_attr = NULL; } +void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr); +void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid, + u32 flow_label, u8 hop_limit, u8 traffic_class, + const struct ib_gid_attr *sgid_attr); +void rdma_copy_ah_attr(struct rdma_ah_attr *dest, + const struct rdma_ah_attr *src); +void rdma_replace_ah_attr(struct rdma_ah_attr *old, + const struct rdma_ah_attr *new); +void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src); + /** * rdma_ah_find_type - Return address handle type. * @@ -4102,6 +4153,20 @@ ib_get_vector_affinity(struct ib_device *device, int comp_vector) } +static inline void ib_set_flow(struct ib_uobject *uobj, struct ib_flow *ibflow, + struct ib_qp *qp, struct ib_device *device) +{ + uobj->object = ibflow; + ibflow->uobject = uobj; + + if (qp) { + atomic_inc(&qp->usecnt); + ibflow->qp = qp; + } + + ibflow->device = device; +} + /** * rdma_roce_rescan_device - Rescan all of the network devices in the system * and add their gids, as needed, to the relevant RoCE devices. @@ -4110,4 +4175,8 @@ ib_get_vector_affinity(struct ib_device *device, int comp_vector) */ void rdma_roce_rescan_device(struct ib_device *ibdev); +struct ib_ucontext *ib_uverbs_get_ucontext(struct ib_uverbs_file *ufile); + +int uverbs_destroy_def_handler(struct ib_uverbs_file *file, + struct uverbs_attr_bundle *attrs); #endif /* IB_VERBS_H */ diff --git a/include/rdma/opa_addr.h b/include/rdma/opa_addr.h index 2bbb7a67e643..66d4393d339c 100644 --- a/include/rdma/opa_addr.h +++ b/include/rdma/opa_addr.h @@ -120,7 +120,7 @@ static inline bool rdma_is_valid_unicast_lid(struct rdma_ah_attr *attr) if (attr->type == RDMA_AH_ATTR_TYPE_IB) { if (!rdma_ah_get_dlid(attr) || rdma_ah_get_dlid(attr) >= - be32_to_cpu(IB_MULTICAST_LID_BASE)) + be16_to_cpu(IB_MULTICAST_LID_BASE)) return false; } else if (attr->type == RDMA_AH_ATTR_TYPE_OPA) { if (!rdma_ah_get_dlid(attr) || diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h index c5c1435c129a..5d71a7f51a9f 100644 --- a/include/rdma/rdma_cm.h +++ b/include/rdma/rdma_cm.h @@ -192,7 +192,7 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr); * @timeout_ms: Time to wait for resolution to complete. */ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, - struct sockaddr *dst_addr, int timeout_ms); + const struct sockaddr *dst_addr, int timeout_ms); /** * rdma_resolve_route - Resolve the RDMA address bound to the RDMA identifier diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h index 1145a4c154b2..927f6d5b6d0f 100644 --- a/include/rdma/rdmavt_qp.h +++ b/include/rdma/rdmavt_qp.h @@ -2,7 +2,7 @@ #define DEF_RDMAVT_INCQP_H /* - * Copyright(c) 2016, 2017 Intel Corporation. + * Copyright(c) 2016 - 2018 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -91,6 +91,7 @@ * RVT_S_WAIT_ACK - waiting for an ACK packet before sending more requests * RVT_S_SEND_ONE - send one packet, request ACK, then wait for ACK * RVT_S_ECN - a BECN was queued to the send engine + * RVT_S_MAX_BIT_MASK - The max bit that can be used by rdmavt */ #define RVT_S_SIGNAL_REQ_WR 0x0001 #define RVT_S_BUSY 0x0002 @@ -103,23 +104,26 @@ #define RVT_S_WAIT_SSN_CREDIT 0x0100 #define RVT_S_WAIT_DMA 0x0200 #define RVT_S_WAIT_PIO 0x0400 -#define RVT_S_WAIT_PIO_DRAIN 0x0800 -#define RVT_S_WAIT_TX 0x1000 -#define RVT_S_WAIT_DMA_DESC 0x2000 -#define RVT_S_WAIT_KMEM 0x4000 -#define RVT_S_WAIT_PSN 0x8000 -#define RVT_S_WAIT_ACK 0x10000 -#define RVT_S_SEND_ONE 0x20000 -#define RVT_S_UNLIMITED_CREDIT 0x40000 -#define RVT_S_AHG_VALID 0x80000 -#define RVT_S_AHG_CLEAR 0x100000 -#define RVT_S_ECN 0x200000 +#define RVT_S_WAIT_TX 0x0800 +#define RVT_S_WAIT_DMA_DESC 0x1000 +#define RVT_S_WAIT_KMEM 0x2000 +#define RVT_S_WAIT_PSN 0x4000 +#define RVT_S_WAIT_ACK 0x8000 +#define RVT_S_SEND_ONE 0x10000 +#define RVT_S_UNLIMITED_CREDIT 0x20000 +#define RVT_S_ECN 0x40000 +#define RVT_S_MAX_BIT_MASK 0x800000 + +/* + * Drivers should use s_flags starting with bit 31 down to the bit next to + * RVT_S_MAX_BIT_MASK + */ /* * Wait flags that would prevent any packet type from being sent. */ #define RVT_S_ANY_WAIT_IO \ - (RVT_S_WAIT_PIO | RVT_S_WAIT_PIO_DRAIN | RVT_S_WAIT_TX | \ + (RVT_S_WAIT_PIO | RVT_S_WAIT_TX | \ RVT_S_WAIT_DMA_DESC | RVT_S_WAIT_KMEM) /* diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index bd6bba3a6e04..9e997c3c2f04 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -61,103 +61,195 @@ enum uverbs_obj_access { UVERBS_ACCESS_DESTROY }; -enum { - UVERBS_ATTR_SPEC_F_MANDATORY = 1U << 0, - /* Support extending attributes by length, validate all unknown size == zero */ - UVERBS_ATTR_SPEC_F_MIN_SZ_OR_ZERO = 1U << 1, -}; - /* Specification of a single attribute inside the ioctl message */ +/* good size 16 */ struct uverbs_attr_spec { + u8 type; + + /* + * Support extending attributes by length. Allow the user to provide + * more bytes than ptr.len, but check that everything after is zero'd + * by the user. + */ + u8 zero_trailing:1; + /* + * Valid only for PTR_IN. Allocate and copy the data inside + * the parser + */ + u8 alloc_and_copy:1; + u8 mandatory:1; + union { - /* Header shared by all following union members - to reduce space. */ - struct { - enum uverbs_attr_type type; - /* Combination of bits from enum UVERBS_ATTR_SPEC_F_XXXX */ - u8 flags; - }; struct { - enum uverbs_attr_type type; - /* Combination of bits from enum UVERBS_ATTR_SPEC_F_XXXX */ - u8 flags; /* Current known size to kernel */ - u16 len; + u16 len; /* User isn't allowed to provide something < min_len */ - u16 min_len; + u16 min_len; } ptr; + struct { - enum uverbs_attr_type type; - /* Combination of bits from enum UVERBS_ATTR_SPEC_F_XXXX */ - u8 flags; /* * higher bits mean the namespace and lower bits mean * the type id within the namespace. */ - u16 obj_type; - u8 access; + u16 obj_type; + u8 access; } obj; + + struct { + u8 num_elems; + } enum_def; + } u; + + /* This weird split of the enum lets us remove some padding */ + union { struct { - enum uverbs_attr_type type; - /* Combination of bits from enum UVERBS_ATTR_SPEC_F_XXXX */ - u8 flags; - u8 num_elems; /* * The enum attribute can select one of the attributes * contained in the ids array. Currently only PTR_IN * attributes are supported in the ids array. */ - const struct uverbs_attr_spec *ids; + const struct uverbs_attr_spec *ids; } enum_def; - }; + } u2; }; -struct uverbs_attr_spec_hash { - size_t num_attrs; - unsigned long *mandatory_attrs_bitmask; - struct uverbs_attr_spec attrs[0]; +/* + * Information about the API is loaded into a radix tree. For IOCTL we start + * with a tuple of: + * object_id, attr_id, method_id + * + * Which is a 48 bit value, with most of the bits guaranteed to be zero. Based + * on the current kernel support this is compressed into 16 bit key for the + * radix tree. Since this compression is entirely internal to the kernel the + * below limits can be revised if the kernel gains additional data. + * + * With 64 leafs per node this is a 3 level radix tree. + * + * The tree encodes multiple types, and uses a scheme where OBJ_ID,0,0 returns + * the object slot, and OBJ_ID,METH_ID,0 and returns the method slot. + */ +enum uapi_radix_data { + UVERBS_API_NS_FLAG = 1U << UVERBS_ID_NS_SHIFT, + + UVERBS_API_ATTR_KEY_BITS = 6, + UVERBS_API_ATTR_KEY_MASK = GENMASK(UVERBS_API_ATTR_KEY_BITS - 1, 0), + UVERBS_API_ATTR_BKEY_LEN = (1 << UVERBS_API_ATTR_KEY_BITS) - 1, + + UVERBS_API_METHOD_KEY_BITS = 5, + UVERBS_API_METHOD_KEY_SHIFT = UVERBS_API_ATTR_KEY_BITS, + UVERBS_API_METHOD_KEY_NUM_CORE = 24, + UVERBS_API_METHOD_KEY_NUM_DRIVER = (1 << UVERBS_API_METHOD_KEY_BITS) - + UVERBS_API_METHOD_KEY_NUM_CORE, + UVERBS_API_METHOD_KEY_MASK = GENMASK( + UVERBS_API_METHOD_KEY_BITS + UVERBS_API_METHOD_KEY_SHIFT - 1, + UVERBS_API_METHOD_KEY_SHIFT), + + UVERBS_API_OBJ_KEY_BITS = 5, + UVERBS_API_OBJ_KEY_SHIFT = + UVERBS_API_METHOD_KEY_BITS + UVERBS_API_METHOD_KEY_SHIFT, + UVERBS_API_OBJ_KEY_NUM_CORE = 24, + UVERBS_API_OBJ_KEY_NUM_DRIVER = + (1 << UVERBS_API_OBJ_KEY_BITS) - UVERBS_API_OBJ_KEY_NUM_CORE, + UVERBS_API_OBJ_KEY_MASK = GENMASK(31, UVERBS_API_OBJ_KEY_SHIFT), + + /* This id guaranteed to not exist in the radix tree */ + UVERBS_API_KEY_ERR = 0xFFFFFFFF, }; -struct uverbs_attr_bundle; -struct ib_uverbs_file; +static inline __attribute_const__ u32 uapi_key_obj(u32 id) +{ + if (id & UVERBS_API_NS_FLAG) { + id &= ~UVERBS_API_NS_FLAG; + if (id >= UVERBS_API_OBJ_KEY_NUM_DRIVER) + return UVERBS_API_KEY_ERR; + id = id + UVERBS_API_OBJ_KEY_NUM_CORE; + } else { + if (id >= UVERBS_API_OBJ_KEY_NUM_CORE) + return UVERBS_API_KEY_ERR; + } -enum { - /* - * Action marked with this flag creates a context (or root for all - * objects). - */ - UVERBS_ACTION_FLAG_CREATE_ROOT = 1U << 0, -}; + return id << UVERBS_API_OBJ_KEY_SHIFT; +} -struct uverbs_method_spec { - /* Combination of bits from enum UVERBS_ACTION_FLAG_XXXX */ - u32 flags; - size_t num_buckets; - size_t num_child_attrs; - int (*handler)(struct ib_device *ib_dev, struct ib_uverbs_file *ufile, - struct uverbs_attr_bundle *ctx); - struct uverbs_attr_spec_hash *attr_buckets[0]; -}; +static inline __attribute_const__ bool uapi_key_is_object(u32 key) +{ + return (key & ~UVERBS_API_OBJ_KEY_MASK) == 0; +} -struct uverbs_method_spec_hash { - size_t num_methods; - struct uverbs_method_spec *methods[0]; -}; +static inline __attribute_const__ u32 uapi_key_ioctl_method(u32 id) +{ + if (id & UVERBS_API_NS_FLAG) { + id &= ~UVERBS_API_NS_FLAG; + if (id >= UVERBS_API_METHOD_KEY_NUM_DRIVER) + return UVERBS_API_KEY_ERR; + id = id + UVERBS_API_METHOD_KEY_NUM_CORE; + } else { + id++; + if (id >= UVERBS_API_METHOD_KEY_NUM_CORE) + return UVERBS_API_KEY_ERR; + } -struct uverbs_object_spec { - const struct uverbs_obj_type *type_attrs; - size_t num_buckets; - struct uverbs_method_spec_hash *method_buckets[0]; -}; + return id << UVERBS_API_METHOD_KEY_SHIFT; +} -struct uverbs_object_spec_hash { - size_t num_objects; - struct uverbs_object_spec *objects[0]; -}; +static inline __attribute_const__ u32 uapi_key_attr_to_method(u32 attr_key) +{ + return attr_key & + (UVERBS_API_OBJ_KEY_MASK | UVERBS_API_METHOD_KEY_MASK); +} -struct uverbs_root_spec { - size_t num_buckets; - struct uverbs_object_spec_hash *object_buckets[0]; -}; +static inline __attribute_const__ bool uapi_key_is_ioctl_method(u32 key) +{ + return (key & UVERBS_API_METHOD_KEY_MASK) != 0 && + (key & UVERBS_API_ATTR_KEY_MASK) == 0; +} + +static inline __attribute_const__ u32 uapi_key_attrs_start(u32 ioctl_method_key) +{ + /* 0 is the method slot itself */ + return ioctl_method_key + 1; +} + +static inline __attribute_const__ u32 uapi_key_attr(u32 id) +{ + /* + * The attr is designed to fit in the typical single radix tree node + * of 64 entries. Since allmost all methods have driver attributes we + * organize things so that the driver and core attributes interleave to + * reduce the length of the attributes array in typical cases. + */ + if (id & UVERBS_API_NS_FLAG) { + id &= ~UVERBS_API_NS_FLAG; + id++; + if (id >= 1 << (UVERBS_API_ATTR_KEY_BITS - 1)) + return UVERBS_API_KEY_ERR; + id = (id << 1) | 0; + } else { + if (id >= 1 << (UVERBS_API_ATTR_KEY_BITS - 1)) + return UVERBS_API_KEY_ERR; + id = (id << 1) | 1; + } + + return id; +} + +static inline __attribute_const__ bool uapi_key_is_attr(u32 key) +{ + return (key & UVERBS_API_METHOD_KEY_MASK) != 0 && + (key & UVERBS_API_ATTR_KEY_MASK) != 0; +} + +/* + * This returns a value in the range [0 to UVERBS_API_ATTR_BKEY_LEN), + * basically it undoes the reservation of 0 in the ID numbering. attr_key + * must already be masked with UVERBS_API_ATTR_KEY_MASK, or be the output of + * uapi_key_attr(). + */ +static inline __attribute_const__ u32 uapi_bkey_attr(u32 attr_key) +{ + return attr_key - 1; +} /* * ======================================= @@ -176,7 +268,7 @@ struct uverbs_method_def { u32 flags; size_t num_attrs; const struct uverbs_attr_def * const (*attrs)[]; - int (*handler)(struct ib_device *ib_dev, struct ib_uverbs_file *ufile, + int (*handler)(struct ib_uverbs_file *ufile, struct uverbs_attr_bundle *ctx); }; @@ -192,196 +284,171 @@ struct uverbs_object_tree_def { const struct uverbs_object_def * const (*objects)[]; }; -#define UA_FLAGS(_flags) .flags = _flags -#define __UVERBS_ATTR0(_id, _type, _fld, _attr, ...) \ - ((const struct uverbs_attr_def) \ - {.id = _id, .attr = {{._fld = {.type = _type, _attr, .flags = 0, } }, } }) -#define __UVERBS_ATTR1(_id, _type, _fld, _attr, _extra1, ...) \ - ((const struct uverbs_attr_def) \ - {.id = _id, .attr = {{._fld = {.type = _type, _attr, _extra1 } },} }) -#define __UVERBS_ATTR2(_id, _type, _fld, _attr, _extra1, _extra2) \ - ((const struct uverbs_attr_def) \ - {.id = _id, .attr = {{._fld = {.type = _type, _attr, _extra1, _extra2 } },} }) -#define __UVERBS_ATTR(_id, _type, _fld, _attr, _extra1, _extra2, _n, ...) \ - __UVERBS_ATTR##_n(_id, _type, _fld, _attr, _extra1, _extra2) +/* + * ======================================= + * Attribute Specifications + * ======================================= + */ -#define UVERBS_ATTR_TYPE(_type) \ - .min_len = sizeof(_type), .len = sizeof(_type) -#define UVERBS_ATTR_STRUCT(_type, _last) \ - .min_len = ((uintptr_t)(&((_type *)0)->_last + 1)), .len = sizeof(_type) #define UVERBS_ATTR_SIZE(_min_len, _len) \ - .min_len = _min_len, .len = _len + .u.ptr.min_len = _min_len, .u.ptr.len = _len + +#define UVERBS_ATTR_NO_DATA() UVERBS_ATTR_SIZE(0, 0) /* - * In new compiler, UVERBS_ATTR could be simplified by declaring it as - * [_id] = {.type = _type, .len = _len, ##__VA_ARGS__} - * But since we support older compilers too, we need the more complex code. + * Specifies a uapi structure that cannot be extended. The user must always + * supply the whole structure and nothing more. The structure must be declared + * in a header under include/uapi/rdma. */ -#define UVERBS_ATTR(_id, _type, _fld, _attr, ...) \ - __UVERBS_ATTR(_id, _type, _fld, _attr, ##__VA_ARGS__, 2, 1, 0) -#define UVERBS_ATTR_PTR_IN_SZ(_id, _len, ...) \ - UVERBS_ATTR(_id, UVERBS_ATTR_TYPE_PTR_IN, ptr, _len, ##__VA_ARGS__) -/* If sizeof(_type) <= sizeof(u64), this will be inlined rather than a pointer */ -#define UVERBS_ATTR_PTR_IN(_id, _type, ...) \ - UVERBS_ATTR_PTR_IN_SZ(_id, _type, ##__VA_ARGS__) -#define UVERBS_ATTR_PTR_OUT_SZ(_id, _len, ...) \ - UVERBS_ATTR(_id, UVERBS_ATTR_TYPE_PTR_OUT, ptr, _len, ##__VA_ARGS__) -#define UVERBS_ATTR_PTR_OUT(_id, _type, ...) \ - UVERBS_ATTR_PTR_OUT_SZ(_id, _type, ##__VA_ARGS__) -#define UVERBS_ATTR_ENUM_IN(_id, _enum_arr, ...) \ - UVERBS_ATTR(_id, UVERBS_ATTR_TYPE_ENUM_IN, enum_def, \ - .ids = (_enum_arr), \ - .num_elems = ARRAY_SIZE(_enum_arr), ##__VA_ARGS__) +#define UVERBS_ATTR_TYPE(_type) \ + .u.ptr.min_len = sizeof(_type), .u.ptr.len = sizeof(_type) +/* + * Specifies a uapi structure where the user must provide at least up to + * member 'last'. Anything after last and up until the end of the structure + * can be non-zero, anything longer than the end of the structure must be + * zero. The structure must be declared in a header under include/uapi/rdma. + */ +#define UVERBS_ATTR_STRUCT(_type, _last) \ + .zero_trailing = 1, \ + UVERBS_ATTR_SIZE(((uintptr_t)(&((_type *)0)->_last + 1)), \ + sizeof(_type)) +/* + * Specifies at least min_len bytes must be passed in, but the amount can be + * larger, up to the protocol maximum size. No check for zeroing is done. + */ +#define UVERBS_ATTR_MIN_SIZE(_min_len) UVERBS_ATTR_SIZE(_min_len, USHRT_MAX) + +/* Must be used in the '...' of any UVERBS_ATTR */ +#define UA_ALLOC_AND_COPY .alloc_and_copy = 1 +#define UA_MANDATORY .mandatory = 1 +#define UA_OPTIONAL .mandatory = 0 + +#define UVERBS_ATTR_IDR(_attr_id, _idr_type, _access, ...) \ + (&(const struct uverbs_attr_def){ \ + .id = _attr_id, \ + .attr = { .type = UVERBS_ATTR_TYPE_IDR, \ + .u.obj.obj_type = _idr_type, \ + .u.obj.access = _access, \ + __VA_ARGS__ } }) + +#define UVERBS_ATTR_FD(_attr_id, _fd_type, _access, ...) \ + (&(const struct uverbs_attr_def){ \ + .id = (_attr_id) + \ + BUILD_BUG_ON_ZERO((_access) != UVERBS_ACCESS_NEW && \ + (_access) != UVERBS_ACCESS_READ), \ + .attr = { .type = UVERBS_ATTR_TYPE_FD, \ + .u.obj.obj_type = _fd_type, \ + .u.obj.access = _access, \ + __VA_ARGS__ } }) + +#define UVERBS_ATTR_PTR_IN(_attr_id, _type, ...) \ + (&(const struct uverbs_attr_def){ \ + .id = _attr_id, \ + .attr = { .type = UVERBS_ATTR_TYPE_PTR_IN, \ + _type, \ + __VA_ARGS__ } }) + +#define UVERBS_ATTR_PTR_OUT(_attr_id, _type, ...) \ + (&(const struct uverbs_attr_def){ \ + .id = _attr_id, \ + .attr = { .type = UVERBS_ATTR_TYPE_PTR_OUT, \ + _type, \ + __VA_ARGS__ } }) + +/* _enum_arry should be a 'static const union uverbs_attr_spec[]' */ +#define UVERBS_ATTR_ENUM_IN(_attr_id, _enum_arr, ...) \ + (&(const struct uverbs_attr_def){ \ + .id = _attr_id, \ + .attr = { .type = UVERBS_ATTR_TYPE_ENUM_IN, \ + .u2.enum_def.ids = _enum_arr, \ + .u.enum_def.num_elems = ARRAY_SIZE(_enum_arr), \ + __VA_ARGS__ }, \ + }) /* - * In new compiler, UVERBS_ATTR_IDR (and FD) could be simplified by declaring - * it as - * {.id = _id, \ - * .attr {.type = __obj_class, \ - * .obj = {.obj_type = _idr_type, \ - * .access = _access \ - * }, ##__VA_ARGS__ } } - * But since we support older compilers too, we need the more complex code. + * An input value that is a bitwise combination of values of _enum_type. + * This permits the flag value to be passed as either a u32 or u64, it must + * be retrieved via uverbs_get_flag(). */ -#define ___UVERBS_ATTR_OBJ0(_id, _obj_class, _obj_type, _access, ...)\ - ((const struct uverbs_attr_def) \ - {.id = _id, \ - .attr = { {.obj = {.type = _obj_class, .obj_type = _obj_type, \ - .access = _access, .flags = 0 } }, } }) -#define ___UVERBS_ATTR_OBJ1(_id, _obj_class, _obj_type, _access, _flags)\ - ((const struct uverbs_attr_def) \ - {.id = _id, \ - .attr = { {.obj = {.type = _obj_class, .obj_type = _obj_type, \ - .access = _access, _flags} }, } }) -#define ___UVERBS_ATTR_OBJ(_id, _obj_class, _obj_type, _access, _flags, \ - _n, ...) \ - ___UVERBS_ATTR_OBJ##_n(_id, _obj_class, _obj_type, _access, _flags) -#define __UVERBS_ATTR_OBJ(_id, _obj_class, _obj_type, _access, ...) \ - ___UVERBS_ATTR_OBJ(_id, _obj_class, _obj_type, _access, \ - ##__VA_ARGS__, 1, 0) -#define UVERBS_ATTR_IDR(_id, _idr_type, _access, ...) \ - __UVERBS_ATTR_OBJ(_id, UVERBS_ATTR_TYPE_IDR, _idr_type, _access,\ - ##__VA_ARGS__) -#define UVERBS_ATTR_FD(_id, _fd_type, _access, ...) \ - __UVERBS_ATTR_OBJ(_id, UVERBS_ATTR_TYPE_FD, _fd_type, \ - (_access) + BUILD_BUG_ON_ZERO( \ - (_access) != UVERBS_ACCESS_NEW && \ - (_access) != UVERBS_ACCESS_READ), \ - ##__VA_ARGS__) -#define DECLARE_UVERBS_ATTR_SPEC(_name, ...) \ - const struct uverbs_attr_def _name = __VA_ARGS__ - -#define DECLARE_UVERBS_ENUM(_name, ...) \ - const struct uverbs_enum_spec _name = { \ - .len = ARRAY_SIZE(((struct uverbs_attr_spec[]){__VA_ARGS__})),\ - .ids = {__VA_ARGS__}, \ +#define UVERBS_ATTR_FLAGS_IN(_attr_id, _enum_type, ...) \ + UVERBS_ATTR_PTR_IN( \ + _attr_id, \ + UVERBS_ATTR_SIZE(sizeof(u32) + BUILD_BUG_ON_ZERO( \ + !sizeof(_enum_type *)), \ + sizeof(u64)), \ + __VA_ARGS__) + +/* + * This spec is used in order to pass information to the hardware driver in a + * legacy way. Every verb that could get driver specific data should get this + * spec. + */ +#define UVERBS_ATTR_UHW() \ + UVERBS_ATTR_PTR_IN(UVERBS_ATTR_UHW_IN, \ + UVERBS_ATTR_MIN_SIZE(0), \ + UA_OPTIONAL), \ + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_UHW_OUT, \ + UVERBS_ATTR_MIN_SIZE(0), \ + UA_OPTIONAL) + +/* + * ======================================= + * Declaration helpers + * ======================================= + */ + +#define DECLARE_UVERBS_OBJECT_TREE(_name, ...) \ + static const struct uverbs_object_def *const _name##_ptr[] = { \ + __VA_ARGS__, \ + }; \ + static const struct uverbs_object_tree_def _name = { \ + .num_objects = ARRAY_SIZE(_name##_ptr), \ + .objects = &_name##_ptr, \ } -#define _UVERBS_METHOD_ATTRS_SZ(...) \ - (sizeof((const struct uverbs_attr_def * const []){__VA_ARGS__}) /\ - sizeof(const struct uverbs_attr_def *)) -#define _UVERBS_METHOD(_id, _handler, _flags, ...) \ - ((const struct uverbs_method_def) { \ - .id = _id, \ - .flags = _flags, \ - .handler = _handler, \ - .num_attrs = _UVERBS_METHOD_ATTRS_SZ(__VA_ARGS__), \ - .attrs = &(const struct uverbs_attr_def * const []){__VA_ARGS__} }) -#define DECLARE_UVERBS_METHOD(_name, _id, _handler, ...) \ - const struct uverbs_method_def _name = \ - _UVERBS_METHOD(_id, _handler, 0, ##__VA_ARGS__) -#define DECLARE_UVERBS_CTX_METHOD(_name, _id, _handler, _flags, ...) \ - const struct uverbs_method_def _name = \ - _UVERBS_METHOD(_id, _handler, \ - UVERBS_ACTION_FLAG_CREATE_ROOT, \ - ##__VA_ARGS__) -#define _UVERBS_OBJECT_METHODS_SZ(...) \ - (sizeof((const struct uverbs_method_def * const []){__VA_ARGS__}) / \ - sizeof(const struct uverbs_method_def *)) -#define _UVERBS_OBJECT(_id, _type_attrs, ...) \ - ((const struct uverbs_object_def) { \ - .id = _id, \ - .type_attrs = _type_attrs, \ - .num_methods = _UVERBS_OBJECT_METHODS_SZ(__VA_ARGS__), \ - .methods = &(const struct uverbs_method_def * const []){__VA_ARGS__} }) -#define DECLARE_UVERBS_OBJECT(_name, _id, _type_attrs, ...) \ - const struct uverbs_object_def _name = \ - _UVERBS_OBJECT(_id, _type_attrs, ##__VA_ARGS__) -#define _UVERBS_TREE_OBJECTS_SZ(...) \ - (sizeof((const struct uverbs_object_def * const []){__VA_ARGS__}) / \ - sizeof(const struct uverbs_object_def *)) -#define _UVERBS_OBJECT_TREE(...) \ - ((const struct uverbs_object_tree_def) { \ - .num_objects = _UVERBS_TREE_OBJECTS_SZ(__VA_ARGS__), \ - .objects = &(const struct uverbs_object_def * const []){__VA_ARGS__} }) -#define DECLARE_UVERBS_OBJECT_TREE(_name, ...) \ - const struct uverbs_object_tree_def _name = \ - _UVERBS_OBJECT_TREE(__VA_ARGS__) /* ================================================= * Parsing infrastructure * ================================================= */ + struct uverbs_ptr_attr { - u64 data; + /* + * If UVERBS_ATTR_SPEC_F_ALLOC_AND_COPY is set then the 'ptr' is + * used. + */ + union { + void *ptr; + u64 data; + }; u16 len; - /* Combination of bits from enum UVERBS_ATTR_F_XXXX */ - u16 flags; + u16 uattr_idx; u8 enum_id; }; struct uverbs_obj_attr { - /* pointer to the kernel descriptor -> type, access, etc */ - const struct uverbs_obj_type *type; struct ib_uobject *uobject; - /* fd or id in idr of this object */ - int id; + const struct uverbs_api_attr *attr_elm; }; struct uverbs_attr { - /* - * pointer to the user-space given attribute, in order to write the - * new uobject's id or update flags. - */ - struct ib_uverbs_attr __user *uattr; union { struct uverbs_ptr_attr ptr_attr; struct uverbs_obj_attr obj_attr; }; }; -struct uverbs_attr_bundle_hash { - /* if bit i is set, it means attrs[i] contains valid information */ - unsigned long *valid_bitmap; - size_t num_attrs; - /* - * arrays of attributes, each element corresponds to the specification - * of the attribute in the same index. - */ - struct uverbs_attr *attrs; -}; - struct uverbs_attr_bundle { - size_t num_buckets; - struct uverbs_attr_bundle_hash hash[]; + struct ib_uverbs_file *ufile; + DECLARE_BITMAP(attr_present, UVERBS_API_ATTR_BKEY_LEN); + struct uverbs_attr attrs[]; }; -static inline bool uverbs_attr_is_valid_in_hash(const struct uverbs_attr_bundle_hash *attrs_hash, - unsigned int idx) -{ - return test_bit(idx, attrs_hash->valid_bitmap); -} - static inline bool uverbs_attr_is_valid(const struct uverbs_attr_bundle *attrs_bundle, unsigned int idx) { - u16 idx_bucket = idx >> UVERBS_ID_NS_SHIFT; - - if (attrs_bundle->num_buckets <= idx_bucket) - return false; - - return uverbs_attr_is_valid_in_hash(&attrs_bundle->hash[idx_bucket], - idx & ~UVERBS_ID_NS_MASK); + return test_bit(uapi_bkey_attr(uapi_key_attr(idx)), + attrs_bundle->attr_present); } #define IS_UVERBS_COPY_ERR(_ret) ((_ret) && (_ret) != -ENOENT) @@ -389,12 +456,10 @@ static inline bool uverbs_attr_is_valid(const struct uverbs_attr_bundle *attrs_b static inline const struct uverbs_attr *uverbs_attr_get(const struct uverbs_attr_bundle *attrs_bundle, u16 idx) { - u16 idx_bucket = idx >> UVERBS_ID_NS_SHIFT; - if (!uverbs_attr_is_valid(attrs_bundle, idx)) return ERR_PTR(-ENOENT); - return &attrs_bundle->hash[idx_bucket].attrs[idx & ~UVERBS_ID_NS_MASK]; + return &attrs_bundle->attrs[uapi_bkey_attr(uapi_key_attr(idx))]; } static inline int uverbs_attr_get_enum_id(const struct uverbs_attr_bundle *attrs_bundle, @@ -431,25 +496,15 @@ static inline struct ib_uobject *uverbs_attr_get_uobject(const struct uverbs_att return attr->obj_attr.uobject; } -static inline int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, const void *from, size_t size) +static inline int +uverbs_attr_get_len(const struct uverbs_attr_bundle *attrs_bundle, u16 idx) { const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle, idx); - u16 flags; - size_t min_size; if (IS_ERR(attr)) return PTR_ERR(attr); - min_size = min_t(size_t, attr->ptr_attr.len, size); - if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) - return -EFAULT; - - flags = attr->ptr_attr.flags | UVERBS_ATTR_F_VALID_OUTPUT; - if (put_user(flags, &attr->uattr->flags)) - return -EFAULT; - - return 0; + return attr->ptr_attr.len; } static inline bool uverbs_attr_ptr_is_inline(const struct uverbs_attr *attr) @@ -457,6 +512,18 @@ static inline bool uverbs_attr_ptr_is_inline(const struct uverbs_attr *attr) return attr->ptr_attr.len <= sizeof(attr->ptr_attr.data); } +static inline void *uverbs_attr_get_alloced_ptr( + const struct uverbs_attr_bundle *attrs_bundle, u16 idx) +{ + const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle, idx); + + if (IS_ERR(attr)) + return (void *)attr; + + return uverbs_attr_ptr_is_inline(attr) ? (void *)&attr->ptr_attr.data : + attr->ptr_attr.ptr; +} + static inline int _uverbs_copy_from(void *to, const struct uverbs_attr_bundle *attrs_bundle, size_t idx, @@ -515,54 +582,54 @@ static inline int _uverbs_copy_from_or_zero(void *to, #define uverbs_copy_from_or_zero(to, attrs_bundle, idx) \ _uverbs_copy_from_or_zero(to, attrs_bundle, idx, sizeof(*to)) -/* ================================================= - * Definitions -> Specs infrastructure - * ================================================= - */ - -/* - * uverbs_alloc_spec_tree - Merges different common and driver specific feature - * into one parsing tree that every uverbs command will be parsed upon. - * - * @num_trees: Number of trees in the array @trees. - * @trees: Array of pointers to tree root definitions to merge. Each such tree - * possibly contains objects, methods and attributes definitions. - * - * Returns: - * uverbs_root_spec *: The root of the merged parsing tree. - * On error, we return an error code. Error is checked via IS_ERR. - * - * The following merges could take place: - * a. Two trees representing the same method with different handler - * -> We take the handler of the tree that its handler != NULL - * and its index in the trees array is greater. The incentive for that - * is that developers are expected to first merge common trees and then - * merge trees that gives specialized the behaviour. - * b. Two trees representing the same object with different - * type_attrs (struct uverbs_obj_type): - * -> We take the type_attrs of the tree that its type_attr != NULL - * and its index in the trees array is greater. This could be used - * in order to override the free function, allocation size, etc. - * c. Two trees representing the same method attribute (same id but possibly - * different attributes): - * -> ERROR (-ENOENT), we believe that's not the programmer's intent. - * - * An object without any methods is considered invalid and will abort the - * function with -ENOENT error. - */ #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) -struct uverbs_root_spec *uverbs_alloc_spec_tree(unsigned int num_trees, - const struct uverbs_object_tree_def **trees); -void uverbs_free_spec_tree(struct uverbs_root_spec *root); -#else -static inline struct uverbs_root_spec *uverbs_alloc_spec_tree(unsigned int num_trees, - const struct uverbs_object_tree_def **trees) +int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 allowed_bits); +int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 allowed_bits); +int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, size_t idx, + const void *from, size_t size); +__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, + gfp_t flags); + +static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle, + size_t size) { - return NULL; + return _uverbs_alloc(bundle, size, GFP_KERNEL); } -static inline void uverbs_free_spec_tree(struct uverbs_root_spec *root) +static inline __malloc void *uverbs_zalloc(struct uverbs_attr_bundle *bundle, + size_t size) +{ + return _uverbs_alloc(bundle, size, GFP_KERNEL | __GFP_ZERO); +} +#else +static inline int +uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 allowed_bits) +{ + return -EINVAL; +} +static inline int +uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 allowed_bits) +{ + return -EINVAL; +} +static inline int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, const void *from, size_t size) +{ + return -EINVAL; +} +static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle, + size_t size) +{ + return ERR_PTR(-EINVAL); +} +static inline __malloc void *uverbs_zalloc(struct uverbs_attr_bundle *bundle, + size_t size) { + return ERR_PTR(-EINVAL); } #endif diff --git a/include/rdma/uverbs_named_ioctl.h b/include/rdma/uverbs_named_ioctl.h index c5bb4ebdb0b0..b3b21733cc55 100644 --- a/include/rdma/uverbs_named_ioctl.h +++ b/include/rdma/uverbs_named_ioctl.h @@ -43,48 +43,89 @@ #define _UVERBS_NAME(x, y) _UVERBS_PASTE(x, y) #define UVERBS_METHOD(id) _UVERBS_NAME(UVERBS_MODULE_NAME, _method_##id) #define UVERBS_HANDLER(id) _UVERBS_NAME(UVERBS_MODULE_NAME, _handler_##id) +#define UVERBS_OBJECT(id) _UVERBS_NAME(UVERBS_MOUDLE_NAME, _object_##id) -#define DECLARE_UVERBS_NAMED_METHOD(id, ...) \ - DECLARE_UVERBS_METHOD(UVERBS_METHOD(id), id, UVERBS_HANDLER(id), ##__VA_ARGS__) +/* These are static so they do not need to be qualified */ +#define UVERBS_METHOD_ATTRS(method_id) _method_attrs_##method_id +#define UVERBS_OBJECT_METHODS(object_id) _object_methods_##object_id -#define DECLARE_UVERBS_NAMED_METHOD_WITH_HANDLER(id, handler, ...) \ - DECLARE_UVERBS_METHOD(UVERBS_METHOD(id), id, handler, ##__VA_ARGS__) +#define DECLARE_UVERBS_NAMED_METHOD(_method_id, ...) \ + static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \ + _method_id)[] = { __VA_ARGS__ }; \ + static const struct uverbs_method_def UVERBS_METHOD(_method_id) = { \ + .id = _method_id, \ + .handler = UVERBS_HANDLER(_method_id), \ + .num_attrs = ARRAY_SIZE(UVERBS_METHOD_ATTRS(_method_id)), \ + .attrs = &UVERBS_METHOD_ATTRS(_method_id), \ + } -#define DECLARE_UVERBS_NAMED_METHOD_NO_OVERRIDE(id, handler, ...) \ - DECLARE_UVERBS_METHOD(UVERBS_METHOD(id), id, NULL, ##__VA_ARGS__) - -#define DECLARE_UVERBS_NAMED_OBJECT(id, ...) \ - DECLARE_UVERBS_OBJECT(UVERBS_OBJECT(id), id, ##__VA_ARGS__) +/* Create a standard destroy method using the default handler. The handle_attr + * argument must be the attribute specifying the handle to destroy, the + * default handler does not support any other attributes. + */ +#define DECLARE_UVERBS_NAMED_METHOD_DESTROY(_method_id, _handle_attr) \ + static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \ + _method_id)[] = { _handle_attr }; \ + static const struct uverbs_method_def UVERBS_METHOD(_method_id) = { \ + .id = _method_id, \ + .handler = uverbs_destroy_def_handler, \ + .num_attrs = ARRAY_SIZE(UVERBS_METHOD_ATTRS(_method_id)), \ + .attrs = &UVERBS_METHOD_ATTRS(_method_id), \ + } -#define _UVERBS_COMP_NAME(x, y, z) _UVERBS_NAME(_UVERBS_NAME(x, y), z) +#define DECLARE_UVERBS_NAMED_OBJECT(_object_id, _type_attrs, ...) \ + static const struct uverbs_method_def *const UVERBS_OBJECT_METHODS( \ + _object_id)[] = { __VA_ARGS__ }; \ + const struct uverbs_object_def UVERBS_OBJECT(_object_id) = { \ + .id = _object_id, \ + .type_attrs = &_type_attrs, \ + .num_methods = ARRAY_SIZE(UVERBS_OBJECT_METHODS(_object_id)), \ + .methods = &UVERBS_OBJECT_METHODS(_object_id) \ + } -#define UVERBS_NO_OVERRIDE NULL +/* + * Declare global methods. These still have a unique object_id because we + * identify all uapi methods with a (object,method) tuple. However, they have + * no type pointer. + */ +#define DECLARE_UVERBS_GLOBAL_METHODS(_object_id, ...) \ + static const struct uverbs_method_def *const UVERBS_OBJECT_METHODS( \ + _object_id)[] = { __VA_ARGS__ }; \ + const struct uverbs_object_def UVERBS_OBJECT(_object_id) = { \ + .id = _object_id, \ + .num_methods = ARRAY_SIZE(UVERBS_OBJECT_METHODS(_object_id)), \ + .methods = &UVERBS_OBJECT_METHODS(_object_id) \ + } -/* This declares a parsing tree with one object and one method. This is usually - * used for merging driver attributes to the common attributes. The driver has - * a chance to override the handler and type attrs of the original object. - * The __VA_ARGS__ just contains a list of attributes. +/* Used by drivers to declare a complete parsing tree for new methods */ -#define ADD_UVERBS_ATTRIBUTES(_name, _object, _method, _type_attrs, _handler, ...) \ -static DECLARE_UVERBS_METHOD(_UVERBS_COMP_NAME(UVERBS_MODULE_NAME, \ - _method_, _name), \ - _method, _handler, ##__VA_ARGS__); \ - \ -static DECLARE_UVERBS_OBJECT(_UVERBS_COMP_NAME(UVERBS_MODULE_NAME, \ - _object_, _name), \ - _object, _type_attrs, \ - &_UVERBS_COMP_NAME(UVERBS_MODULE_NAME, \ - _method_, _name)); \ - \ -static DECLARE_UVERBS_OBJECT_TREE(_name, \ - &_UVERBS_COMP_NAME(UVERBS_MODULE_NAME, \ - _object_, _name)) +#define ADD_UVERBS_METHODS(_name, _object_id, ...) \ + static const struct uverbs_method_def *const UVERBS_OBJECT_METHODS( \ + _object_id)[] = { __VA_ARGS__ }; \ + static const struct uverbs_object_def _name##_struct = { \ + .id = _object_id, \ + .num_methods = ARRAY_SIZE(UVERBS_OBJECT_METHODS(_object_id)), \ + .methods = &UVERBS_OBJECT_METHODS(_object_id) \ + }; \ + static const struct uverbs_object_def *const _name##_ptrs[] = { \ + &_name##_struct, \ + }; \ + static const struct uverbs_object_tree_def _name = { \ + .num_objects = 1, \ + .objects = &_name##_ptrs, \ + } -/* A very common use case is that the driver doesn't override the handler and - * type_attrs. Therefore, we provide a simplified macro for this common case. +/* Used by drivers to declare a complete parsing tree for a single method that + * differs only in having additional driver specific attributes. */ -#define ADD_UVERBS_ATTRIBUTES_SIMPLE(_name, _object, _method, ...) \ - ADD_UVERBS_ATTRIBUTES(_name, _object, _method, UVERBS_NO_OVERRIDE, \ - UVERBS_NO_OVERRIDE, ##__VA_ARGS__) +#define ADD_UVERBS_ATTRIBUTES_SIMPLE(_name, _object_id, _method_id, ...) \ + static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \ + _method_id)[] = { __VA_ARGS__ }; \ + static const struct uverbs_method_def UVERBS_METHOD(_method_id) = { \ + .id = _method_id, \ + .num_attrs = ARRAY_SIZE(UVERBS_METHOD_ATTRS(_method_id)), \ + .attrs = &UVERBS_METHOD_ATTRS(_method_id), \ + }; \ + ADD_UVERBS_METHODS(_name, _object_id, &UVERBS_METHOD(_method_id)) #endif diff --git a/include/rdma/uverbs_std_types.h b/include/rdma/uverbs_std_types.h index 9d56cdb84655..3b00231cc084 100644 --- a/include/rdma/uverbs_std_types.h +++ b/include/rdma/uverbs_std_types.h @@ -37,8 +37,6 @@ #include <rdma/uverbs_ioctl.h> #include <rdma/ib_user_ioctl_verbs.h> -#define UVERBS_OBJECT(id) uverbs_object_##id - #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) const struct uverbs_object_tree_def *uverbs_default_get_objects(void); #else @@ -48,34 +46,61 @@ static inline const struct uverbs_object_tree_def *uverbs_default_get_objects(vo } #endif -static inline struct ib_uobject *__uobj_get(const struct uverbs_obj_type *type, - bool write, - struct ib_ucontext *ucontext, - int id) +/* Returns _id, or causes a compile error if _id is not a u32. + * + * The uobj APIs should only be used with the write based uAPI to access + * object IDs. The write API must use a u32 for the object handle, which is + * checked by this macro. + */ +#define _uobj_check_id(_id) ((_id) * typecheck(u32, _id)) + +#define uobj_get_type(_ufile, _object) \ + uapi_get_object((_ufile)->device->uapi, _object) + +#define uobj_get_read(_type, _id, _ufile) \ + rdma_lookup_get_uobject(uobj_get_type(_ufile, _type), _ufile, \ + _uobj_check_id(_id), UVERBS_LOOKUP_READ) + +#define ufd_get_read(_type, _fdnum, _ufile) \ + rdma_lookup_get_uobject(uobj_get_type(_ufile, _type), _ufile, \ + (_fdnum)*typecheck(s32, _fdnum), \ + UVERBS_LOOKUP_READ) + +static inline void *_uobj_get_obj_read(struct ib_uobject *uobj) { - return rdma_lookup_get_uobject(type, ucontext, id, write); + if (IS_ERR(uobj)) + return NULL; + return uobj->object; } +#define uobj_get_obj_read(_object, _type, _id, _ufile) \ + ((struct ib_##_object *)_uobj_get_obj_read( \ + uobj_get_read(_type, _id, _ufile))) -#define uobj_get_type(_object) UVERBS_OBJECT(_object).type_attrs +#define uobj_get_write(_type, _id, _ufile) \ + rdma_lookup_get_uobject(uobj_get_type(_ufile, _type), _ufile, \ + _uobj_check_id(_id), UVERBS_LOOKUP_WRITE) -#define uobj_get_read(_type, _id, _ucontext) \ - __uobj_get(uobj_get_type(_type), false, _ucontext, _id) +int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id, + struct ib_uverbs_file *ufile, int success_res); +#define uobj_perform_destroy(_type, _id, _ufile, _success_res) \ + __uobj_perform_destroy(uobj_get_type(_ufile, _type), \ + _uobj_check_id(_id), _ufile, _success_res) -#define uobj_get_obj_read(_object, _type, _id, _ucontext) \ -({ \ - struct ib_uobject *__uobj = \ - __uobj_get(uobj_get_type(_type), \ - false, _ucontext, _id); \ - \ - (struct ib_##_object *)(IS_ERR(__uobj) ? NULL : __uobj->object);\ -}) +struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj, + u32 id, struct ib_uverbs_file *ufile); -#define uobj_get_write(_type, _id, _ucontext) \ - __uobj_get(uobj_get_type(_type), true, _ucontext, _id) +#define uobj_get_destroy(_type, _id, _ufile) \ + __uobj_get_destroy(uobj_get_type(_ufile, _type), _uobj_check_id(_id), \ + _ufile) + +static inline void uobj_put_destroy(struct ib_uobject *uobj) +{ + rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE); +} static inline void uobj_put_read(struct ib_uobject *uobj) { - rdma_lookup_put_uobject(uobj, false); + rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_READ); } #define uobj_put_obj_read(_obj) \ @@ -83,17 +108,17 @@ static inline void uobj_put_read(struct ib_uobject *uobj) static inline void uobj_put_write(struct ib_uobject *uobj) { - rdma_lookup_put_uobject(uobj, true); + rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE); } -static inline int __must_check uobj_remove_commit(struct ib_uobject *uobj) +static inline int __must_check uobj_alloc_commit(struct ib_uobject *uobj, + int success_res) { - return rdma_remove_commit_uobject(uobj); -} + int ret = rdma_alloc_commit_uobject(uobj); -static inline void uobj_alloc_commit(struct ib_uobject *uobj) -{ - rdma_alloc_commit_uobject(uobj); + if (ret) + return ret; + return success_res; } static inline void uobj_alloc_abort(struct ib_uobject *uobj) @@ -101,14 +126,19 @@ static inline void uobj_alloc_abort(struct ib_uobject *uobj) rdma_alloc_abort_uobject(uobj); } -static inline struct ib_uobject *__uobj_alloc(const struct uverbs_obj_type *type, - struct ib_ucontext *ucontext) +static inline struct ib_uobject * +__uobj_alloc(const struct uverbs_api_object *obj, struct ib_uverbs_file *ufile, + struct ib_device **ib_dev) { - return rdma_alloc_begin_uobject(type, ucontext); + struct ib_uobject *uobj = rdma_alloc_begin_uobject(obj, ufile); + + if (!IS_ERR(uobj)) + *ib_dev = uobj->context->device; + return uobj; } -#define uobj_alloc(_type, ucontext) \ - __uobj_alloc(uobj_get_type(_type), ucontext) +#define uobj_alloc(_type, _ufile, _ib_dev) \ + __uobj_alloc(uobj_get_type(_ufile, _type), _ufile, _ib_dev) #endif diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index cc04ec65588d..acb1bfa3cc99 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -37,63 +37,72 @@ #include <rdma/ib_verbs.h> struct uverbs_obj_type; +struct uverbs_api_object; -struct uverbs_obj_type_class { +enum rdma_lookup_mode { + UVERBS_LOOKUP_READ, + UVERBS_LOOKUP_WRITE, /* - * Get an ib_uobject that corresponds to the given id from ucontext, - * These functions could create or destroy objects if required. - * The action will be finalized only when commit, abort or put fops are - * called. - * The flow of the different actions is: - * [alloc]: Starts with alloc_begin. The handlers logic is than - * executed. If the handler is successful, alloc_commit - * is called and the object is inserted to the repository. - * Once alloc_commit completes the object is visible to - * other threads and userspace. - e Otherwise, alloc_abort is called and the object is - * destroyed. - * [lookup]: Starts with lookup_get which fetches and locks the - * object. After the handler finished using the object, it - * needs to call lookup_put to unlock it. The exclusive - * flag indicates if the object is locked for exclusive - * access. - * [remove]: Starts with lookup_get with exclusive flag set. This - * locks the object for exclusive access. If the handler - * code completed successfully, remove_commit is called - * and the ib_uobject is removed from the context's - * uobjects repository and put. The object itself is - * destroyed as well. Once remove succeeds new krefs to - * the object cannot be acquired by other threads or - * userspace and the hardware driver is removed from the - * object. Other krefs on the object may still exist. - * If the handler code failed, lookup_put should be - * called. This callback is used when the context - * is destroyed as well (process termination, - * reset flow). + * Destroy is like LOOKUP_WRITE, except that the uobject is not + * locked. uobj_destroy is used to convert a LOOKUP_DESTROY lock into + * a LOOKUP_WRITE lock. */ - struct ib_uobject *(*alloc_begin)(const struct uverbs_obj_type *type, - struct ib_ucontext *ucontext); - void (*alloc_commit)(struct ib_uobject *uobj); + UVERBS_LOOKUP_DESTROY, +}; + +/* + * The following sequences are valid: + * Success flow: + * alloc_begin + * alloc_commit + * [..] + * Access flow: + * lookup_get(exclusive=false) & uverbs_try_lock_object + * lookup_put(exclusive=false) via rdma_lookup_put_uobject + * Destruction flow: + * lookup_get(exclusive=true) & uverbs_try_lock_object + * remove_commit + * remove_handle (optional) + * lookup_put(exclusive=true) via rdma_lookup_put_uobject + * + * Allocate Error flow #1 + * alloc_begin + * alloc_abort + * Allocate Error flow #2 + * alloc_begin + * remove_commit + * alloc_abort + * Allocate Error flow #3 + * alloc_begin + * alloc_commit (fails) + * remove_commit + * alloc_abort + * + * In all cases the caller must hold the ufile kref until alloc_commit or + * alloc_abort returns. + */ +struct uverbs_obj_type_class { + struct ib_uobject *(*alloc_begin)(const struct uverbs_api_object *obj, + struct ib_uverbs_file *ufile); + /* This consumes the kref on uobj */ + int (*alloc_commit)(struct ib_uobject *uobj); + /* This does not consume the kref on uobj */ void (*alloc_abort)(struct ib_uobject *uobj); - struct ib_uobject *(*lookup_get)(const struct uverbs_obj_type *type, - struct ib_ucontext *ucontext, int id, - bool exclusive); - void (*lookup_put)(struct ib_uobject *uobj, bool exclusive); - /* - * Must be called with the exclusive lock held. If successful uobj is - * invalid on return. On failure uobject is left completely - * unchanged - */ - int __must_check (*remove_commit)(struct ib_uobject *uobj, - enum rdma_remove_reason why); + struct ib_uobject *(*lookup_get)(const struct uverbs_api_object *obj, + struct ib_uverbs_file *ufile, s64 id, + enum rdma_lookup_mode mode); + void (*lookup_put)(struct ib_uobject *uobj, enum rdma_lookup_mode mode); + /* This does not consume the kref on uobj */ + int __must_check (*destroy_hw)(struct ib_uobject *uobj, + enum rdma_remove_reason why); + void (*remove_handle)(struct ib_uobject *uobj); u8 needs_kfree_rcu; }; struct uverbs_obj_type { const struct uverbs_obj_type_class * const type_class; size_t obj_size; - unsigned int destroy_order; }; /* @@ -120,16 +129,15 @@ struct uverbs_obj_idr_type { enum rdma_remove_reason why); }; -struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_obj_type *type, - struct ib_ucontext *ucontext, - int id, bool exclusive); -void rdma_lookup_put_uobject(struct ib_uobject *uobj, bool exclusive); -struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_obj_type *type, - struct ib_ucontext *ucontext); +struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj, + struct ib_uverbs_file *ufile, s64 id, + enum rdma_lookup_mode mode); +void rdma_lookup_put_uobject(struct ib_uobject *uobj, + enum rdma_lookup_mode mode); +struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, + struct ib_uverbs_file *ufile); void rdma_alloc_abort_uobject(struct ib_uobject *uobj); -int __must_check rdma_remove_commit_uobject(struct ib_uobject *uobj); -int rdma_alloc_commit_uobject(struct ib_uobject *uobj); -int rdma_explicit_destroy(struct ib_uobject *uobject); +int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj); struct uverbs_obj_fd_type { /* @@ -140,7 +148,7 @@ struct uverbs_obj_fd_type { * the driver is removed or the process terminated. */ struct uverbs_obj_type type; - int (*context_closed)(struct ib_uobject_file *uobj_file, + int (*context_closed)(struct ib_uobject *uobj, enum rdma_remove_reason why); const struct file_operations *fops; const char *name; @@ -152,30 +160,29 @@ extern const struct uverbs_obj_type_class uverbs_fd_class; #define UVERBS_BUILD_BUG_ON(cond) (sizeof(char[1 - 2 * !!(cond)]) - \ sizeof(char)) -#define UVERBS_TYPE_ALLOC_FD(_order, _obj_size, _context_closed, _fops, _name, _flags)\ +#define UVERBS_TYPE_ALLOC_FD(_obj_size, _context_closed, _fops, _name, _flags)\ ((&((const struct uverbs_obj_fd_type) \ {.type = { \ - .destroy_order = _order, \ .type_class = &uverbs_fd_class, \ .obj_size = (_obj_size) + \ - UVERBS_BUILD_BUG_ON((_obj_size) < sizeof(struct ib_uobject_file)), \ + UVERBS_BUILD_BUG_ON((_obj_size) < \ + sizeof(struct ib_uobject)), \ }, \ .context_closed = _context_closed, \ .fops = _fops, \ .name = _name, \ .flags = _flags}))->type) -#define UVERBS_TYPE_ALLOC_IDR_SZ(_size, _order, _destroy_object) \ +#define UVERBS_TYPE_ALLOC_IDR_SZ(_size, _destroy_object) \ ((&((const struct uverbs_obj_idr_type) \ {.type = { \ - .destroy_order = _order, \ .type_class = &uverbs_idr_class, \ .obj_size = (_size) + \ UVERBS_BUILD_BUG_ON((_size) < \ sizeof(struct ib_uobject)) \ }, \ .destroy_object = _destroy_object,}))->type) -#define UVERBS_TYPE_ALLOC_IDR(_order, _destroy_object) \ - UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uobject), _order, \ +#define UVERBS_TYPE_ALLOC_IDR(_destroy_object) \ + UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uobject), \ _destroy_object) #endif diff --git a/include/uapi/rdma/cxgb4-abi.h b/include/uapi/rdma/cxgb4-abi.h index a159ba8dcf8f..f85ec1a3f727 100644 --- a/include/uapi/rdma/cxgb4-abi.h +++ b/include/uapi/rdma/cxgb4-abi.h @@ -44,6 +44,16 @@ * In particular do not use pointer types -- pass pointers in __aligned_u64 * instead. */ + +enum { + C4IW_64B_CQE = (1 << 0) +}; + +struct c4iw_create_cq { + __u32 flags; + __u32 reserved; +}; + struct c4iw_create_cq_resp { __aligned_u64 key; __aligned_u64 gts_key; @@ -51,11 +61,12 @@ struct c4iw_create_cq_resp { __u32 cqid; __u32 size; __u32 qid_mask; - __u32 reserved; /* explicit padding (optional for i386) */ + __u32 flags; }; enum { - C4IW_QPF_ONCHIP = (1 << 0) + C4IW_QPF_ONCHIP = (1 << 0), + C4IW_QPF_WRITE_W_IMM = (1 << 1) }; struct c4iw_create_qp_resp { @@ -74,6 +85,23 @@ struct c4iw_create_qp_resp { __u32 flags; }; +struct c4iw_create_srq_resp { + __aligned_u64 srq_key; + __aligned_u64 srq_db_gts_key; + __aligned_u64 srq_memsize; + __u32 srqid; + __u32 srq_size; + __u32 rqt_abs_idx; + __u32 qid_mask; + __u32 flags; + __u32 reserved; /* explicit padding */ +}; + +enum { + /* HW supports SRQ_LIMIT_REACHED event */ + T4_SRQ_LIMIT_SUPPORT = 1 << 0, +}; + struct c4iw_alloc_ucontext_resp { __aligned_u64 status_page_key; __u32 status_page_size; diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index 78613b609fa8..c1f87735514f 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -53,6 +53,7 @@ struct hns_roce_ib_create_qp { __u8 log_sq_stride; __u8 sq_no_prefetch; __u8 reserved[5]; + __aligned_u64 sdb_addr; }; struct hns_roce_ib_create_qp_resp { diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index 888ac5975a6c..2c881aaf05c2 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -79,7 +79,7 @@ enum uverbs_attrs_destroy_cq_cmd_attr_ids { }; enum uverbs_attrs_create_flow_action_esp { - UVERBS_ATTR_FLOW_ACTION_ESP_HANDLE, + UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE, UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS, UVERBS_ATTR_FLOW_ACTION_ESP_ESN, UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT, @@ -87,6 +87,11 @@ enum uverbs_attrs_create_flow_action_esp { UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP, }; +enum uverbs_attrs_modify_flow_action_esp { + UVERBS_ATTR_MODIFY_FLOW_ACTION_ESP_HANDLE = + UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE, +}; + enum uverbs_attrs_destroy_flow_action_esp { UVERBS_ATTR_DESTROY_FLOW_ACTION_HANDLE, }; diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h index 625545d862d7..6cdf192070a2 100644 --- a/include/uapi/rdma/ib_user_ioctl_verbs.h +++ b/include/uapi/rdma/ib_user_ioctl_verbs.h @@ -40,6 +40,59 @@ #define RDMA_UAPI_PTR(_type, _name) __aligned_u64 _name #endif +enum ib_uverbs_access_flags { + IB_UVERBS_ACCESS_LOCAL_WRITE = 1 << 0, + IB_UVERBS_ACCESS_REMOTE_WRITE = 1 << 1, + IB_UVERBS_ACCESS_REMOTE_READ = 1 << 2, + IB_UVERBS_ACCESS_REMOTE_ATOMIC = 1 << 3, + IB_UVERBS_ACCESS_MW_BIND = 1 << 4, + IB_UVERBS_ACCESS_ZERO_BASED = 1 << 5, + IB_UVERBS_ACCESS_ON_DEMAND = 1 << 6, + IB_UVERBS_ACCESS_HUGETLB = 1 << 7, +}; + +enum ib_uverbs_query_port_cap_flags { + IB_UVERBS_PCF_SM = 1 << 1, + IB_UVERBS_PCF_NOTICE_SUP = 1 << 2, + IB_UVERBS_PCF_TRAP_SUP = 1 << 3, + IB_UVERBS_PCF_OPT_IPD_SUP = 1 << 4, + IB_UVERBS_PCF_AUTO_MIGR_SUP = 1 << 5, + IB_UVERBS_PCF_SL_MAP_SUP = 1 << 6, + IB_UVERBS_PCF_MKEY_NVRAM = 1 << 7, + IB_UVERBS_PCF_PKEY_NVRAM = 1 << 8, + IB_UVERBS_PCF_LED_INFO_SUP = 1 << 9, + IB_UVERBS_PCF_SM_DISABLED = 1 << 10, + IB_UVERBS_PCF_SYS_IMAGE_GUID_SUP = 1 << 11, + IB_UVERBS_PCF_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12, + IB_UVERBS_PCF_EXTENDED_SPEEDS_SUP = 1 << 14, + IB_UVERBS_PCF_CM_SUP = 1 << 16, + IB_UVERBS_PCF_SNMP_TUNNEL_SUP = 1 << 17, + IB_UVERBS_PCF_REINIT_SUP = 1 << 18, + IB_UVERBS_PCF_DEVICE_MGMT_SUP = 1 << 19, + IB_UVERBS_PCF_VENDOR_CLASS_SUP = 1 << 20, + IB_UVERBS_PCF_DR_NOTICE_SUP = 1 << 21, + IB_UVERBS_PCF_CAP_MASK_NOTICE_SUP = 1 << 22, + IB_UVERBS_PCF_BOOT_MGMT_SUP = 1 << 23, + IB_UVERBS_PCF_LINK_LATENCY_SUP = 1 << 24, + IB_UVERBS_PCF_CLIENT_REG_SUP = 1 << 25, + /* + * IsOtherLocalChangesNoticeSupported is aliased by IP_BASED_GIDS and + * is inaccessible + */ + IB_UVERBS_PCF_LINK_SPEED_WIDTH_TABLE_SUP = 1 << 27, + IB_UVERBS_PCF_VENDOR_SPECIFIC_MADS_TABLE_SUP = 1 << 28, + IB_UVERBS_PCF_MCAST_PKEY_TRAP_SUPPRESSION_SUP = 1 << 29, + IB_UVERBS_PCF_MCAST_FDB_TOP_SUP = 1 << 30, + IB_UVERBS_PCF_HIERARCHY_INFO_SUP = 1ULL << 31, + + /* NOTE this is an internal flag, not an IBA flag */ + IB_UVERBS_PCF_IP_BASED_GIDS = 1 << 26, +}; + +enum ib_uverbs_query_port_flags { + IB_UVERBS_QPF_GRH_REQUIRED = 1 << 0, +}; + enum ib_uverbs_flow_action_esp_keymat { IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM, }; @@ -99,4 +152,9 @@ struct ib_uverbs_flow_action_esp { __aligned_u64 hard_limit_pkts; }; +enum ib_uverbs_read_counters_flags { + /* prefer read values from driver cache */ + IB_UVERBS_READ_COUNTERS_PREFER_CACHED = 1 << 0, +}; + #endif diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h index 4f9991de8e3a..25a16760de2a 100644 --- a/include/uapi/rdma/ib_user_verbs.h +++ b/include/uapi/rdma/ib_user_verbs.h @@ -279,7 +279,7 @@ struct ib_uverbs_query_port { }; struct ib_uverbs_query_port_resp { - __u32 port_cap_flags; + __u32 port_cap_flags; /* see ib_uverbs_query_port_cap_flags */ __u32 max_msg_sz; __u32 bad_pkey_cntr; __u32 qkey_viol_cntr; @@ -299,7 +299,8 @@ struct ib_uverbs_query_port_resp { __u8 active_speed; __u8 phys_state; __u8 link_layer; - __u8 reserved[2]; + __u8 flags; /* see ib_uverbs_query_port_flags */ + __u8 reserved; }; struct ib_uverbs_alloc_pd { diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h index 8daec1fa49cf..addbb9c4529e 100644 --- a/include/uapi/rdma/mlx5-abi.h +++ b/include/uapi/rdma/mlx5-abi.h @@ -76,6 +76,9 @@ enum mlx5_lib_caps { MLX5_LIB_CAP_4K_UAR = (__u64)1 << 0, }; +enum mlx5_ib_alloc_uctx_v2_flags { + MLX5_IB_ALLOC_UCTX_DEVX = 1 << 0, +}; struct mlx5_ib_alloc_ucontext_req_v2 { __u32 total_num_bfregs; __u32 num_low_latency_bfregs; @@ -90,6 +93,7 @@ struct mlx5_ib_alloc_ucontext_req_v2 { enum mlx5_ib_alloc_ucontext_resp_mask { MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET = 1UL << 0, + MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY = 1UL << 1, }; enum mlx5_user_cmds_supp_uhw { @@ -138,7 +142,7 @@ struct mlx5_ib_alloc_ucontext_resp { __u32 log_uar_size; __u32 num_uars_per_page; __u32 num_dyn_bfregs; - __u32 reserved3; + __u32 dump_fill_mkey; }; struct mlx5_ib_alloc_pd_resp { diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h index f7d685ef2d1f..9c51801b9e64 100644 --- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h +++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h @@ -33,6 +33,7 @@ #ifndef MLX5_USER_IOCTL_CMDS_H #define MLX5_USER_IOCTL_CMDS_H +#include <linux/types.h> #include <rdma/ib_user_ioctl_cmds.h> enum mlx5_ib_create_flow_action_attrs { @@ -45,4 +46,124 @@ enum mlx5_ib_alloc_dm_attrs { MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX, }; +enum mlx5_ib_devx_methods { + MLX5_IB_METHOD_DEVX_OTHER = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_METHOD_DEVX_QUERY_UAR, + MLX5_IB_METHOD_DEVX_QUERY_EQN, +}; + +enum mlx5_ib_devx_other_attrs { + MLX5_IB_ATTR_DEVX_OTHER_CMD_IN = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, +}; + +enum mlx5_ib_devx_obj_create_attrs { + MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN, + MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, +}; + +enum mlx5_ib_devx_query_uar_attrs { + MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX, +}; + +enum mlx5_ib_devx_obj_destroy_attrs { + MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum mlx5_ib_devx_obj_modify_attrs { + MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN, + MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT, +}; + +enum mlx5_ib_devx_obj_query_attrs { + MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN, + MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT, +}; + +enum mlx5_ib_devx_query_eqn_attrs { + MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, +}; + +enum mlx5_ib_devx_obj_methods { + MLX5_IB_METHOD_DEVX_OBJ_CREATE = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_METHOD_DEVX_OBJ_DESTROY, + MLX5_IB_METHOD_DEVX_OBJ_MODIFY, + MLX5_IB_METHOD_DEVX_OBJ_QUERY, +}; + +enum mlx5_ib_devx_umem_reg_attrs { + MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR, + MLX5_IB_ATTR_DEVX_UMEM_REG_LEN, + MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, + MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, +}; + +enum mlx5_ib_devx_umem_dereg_attrs { + MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum mlx5_ib_devx_umem_methods { + MLX5_IB_METHOD_DEVX_UMEM_REG = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_METHOD_DEVX_UMEM_DEREG, +}; + +enum mlx5_ib_objects { + MLX5_IB_OBJECT_DEVX = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_OBJECT_DEVX_OBJ, + MLX5_IB_OBJECT_DEVX_UMEM, + MLX5_IB_OBJECT_FLOW_MATCHER, +}; + +enum mlx5_ib_flow_matcher_create_attrs { + MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, + MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, + MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, +}; + +enum mlx5_ib_flow_matcher_destroy_attrs { + MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum mlx5_ib_flow_matcher_methods { + MLX5_IB_METHOD_FLOW_MATCHER_CREATE = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, +}; + +#define MLX5_IB_DW_MATCH_PARAM 0x80 + +struct mlx5_ib_match_params { + __u32 match_params[MLX5_IB_DW_MATCH_PARAM]; +}; + +enum mlx5_ib_flow_type { + MLX5_IB_FLOW_TYPE_NORMAL, + MLX5_IB_FLOW_TYPE_SNIFFER, + MLX5_IB_FLOW_TYPE_ALL_DEFAULT, + MLX5_IB_FLOW_TYPE_MC_DEFAULT, +}; + +enum mlx5_ib_create_flow_attrs { + MLX5_IB_ATTR_CREATE_FLOW_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, + MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, + MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, + MLX5_IB_ATTR_CREATE_FLOW_MATCHER, +}; + +enum mlx5_ib_destoy_flow_attrs { + MLX5_IB_ATTR_DESTROY_FLOW_HANDLE = (1U << UVERBS_ID_NS_SHIFT), +}; + +enum mlx5_ib_flow_methods { + MLX5_IB_METHOD_CREATE_FLOW = (1U << UVERBS_ID_NS_SHIFT), + MLX5_IB_METHOD_DESTROY_FLOW, +}; + #endif diff --git a/include/uapi/rdma/qedr-abi.h b/include/uapi/rdma/qedr-abi.h index 24c658b3c790..7a10b3a325fa 100644 --- a/include/uapi/rdma/qedr-abi.h +++ b/include/uapi/rdma/qedr-abi.h @@ -111,4 +111,21 @@ struct qedr_create_qp_uresp { __u32 reserved; }; +struct qedr_create_srq_ureq { + /* user space virtual address of producer pair */ + __aligned_u64 prod_pair_addr; + + /* user space virtual address of SRQ buffer */ + __aligned_u64 srq_addr; + + /* length of SRQ buffer */ + __aligned_u64 srq_len; +}; + +struct qedr_create_srq_uresp { + __u16 srq_id; + __u16 reserved0; + __u32 reserved1; +}; + #endif /* __QEDR_USER_H__ */ diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h index 1da5a1e1f3a8..24800c6c1f32 100644 --- a/include/uapi/rdma/rdma_user_ioctl_cmds.h +++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h @@ -62,7 +62,12 @@ struct ib_uverbs_attr { } enum_data; __u16 reserved; } attr_data; - __aligned_u64 data; /* ptr to command, inline data or idr/fd */ + union { + /* Used by PTR_IN/OUT, ENUM_IN and IDR */ + __aligned_u64 data; + /* Used by FD_IN and FD_OUT */ + __s64 data_s64; + }; }; struct ib_uverbs_ioctl_hdr { |