bbdev: add operation for FFT processing

Extended bbdev operations to support FFT based operations.

Signed-off-by: Nicolas Chautru <[email protected]>
Acked-by: Hemant Agrawal <[email protected]>
Acked-by: Maxime Coquelin <[email protected]>
Acked-by: Akhil Goyal <[email protected]>

4 files changed, 244 insertions, 8 deletions

diff --git a/lib/bbdev/rte_bbdev.c b/lib/bbdev/rte_bbdev.c
index d74c68377f..d2380ad2a8 100644
--- a/lib/bbdev/rte_bbdev.c
+++ b/lib/bbdev/rte_bbdev.c
@@ -24,7 +24,7 @@
 #define DEV_NAME "BBDEV"
 
 /* Number of supported operation types in *rte_bbdev_op_type*. */
-#define BBDEV_OP_TYPE_COUNT 5
+#define BBDEV_OP_TYPE_COUNT 6
 
 /* BBDev library logging ID */
 RTE_LOG_REGISTER_DEFAULT(bbdev_logtype, NOTICE);
@@ -852,6 +852,9 @@ get_bbdev_op_size(enum rte_bbdev_op_type type)
 	case RTE_BBDEV_OP_LDPC_ENC:
 		result = sizeof(struct rte_bbdev_enc_op);
 		break;
+	case RTE_BBDEV_OP_FFT:
+		result = sizeof(struct rte_bbdev_fft_op);
+		break;
 	default:
 		break;
 	}
@@ -875,6 +878,10 @@ bbdev_op_init(struct rte_mempool *mempool, void *arg, void *element,
 		struct rte_bbdev_enc_op *op = element;
 		memset(op, 0, mempool->elt_size);
 		op->mempool = mempool;
+	} else if (type == RTE_BBDEV_OP_FFT) {
+		struct rte_bbdev_fft_op *op = element;
+		memset(op, 0, mempool->elt_size);
+		op->mempool = mempool;
 	}
 }
 
@@ -1125,6 +1132,7 @@ rte_bbdev_op_type_str(enum rte_bbdev_op_type op_type)
 		"RTE_BBDEV_OP_TURBO_ENC",
 		"RTE_BBDEV_OP_LDPC_DEC",
 		"RTE_BBDEV_OP_LDPC_ENC",
+		"RTE_BBDEV_OP_FFT",
 	};
 
 	if (op_type < BBDEV_OP_TYPE_COUNT)
diff --git a/lib/bbdev/rte_bbdev.h b/lib/bbdev/rte_bbdev.h
index a7dae53aac..0fe41ab8ce 100644
--- a/lib/bbdev/rte_bbdev.h
+++ b/lib/bbdev/rte_bbdev.h
@@ -401,6 +401,12 @@ typedef uint16_t (*rte_bbdev_enqueue_dec_ops_t)(
 		struct rte_bbdev_dec_op **ops,
 		uint16_t num);
 
+/** @internal Enqueue FFT operations for processing on queue of a device. */
+typedef uint16_t (*rte_bbdev_enqueue_fft_ops_t)(
+		struct rte_bbdev_queue_data *q_data,
+		struct rte_bbdev_fft_op **ops,
+		uint16_t num);
+
 /** @internal Dequeue encode operations from a queue of a device. */
 typedef uint16_t (*rte_bbdev_dequeue_enc_ops_t)(
 		struct rte_bbdev_queue_data *q_data,
@@ -411,6 +417,11 @@ typedef uint16_t (*rte_bbdev_dequeue_dec_ops_t)(
 		struct rte_bbdev_queue_data *q_data,
 		struct rte_bbdev_dec_op **ops, uint16_t num);
 
+/** @internal Dequeue FFT operations from a queue of a device. */
+typedef uint16_t (*rte_bbdev_dequeue_fft_ops_t)(
+		struct rte_bbdev_queue_data *q_data,
+		struct rte_bbdev_fft_op **ops, uint16_t num);
+
 #define RTE_BBDEV_NAME_MAX_LEN  64  /**< Max length of device name */
 
 /**
@@ -459,6 +470,10 @@ struct __rte_cache_aligned rte_bbdev {
 	rte_bbdev_dequeue_enc_ops_t dequeue_ldpc_enc_ops;
 	/** Dequeue decode function */
 	rte_bbdev_dequeue_dec_ops_t dequeue_ldpc_dec_ops;
+	/** Enqueue FFT function */
+	rte_bbdev_enqueue_fft_ops_t enqueue_fft_ops;
+	/** Dequeue FFT function */
+	rte_bbdev_dequeue_fft_ops_t dequeue_fft_ops;
 	const struct rte_bbdev_ops *dev_ops;  /**< Functions exported by PMD */
 	struct rte_bbdev_data *data;  /**< Pointer to device data */
 	enum rte_bbdev_state state;  /**< If device is currently used or not */
@@ -591,11 +606,41 @@ rte_bbdev_enqueue_ldpc_dec_ops(uint16_t dev_id, uint16_t queue_id,
 	return dev->enqueue_ldpc_dec_ops(q_data, ops, num_ops);
 }
 
+/**
+ * Enqueue a burst of FFT operations to a queue of the device.
+ * This functions only enqueues as many operations as currently possible and
+ * does not block until @p num_ops entries in the queue are available.
+ * This function does not provide any error notification to avoid the
+ * corresponding overhead.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param queue_id
+ *   The index of the queue.
+ * @param ops
+ *   Pointer array containing operations to be enqueued.
+ *   Must have at least @p num_ops entries.
+ * @param num_ops
+ *   The maximum number of operations to enqueue.
+ *
+ * @return
+ *   The number of operations actually enqueued.
+ *   (This is the number of processed entries in the @p ops array.)
+ */
+__rte_experimental
+static inline uint16_t
+rte_bbdev_enqueue_fft_ops(uint16_t dev_id, uint16_t queue_id,
+		struct rte_bbdev_fft_op **ops, uint16_t num_ops)
+{
+	struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
+	struct rte_bbdev_queue_data *q_data = &dev->data->queues[queue_id];
+	return dev->enqueue_fft_ops(q_data, ops, num_ops);
+}
 
 /**
  * Dequeue a burst of processed encode operations from a queue of the device.
- * This functions returns only the current contents of the queue, and does not
- * block until @ num_ops is available.
+ * This functions returns only the current contents of the queue,
+ * and does not block until @ num_ops is available.
  * This function does not provide any error notification to avoid the
  * corresponding overhead.
  *
@@ -604,15 +649,15 @@ rte_bbdev_enqueue_ldpc_dec_ops(uint16_t dev_id, uint16_t queue_id,
  * @param queue_id
  *   The index of the queue.
  * @param ops
- *   Pointer array where operations will be dequeued to. Must have at least
- *   @p num_ops entries
- *   ie. A pointer to a table of void * pointers (ops) that will be filled.
+ *   Pointer array where operations will be dequeued to.
+ *   Must have at least @p num_ops entries, i.e.
+ *   a pointer to a table of void * pointers (ops) that will be filled.
  * @param num_ops
  *   The maximum number of operations to dequeue.
  *
  * @return
- *   The number of operations actually dequeued (this is the number of entries
- *   copied into the @p ops array).
+ *   The number of operations actually dequeued.
+ *   (This is the number of entries copied into the @p ops array.)
  */
 static inline uint16_t
 rte_bbdev_dequeue_enc_ops(uint16_t dev_id, uint16_t queue_id,
@@ -716,6 +761,37 @@ rte_bbdev_dequeue_ldpc_dec_ops(uint16_t dev_id, uint16_t queue_id,
 	return dev->dequeue_ldpc_dec_ops(q_data, ops, num_ops);
 }
 
+/**
+ * Dequeue a burst of FFT operations from a queue of the device.
+ * This functions returns only the current contents of the queue, and does not
+ * block until @ num_ops is available.
+ * This function does not provide any error notification to avoid the
+ * corresponding overhead.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param queue_id
+ *   The index of the queue.
+ * @param ops
+ *   Pointer array where operations will be dequeued to. Must have at least
+ *   @p num_ops entries
+ * @param num_ops
+ *   The maximum number of operations to dequeue.
+ *
+ * @return
+ *   The number of operations actually dequeued (this is the number of entries
+ *   copied into the @p ops array).
+ */
+__rte_experimental
+static inline uint16_t
+rte_bbdev_dequeue_fft_ops(uint16_t dev_id, uint16_t queue_id,
+		struct rte_bbdev_fft_op **ops, uint16_t num_ops)
+{
+	struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
+	struct rte_bbdev_queue_data *q_data = &dev->data->queues[queue_id];
+	return dev->dequeue_fft_ops(q_data, ops, num_ops);
+}
+
 /** Definitions of device event types */
 enum rte_bbdev_event_type {
 	RTE_BBDEV_EVENT_UNKNOWN,  /**< unknown event type */
diff --git a/lib/bbdev/rte_bbdev_op.h b/lib/bbdev/rte_bbdev_op.h
index 99a7552779..3a00ef76ba 100644
--- a/lib/bbdev/rte_bbdev_op.h
+++ b/lib/bbdev/rte_bbdev_op.h
@@ -47,6 +47,8 @@ extern "C" {
 #define RTE_BBDEV_TURBO_MAX_CODE_BLOCKS (64)
 /* LDPC:  Maximum number of Code Blocks in Transport Block.*/
 #define RTE_BBDEV_LDPC_MAX_CODE_BLOCKS (256)
+/* 12 CS maximum */
+#define RTE_BBDEV_MAX_CS_2 (6)
 
 /*
  * Maximum size to be used to manage the enum rte_bbdev_op_type
@@ -218,6 +220,26 @@ enum rte_bbdev_op_ldpcenc_flag_bitmasks {
 	RTE_BBDEV_LDPC_ENC_CONCATENATION = (1ULL << 7)
 };
 
+/** Flags for FFT operation and capability structure. */
+enum rte_bbdev_op_fft_flag_bitmasks {
+	/** Flexible windowing capability. */
+	RTE_BBDEV_FFT_WINDOWING = (1ULL << 0),
+	/** Flexible adjustment of Cyclic Shift time offset. */
+	RTE_BBDEV_FFT_CS_ADJUSTMENT = (1ULL << 1),
+	/** Set for bypass the DFT and get directly into iDFT input. */
+	RTE_BBDEV_FFT_DFT_BYPASS = (1ULL << 2),
+	/** Set for bypass the IDFT and get directly the DFT output. */
+	RTE_BBDEV_FFT_IDFT_BYPASS = (1ULL << 3),
+	/** Set for bypass time domain windowing. */
+	RTE_BBDEV_FFT_WINDOWING_BYPASS = (1ULL << 4),
+	/** Set for optional power measurement on DFT output. */
+	RTE_BBDEV_FFT_POWER_MEAS = (1ULL << 5),
+	/** Set if the input data used FP16 format. */
+	RTE_BBDEV_FFT_FP16_INPUT = (1ULL << 6),
+	/** Set if the output data uses FP16 format. */
+	RTE_BBDEV_FFT_FP16_OUTPUT = (1ULL << 7)
+};
+
 /** Flags for the Code Block/Transport block mode  */
 enum rte_bbdev_op_cb_mode {
 	/** One operation is one or fraction of one transport block  */
@@ -696,6 +718,55 @@ struct rte_bbdev_op_ldpc_enc {
 	};
 };
 
+/** Operation structure for FFT processing.
+ *
+ * The operation processes the data for multiple antennas in a single call
+ * (i.e. for all the REs belonging to a given SRS sequence for instance).
+ *
+ * The output mbuf data structure is expected to be allocated by the
+ * application with enough room for the output data.
+ */
+struct rte_bbdev_op_fft {
+	/** Input data starting from first antenna. */
+	struct rte_bbdev_op_data base_input;
+	/** Output data starting from first antenna and first cyclic shift. */
+	struct rte_bbdev_op_data base_output;
+	/** Optional power measurement output data. */
+	struct rte_bbdev_op_data power_meas_output;
+	/** Flags from rte_bbdev_op_fft_flag_bitmasks. */
+	uint32_t op_flags;
+	/** Input sequence size in 32-bits points. */
+	uint16_t input_sequence_size;
+	/** Padding at the start of the sequence. */
+	uint16_t input_leading_padding;
+	/** Output sequence size in 32-bits points. */
+	uint16_t output_sequence_size;
+	/** Depadding at the start of the DFT output. */
+	uint16_t output_leading_depadding;
+	/** Window index being used for each cyclic shift output. */
+	uint8_t window_index[RTE_BBDEV_MAX_CS_2];
+	/** Bitmap of the cyclic shift output requested. */
+	uint16_t cs_bitmap;
+	/** Number of antennas as a log2 – 8 to 128. */
+	uint8_t num_antennas_log2;
+	/** iDFT size as a log2 - 32 to 2048. */
+	uint8_t idft_log2;
+	/** DFT size as a log2 - 8 to 2048. */
+	uint8_t dft_log2;
+	/** Adjustment of position of the cyclic shifts - -31 to 31. */
+	int8_t cs_time_adjustment;
+	/** iDFT shift down. */
+	int8_t idft_shift;
+	/** DFT shift down. */
+	int8_t dft_shift;
+	/** NCS reciprocal factor. */
+	uint16_t ncs_reciprocal;
+	/** Power measurement out shift down. */
+	uint16_t power_shift;
+	/** Adjust the FP6 exponent for INT<->FP16 conversion. */
+	uint16_t fp16_exp_adjust;
+};
+
 /** List of the capabilities for the Turbo Decoder */
 struct rte_bbdev_op_cap_turbo_dec {
 	/** Flags from rte_bbdev_op_td_flag_bitmasks */
@@ -748,6 +819,16 @@ struct rte_bbdev_op_cap_ldpc_enc {
 	uint16_t num_buffers_dst;
 };
 
+/** List of the capabilities for the FFT. */
+struct rte_bbdev_op_cap_fft {
+	/** Flags from *rte_bbdev_op_fft_flag_bitmasks*. */
+	uint32_t capability_flags;
+	/** Number of input code block buffers. */
+	uint16_t num_buffers_src;
+	/** Number of output code block buffers. */
+	uint16_t num_buffers_dst;
+};
+
 /** Different operation types supported by the device.
  *  The related macro RTE_BBDEV_OP_TYPE_SIZE_MAX can be used as an absolute maximum for
  *  notably sizing array while allowing for future enumeration insertion.
@@ -758,6 +839,7 @@ enum rte_bbdev_op_type {
 	RTE_BBDEV_OP_TURBO_ENC,  /**< Turbo encode */
 	RTE_BBDEV_OP_LDPC_DEC,  /**< LDPC decode */
 	RTE_BBDEV_OP_LDPC_ENC,  /**< LDPC encode */
+	RTE_BBDEV_OP_FFT,  /**< FFT */
 	/* Note: RTE_BBDEV_OP_TYPE_SIZE_MAX must be larger or equal to maximum enum value */
 };
 
@@ -801,6 +883,18 @@ struct rte_bbdev_dec_op {
 	};
 };
 
+/** Structure specifying a single FFT operation. */
+struct rte_bbdev_fft_op {
+	/** Status of operation performed. */
+	int status;
+	/** Mempool used for op instance. */
+	struct rte_mempool *mempool;
+	/** Opaque pointer for user data. */
+	void *opaque_data;
+	/** Contains turbo decoder specific parameters. */
+	struct rte_bbdev_op_fft fft;
+};
+
 /** Operation capabilities supported by a device */
 struct rte_bbdev_op_cap {
 	enum rte_bbdev_op_type type;  /**< Type of operation */
@@ -809,6 +903,7 @@ struct rte_bbdev_op_cap {
 		struct rte_bbdev_op_cap_turbo_enc turbo_enc;
 		struct rte_bbdev_op_cap_ldpc_dec ldpc_dec;
 		struct rte_bbdev_op_cap_ldpc_enc ldpc_enc;
+		struct rte_bbdev_op_cap_fft fft;
 	} cap;  /**< Operation-type specific capabilities */
 };
 
@@ -928,6 +1023,41 @@ rte_bbdev_dec_op_alloc_bulk(struct rte_mempool *mempool,
 }
 
 /**
+ * Bulk allocate FFT operations from a mempool with default parameters.
+ *
+ * @param mempool
+ *   Operation mempool, created by *rte_bbdev_op_pool_create*.
+ * @param ops
+ *   Output array to place allocated operations.
+ * @param num_ops
+ *   Number of operations to allocate.
+ *
+ * @returns
+ *   - 0 on success.
+ *   - EINVAL if invalid mempool is provided.
+ */
+__rte_experimental
+static inline int
+rte_bbdev_fft_op_alloc_bulk(struct rte_mempool *mempool,
+		struct rte_bbdev_fft_op **ops, uint16_t num_ops)
+{
+	struct rte_bbdev_op_pool_private *priv;
+	int ret;
+
+	/* Check type */
+	priv = (struct rte_bbdev_op_pool_private *)rte_mempool_get_priv(mempool);
+	if (unlikely(priv->type != RTE_BBDEV_OP_FFT))
+		return -EINVAL;
+
+	/* Get elements */
+	ret = rte_mempool_get_bulk(mempool, (void **)ops, num_ops);
+	if (unlikely(ret < 0))
+		return ret;
+
+	return 0;
+}
+
+/**
  * Free decode operation structures that were allocated by
  * rte_bbdev_dec_op_alloc_bulk().
  * All structures must belong to the same mempool.
@@ -961,6 +1091,24 @@ rte_bbdev_enc_op_free_bulk(struct rte_bbdev_enc_op **ops, unsigned int num_ops)
 		rte_mempool_put_bulk(ops[0]->mempool, (void **)ops, num_ops);
 }
 
+/**
+ * Free encode operation structures that were allocated by
+ * *rte_bbdev_fft_op_alloc_bulk*.
+ * All structures must belong to the same mempool.
+ *
+ * @param ops
+ *   Operation structures.
+ * @param num_ops
+ *   Number of structures.
+ */
+__rte_experimental
+static inline void
+rte_bbdev_fft_op_free_bulk(struct rte_bbdev_fft_op **ops, unsigned int num_ops)
+{
+	if (num_ops > 0)
+		rte_mempool_put_bulk(ops[0]->mempool, (void **)ops, num_ops);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/bbdev/version.map b/lib/bbdev/version.map
index addea05f00..db178917e2 100644
--- a/lib/bbdev/version.map
+++ b/lib/bbdev/version.map
@@ -44,5 +44,9 @@ EXPERIMENTAL {
 	global:
 
 	# added in 22.11
+	rte_bbdev_dequeue_fft_ops;
 	rte_bbdev_device_status_str;
+	rte_bbdev_enqueue_fft_ops;
+	rte_bbdev_fft_op_alloc_bulk;
+	rte_bbdev_fft_op_free_bulk;
 };