diff --git a/docs/api-reference/configuration.md b/docs/api-reference/configuration.md index e79a815d..b27ba8a3 100644 --- a/docs/api-reference/configuration.md +++ b/docs/api-reference/configuration.md @@ -349,6 +349,10 @@ a template table. When enabled, DAQIRI requires hardware timestamp support from the NIC and driver. Timestamps returned by `get_packet_rx_timestamp()` are unsigned 64-bit PTP epoch nanoseconds in the same clock domain as a PTP-synchronized `CLOCK_REALTIME`. +The raw ibverbs engine requests the mlx5 real-time CQ timestamp format only when +the device advertises it; otherwise it uses the default mlx5 device-clock CQ +format and converts those ticks to nanoseconds internally. Device-clock ticks are +not exposed by the public API. **WARNING: PTP synchronization is required.** DAQIRI does not validate the NIC or system clock configuration. Timestamp values are invalid if the clocks are not PTP-synchronized. diff --git a/docs/api-reference/cpp.md b/docs/api-reference/cpp.md index 5beccd85..a322e8c8 100644 --- a/docs/api-reference/cpp.md +++ b/docs/api-reference/cpp.md @@ -121,6 +121,10 @@ RX hardware timestamps are available only when DAQIRI is configured with `rx.hardware_timestamps: true` and the NIC and driver support hardware timestamps. DAQIRI returns unsigned 64-bit PTP epoch nanoseconds in the same clock domain as a PTP-synchronized `CLOCK_REALTIME`. Device-clock ticks are not part of the public API. +On the raw ibverbs engine, DAQIRI requests mlx5 real-time CQ timestamps only when +the device advertises that format. Devices without that capability stay on the +default mlx5 device-clock CQ format, and DAQIRI converts those raw ticks to +nanoseconds before returning them. **WARNING: PTP synchronization is required.** DAQIRI does not validate the NIC or system clock configuration. Timestamp values are invalid if the clocks are not PTP-synchronized. For reordered aggregate bursts, diff --git a/docs/api-reference/python.md b/docs/api-reference/python.md index 28f3df67..79586c96 100644 --- a/docs/api-reference/python.md +++ b/docs/api-reference/python.md @@ -219,6 +219,10 @@ RX hardware timestamps are available only when DAQIRI is configured with `rx.hardware_timestamps: true` and the NIC and driver support hardware timestamps. DAQIRI returns unsigned 64-bit PTP epoch nanoseconds in the same clock domain as a PTP-synchronized `CLOCK_REALTIME`. Device-clock ticks are not part of the public API. +On the raw ibverbs engine, DAQIRI requests mlx5 real-time CQ timestamps only when +the device advertises that format. Devices without that capability stay on the +default mlx5 device-clock CQ format, and DAQIRI converts those raw ticks to +nanoseconds before returning them. **WARNING: PTP synchronization is required.** DAQIRI does not validate the NIC or system clock configuration. Timestamp values are invalid if the clocks are not PTP-synchronized. diff --git a/docs/concepts.md b/docs/concepts.md index 7cc1a787..756ce65d 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -365,6 +365,10 @@ transmission and receive timestamps. When hardware receive timestamps are enabled, applications use `get_packet_rx_timestamp()` to read each packet's PTP epoch-nanosecond timestamp. +Raw ibverbs devices that support mlx5 real-time CQ timestamps produce that +representation directly; devices without that capability use the default mlx5 +device-clock CQ format, which DAQIRI converts internally before returning the +timestamp. Applications never handle raw device-clock ticks. For timed transmission, applications pass the desired PTP epoch-nanosecond value directly to `set_packet_tx_time()`. diff --git a/include/daqiri/common.h b/include/daqiri/common.h index 7cd5d95b..364cf3c8 100644 --- a/include/daqiri/common.h +++ b/include/daqiri/common.h @@ -246,8 +246,9 @@ Status poll_flow_op(FlowOpResult *result); * * Retrieves the 64-bit receive timestamp for a packet when DAQIRI is configured * with rx.hardware_timestamps enabled and the NIC and driver provide hardware - * timestamps as PTP epoch nanoseconds. DAQIRI assumes that the - * NIC clock and CLOCK_REALTIME are PTP-synchronized and does not validate + * timestamps. DAQIRI returns PTP epoch nanoseconds; engines that receive raw + * device-clock ticks convert them before returning. DAQIRI assumes that the NIC + * clock and CLOCK_REALTIME are PTP-synchronized and does not validate * synchronization. The result is invalid without working PTP. * * @param burst Burst structure containing packets diff --git a/src/engines/ibverbs/daqiri_ibverbs_engine.cpp b/src/engines/ibverbs/daqiri_ibverbs_engine.cpp index a89e628a..bece4e1a 100644 --- a/src/engines/ibverbs/daqiri_ibverbs_engine.cpp +++ b/src/engines/ibverbs/daqiri_ibverbs_engine.cpp @@ -997,17 +997,25 @@ Status IbverbsEngine::devx_create_rq(IbvRxQueue& q, uint32_t stride_log, uint32_ DEVX_SET(rqc, rqc, cqn, q.dv_cq.cqn); DEVX_SET(rqc, rqc, flush_in_error_en, 1); DEVX_SET(rqc, rqc, vsd, 1); // do not strip VLAN - // The public timestamp contract is PTP epoch nanoseconds. Request mlx5 - // real-time CQEs, whose timestamp field uses UTC encoding - // (seconds << 32 | nanoseconds). - DEVX_SET(rqc, rqc, ts_format, MLX5_RQC_TIMESTAMP_FORMAT_REAL_TIME); + struct mlx5dv_context dv_ctx {}; + q.realtime_timestamps = mlx5dv_query_device(q.ctx, &dv_ctx) == 0 && + (dv_ctx.flags & MLX5DV_CONTEXT_FLAGS_REAL_TIME_TS) != 0; + if (q.realtime_timestamps) { + // Real-time CQEs use UTC encoding (seconds << 32 | nanoseconds), matching + // the public PTP epoch-nanosecond timestamp contract. + DEVX_SET(rqc, rqc, ts_format, MLX5_RQC_TIMESTAMP_FORMAT_REAL_TIME); + } else { + DAQIRI_LOG_CRITICAL( + "RX queue {}: real-time CQ timestamps are unsupported; using the device clock format", + q.queue_id); + } DEVX_SET(wq, wq, wq_type, q.striding ? MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC); DEVX_SET(wq, wq, log_wq_stride, log2_floor(q.wqe_stride)); DEVX_SET(wq, wq, log_wq_sz, log2_floor(q.num_wqe)); DEVX_SET(wq, wq, pd, dvpd.pdn); - // WQ page size is encoded relative to the mlx5 4 KiB adapter page, not the - // operating-system page size (which is 64 KiB on GH200). + // The PRM field is relative to the mlx5 4 KiB adapter page, independent of + // the operating-system page size used to register the UMEM. DEVX_SET(wq, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SIZE_4_KIB); DEVX_SET64(wq, wq, dbr_addr, dbr_off); DEVX_SET(wq, wq, dbr_umem_id, q.wq_umem->umem_id); @@ -3748,6 +3756,10 @@ Status IbverbsEngine::get_packet_rx_timestamp(BurstParams* burst, int idx, uint6 return Status::INVALID_PARAMETER; } const uint64_t raw_timestamp = burst_ts_arr(burst)[idx]; + if (!q->realtime_timestamps) { + *timestamp_ns = ts_to_ns(q->ctx, raw_timestamp); + return Status::SUCCESS; + } // mlx5 real-time CQEs use UTC encoding; expose the public API's linear PTP // epoch-nanosecond representation. const uint64_t seconds = raw_timestamp >> 32; diff --git a/src/engines/ibverbs/daqiri_ibverbs_engine.h b/src/engines/ibverbs/daqiri_ibverbs_engine.h index cc1b6197..411baff9 100644 --- a/src/engines/ibverbs/daqiri_ibverbs_engine.h +++ b/src/engines/ibverbs/daqiri_ibverbs_engine.h @@ -198,6 +198,7 @@ struct IbvRxQueue { // Direct mlx5dv views (we own the CQ consumer index; rdma-core owns the RQ). struct mlx5dv_cq dv_cq {}; uint32_t cq_ci = 0; // CQ consumer index (monotonic) + bool realtime_timestamps = false; // Per-WQE stride accounting for the reclaim path. Indexed by WQE/region. std::vector consumed_strides; // strides handed to the app (verbs path)