ptp: add ptp sync notify callback

app can use this notify for some related monitor task.

Signed-off-by: Frank Du <[email protected]>

app/src/app_base.h

@@ -504,6 +504,12 @@ struct st_app_context {
   bool tx_copy_once;
   bool app_thread;
 
+  bool ptp_systime_sync;
+  int ptp_sync_cnt;
+  int64_t ptp_sync_delta_sum;
+  int64_t ptp_sync_delta_max;
+  int64_t ptp_sync_delta_min;
+
   char tx_video_url[ST_APP_URL_MAX_LEN]; /* send video content url*/
   struct st_app_tx_video_session* tx_video_sessions;
   int tx_video_session_cnt;
@@ -583,12 +589,21 @@ static inline void* st_app_zmalloc(size_t sz) {
 
 static inline void st_app_free(void* p) { free(p); }
 
+static inline uint64_t st_timespec_to_ns(const struct timespec* ts) {
+  return ((uint64_t)ts->tv_sec * NS_PER_S) + ts->tv_nsec;
+}
+
+static inline void st_ns_to_timespec(uint64_t ns, struct timespec* ts) {
+  ts->tv_sec = ns / NS_PER_S;
+  ts->tv_nsec = ns % NS_PER_S;
+}
+
 /* Monotonic time (in nanoseconds) since some unspecified starting point. */
 static inline uint64_t st_app_get_monotonic_time() {
   struct timespec ts;
 
   clock_gettime(ST_CLOCK_MONOTONIC_ID, &ts);
-  return ((uint64_t)ts.tv_sec * NS_PER_S) + ts.tv_nsec;
+  return st_timespec_to_ns(&ts);
 }
 
 int st_app_video_get_lcore(struct st_app_context* ctx, int sch_idx, bool rtp,

app/src/app_platform.h

@@ -150,7 +150,7 @@ static inline void st_usleep(
 #endif
 }
 
-static inline void st_getrealtime(struct timespec* pspec) {
+static inline int st_get_real_time(struct timespec* ts) {
 #ifdef WINDOWSENV
   unsigned __int64 t;
   union {
@@ -159,10 +159,38 @@ static inline void st_getrealtime(struct timespec* pspec) {
   } ct;
   GetSystemTimePreciseAsFileTime(&ct.ft);
   t = ct.u64 - INT64_C(116444736000000000);
-  pspec->tv_sec = t / 10000000;
-  pspec->tv_nsec = ((int)(t % 10000000)) * 100;
+  ts->tv_sec = t / 10000000;
+  ts->tv_nsec = ((int)(t % 10000000)) * 100;
+  return 0;
 #else
-  clock_gettime(CLOCK_REALTIME, pspec);
+  return clock_gettime(CLOCK_REALTIME, ts);
+#endif
+}
+
+static inline int st_set_real_time(struct timespec* ts) {
+#ifdef WINDOWSENV
+  time_t secs = ts->tv_sec;
+  WORD milliseconds = ts->tv_nsec / 1000000;
+
+  struct tm* tm;
+  tm = gmtime(&secs);
+
+  st->wYear = tm->tm_year + 1900;
+  st->wMonth = tm->tm_mon + 1;
+  st->wDayOfWeek = tm->tm_wday;
+  st->wDay = tm->tm_mday;
+  st->wHour = tm->tm_hour;
+  st->wMinute = tm->tm_min;
+  st->wSecond = tm->tm_sec;
+  st->wMilliseconds = milliseconds;
+
+  if (!SetSystemTime(&st)) {
+    /* set failed */
+    return -EPERM;
+  }
+  return 0;
+#else
+  return clock_settime(CLOCK_REALTIME, ts);
 #endif
 }
 

app/src/args.c

@@ -67,6 +67,7 @@ enum st_args_cmd {
   ST_ARG_NIC_RX_PROMISCUOUS,
   ST_ARG_LIB_PTP,
   ST_ARG_LIB_PHC2SYS,
+  ST_ARG_LIB_PTP_SYNC_SYS,
   ST_ARG_RX_MONO_POOL,
   ST_ARG_TX_MONO_POOL,
   ST_ARG_MONO_POOL,
@@ -182,6 +183,7 @@ static struct option st_app_args_options[] = {
     {"log_file", required_argument, 0, ST_ARG_LOG_FILE},
     {"ptp", no_argument, 0, ST_ARG_LIB_PTP},
     {"phc2sys", no_argument, 0, ST_ARG_LIB_PHC2SYS},
+    {"ptp_sync_sys", no_argument, 0, ST_ARG_LIB_PTP_SYNC_SYS},
     {"rx_mono_pool", no_argument, 0, ST_ARG_RX_MONO_POOL},
     {"tx_mono_pool", no_argument, 0, ST_ARG_TX_MONO_POOL},
     {"mono_pool", no_argument, 0, ST_ARG_MONO_POOL},
@@ -541,6 +543,11 @@ int st_app_parse_args(struct st_app_context* ctx, struct mtl_init_params* p, int
         p->flags |= MTL_FLAG_PTP_ENABLE;
         p->ptp_get_time_fn = NULL; /* clear the user ptp func */
         break;
+      case ST_ARG_LIB_PTP_SYNC_SYS:
+        p->flags |= MTL_FLAG_PTP_ENABLE; /* enable built-in ptp */
+        p->ptp_get_time_fn = NULL;       /* clear the user ptp func */
+        ctx->ptp_systime_sync = true;
+        break;
       case ST_ARG_LIB_PHC2SYS:
         p->flags |= MTL_FLAG_PHC2SYS_ENABLE;
         break;

app/src/rxtx_app.c

@@ -58,6 +58,17 @@ static int app_dump_io_stat(struct st_app_context* ctx) {
   return 0;
 }
 
+static int app_dump_ptp_sync_stat(struct st_app_context* ctx) {
+  info("%s, cnt %d max %" PRId64 " min %" PRId64 " average %fus\n", __func__,
+       ctx->ptp_sync_cnt, ctx->ptp_sync_delta_max, ctx->ptp_sync_delta_min,
+       (float)ctx->ptp_sync_delta_sum / ctx->ptp_sync_cnt / NS_PER_US);
+  ctx->ptp_sync_delta_sum = 0;
+  ctx->ptp_sync_cnt = 0;
+  ctx->ptp_sync_delta_max = INT64_MIN;
+  ctx->ptp_sync_delta_min = INT64_MAX;
+  return 0;
+}
+
 static void app_stat(void* priv) {
   struct st_app_context* ctx = priv;
 
@@ -74,17 +85,53 @@ static void app_stat(void* priv) {
   st_app_rx_st20p_sessions_stat(ctx);
   st_app_rx_st20r_sessions_stat(ctx);
 
+  if (ctx->ptp_systime_sync) {
+    app_dump_ptp_sync_stat(ctx);
+  }
+
   ctx->last_stat_time_ns = st_app_get_monotonic_time();
 }
 
+static void app_ptp_sync_notify(void* priv, struct mtl_ptp_sync_notify_meta* meta) {
+  struct st_app_context* ctx = priv;
+  if (!ctx->ptp_systime_sync) return;
+
+  /* sync raw ptp to sys time */
+  uint64_t to_ns = mtl_ptp_read_time_raw(ctx->st);
+  int ret;
+  struct timespec from_ts, to_ts;
+  st_get_real_time(&from_ts);
+  from_ts.tv_sec += meta->master_utc_offset; /* utc offset */
+  uint64_t from_ns = st_timespec_to_ns(&from_ts);
+
+  /* record the sync delta */
+  int64_t delta = to_ns - from_ns;
+  ctx->ptp_sync_cnt++;
+  ctx->ptp_sync_delta_sum += delta;
+  if (delta > ctx->ptp_sync_delta_max) ctx->ptp_sync_delta_max = delta;
+  if (delta < ctx->ptp_sync_delta_min) ctx->ptp_sync_delta_min = delta;
+
+  /* sample just offset the system time delta, better to calibrate as phc2sys way which
+   * adjust the time frequency also  */
+  st_ns_to_timespec(to_ns, &to_ts);
+  to_ts.tv_sec -= meta->master_utc_offset; /* utc offset */
+  ret = st_set_real_time(&to_ts);
+  if (ret < 0)
+    err("%s, set real time to %" PRIu64 " fail, delta %" PRId64 "\n", __func__, to_ns,
+        delta);
+  dbg("%s, from_ns %" PRIu64 " to_ns %" PRIu64 " delta %" PRId64 " done\n", __func__,
+      from_ns, to_ns, delta);
+  return;
+}
+
 void app_set_log_level(enum mtl_log_level level) { app_log_level = level; }
 
 enum mtl_log_level app_get_log_level(void) { return app_log_level; }
 
 static uint64_t app_ptp_from_tai_time(void* priv) {
   struct st_app_context* ctx = priv;
   struct timespec spec;
-  st_getrealtime(&spec);
+  st_get_real_time(&spec);
   spec.tv_sec -= ctx->utc_offset;
   return ((uint64_t)spec.tv_sec * NS_PER_S) + spec.tv_nsec;
 }
@@ -144,6 +191,9 @@ static void st_app_ctx_init(struct st_app_context* ctx) {
 
   ctx->utc_offset = UTC_OFFSET;
 
+  ctx->ptp_sync_delta_min = INT64_MAX;
+  ctx->ptp_sync_delta_max = INT64_MIN;
+
   /* init lcores and sch */
   for (int i = 0; i < ST_APP_MAX_LCORES; i++) {
     ctx->lcore[i] = -1;
@@ -352,6 +402,8 @@ int main(int argc, char** argv) {
     ctx->para.nb_rx_hdr_split_queues = ctx->rx_video_session_cnt;
   }
 
+  if (ctx->ptp_systime_sync) ctx->para.ptp_sync_notify = app_ptp_sync_notify;
+
   ctx->st = mtl_init(&ctx->para);
   if (!ctx->st) {
     err("%s, mtl_init fail\n", __func__);

doc/run.md

@@ -343,6 +343,7 @@ packet egresses from the sender.
 --audio_fifo_size <count>            : debug option, the audio fifo size between packet builder and pacing.
 --dhcp                               : debug option, enable DHCP for all ports.
 --virtio_user                        : debug option, enable virtio_user ports for control plane packets. Linux only, need to set capability for the app before running, `sudo setcap 'cap_net_admin+ep' ./build/app/RxTxApp`.
+--ptp_sync_sys                       : debug option, enable sync the PTP time from MTL to system time, it required root permission since set system time time is a privilege operation.
 ```
 
 ## 6. Tests

include/mtl_api.h

@@ -359,6 +359,21 @@ enum st21_tx_pacing_way {
  * Enable shared queue for rx.
  */
 #define MTL_FLAG_SHARED_RX_QUEUE (MTL_BIT64(14))
+/**
+ * Flag bit in flags of struct mtl_init_params.
+ * Enable built-in PHC2SYS implementation.
+ */
+#define MTL_FLAG_PHC2SYS_ENABLE (MTL_BIT64(15))
+/**
+ * Flag bit in flags of struct mtl_init_params.
+ * Enable virtio_user as exception path.
+ */
+#define MTL_FLAG_VIRTIO_USER (MTL_BIT64(16))
+/**
+ * Flag bit in flags of struct mtl_init_params.
+ * Do mtl_start in mtl_init, mtl_stop in mtl_uninit, and skip the mtl_start/mtl_stop
+ */
+#define MTL_FLAG_DEV_AUTO_START_STOP (MTL_BIT64(17))
 
 /**
  * Flag bit in flags of struct mtl_init_params, debug usage only.
@@ -385,11 +400,6 @@ enum st21_tx_pacing_way {
  * Mono memory pool for all rx queue(sessions)
  */
 #define MTL_FLAG_RX_MONO_POOL (MTL_BIT64(36))
-/**
- * Flag bit in flags of struct mtl_init_params, debug usage only.
- * Do mtl_start in mtl_init, mtl_stop in mtl_uninit, and skip the mtl_start/mtl_stop
- */
-#define MTL_FLAG_DEV_AUTO_START_STOP (MTL_BIT64(37))
 /**
  * Flag bit in flags of struct mtl_init_params, debug usage only.
  * Enable tasklet time measurement, report status if tasklet run time longer than
@@ -432,16 +442,6 @@ enum st21_tx_pacing_way {
  * Use CNI based queue for RX.
  */
 #define MTL_FLAG_RX_USE_CNI (MTL_BIT64(45))
-/**
- * Flag bit in flags of struct mtl_init_params.
- * Enable built-in PHC2SYS implementation.
- */
-#define MTL_FLAG_PHC2SYS_ENABLE (MTL_BIT64(46))
-/**
- * Flag bit in flags of struct mtl_init_params.
- * Enable virtio_user as exception path.
- */
-#define MTL_FLAG_VIRTIO_USER (MTL_BIT64(47))
 
 /**
  * The structure describing how to init af_xdp interface.
@@ -452,6 +452,13 @@ struct mtl_af_xdp_params {
   uint8_t start_queue;
 };
 
+struct mtl_ptp_sync_notify_meta {
+  /** offset to UTC of current master PTP */
+  int16_t master_utc_offset;
+  /* phc delta of current sync */
+  int64_t delta;
+};
+
 /**
  * The structure describing how to init the mtl context.
  * Include the PCIE port and other required info.
@@ -559,6 +566,9 @@ struct mtl_init_params {
    * system time if built-in ptp4l is not enabled.
    */
   uint64_t (*ptp_get_time_fn)(void* priv);
+  /** Optional for MTL_FLAG_PTP_ENABLE. The callback is notified every time the built-in
+   * PTP protocol receives a valid PTP_DELAY_RESP message from the PTP grandmaster. */
+  void (*ptp_sync_notify)(void* priv, struct mtl_ptp_sync_notify_meta* meta);
 
   /** Optional. Stats dump period in seconds, zero means determined by lib(10s default) */
   uint16_t dump_period_s;
@@ -940,7 +950,7 @@ int mtl_put_lcore(mtl_handle mt, unsigned int lcore);
 void* mtl_memcpy(void* dest, const void* src, size_t n);
 
 /**
- * Read current time from ptp source.
+ * Read current cache time from ptp source.
  *
  * @param mt
  *   The handle to the MTL transport device context.
@@ -949,6 +959,16 @@ void* mtl_memcpy(void* dest, const void* src, size_t n);
  */
 uint64_t mtl_ptp_read_time(mtl_handle mt);
 
+/**
+ * Read raw time from ptp source.
+ *
+ * @param mt
+ *   The handle to the MTL transport device context.
+ * @return
+ *   - The time in nanoseconds in current ptp system
+ */
+uint64_t mtl_ptp_read_time_raw(mtl_handle mt);
+
 /**
  * Allocate memory from the huge-page area of memory. The memory is not cleared.
  * In NUMA systems, the memory allocated from the same NUMA socket of the port.

lib/src/mt_main.c

@@ -1086,6 +1086,18 @@ uint64_t mtl_ptp_read_time(mtl_handle mt) {
   return ptp;
 }
 
+uint64_t mtl_ptp_read_time_raw(mtl_handle mt) {
+  struct mtl_main_impl* impl = mt;
+  enum mtl_port port = MTL_PORT_P;
+
+  if (impl->type != MT_HANDLE_MAIN) {
+    err("%s, invalid type %d\n", __func__, impl->type);
+    return 0;
+  }
+
+  return mt_get_ptp_time(impl, port);
+}
+
 mtl_udma_handle mtl_udma_create(mtl_handle mt, uint16_t nb_desc, enum mtl_port port) {
   struct mtl_main_impl* impl = mt;
   struct mt_dma_request_req req;

lib/src/mt_ptp.c

@@ -591,6 +591,7 @@ static void ptp_sync_timeout_handler(void* param) {
 }
 
 static int ptp_parse_result(struct mt_ptp_impl* ptp) {
+  struct mtl_main_impl* impl = ptp->impl;
   int64_t delta = ((int64_t)ptp->t4 - ptp->t3) - ((int64_t)ptp->t2 - ptp->t1);
   int64_t path_delay = ((int64_t)ptp->t2 - ptp->t1) + ((int64_t)ptp->t4 - ptp->t3);
   uint64_t abs_delta, expect_delta;
@@ -664,6 +665,15 @@ static int ptp_parse_result(struct mt_ptp_impl* ptp) {
   ptp_t_result_clear(ptp);
   ptp->connected = true;
 
+  /* notify the sync event if ptp_sync_notify is enabled */
+  struct mtl_init_params* p = mt_get_user_params(impl);
+  if (p->ptp_sync_notify && (MTL_PORT_P == ptp->port)) {
+    struct mtl_ptp_sync_notify_meta meta;
+    meta.master_utc_offset = ptp->master_utc_offset;
+    meta.delta = delta;
+    p->ptp_sync_notify(p->priv, &meta);
+  }
+
   if (ptp->delta_result_cnt > 10) {
     if (labs(delta) < 30000) {
       ptp->expect_result_cnt++;