From e9b1ea531c2c18dd9b57ceb279720029e30f8eb5 Mon Sep 17 00:00:00 2001 From: Frank Du Date: Thu, 12 Oct 2023 14:49:14 +0800 Subject: [PATCH] ptp: add ptp sync notify callback app can use this notify for some related monitor task. Signed-off-by: Frank Du --- app/src/app_base.h | 17 ++++++++++++- app/src/app_platform.h | 36 ++++++++++++++++++++++++---- app/src/args.c | 7 ++++++ app/src/rxtx_app.c | 54 +++++++++++++++++++++++++++++++++++++++++- doc/run.md | 1 + include/mtl_api.h | 52 +++++++++++++++++++++++++++------------- lib/src/mt_main.c | 12 ++++++++++ lib/src/mt_ptp.c | 10 ++++++++ 8 files changed, 167 insertions(+), 22 deletions(-) diff --git a/app/src/app_base.h b/app/src/app_base.h index 7e5dd376f..862e87a49 100644 --- a/app/src/app_base.h +++ b/app/src/app_base.h @@ -504,6 +504,12 @@ struct st_app_context { bool tx_copy_once; bool app_thread; + bool ptp_systime_sync; + int ptp_sync_cnt; + int64_t ptp_sync_delta_sum; + int64_t ptp_sync_delta_max; + int64_t ptp_sync_delta_min; + char tx_video_url[ST_APP_URL_MAX_LEN]; /* send video content url*/ struct st_app_tx_video_session* tx_video_sessions; int tx_video_session_cnt; @@ -583,12 +589,21 @@ static inline void* st_app_zmalloc(size_t sz) { static inline void st_app_free(void* p) { free(p); } +static inline uint64_t st_timespec_to_ns(const struct timespec* ts) { + return ((uint64_t)ts->tv_sec * NS_PER_S) + ts->tv_nsec; +} + +static inline void st_ns_to_timespec(uint64_t ns, struct timespec* ts) { + ts->tv_sec = ns / NS_PER_S; + ts->tv_nsec = ns % NS_PER_S; +} + /* Monotonic time (in nanoseconds) since some unspecified starting point. */ static inline uint64_t st_app_get_monotonic_time() { struct timespec ts; clock_gettime(ST_CLOCK_MONOTONIC_ID, &ts); - return ((uint64_t)ts.tv_sec * NS_PER_S) + ts.tv_nsec; + return st_timespec_to_ns(&ts); } int st_app_video_get_lcore(struct st_app_context* ctx, int sch_idx, bool rtp, diff --git a/app/src/app_platform.h b/app/src/app_platform.h index 597808b47..b2374f0a2 100644 --- a/app/src/app_platform.h +++ b/app/src/app_platform.h @@ -150,7 +150,7 @@ static inline void st_usleep( #endif } -static inline void st_getrealtime(struct timespec* pspec) { +static inline int st_get_real_time(struct timespec* ts) { #ifdef WINDOWSENV unsigned __int64 t; union { @@ -159,10 +159,38 @@ static inline void st_getrealtime(struct timespec* pspec) { } ct; GetSystemTimePreciseAsFileTime(&ct.ft); t = ct.u64 - INT64_C(116444736000000000); - pspec->tv_sec = t / 10000000; - pspec->tv_nsec = ((int)(t % 10000000)) * 100; + ts->tv_sec = t / 10000000; + ts->tv_nsec = ((int)(t % 10000000)) * 100; + return 0; #else - clock_gettime(CLOCK_REALTIME, pspec); + return clock_gettime(CLOCK_REALTIME, ts); +#endif +} + +static inline int st_set_real_time(struct timespec* ts) { +#ifdef WINDOWSENV + time_t secs = ts->tv_sec; + WORD milliseconds = ts->tv_nsec / 1000000; + + struct tm* tm; + tm = gmtime(&secs); + + st->wYear = tm->tm_year + 1900; + st->wMonth = tm->tm_mon + 1; + st->wDayOfWeek = tm->tm_wday; + st->wDay = tm->tm_mday; + st->wHour = tm->tm_hour; + st->wMinute = tm->tm_min; + st->wSecond = tm->tm_sec; + st->wMilliseconds = milliseconds; + + if (!SetSystemTime(&st)) { + /* set failed */ + return -EPERM; + } + return 0; +#else + return clock_settime(CLOCK_REALTIME, ts); #endif } diff --git a/app/src/args.c b/app/src/args.c index 6e4b6d02f..12c0a2ca2 100644 --- a/app/src/args.c +++ b/app/src/args.c @@ -67,6 +67,7 @@ enum st_args_cmd { ST_ARG_NIC_RX_PROMISCUOUS, ST_ARG_LIB_PTP, ST_ARG_LIB_PHC2SYS, + ST_ARG_LIB_PTP_SYNC_SYS, ST_ARG_RX_MONO_POOL, ST_ARG_TX_MONO_POOL, ST_ARG_MONO_POOL, @@ -182,6 +183,7 @@ static struct option st_app_args_options[] = { {"log_file", required_argument, 0, ST_ARG_LOG_FILE}, {"ptp", no_argument, 0, ST_ARG_LIB_PTP}, {"phc2sys", no_argument, 0, ST_ARG_LIB_PHC2SYS}, + {"ptp_sync_sys", no_argument, 0, ST_ARG_LIB_PTP_SYNC_SYS}, {"rx_mono_pool", no_argument, 0, ST_ARG_RX_MONO_POOL}, {"tx_mono_pool", no_argument, 0, ST_ARG_TX_MONO_POOL}, {"mono_pool", no_argument, 0, ST_ARG_MONO_POOL}, @@ -541,6 +543,11 @@ int st_app_parse_args(struct st_app_context* ctx, struct mtl_init_params* p, int p->flags |= MTL_FLAG_PTP_ENABLE; p->ptp_get_time_fn = NULL; /* clear the user ptp func */ break; + case ST_ARG_LIB_PTP_SYNC_SYS: + p->flags |= MTL_FLAG_PTP_ENABLE; /* enable built-in ptp */ + p->ptp_get_time_fn = NULL; /* clear the user ptp func */ + ctx->ptp_systime_sync = true; + break; case ST_ARG_LIB_PHC2SYS: p->flags |= MTL_FLAG_PHC2SYS_ENABLE; break; diff --git a/app/src/rxtx_app.c b/app/src/rxtx_app.c index 23d6e5744..7315a61f4 100644 --- a/app/src/rxtx_app.c +++ b/app/src/rxtx_app.c @@ -58,6 +58,17 @@ static int app_dump_io_stat(struct st_app_context* ctx) { return 0; } +static int app_dump_ptp_sync_stat(struct st_app_context* ctx) { + info("%s, cnt %d max %" PRId64 " min %" PRId64 " average %fus\n", __func__, + ctx->ptp_sync_cnt, ctx->ptp_sync_delta_max, ctx->ptp_sync_delta_min, + (float)ctx->ptp_sync_delta_sum / ctx->ptp_sync_cnt / NS_PER_US); + ctx->ptp_sync_delta_sum = 0; + ctx->ptp_sync_cnt = 0; + ctx->ptp_sync_delta_max = INT64_MIN; + ctx->ptp_sync_delta_min = INT64_MAX; + return 0; +} + static void app_stat(void* priv) { struct st_app_context* ctx = priv; @@ -74,9 +85,45 @@ static void app_stat(void* priv) { st_app_rx_st20p_sessions_stat(ctx); st_app_rx_st20r_sessions_stat(ctx); + if (ctx->ptp_systime_sync) { + app_dump_ptp_sync_stat(ctx); + } + ctx->last_stat_time_ns = st_app_get_monotonic_time(); } +static void app_ptp_sync_notify(void* priv, struct mtl_ptp_sync_notify_meta* meta) { + struct st_app_context* ctx = priv; + if (!ctx->ptp_systime_sync) return; + + /* sync raw ptp to sys time */ + uint64_t to_ns = mtl_ptp_read_time_raw(ctx->st); + int ret; + struct timespec from_ts, to_ts; + st_get_real_time(&from_ts); + from_ts.tv_sec += meta->master_utc_offset; /* utc offset */ + uint64_t from_ns = st_timespec_to_ns(&from_ts); + + /* record the sync delta */ + int64_t delta = to_ns - from_ns; + ctx->ptp_sync_cnt++; + ctx->ptp_sync_delta_sum += delta; + if (delta > ctx->ptp_sync_delta_max) ctx->ptp_sync_delta_max = delta; + if (delta < ctx->ptp_sync_delta_min) ctx->ptp_sync_delta_min = delta; + + /* sample just offset the system time delta, better to calibrate as phc2sys way which + * adjust the time frequency also */ + st_ns_to_timespec(to_ns, &to_ts); + to_ts.tv_sec -= meta->master_utc_offset; /* utc offset */ + ret = st_set_real_time(&to_ts); + if (ret < 0) + err("%s, set real time to %" PRIu64 " fail, delta %" PRId64 "\n", __func__, to_ns, + delta); + dbg("%s, from_ns %" PRIu64 " to_ns %" PRIu64 " delta %" PRId64 " done\n", __func__, + from_ns, to_ns, delta); + return; +} + void app_set_log_level(enum mtl_log_level level) { app_log_level = level; } enum mtl_log_level app_get_log_level(void) { return app_log_level; } @@ -84,7 +131,7 @@ enum mtl_log_level app_get_log_level(void) { return app_log_level; } static uint64_t app_ptp_from_tai_time(void* priv) { struct st_app_context* ctx = priv; struct timespec spec; - st_getrealtime(&spec); + st_get_real_time(&spec); spec.tv_sec -= ctx->utc_offset; return ((uint64_t)spec.tv_sec * NS_PER_S) + spec.tv_nsec; } @@ -144,6 +191,9 @@ static void st_app_ctx_init(struct st_app_context* ctx) { ctx->utc_offset = UTC_OFFSET; + ctx->ptp_sync_delta_min = INT64_MAX; + ctx->ptp_sync_delta_max = INT64_MIN; + /* init lcores and sch */ for (int i = 0; i < ST_APP_MAX_LCORES; i++) { ctx->lcore[i] = -1; @@ -352,6 +402,8 @@ int main(int argc, char** argv) { ctx->para.nb_rx_hdr_split_queues = ctx->rx_video_session_cnt; } + if (ctx->ptp_systime_sync) ctx->para.ptp_sync_notify = app_ptp_sync_notify; + ctx->st = mtl_init(&ctx->para); if (!ctx->st) { err("%s, mtl_init fail\n", __func__); diff --git a/doc/run.md b/doc/run.md index 9bd1fcb15..adb6a85cc 100644 --- a/doc/run.md +++ b/doc/run.md @@ -343,6 +343,7 @@ packet egresses from the sender. --audio_fifo_size : debug option, the audio fifo size between packet builder and pacing. --dhcp : debug option, enable DHCP for all ports. --virtio_user : debug option, enable virtio_user ports for control plane packets. Linux only, need to set capability for the app before running, `sudo setcap 'cap_net_admin+ep' ./build/app/RxTxApp`. +--ptp_sync_sys : debug option, enable sync the PTP time from MTL to system time, it required root permission since set system time time is a privilege operation. ``` ## 6. Tests diff --git a/include/mtl_api.h b/include/mtl_api.h index 83cb0e18c..6ee85d419 100644 --- a/include/mtl_api.h +++ b/include/mtl_api.h @@ -359,6 +359,21 @@ enum st21_tx_pacing_way { * Enable shared queue for rx. */ #define MTL_FLAG_SHARED_RX_QUEUE (MTL_BIT64(14)) +/** + * Flag bit in flags of struct mtl_init_params. + * Enable built-in PHC2SYS implementation. + */ +#define MTL_FLAG_PHC2SYS_ENABLE (MTL_BIT64(15)) +/** + * Flag bit in flags of struct mtl_init_params. + * Enable virtio_user as exception path. + */ +#define MTL_FLAG_VIRTIO_USER (MTL_BIT64(16)) +/** + * Flag bit in flags of struct mtl_init_params. + * Do mtl_start in mtl_init, mtl_stop in mtl_uninit, and skip the mtl_start/mtl_stop + */ +#define MTL_FLAG_DEV_AUTO_START_STOP (MTL_BIT64(17)) /** * Flag bit in flags of struct mtl_init_params, debug usage only. @@ -385,11 +400,6 @@ enum st21_tx_pacing_way { * Mono memory pool for all rx queue(sessions) */ #define MTL_FLAG_RX_MONO_POOL (MTL_BIT64(36)) -/** - * Flag bit in flags of struct mtl_init_params, debug usage only. - * Do mtl_start in mtl_init, mtl_stop in mtl_uninit, and skip the mtl_start/mtl_stop - */ -#define MTL_FLAG_DEV_AUTO_START_STOP (MTL_BIT64(37)) /** * Flag bit in flags of struct mtl_init_params, debug usage only. * Enable tasklet time measurement, report status if tasklet run time longer than @@ -432,16 +442,6 @@ enum st21_tx_pacing_way { * Use CNI based queue for RX. */ #define MTL_FLAG_RX_USE_CNI (MTL_BIT64(45)) -/** - * Flag bit in flags of struct mtl_init_params. - * Enable built-in PHC2SYS implementation. - */ -#define MTL_FLAG_PHC2SYS_ENABLE (MTL_BIT64(46)) -/** - * Flag bit in flags of struct mtl_init_params. - * Enable virtio_user as exception path. - */ -#define MTL_FLAG_VIRTIO_USER (MTL_BIT64(47)) /** * The structure describing how to init af_xdp interface. @@ -452,6 +452,13 @@ struct mtl_af_xdp_params { uint8_t start_queue; }; +struct mtl_ptp_sync_notify_meta { + /** offset to UTC of current master PTP */ + int16_t master_utc_offset; + /* phc delta of current sync */ + int64_t delta; +}; + /** * The structure describing how to init the mtl context. * Include the PCIE port and other required info. @@ -559,6 +566,9 @@ struct mtl_init_params { * system time if built-in ptp4l is not enabled. */ uint64_t (*ptp_get_time_fn)(void* priv); + /** Optional for MTL_FLAG_PTP_ENABLE. The callback is notified every time the built-in + * PTP protocol receives a valid PTP_DELAY_RESP message from the PTP grandmaster. */ + void (*ptp_sync_notify)(void* priv, struct mtl_ptp_sync_notify_meta* meta); /** Optional. Stats dump period in seconds, zero means determined by lib(10s default) */ uint16_t dump_period_s; @@ -940,7 +950,7 @@ int mtl_put_lcore(mtl_handle mt, unsigned int lcore); void* mtl_memcpy(void* dest, const void* src, size_t n); /** - * Read current time from ptp source. + * Read current cache time from ptp source. * * @param mt * The handle to the MTL transport device context. @@ -949,6 +959,16 @@ void* mtl_memcpy(void* dest, const void* src, size_t n); */ uint64_t mtl_ptp_read_time(mtl_handle mt); +/** + * Read raw time from ptp source. + * + * @param mt + * The handle to the MTL transport device context. + * @return + * - The time in nanoseconds in current ptp system + */ +uint64_t mtl_ptp_read_time_raw(mtl_handle mt); + /** * Allocate memory from the huge-page area of memory. The memory is not cleared. * In NUMA systems, the memory allocated from the same NUMA socket of the port. diff --git a/lib/src/mt_main.c b/lib/src/mt_main.c index a32cb365e..069b8137b 100644 --- a/lib/src/mt_main.c +++ b/lib/src/mt_main.c @@ -1086,6 +1086,18 @@ uint64_t mtl_ptp_read_time(mtl_handle mt) { return ptp; } +uint64_t mtl_ptp_read_time_raw(mtl_handle mt) { + struct mtl_main_impl* impl = mt; + enum mtl_port port = MTL_PORT_P; + + if (impl->type != MT_HANDLE_MAIN) { + err("%s, invalid type %d\n", __func__, impl->type); + return 0; + } + + return mt_get_ptp_time(impl, port); +} + mtl_udma_handle mtl_udma_create(mtl_handle mt, uint16_t nb_desc, enum mtl_port port) { struct mtl_main_impl* impl = mt; struct mt_dma_request_req req; diff --git a/lib/src/mt_ptp.c b/lib/src/mt_ptp.c index f0b16880e..149a77c99 100644 --- a/lib/src/mt_ptp.c +++ b/lib/src/mt_ptp.c @@ -591,6 +591,7 @@ static void ptp_sync_timeout_handler(void* param) { } static int ptp_parse_result(struct mt_ptp_impl* ptp) { + struct mtl_main_impl* impl = ptp->impl; int64_t delta = ((int64_t)ptp->t4 - ptp->t3) - ((int64_t)ptp->t2 - ptp->t1); int64_t path_delay = ((int64_t)ptp->t2 - ptp->t1) + ((int64_t)ptp->t4 - ptp->t3); uint64_t abs_delta, expect_delta; @@ -664,6 +665,15 @@ static int ptp_parse_result(struct mt_ptp_impl* ptp) { ptp_t_result_clear(ptp); ptp->connected = true; + /* notify the sync event if ptp_sync_notify is enabled */ + struct mtl_init_params* p = mt_get_user_params(impl); + if (p->ptp_sync_notify && (MTL_PORT_P == ptp->port)) { + struct mtl_ptp_sync_notify_meta meta; + meta.master_utc_offset = ptp->master_utc_offset; + meta.delta = delta; + p->ptp_sync_notify(p->priv, &meta); + } + if (ptp->delta_result_cnt > 10) { if (labs(delta) < 30000) { ptp->expect_result_cnt++;