From 2088708baec5cc6fec3173aa98d3e6005796a9d6 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 9 Dec 2024 17:14:33 +0200 Subject: [PATCH 1/2] linux-gen: sched: remove unused function prototypes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove unused implementation internal scheduler interface function prototypes. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- platform/linux-generic/include/odp_schedule_if.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 7a61b1dc0b..17d12600da 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -54,8 +54,6 @@ typedef int (*schedule_term_local_fn_t)(void); typedef void (*schedule_order_lock_fn_t)(void); typedef void (*schedule_order_unlock_fn_t)(void); typedef void (*schedule_order_unlock_lock_fn_t)(void); -typedef void (*schedule_order_lock_start_fn_t)(void); -typedef void (*schedule_order_lock_wait_fn_t)(void); typedef uint32_t (*schedule_max_ordered_locks_fn_t)(void); typedef void (*schedule_get_config_fn_t)(schedule_config_t *config); typedef const _odp_schedule_api_fn_t *(*schedule_sched_api_fn_t)(void); @@ -75,9 +73,7 @@ typedef struct schedule_fn_t { schedule_term_local_fn_t term_local; schedule_order_lock_fn_t order_lock; schedule_order_unlock_fn_t order_unlock; - schedule_order_lock_start_fn_t start_order_lock; - schedule_order_lock_wait_fn_t wait_order_lock; - schedule_order_unlock_lock_fn_t order_unlock_lock; + schedule_order_unlock_lock_fn_t order_unlock_lock; schedule_max_ordered_locks_fn_t max_ordered_locks; schedule_get_config_fn_t get_config; schedule_sched_api_fn_t sched_api; From 252d6ea80e13592e47ee63e9dd594ccb11112623 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 9 Dec 2024 16:21:58 +0200 Subject: [PATCH 2/2] linux-gen: sched: remove scalable scheduler implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove unused and bug prone scalable scheduler implementation. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- .github/workflows/ci-pipeline-arm64.yml | 11 - .github/workflows/ci-pipeline.yml | 9 - platform/linux-generic/Makefile.am | 26 +- platform/linux-generic/arch/aarch64/odp_cpu.h | 8 - .../arch/aarch64/odp_wait_until.h | 100 - platform/linux-generic/arch/arm/odp_cpu.h | 8 - platform/linux-generic/arch/default/odp_cpu.h | 1 - .../arch/default/odp_wait_until.h | 53 - platform/linux-generic/include/odp_bitset.h | 93 - .../include/odp_ishmpool_internal.h | 52 - platform/linux-generic/include/odp_llqueue.h | 321 --- .../include/odp_queue_scalable_internal.h | 101 - .../linux-generic/include/odp_schedule_if.h | 1 - .../include/odp_schedule_scalable.h | 148 -- .../include/odp_schedule_scalable_config.h | 52 - .../include/odp_schedule_scalable_ordered.h | 123 - platform/linux-generic/odp_ishm.c | 5 +- platform/linux-generic/odp_ishmpool.c | 657 ----- platform/linux-generic/odp_queue_if.c | 6 - platform/linux-generic/odp_queue_scalable.c | 1199 --------- platform/linux-generic/odp_schedule_if.c | 4 - .../linux-generic/odp_schedule_scalable.c | 2210 ----------------- .../odp_schedule_scalable_ordered.c | 367 --- scripts/ci/check_inline_timer.sh | 1 - scripts/ci/coverage.sh | 3 +- 25 files changed, 7 insertions(+), 5552 deletions(-) delete mode 100644 platform/linux-generic/arch/aarch64/odp_wait_until.h delete mode 100644 platform/linux-generic/arch/default/odp_wait_until.h delete mode 100644 platform/linux-generic/include/odp_bitset.h delete mode 100644 platform/linux-generic/include/odp_ishmpool_internal.h delete mode 100644 platform/linux-generic/include/odp_llqueue.h delete mode 100644 platform/linux-generic/include/odp_queue_scalable_internal.h delete mode 100644 platform/linux-generic/include/odp_schedule_scalable.h delete mode 100644 platform/linux-generic/include/odp_schedule_scalable_config.h delete mode 100644 platform/linux-generic/include/odp_schedule_scalable_ordered.h delete mode 100644 platform/linux-generic/odp_ishmpool.c delete mode 100644 platform/linux-generic/odp_queue_scalable.c delete mode 100644 platform/linux-generic/odp_schedule_scalable.c delete mode 100644 platform/linux-generic/odp_schedule_scalable_ordered.c diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index acc3203f0c..828e5ff044 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -266,17 +266,6 @@ jobs: - if: ${{ failure() }} uses: ./.github/actions/run-failure-log - Run_scheduler_scalable: - if: ${{ github.repository == 'OpenDataPlane/odp' }} - runs-on: ah-ubuntu_22_04-c7g_2x-50 - steps: - - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v4 - - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" - -e CONF="${CONF}" -e ODP_SCHEDULER=scalable -e CI_SKIP=pktio_test_pktin_event_sched $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-graviton3 /odp/scripts/ci/check.sh - - if: ${{ failure() }} - uses: ./.github/actions/run-failure-log - Run_process_mode: if: ${{ github.repository == 'OpenDataPlane/odp' }} runs-on: ah-ubuntu_22_04-c7g_2x-50 diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 0fb19f7cd9..e74f53d291 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -348,15 +348,6 @@ jobs: - if: ${{ failure() }} uses: ./.github/actions/run-failure-log - Run_scheduler_scalable: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" - -e CONF="${CONF}" -e ODP_SCHEDULER=scalable -e CI_SKIP=pktio_test_pktin_event_sched $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - - if: ${{ failure() }} - uses: ./.github/actions/run-failure-log - Run_process_mode: runs-on: ubuntu-22.04 steps: diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index b1130d3655..e5422c1ac1 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -130,7 +130,6 @@ endif noinst_HEADERS = \ include/odp_atomic_internal.h \ - include/odp_bitset.h \ include/odp_buffer_internal.h \ include/odp_chksum_internal.h \ include/odp_classification_datamodel.h \ @@ -147,9 +146,7 @@ noinst_HEADERS = \ include/odp_init_internal.h \ include/odp_ipsec_internal.h \ include/odp_ishmphy_internal.h \ - include/odp_ishmpool_internal.h \ include/odp_libconfig_internal.h \ - include/odp_llqueue.h \ include/odp_macros_internal.h \ include/odp_name_table_internal.h \ include/odp_packet_dpdk.h \ @@ -170,7 +167,6 @@ noinst_HEADERS = \ include/odp_queue_if.h \ include/odp_queue_basic_internal.h \ include/odp_queue_lf.h \ - include/odp_queue_scalable_internal.h \ include/odp_random_std_internal.h \ include/odp_random_openssl_internal.h \ include/odp_ring_common.h \ @@ -184,9 +180,6 @@ noinst_HEADERS = \ include/odp_ring_u32_internal.h \ include/odp_ring_u64_internal.h \ include/odp_schedule_if.h \ - include/odp_schedule_scalable_config.h \ - include/odp_schedule_scalable.h \ - include/odp_schedule_scalable_ordered.h \ include/odp_shm_internal.h \ include/odp_sorted_list_internal.h \ include/odp_sysinfo_internal.h \ @@ -230,7 +223,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_ipsec_sad.c \ odp_ishm.c \ odp_ishmphy.c \ - odp_ishmpool.c \ odp_libconfig.c \ odp_ml_fp16.c \ odp_ml_quantize.c \ @@ -246,15 +238,12 @@ __LIB__libodp_linux_la_SOURCES = \ odp_queue_basic.c \ odp_queue_if.c \ odp_queue_lf.c \ - odp_queue_scalable.c \ odp_queue_spsc.c \ odp_random.c \ odp_random_std.c \ odp_random_openssl.c \ odp_schedule_basic.c \ odp_schedule_if.c \ - odp_schedule_scalable.c \ - odp_schedule_scalable_ordered.c \ odp_schedule_sp.c \ odp_shared_memory.c \ odp_sorted_list.c \ @@ -363,8 +352,7 @@ endif noinst_HEADERS += arch/arm/odp_cpu.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_random.h \ - arch/default/odp_wait_until.h + arch/default/odp_random.h endif if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ @@ -392,8 +380,7 @@ endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ arch/aarch64/cpu_flags.h \ - arch/aarch64/odp_random.h \ - arch/aarch64/odp_wait_until.h + arch/aarch64/odp_random.h endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -416,8 +403,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_random.h \ - arch/default/odp_wait_until.h + arch/default/odp_random.h endif if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -440,8 +426,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_random.h \ - arch/default/odp_wait_until.h + arch/default/odp_random.h endif if ARCH_IS_X86 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -470,8 +455,7 @@ noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_cpu.h \ arch/x86/odp_random.h \ arch/default/odp_atomic.h \ - arch/default/odp_cpu.h \ - arch/default/odp_wait_until.h + arch/default/odp_cpu.h endif if ODP_PKTIO_PCAP diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h index cd15cda2d7..86e8ca5a4f 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu.h @@ -13,13 +13,6 @@ #include #include -/* - * Use LLD/SCD atomic primitives instead of lock-based code path in llqueue - * LLD/SCD is on ARM the fastest way to enqueue and dequeue elements from a - * linked list queue. - */ -#define CONFIG_LLDSCD - /* * Use DMB;STR instead of STRL on ARM * On early ARMv8 implementations (e.g. Cortex-A57) this is noticeably more @@ -188,7 +181,6 @@ static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) } #include "odp_atomic.h" -#include "odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED #define _ODP_UNALIGNED 1 diff --git a/platform/linux-generic/arch/aarch64/odp_wait_until.h b/platform/linux-generic/arch/aarch64/odp_wait_until.h deleted file mode 100644 index eca3f9ce5e..0000000000 --- a/platform/linux-generic/arch/aarch64/odp_wait_until.h +++ /dev/null @@ -1,100 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - * Copyright (c) 2024 Nokia - */ - -#ifndef ODP_AARCH64_WAIT_UNTIL_H_ -#define ODP_AARCH64_WAIT_UNTIL_H_ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#include - -#include - -static inline void _odp_sevl(void) -{ - __asm__ volatile("sevl" : : : ); -} - -static inline int _odp_wfe(void) -{ - __asm__ volatile("wfe" : : : "memory"); - return 1; -} - -#define _odp_monitor_u8(addr, mo) ll8((addr), (mo)) -#define _odp_monitor_u32(addr, mo) ll32((addr), (mo)) -#define _odp_monitor_u64(addr, mo) ll64((addr), (mo)) -#define _odp_monitor_u128(addr, mo) lld((addr), (mo)) - -#if ATOM_BITSET_SIZE <= 32 -static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) -{ - return _odp_monitor_u32(bs, mo); -} -#elif ATOM_BITSET_SIZE <= 64 -static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) -{ - return _odp_monitor_u64(bs, mo); -} -#elif ATOM_BITSET_SIZE <= 128 -static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) -{ - return _odp_monitor_u128(bs, mo); -} -#else -#error Unsupported size of bit sets (ATOM_BITSET_SIZE) -#endif - -/** - * The _odp_wait_until_eq_*() functions defined in this header are intended to - * be used only with the scalable scheduler and queue implementations. Even - * though these functions use standard non-atomic parameter types, the - * parameters must only be operated using atomic operations. If new functions - * are added to this file, they should use _odp_wait_until_equal_*() prefix and - * atomic parameter types. - */ - -static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) -{ - _odp_sevl(); - while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_RELAXED) != expected) - odp_cpu_pause(); -} - -static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) -{ - _odp_sevl(); - while (_odp_wfe() && _odp_bitset_monitor(val, __ATOMIC_RELAXED != expected)) - odp_cpu_pause(); -} - -static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) -{ - _odp_sevl(); - while (_odp_wfe() && _odp_monitor_u8(val, __ATOMIC_ACQUIRE) != expected) - odp_cpu_pause(); -} - -static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) -{ - _odp_sevl(); - while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_ACQUIRE) != expected) - odp_cpu_pause(); -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h index f8af548009..4ced9cd40d 100644 --- a/platform/linux-generic/arch/arm/odp_cpu.h +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -12,13 +12,6 @@ #include -/* - * Use LLD/SCD atomic primitives instead of lock-based code path in llqueue - * LLD/SCD is on ARM the fastest way to enqueue and dequeue elements from a - * linked list queue. - */ -#define CONFIG_LLDSCD - /* * Use DMB;STR instead of STRL on ARM * On early ARMv8 implementations (e.g. Cortex-A57) this is noticeably more @@ -73,7 +66,6 @@ do { \ #endif /* CONFIG_DMBSTR */ #include "../default/odp_atomic.h" -#include "../default/odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED #define _ODP_UNALIGNED 1 diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h index e9d88c7918..fa6580054c 100644 --- a/platform/linux-generic/arch/default/odp_cpu.h +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -18,6 +18,5 @@ __atomic_store_n(loc, val, __ATOMIC_RELEASE) #include "odp_atomic.h" -#include "odp_wait_until.h" #endif diff --git a/platform/linux-generic/arch/default/odp_wait_until.h b/platform/linux-generic/arch/default/odp_wait_until.h deleted file mode 100644 index c51f4355e5..0000000000 --- a/platform/linux-generic/arch/default/odp_wait_until.h +++ /dev/null @@ -1,53 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2024 Nokia - */ - -#ifndef ODP_DEFAULT_WAIT_UNTIL_H_ -#define ODP_DEFAULT_WAIT_UNTIL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#include - -/** - * The _odp_wait_until_eq_*() functions defined in this header are intended to - * be used only with the scalable scheduler and queue implementations. Even - * though these functions use standard non-atomic parameter types, the - * parameters must only be operated using atomic operations. If new functions - * are added to this file, they should use _odp_wait_until_equal_*() prefix and - * atomic parameter types. - */ - -static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) -{ - while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) - odp_cpu_pause(); -} - -static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) -{ - while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) - odp_cpu_pause(); -} - -static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) -{ - while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) - odp_cpu_pause(); -} - -static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) -{ - while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) - odp_cpu_pause(); -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/include/odp_bitset.h b/platform/linux-generic/include/odp_bitset.h deleted file mode 100644 index 92212330bc..0000000000 --- a/platform/linux-generic/include/odp_bitset.h +++ /dev/null @@ -1,93 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#ifndef _ODP_BITSET_H_ -#define _ODP_BITSET_H_ - -#include - -#include - -/****************************************************************************** - * bitset abstract data type - *****************************************************************************/ -/* This could be a struct of scalars to support larger bit sets */ - -/* - * Size of atomic bit set. This limits the max number of threads, - * scheduler groups and reorder windows. On ARMv8/64-bit and x86-64, the - * (lock-free) max is 128 - */ - -#if ATOM_BITSET_SIZE <= 32 - -/* Return first-bit-set with StdC ffs() semantics */ -static inline uint32_t bitset_ffs(bitset_t b) -{ - return __builtin_ffsl(b); -} - -#elif ATOM_BITSET_SIZE <= 64 - -/* Return first-bit-set with StdC ffs() semantics */ -static inline uint32_t bitset_ffs(bitset_t b) -{ - return __builtin_ffsll(b); -} - -#elif ATOM_BITSET_SIZE <= 128 - -/* Return first-bit-set with StdC ffs() semantics */ -static inline uint32_t bitset_ffs(bitset_t b) -{ - if ((uint64_t)b != 0) - return __builtin_ffsll((uint64_t)b); - else if ((b >> 64) != 0) - return __builtin_ffsll((uint64_t)(b >> 64)) + 64; - else - return 0; -} - -#else -#error Unsupported size of bit sets (ATOM_BITSET_SIZE) -#endif - -/* Return a & ~b */ -static inline bitset_t bitset_andn(bitset_t a, bitset_t b) -{ - return a & ~b; -} - -static inline bool bitset_is_eql(bitset_t a, bitset_t b) -{ - return a == b; -} - -static inline bitset_t bitset_clr(bitset_t bs, uint32_t bit) -{ - return bs & ~bitset_mask(bit); -} - -static inline bitset_t bitset_set(bitset_t bs, uint32_t bit) -{ - return bs | bitset_mask(bit); -} - -static inline bitset_t bitset_null(void) -{ - return 0U; -} - -static inline bool bitset_is_null(bitset_t a) -{ - return a == 0U; -} - -static inline bool bitset_is_set(bitset_t a, uint32_t bit) -{ - return (a & bitset_mask(bit)) != 0; -} - -#endif diff --git a/platform/linux-generic/include/odp_ishmpool_internal.h b/platform/linux-generic/include/odp_ishmpool_internal.h deleted file mode 100644 index 4c1e071f59..0000000000 --- a/platform/linux-generic/include/odp_ishmpool_internal.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017-2018 Linaro Limited - */ - -#ifndef ODP_ISHMBUDDY_INTERNAL_H_ -#define ODP_ISHMBUDDY_INTERNAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -typedef struct _odp_ishm_pool_ctrl_t { - uint32_t element_sz; /* 0 for buddy pools, >0 for slab. */ - int ishm_blk_idx; /* the block index returned by _ishm_resrve()*/ - odp_spinlock_t lock; /* for pool access mutex */ - void *user_addr; /* user pool area ('real user pool') */ - union { - struct { /* things needed for buddy pools: */ - uint8_t order; /* pool is 2^order bytes long */ - uint8_t min_order; /*alloc won't go below 2^min_order*/ - void **free_heads; /* 'order' free list heads. */ - uint8_t *alloced_order; /* size of blocks, 0=free */ - }; - struct { /* things needed for slab pools: */ - void *free_head; /* free element list head */ - uint64_t nb_elem;/* total number of elements in pool */ - }; - }; -} _odp_ishm_pool_ctrl_t; - -typedef struct _odp_ishm_pool_t { - _odp_ishm_pool_ctrl_t ctrl; /* control part */ - uint8_t mem[1]; /* area for heads, saved alloc'd orders, data*/ -} _odp_ishm_pool_t; - -_odp_ishm_pool_t *_odp_ishm_pool_create(const char *pool_name, - uint64_t size, - uint64_t min_alloc, - uint64_t max_alloc, int flags); -int _odp_ishm_pool_destroy(_odp_ishm_pool_t *pool); -void *_odp_ishm_pool_alloc(_odp_ishm_pool_t *pool, uint64_t size); -int _odp_ishm_pool_free(_odp_ishm_pool_t *pool, void *addr); -void _odp_ishm_pool_init(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h deleted file mode 100644 index 40e2b2dda5..0000000000 --- a/platform/linux-generic/include/odp_llqueue.h +++ /dev/null @@ -1,321 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#ifndef ODP_LLQUEUE_H_ -#define ODP_LLQUEUE_H_ - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -/****************************************************************************** - * Linked list queues - *****************************************************************************/ - -struct llqueue; -struct llnode; - -static struct llnode *llq_head(struct llqueue *llq); -static void llqueue_init(struct llqueue *llq); -static void llq_enqueue(struct llqueue *llq, struct llnode *node); -static struct llnode *llq_dequeue(struct llqueue *llq); -static odp_bool_t llq_dequeue_cond(struct llqueue *llq, struct llnode *exp); -static odp_bool_t llq_cond_rotate(struct llqueue *llq, struct llnode *node); -static odp_bool_t llq_on_queue(struct llnode *node); - -/****************************************************************************** - * The implementation(s) - *****************************************************************************/ - -#define SENTINEL ((void *)~(uintptr_t)0) -#define MAX_SPIN_COUNT 1000 - -#ifdef CONFIG_LLDSCD -/* Implement queue operations using double-word LL/SC */ - -/* The scalar equivalent of a double pointer */ -#if __SIZEOF_PTRDIFF_T__ == 4 -typedef uint64_t dintptr_t; -#endif -#if __SIZEOF_PTRDIFF_T__ == 8 -typedef _odp_u128_t dintptr_t; -#endif - -struct llnode { - struct llnode *next; -}; - -union llht { - struct { - struct llnode *head, *tail; - } st; - dintptr_t ui; -}; - -struct llqueue { - union llht u; -}; - -static inline struct llnode *llq_head(struct llqueue *llq) -{ - return __atomic_load_n(&llq->u.st.head, __ATOMIC_RELAXED); -} - -static inline void llqueue_init(struct llqueue *llq) -{ - llq->u.st.head = NULL; - llq->u.st.tail = NULL; -} - -static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) -{ - union llht old, neu; - - _ODP_ASSERT(node->next == NULL); - node->next = SENTINEL; - do { - old.ui = lld(&llq->u.ui, __ATOMIC_RELAXED); - neu.st.head = old.st.head == NULL ? node : old.st.head; - neu.st.tail = node; - } while (odp_unlikely(scd(&llq->u.ui, neu.ui, __ATOMIC_RELEASE))); - if (old.st.tail != NULL) { - /* List was not empty */ - _ODP_ASSERT(old.st.tail->next == SENTINEL); - old.st.tail->next = node; - } -} - -static inline struct llnode *llq_dequeue(struct llqueue *llq) -{ - struct llnode *head; - union llht old, neu; - - /* llq_dequeue() may be used in a busy-waiting fashion - * Read head using plain load to avoid disturbing remote LL/SC - */ - head = __atomic_load_n(&llq->u.st.head, __ATOMIC_ACQUIRE); - if (head == NULL) - return NULL; - /* Read head->next before LL to minimize cache miss latency - * in LL/SC below - */ - (void)__atomic_load_n(&head->next, __ATOMIC_RELAXED); - - do { -restart_loop: - old.ui = lld(&llq->u.ui, __ATOMIC_RELAXED); - if (odp_unlikely(old.st.head == NULL)) { - /* Empty list */ - return NULL; - } else if (odp_unlikely(old.st.head == old.st.tail)) { - /* Single-element in list */ - neu.st.head = NULL; - neu.st.tail = NULL; - } else { - /* Multi-element list, dequeue head */ - struct llnode *next; - int spin_count = 0; - - /* Wait until llq_enqueue() has written true next - * pointer - */ - while ((next = __atomic_load_n(&old.st.head->next, - __ATOMIC_RELAXED)) == - SENTINEL) { - odp_cpu_pause(); - if (++spin_count >= MAX_SPIN_COUNT) - goto restart_loop; - } - neu.st.head = next; - neu.st.tail = old.st.tail; - } - } while (odp_unlikely(scd(&llq->u.ui, neu.ui, __ATOMIC_RELAXED))); - old.st.head->next = NULL; - return old.st.head; -} - -static inline odp_bool_t llq_dequeue_cond(struct llqueue *llq, - struct llnode *exp) -{ - union llht old, neu; - - do { -restart_loop: - old.ui = lld(&llq->u.ui, __ATOMIC_ACQUIRE); - if (odp_unlikely(old.st.head == NULL || old.st.head != exp)) { - /* Empty list or wrong head */ - return false; - } else if (odp_unlikely(old.st.head == old.st.tail)) { - /* Single-element in list */ - neu.st.head = NULL; - neu.st.tail = NULL; - } else { - /* Multi-element list, dequeue head */ - struct llnode *next; - int spin_count = 0; - - /* Wait until llq_enqueue() has written true next - * pointer */ - while ((next = __atomic_load_n(&old.st.head->next, - __ATOMIC_RELAXED)) == - SENTINEL) { - odp_cpu_pause(); - if (++spin_count >= MAX_SPIN_COUNT) - goto restart_loop; - } - - neu.st.head = next; - neu.st.tail = old.st.tail; - } - } while (odp_unlikely(scd(&llq->u.ui, neu.ui, __ATOMIC_RELAXED))); - old.st.head->next = NULL; - return true; -} - -/* If 'node' is a head of llq then move it to tail */ -static inline odp_bool_t llq_cond_rotate(struct llqueue *llq, - struct llnode *node) -{ - /* Difficult to make this into a single atomic operation - * Instead use existing primitives. - */ - if (odp_likely(llq_dequeue_cond(llq, node))) { - llq_enqueue(llq, node); - return true; - } - return false; -} - -static inline odp_bool_t llq_on_queue(struct llnode *node) -{ - return node->next != NULL; -} - -#else -/* Implement queue operations protected by a spin lock */ - -struct llnode { - struct llnode *next; -}; - -struct llqueue { - struct llnode *head, *tail; - odp_spinlock_t lock; -}; - -static inline struct llnode *llq_head(struct llqueue *llq) -{ - return __atomic_load_n(&llq->head, __ATOMIC_RELAXED); -} - -static inline void llqueue_init(struct llqueue *llq) -{ - llq->head = NULL; - llq->tail = NULL; - odp_spinlock_init(&llq->lock); -} - -static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) -{ - _ODP_ASSERT(node->next == NULL); - node->next = SENTINEL; - - odp_spinlock_lock(&llq->lock); - if (llq->head == NULL) { - llq->head = node; - llq->tail = node; - } else { - llq->tail->next = node; - llq->tail = node; - } - odp_spinlock_unlock(&llq->lock); -} - -static inline struct llnode *llq_dequeue(struct llqueue *llq) -{ - struct llnode *head; - struct llnode *node = NULL; - - head = __atomic_load_n(&llq->head, __ATOMIC_RELAXED); - if (head == NULL) - return NULL; - - odp_spinlock_lock(&llq->lock); - if (llq->head != NULL) { - node = llq->head; - if (llq->head == llq->tail) { - _ODP_ASSERT(node->next == SENTINEL); - llq->head = NULL; - llq->tail = NULL; - } else { - _ODP_ASSERT(node->next != SENTINEL); - llq->head = node->next; - } - node->next = NULL; - } - odp_spinlock_unlock(&llq->lock); - return node; -} - -static inline odp_bool_t llq_dequeue_cond(struct llqueue *llq, - struct llnode *node) -{ - odp_bool_t success = false; - - odp_spinlock_lock(&llq->lock); - if (odp_likely(llq->head != NULL && llq->head == node)) { - success = true; - if (llq->head == llq->tail) { - _ODP_ASSERT(node->next == SENTINEL); - llq->head = NULL; - llq->tail = NULL; - } else { - _ODP_ASSERT(node->next != SENTINEL); - llq->head = node->next; - } - node->next = NULL; - } - odp_spinlock_unlock(&llq->lock); - return success; -} - -/* If 'node' is a head of llq then move it to tail */ -static inline odp_bool_t llq_cond_rotate(struct llqueue *llq, - struct llnode *node) -{ - odp_bool_t success = false; - - odp_spinlock_lock(&llq->lock); - if (odp_likely(llq->head == node)) { - success = true; - if (llq->tail != node) { - _ODP_ASSERT(node->next != SENTINEL); - llq->head = node->next; - llq->tail->next = node; - llq->tail = node; - node->next = SENTINEL; - } - /* Else 'node' is only element on list => nothing to do */ - } - odp_spinlock_unlock(&llq->lock); - return success; -} - -static inline odp_bool_t llq_on_queue(struct llnode *node) -{ - return node->next != NULL; -} - -#endif - -#endif diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h deleted file mode 100644 index 7ba668f5f2..0000000000 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ /dev/null @@ -1,101 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#ifndef ODP_QUEUE_SCALABLE_INTERNAL_H_ -#define ODP_QUEUE_SCALABLE_INTERNAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define QUEUE_STATUS_FREE 0 -#define QUEUE_STATUS_DESTROYED 1 -#define QUEUE_STATUS_READY 2 - -struct ODP_ALIGNED_CACHE queue_entry_s { - sched_elem_t sched_elem; - - odp_ticketlock_t lock ODP_ALIGNED_CACHE; - odp_atomic_u64_t num_timers; - int status; - - queue_enq_fn_t enqueue ODP_ALIGNED_CACHE; - queue_deq_fn_t dequeue; - queue_enq_multi_fn_t enqueue_multi; - queue_deq_multi_fn_t dequeue_multi; - queue_deq_multi_fn_t orig_dequeue_multi; - - uint32_t index; - odp_queue_t handle; - odp_queue_type_t type; - odp_queue_param_t param; - odp_pktin_queue_t pktin; - odp_pktout_queue_t pktout; - char name[ODP_QUEUE_NAME_LEN]; -}; - -int _odp_queue_deq(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num); -int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num); -int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num); -int _odp_queue_enq_sp(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num); -queue_entry_t *_odp_qentry_from_ext(odp_queue_t handle); - -/* Round up memory size to next cache line size to - * align all memory addresses on cache line boundary. - */ -static inline void *shm_pool_alloc_align(_odp_ishm_pool_t *pool, uint32_t size) -{ - void *addr; - - addr = _odp_ishm_pool_alloc(pool, _ODP_ROUNDUP_CACHE_LINE(size)); - _ODP_ASSERT(((uintptr_t)addr & (ODP_CACHE_LINE_SIZE - 1)) == 0); - - return addr; -} - -static inline uint32_t queue_to_id(odp_queue_t handle) -{ - return _odp_qentry_from_ext(handle)->index; -} - -static inline queue_entry_t *qentry_from_int(odp_queue_t handle) -{ - return (queue_entry_t *)(uintptr_t)handle; -} - -static inline odp_queue_t qentry_to_int(queue_entry_t *qentry) -{ - return (odp_queue_t)qentry; -} - -static inline odp_queue_t queue_get_handle(queue_entry_t *queue) -{ - return queue->handle; -} - -static inline reorder_window_t *queue_get_rwin(queue_entry_t *queue) -{ - return queue->sched_elem.rwin; -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 17d12600da..e4c90ea396 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -19,7 +19,6 @@ extern "C" { #define _ODP_SCHED_ID_BASIC 0 #define _ODP_SCHED_ID_SP 1 -#define _ODP_SCHED_ID_SCALABLE 2 /* Scheduler identifier */ extern int _odp_sched_id; diff --git a/platform/linux-generic/include/odp_schedule_scalable.h b/platform/linux-generic/include/odp_schedule_scalable.h deleted file mode 100644 index a2db327dfc..0000000000 --- a/platform/linux-generic/include/odp_schedule_scalable.h +++ /dev/null @@ -1,148 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#ifndef ODP_SCHEDULE_SCALABLE_H -#define ODP_SCHEDULE_SCALABLE_H - -#include -#include -#include - -#include -#include -#include -#include - -/* - * Define scalable scheduler internal maximum priority count - * ODP_SCHED_PRIO_NUM as it is not compile-time constant at API - * level. The current API for this is odp_schedule_num_prio(). - * The other schedulers also define this internally as NUM_PRIO. - * - * One additional priority level for idle pktin queues. - * This is only for internal use and not visible to the user. - */ -#define ODP_SCHED_PRIO_PKTIN 8 -#define ODP_SCHED_PRIO_NUM 9 - -typedef struct ODP_ALIGNED_CACHE { - union { - struct { - struct llqueue llq; - uint32_t prio; - }; - char line[ODP_CACHE_LINE_SIZE]; - }; -} sched_queue_t; - -#define TICKET_INVALID (uint16_t)(~0U) - -typedef struct ODP_ALIGNED(sizeof(uint64_t)) { - int32_t numevts; - uint16_t wrr_budget; - uint8_t cur_ticket; - uint8_t nxt_ticket; -} qschedstate_t; - -typedef uint32_t ringidx_t; - -#ifdef CONFIG_SPLIT_PRODCONS -#define SPLIT_PC ODP_ALIGNED_CACHE -#else -#define SPLIT_PC -#endif - -#define ODP_NO_SCHED_QUEUE (ODP_SCHED_SYNC_ORDERED + 1) - -typedef struct ODP_ALIGNED_CACHE { - struct llnode node; - sched_queue_t *schedq; -#ifdef CONFIG_QSCHST_LOCK - odp_ticketlock_t qschlock; -#endif - qschedstate_t qschst; - uint8_t pop_deficit; - uint8_t qschst_type; - uint8_t pktio_idx; - uint8_t rx_queue; - uint16_t xoffset; - uint8_t sched_prio; - ringidx_t prod_read SPLIT_PC; - ringidx_t prod_write; - ringidx_t prod_mask; - _odp_event_hdr_t **prod_ring; - ringidx_t cons_write SPLIT_PC; - ringidx_t cons_read; - reorder_window_t *rwin; - void *user_ctx; -#ifdef CONFIG_SPLIT_PRODCONS - _odp_event_hdr_t **cons_ring; - ringidx_t cons_mask; - uint16_t cons_type; -#else -#define cons_mask prod_mask -#define cons_ring prod_ring -#define cons_type qschst_type -#endif - odp_schedule_group_t sched_grp; - uint32_t loop_check[CONFIG_NUM_CPU_IDS]; -} sched_elem_t; - -/* Number of scheduling groups */ -#define MAX_SCHED_GROUP (sizeof(sched_group_mask_t) * CHAR_BIT) - -typedef bitset_t sched_group_mask_t; - -typedef struct { - /* Threads currently associated with the sched group */ - bitset_t thr_actual[ODP_SCHED_PRIO_NUM] ODP_ALIGNED_CACHE; - bitset_t thr_wanted; - /* Used to spread queues over schedq's */ - uint32_t xcount[ODP_SCHED_PRIO_NUM]; - /* Number of schedq's per prio */ - uint32_t xfactor; - char name[ODP_SCHED_GROUP_NAME_LEN]; - /* ODP_SCHED_PRIO_NUM * xfactor. Must be last. */ - sched_queue_t schedq[1] ODP_ALIGNED_CACHE; -} sched_group_t; - -/* Number of reorder contexts per thread */ -#define TS_RVEC_SIZE 16 - -typedef struct ODP_ALIGNED_CACHE { - /* Atomic queue currently being processed or NULL */ - sched_elem_t *atomq; - /* Schedq the currently processed queue was popped from */ - sched_queue_t *src_schedq; - /* Current reorder context or NULL */ - reorder_context_t *rctx; - uint8_t pause; - uint8_t out_of_order; - uint8_t tidx; - uint8_t pad; - uint32_t dequeued; /* Number of events dequeued from atomic queue */ - uint16_t ticket; /* Ticket for atomic queue or TICKET_INVALID */ - uint16_t num_schedq; - uint16_t sg_sem; /* Set when sg_wanted is modified by other thread */ -#define SCHEDQ_PER_THREAD (MAX_SCHED_GROUP * ODP_SCHED_PRIO_NUM) - sched_queue_t *schedq_list[SCHEDQ_PER_THREAD]; - /* Current sched_group membership */ - sched_group_mask_t sg_actual[ODP_SCHED_PRIO_NUM]; - /* Future sched_group membership. */ - sched_group_mask_t sg_wanted[ODP_SCHED_PRIO_NUM]; - bitset_t priv_rvec_free; - /* Bitset of free entries in rvec[] */ - bitset_t rvec_free ODP_ALIGNED_CACHE; - /* Reordering contexts to allocate from */ - reorder_context_t rvec[TS_RVEC_SIZE] ODP_ALIGNED_CACHE; - uint32_t loop_cnt; /*Counter to check pktio ingress queue dead loop */ -} sched_scalable_thread_state_t; - -void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual); -void _odp_sched_update_enq_sp(sched_elem_t *q, uint32_t actual); -sched_queue_t *_odp_sched_queue_add(odp_schedule_group_t grp, uint32_t prio); -void _odp_sched_queue_rem(odp_schedule_group_t grp, uint32_t prio); - -#endif /* ODP_SCHEDULE_SCALABLE_H */ diff --git a/platform/linux-generic/include/odp_schedule_scalable_config.h b/platform/linux-generic/include/odp_schedule_scalable_config.h deleted file mode 100644 index 1133d49aae..0000000000 --- a/platform/linux-generic/include/odp_schedule_scalable_config.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#ifndef ODP_SCHEDULE_SCALABLE_CONFIG_H_ -#define ODP_SCHEDULE_SCALABLE_CONFIG_H_ - -/* Maximum number of events that can be stored in a queue */ -#define CONFIG_SCAL_QUEUE_SIZE 4096 - -/* - * Default scaling factor for the scheduler group - * - * This scaling factor is used when the application creates a scheduler - * group with no worker threads. - */ -#define CONFIG_DEFAULT_XFACTOR 4 - -/* - * Default weight (in events) for WRR in scalable scheduler - * - * This controls the per-queue weight for WRR between queues of the same - * priority in the scalable scheduler - * A higher value improves throughput while a lower value increases fairness - * and thus likely decreases latency - * - * If WRR is undesired, set the value to ~0 which will use the largest possible - * weight - * - * Note: an API for specifying this on a per-queue basis would be useful but is - * not yet available - */ -#define CONFIG_WRR_WEIGHT 64 - -/* - * Split queue producer/consumer metadata into separate cache lines. - * This is beneficial on e.g. Cortex-A57 but not so much on A53. - */ -#define CONFIG_SPLIT_PRODCONS - -/* - * Use locks to protect queue (ring buffer) and scheduler state updates - * On x86, this decreases overhead noticeably. - */ -#if !defined(__arm__) && !defined(__aarch64__) -#define CONFIG_QSCHST_LOCK -/* Keep all ring buffer/qschst data together when using locks */ -#undef CONFIG_SPLIT_PRODCONS -#endif - -#endif /* ODP_SCHEDULE_SCALABLE_CONFIG_H_ */ diff --git a/platform/linux-generic/include/odp_schedule_scalable_ordered.h b/platform/linux-generic/include/odp_schedule_scalable_ordered.h deleted file mode 100644 index ad2dfc8eef..0000000000 --- a/platform/linux-generic/include/odp_schedule_scalable_ordered.h +++ /dev/null @@ -1,123 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#ifndef ODP_SCHEDULE_SCALABLE_ORDERED_H -#define ODP_SCHEDULE_SCALABLE_ORDERED_H - -#include -#include - -#include -#include -#include -#include - -/* High level functioning of reordering - * Datastructures - - * Reorder Window - Every ordered queue is associated with a reorder window. - * Reorder window stores reorder contexts from threads that - * have completed processing out-of-order. - * Reorder Context - Reorder context consists of events that a thread - * wants to enqueue while processing a batch of events - * from an ordered queue. - * - * Algorithm - - * 1) Thread identifies the ordered queue. - * 2) It 'reserves a slot in the reorder window and dequeues the - * events' atomically. Atomicity is achieved by using a ticket-lock - * like design where the reorder window slot is the ticket. - * 3a) Upon order-release/next schedule call, the thread - * checks if it's slot (ticket) equals the head of the reorder window. - * If yes, enqueues the events to the destination queue till - * i) the reorder window is empty or - * ii) there is a gap in the reorder window - * If no, the reorder context is stored in the reorder window at - * the reserved slot. - * 3b) Upon the first enqueue, the thread checks if it's slot (ticket) - * equals the head of the reorder window. - * If yes, enqueues the events immediately to the destination queue - * If no, these (and subsequent) events are stored in the reorder context - * (in the application given order) - */ - -/* Head and change indicator variables are used to synchronise between - * concurrent insert operations in the reorder window. A thread performing - * an in-order insertion must be notified about the newly inserted - * reorder contexts so that it doesn’t halt the retire process too early. - * A thread performing an out-of-order insertion must correspondingly - * notify the thread doing in-order insertion of the new waiting reorder - * context, which may need to be handled by that thread. - * - * Also, an out-of-order insertion may become an in-order insertion if the - * thread doing an in-order insertion completes before this thread completes. - * We need a point of synchronisation where this knowledge and potential state - * change can be transferred between threads. - */ -typedef struct ODP_ALIGNED(sizeof(uint64_t)) hc { - /* First missing context */ - uint32_t head; - /* Change indicator */ - uint32_t chgi; -} hc_t; - -/* Number of reorder contects in the reorder window. - * Should be at least one per CPU. - */ -#define RWIN_SIZE 32 -ODP_STATIC_ASSERT(_ODP_CHECK_IS_POWER2(RWIN_SIZE), "RWIN_SIZE is not a power of 2"); - -typedef struct reorder_context reorder_context_t; - -typedef struct reorder_window { - /* head and change indicator */ - hc_t hc; - uint32_t winmask; - uint32_t tail; - uint32_t turn; - uint32_t olock[CONFIG_QUEUE_MAX_ORD_LOCKS]; - uint32_t lock_count; - /* Reorder contexts in this window */ - reorder_context_t *ring[RWIN_SIZE]; -} reorder_window_t; - -/* Number of events that can be stored in a reorder context. - * This size is chosen so that there is no space left unused at the end - * of the last cache line (for 64b architectures and 64b handles). - */ -#define RC_EVT_SIZE 18 - -struct ODP_ALIGNED_CACHE reorder_context { - /* Reorder window to which this context belongs */ - reorder_window_t *rwin; - /* Pointer to TS->rvec_free */ - bitset_t *rvec_free; - /* Our slot number in the reorder window */ - uint32_t sn; - uint8_t olock_flags; - /* Our index in thread_state rvec array */ - uint8_t idx; - /* Use to link reorder contexts together */ - uint8_t next_idx; - /* Current reorder context to save events in */ - uint8_t cur_idx; - /* Number of events stored in this reorder context */ - uint8_t numevts; - /* Events stored in this context */ - _odp_event_hdr_t *events[RC_EVT_SIZE]; - queue_entry_t *destq[RC_EVT_SIZE]; -}; - -reorder_window_t *_odp_rwin_alloc(_odp_ishm_pool_t *pool, - unsigned int lock_count); -int _odp_rwin_free(_odp_ishm_pool_t *pool, reorder_window_t *rwin); -bool _odp_rwin_reserve(reorder_window_t *rwin, uint32_t *sn); -bool _odp_rwin_reserve_sc(reorder_window_t *rwin, uint32_t *sn); -void _odp_rwin_unreserve_sc(reorder_window_t *rwin, uint32_t sn); -void _odp_rctx_init(reorder_context_t *rctx, uint16_t idx, - reorder_window_t *rwin, uint32_t sn); -void _odp_rctx_release(reorder_context_t *rctx); -int _odp_rctx_save(queue_entry_t *queue, _odp_event_hdr_t *event_hdr[], int num); - -#endif /* ODP_SCHEDULE_SCALABLE_ORDERED_H */ diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 84663078c3..ad160ac8de 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -45,9 +45,9 @@ #include #include #include -#include #include #include + #include #include #include @@ -1787,9 +1787,6 @@ int _odp_ishm_init_global(const odp_init_t *init) goto init_glob_err4; } - /* get ready to create pools: */ - _odp_ishm_pool_init(); - /* init cache files */ hp_init(); diff --git a/platform/linux-generic/odp_ishmpool.c b/platform/linux-generic/odp_ishmpool.c deleted file mode 100644 index fb2cf43c4f..0000000000 --- a/platform/linux-generic/odp_ishmpool.c +++ /dev/null @@ -1,657 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017-2018 Linaro Limited - */ - -/* This file gathers the buddy and slab allocation functionality provided - * by _ishm. - * _odp_ishmpool_create() can be used to create a pool for buddy/slab - * allocation. _odp_ishmpool_create() will allocate a memory area using - * ishm_reserve() for both the control part (needed for tracking - * allocation/free...) and the user memory itself (part of which will be given - * at each ishmpool_alloc()). - * The element size provided at pool creation time determines whether - * to pool will of type buddy or slab. - * For buddy, all allocations are rounded to the nearest power of 2. - * - * The implementation of the buddy allocator is very traditional: it - * maintains N lists of free buffers. - * The control part actually contains these N queue heads, (N-M are actually - * used), the free buffers themselves being used for chaining (the chaining info - * is in the buffers: as they are "free" they should not be touched by the - * user). The control part also contains a array of bytes for remembering - * the size (actually the order) of the allocated buffers: - * There are 2^(N-M) such bytes, this number being the maximum number of - * allocated buffers (when all allocation are <= 2^M bytes) - * Buddy allocators handle fragmentation by splitting or merging blocks by 2. - * They guarantee a minimum efficiency of 50%, at worse case fragmentation. - * - * Slab implementation is even simpler, all free elements being queued in - * one single queue at init, taken from this queue when allocated and - * returned to this same queue when freed. - * - * The reason for not using malloc() is that malloc does not guarantee - * memory sharability between ODP threads (regardless of their implementation) - * which ishm_reserve() can do. see the comments around - * _odp_ishmbud_pool_create() and ishm_reserve() for more details. - * - * This file is divided in 3 sections: the first one regroups functions - * needed by the buddy allocation. - * The second one regroups the functions needed by the slab allocator. - * The third section regroups the common functions exported externally. - */ - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#define BUDDY_MIN_SIZE 32 /* minimal buddy allocation size */ - -typedef _odp_ishm_pool_t pool_t; /* for shorter writing */ - -/* array of ishm block index used for pools. only used for pool - * lookup by name */ -#define MAX_NB_POOL 100 -static int pool_blk_idx[MAX_NB_POOL]; - -/* section 1: functions for buddy allocation: */ - -/* free buddy blocks contains the following structure, used to link the - * free blocks together. - */ -typedef struct bblock_t { - struct bblock_t *next; - uint32_t order; -} bblock_t; - -/* value set in the 'order' table when the block is not allocated: */ -#define BBLOCK_FREE 0 - -/* compute ceil(log2(size)) */ -static uint8_t clog2(uint64_t size) -{ - uint64_t sz; - uint32_t bit; - uint8_t res; - - sz = size; /* we start by computing res = log2(sz)... */ - res = 0; - for (bit = 32; bit ; bit >>= 1) { - if (sz >= ((uint64_t)1 << bit)) { - sz >>= bit; - res += bit; - } - } - if (((uint64_t)1 << res) < size) /* ...and then ceil(x) */ - res++; - - return res; -} - -/* - * given a bblock address, and an order value, returns the address - * of the buddy bblock (the other "half") - */ -static inline bblock_t *get_bblock_buddy(pool_t *bpool, bblock_t *addr, - uint8_t order) -{ - uintptr_t b; - - b = ((uintptr_t)addr - (uintptr_t)bpool->ctrl.user_addr); - b ^= 1 << order; - return (void *)(b + (uintptr_t)bpool->ctrl.user_addr); -} - -/* - * given a buddy block address, return its number (used for busy flags): - */ -static inline uintptr_t get_bblock_nr(pool_t *bpool, void *addr) -{ - uintptr_t b; - uint8_t min_order; - - min_order = bpool->ctrl.min_order; - b = ((uintptr_t)addr - (uintptr_t)bpool->ctrl.user_addr) >> min_order; - return b; -} - -/* remove bblock from the list for bblocks of rank order. The bblock to be - * removed is really expected to be on the list: not finding it is an error */ -static inline void remove_from_list(pool_t *bpool, uint8_t order, - bblock_t *bblock) -{ - bblock_t *curr; - bblock_t *prev = NULL; - - curr = bpool->ctrl.free_heads[order]; - if (!curr) - goto remove_from_list_error; - - if (curr == bblock) { - bpool->ctrl.free_heads[order] = curr->next; - return; - } - - while (curr) { - if (curr == bblock) { - prev->next = curr->next; - return; - } - prev = curr; - curr = curr->next; - } - -remove_from_list_error: - _ODP_ERR("List corrupted\n"); -} - -/* - * create a buddy memory pool of given size (actually nearest power of 2), - * where allocation will never be smaller than min_alloc. - * returns a pointer to the created buddy_pool - * The allocated area contains: - * - The _odp_ishm_pool_ctrl_t structure - * - The array of ((order - min_order) of free list heads - * - The array of 'order' values, remembering sizes of allocated bblocks - * - alignment to cache line - * - The user memory - */ -static pool_t *_odp_ishmbud_pool_create(const char *pool_name, int store_idx, - uint64_t size, - uint64_t min_alloc, int flags) -{ - uint8_t order; /* pool order = ceil(log2(size)) */ - uint8_t min_order; /* pool min_order = ceil(log2(min_alloc))*/ - uint32_t max_nb_bblock; /* max number of bblock, when smallest */ - uint32_t control_sz; /* size of control area */ - uint32_t free_head_sz; /* mem area needed for list heads */ - uint32_t saved_order_sz; /* mem area to remember given sizes */ - uint64_t user_sz; /* 2^order bytes */ - uint64_t total_sz; /* total size to request */ - int blk_idx; /* as returned by _ishm_resrve() */ - pool_t *bpool; - int i; - bblock_t *first_block; - - /* a bblock_t must fit in the buffers for linked chain! */ - if (min_alloc < sizeof(bblock_t)) - min_alloc = sizeof(bblock_t); - - /* pool order is such that 2^order = size. same for min_order */ - order = clog2(size); - min_order = clog2(min_alloc); - - /* check parameters obvious wishes: */ - if (order >= 64) - return NULL; - if (order < min_order) - return NULL; - - /* at worst case, all bblocks have smallest (2^min_order) size */ - max_nb_bblock = (1 << (order - min_order)); - - /* space needed for the control area (padded to cache line size)*/ - control_sz = _ODP_ROUNDUP_CACHE_LINE(sizeof(_odp_ishm_pool_ctrl_t)); - - /* space needed for 'order' free bblock list heads: */ - /* Note that only lists from min_order to order are really used.*/ - free_head_sz = _ODP_ROUNDUP_CACHE_LINE(sizeof(void *) * (order + 1)); - - /* space needed for order -i.e. size- storage of alloc'd bblock:*/ - saved_order_sz = _ODP_ROUNDUP_CACHE_LINE(max_nb_bblock * sizeof(uint8_t)); - - /* space needed for user area is 2^order bytes: */ - user_sz = 1ULL << order; - - total_sz = control_sz + - free_head_sz + - saved_order_sz + - user_sz; - - /* allocate required memory: */ - blk_idx = _odp_ishm_reserve(pool_name, total_sz, -1, - ODP_CACHE_LINE_SIZE, 0, flags, 0); - if (blk_idx < 0) { - _ODP_ERR("_odp_ishm_reserve failed."); - return NULL; - } - - bpool = _odp_ishm_address(blk_idx); - if (bpool == NULL) { - _ODP_ERR("_odp_ishm_address failed."); - return NULL; - } - - /* store in pool array (needed for look up): */ - pool_blk_idx[store_idx] = blk_idx; - - /* remember block index, needed when pool is destroyed */ - bpool->ctrl.ishm_blk_idx = blk_idx; - - /* remember element size: 0 means unknown size, i.e. buddy alloation*/ - bpool->ctrl.element_sz = 0; - - /* prepare mutex: */ - odp_spinlock_init(&bpool->ctrl.lock); - - /* initialise pointers and things... */ - bpool->ctrl.order = order; - bpool->ctrl.min_order = min_order; - bpool->ctrl.free_heads = - (void *)((uintptr_t)bpool + control_sz); - bpool->ctrl.alloced_order = - (uint8_t *)((uintptr_t)bpool->ctrl.free_heads + free_head_sz); - bpool->ctrl.user_addr = - (void *)((uintptr_t)bpool->ctrl.alloced_order + saved_order_sz); - - /* initialize all free list to NULL, except the top biggest element:*/ - for (i = 0; i < (order - min_order); i++) - bpool->ctrl.free_heads[i] = NULL; - bpool->ctrl.free_heads[order] = bpool->ctrl.user_addr; - first_block = (bblock_t *)bpool->ctrl.user_addr; - first_block->next = NULL; - first_block->order = order; - - /* set all 'order' of allocated bblocks to free: */ - memset(bpool->ctrl.alloced_order, BBLOCK_FREE, saved_order_sz); - - return bpool; -} - -/* allocated memory from the given buddy pool */ -static void *_odp_ishmbud_alloc(pool_t *bpool, uint64_t size) -{ - uint32_t rq_order; /* requested order */ - uint32_t try_order; - bblock_t *bblock; - bblock_t *buddy; - uintptr_t nr; - - /* if size is zero or too big reject: */ - if ((!size) && (size > (1ULL << bpool->ctrl.order))) { - _ODP_ERR("Invalid alloc size (0 or larger than whole pool)\n"); - return NULL; - } - - /* compute ceil(log2(size)), to get the requested block order: */ - rq_order = clog2(size); - - /* make sure the requested order is bigger (or same) as minimum! */ - if (rq_order < bpool->ctrl.min_order) - rq_order = bpool->ctrl.min_order; - - /* mutex from here: */ - odp_spinlock_lock(&bpool->ctrl.lock); - - /* now, start trying to allocate a bblock of rq_order. If that - * fails keep trying larger orders until pool order is reached */ - bblock = NULL; - for (try_order = rq_order; try_order <= bpool->ctrl.order; - try_order++) { - if (bpool->ctrl.free_heads[try_order]) { - /* remove from list: */ - bblock = - (bblock_t *)(bpool->ctrl.free_heads[try_order]); - bpool->ctrl.free_heads[try_order] = bblock->next; - break; - } - } - - if (!bblock) { - odp_spinlock_unlock(&bpool->ctrl.lock); - _ODP_ERR("Out of memory. (Buddy pool full)\n"); - return NULL; - } - - /* OK: we got a block, but possibbly too large (if try_order>rq_order) - * return the extra halves to the pool hence splitting the bblock at - * each 'extra' order: */ - while (try_order-- > rq_order) { - /* split: */ - buddy = (bblock_t *)((uintptr_t)bblock + (1 << try_order)); - buddy->order = try_order; - /* add to list: */ - buddy->next = bpool->ctrl.free_heads[try_order]; - bpool->ctrl.free_heads[try_order] = buddy; - /* mark as free (non allocated block get size 0): */ - nr = get_bblock_nr(bpool, buddy); - bpool->ctrl.alloced_order[nr] = BBLOCK_FREE; - } - - /* remember the size if the allocated block: */ - nr = get_bblock_nr(bpool, bblock); - bpool->ctrl.alloced_order[nr] = rq_order; - - /* and return the allocated block! */ - odp_spinlock_unlock(&bpool->ctrl.lock); - return (void *)bblock; -} - -/* free a previously allocated buffer from a given buddy pool */ -static int _odp_ishmbud_free(pool_t *bpool, void *addr) -{ - uintptr_t user_start; /* start of user area */ - uintptr_t user_stop; /* stop of user area */ - uintptr_t mask; /* 2^min_order - 1 */ - bblock_t *bblock; /* bblock being freed */ - bblock_t *buddy; /* buddy bblock of bblock being freed */ - uint8_t order; /* order of block being freed */ - uintptr_t nr; /* block number */ - - /* freeing NULL is regarded as OK, though without any effect: */ - if (!addr) - return 0; - - user_start = (uintptr_t)bpool->ctrl.user_addr; - user_stop = user_start + ((uintptr_t)1 << bpool->ctrl.order); - mask = ((uintptr_t)1 << bpool->ctrl.min_order) - 1; - - /* some sanity checks: check that given address is within pool and - * that relative address has 2^min_order granularity: */ - if (((uintptr_t)addr < user_start) || - ((uintptr_t)addr > user_stop) || - (((uintptr_t)addr - user_start) & mask)) { - _ODP_ERR("Invalid address to be freed\n"); - return -1; - } - - /* mutex from here: */ - odp_spinlock_lock(&bpool->ctrl.lock); - - /* collect saved block order and make sure bblock was allocated */ - bblock = (bblock_t *)addr; - nr = get_bblock_nr(bpool, bblock); - order = bpool->ctrl.alloced_order[nr]; - if (order == BBLOCK_FREE) { - _ODP_ERR("Double free error\n"); - odp_spinlock_unlock(&bpool->ctrl.lock); - return -1; - } - - /* this looks like a valid free, mark at least this as free: */ - bpool->ctrl.alloced_order[nr] = BBLOCK_FREE; - - /* go up in orders, trying to merge buddies... */ - while (order < bpool->ctrl.order) { - buddy = get_bblock_buddy(bpool, bblock, order); - /*if buddy is not free: no further merge possible */ - nr = get_bblock_nr(bpool, buddy); - if (bpool->ctrl.alloced_order[nr] != BBLOCK_FREE) - break; - /*merge only bblock of same order:*/ - if (buddy->order != order) - break; - /*merge: remove buddy from free list: */ - remove_from_list(bpool, order, buddy); - /*merge: make sure we point at start of block: */ - if (bblock > buddy) - bblock = buddy; - /*merge: size of block has doubled: increase order: */ - order++; - } - - /* insert the bblock into its correct free block list: */ - bblock->next = bpool->ctrl.free_heads[order]; - bpool->ctrl.free_heads[order] = bblock; - - /* remember the (possibly now merged) block order: */ - bblock->order = order; - - odp_spinlock_unlock(&bpool->ctrl.lock); - return 0; -} - -/* section 2: functions for slab allocation: */ - -/* free slab blocks contains the following structure, used to link the - * free blocks together. - */ -typedef struct sblock_t { - struct sblock_t *next; -} sblock_t; - -/* - * create a slab memory pool of given size (rounded up to the nearest integer - * number of element, where each element has size 'elt_size'). - * returns a pointer to the created slab pool. - * The allocated area contains: - * - The _odp_ishm_pool_ctrl_t structure - * - alignment to cache line - * - The user memory - */ -static pool_t *_odp_ishmslab_pool_create(const char *pool_name, int store_idx, - uint64_t size, - uint64_t elt_size, int flags) -{ - uint32_t nb_sblock; /* number of elements in the pool */ - uint32_t control_sz; /* size of control area */ - uint64_t total_sz; /* total size to request */ - uint64_t user_sz; /* 2^order bytes */ - int blk_idx; /* as returned by _ishm_reserve() */ - pool_t *spool; - unsigned int i; - sblock_t *block; - - /* a sblock_t must fit in the buffers for linked chain! */ - if (elt_size < sizeof(bblock_t)) { - elt_size = sizeof(bblock_t); - size = size * (sizeof(bblock_t) / elt_size + - ((sizeof(bblock_t) % elt_size) ? 1 : 0)); - } - - /* nb of element fitting in the pool is just ceil(size/elt_size)*/ - nb_sblock = (size / elt_size) + ((size % elt_size) ? 1 : 0); - - /* space needed for the control area (padded to cache line size)*/ - control_sz = _ODP_ROUNDUP_CACHE_LINE(sizeof(_odp_ishm_pool_ctrl_t)); - - /* space needed for user area is : */ - user_sz = nb_sblock * elt_size; - - total_sz = control_sz + - user_sz; - - /* allocate required memory: */ - blk_idx = _odp_ishm_reserve(pool_name, total_sz, -1, - ODP_CACHE_LINE_SIZE, 0, flags, 0); - if (blk_idx < 0) { - _ODP_ERR("_odp_ishm_reserve failed."); - return NULL; - } - - spool = _odp_ishm_address(blk_idx); - if (spool == NULL) { - _ODP_ERR("_odp_ishm_address failed."); - return NULL; - } - - /* store in pool array (needed for look up): */ - pool_blk_idx[store_idx] = blk_idx; - - /* remember block index, needed when pool is destroyed */ - spool->ctrl.ishm_blk_idx = blk_idx; - - /* remember element (sblock) size and their number: */ - spool->ctrl.element_sz = elt_size; - spool->ctrl.nb_elem = nb_sblock; - - /* prepare mutex: */ - odp_spinlock_init(&spool->ctrl.lock); - - /* initialise pointers and things... */ - spool->ctrl.user_addr = - (void *)((uintptr_t)spool + control_sz); - - /* initialise the free list with the list of all elements:*/ - spool->ctrl.free_head = spool->ctrl.user_addr; - for (i = 0; i < nb_sblock - 1; i++) { - block = (sblock_t *)((uintptr_t)spool->ctrl.user_addr + - i * (uintptr_t)elt_size); - block->next = (sblock_t *)((uintptr_t)block + - (uintptr_t)elt_size); - } - block = (sblock_t *)((uintptr_t)spool->ctrl.user_addr + - (nb_sblock - 1) * (uintptr_t)elt_size); - block->next = NULL; - - return spool; -} - -/* allocated memory from the given slab pool */ -static void *_odp_ishmslab_alloc(pool_t *spool, uint64_t size) -{ - void *ret; - sblock_t *block; - - if (size > spool->ctrl.element_sz) - return NULL; - - odp_spinlock_lock(&spool->ctrl.lock); - ret = spool->ctrl.free_head; - if (!ret) { - odp_spinlock_unlock(&spool->ctrl.lock); - _ODP_ERR("Out of memory. (Slab pool full)\n"); - return NULL; - } - - block = (sblock_t *)ret; - spool->ctrl.free_head = block->next; - - odp_spinlock_unlock(&spool->ctrl.lock); - return ret; -} - -/* free a previously allocated buffer from a given slab pool */ -static int _odp_ishmslab_free(pool_t *spool, void *addr) -{ - uintptr_t user_start; /* start of user area */ - uintptr_t user_stop; /* stop of user area */ - sblock_t *block; - - /* freeing NULL is regarded as OK, though without any effect: */ - if (!addr) - return 0; - - user_start = (uintptr_t)spool->ctrl.user_addr; - user_stop = user_start + spool->ctrl.element_sz * spool->ctrl.nb_elem; - - /* some sanity checks: check that given address is within pool and - * that relative address has element_sz granularity: */ - if (((uintptr_t)addr < user_start) || - ((uintptr_t)addr > user_stop) || - (((uintptr_t)addr - user_start) % spool->ctrl.element_sz)) { - _ODP_ERR("Invalid address to be freed\n"); - return -1; - } - - odp_spinlock_lock(&spool->ctrl.lock); - block = (sblock_t *)addr; - block->next = (sblock_t *)spool->ctrl.free_head; - spool->ctrl.free_head = addr; - odp_spinlock_unlock(&spool->ctrl.lock); - - return 0; -} - -/* section 3: common, external functions: */ - -/* create a pool: either with fixed alloc size (if max_alloc/min_alloc<2) or - * of variable block size (if max_alloc == 0) */ -pool_t *_odp_ishm_pool_create(const char *pool_name, uint64_t size, - uint64_t min_alloc, uint64_t max_alloc, int flags) -{ - int store_idx; - uint64_t real_pool_sz; - - if (min_alloc > max_alloc) { - _ODP_ERR("invalid parameter: min_alloc > max_alloc"); - return NULL; - } - - /* search for a free index in pool_blk_idx for the pool */ - for (store_idx = 0; store_idx < MAX_NB_POOL; store_idx++) { - if (pool_blk_idx[store_idx] < 0) - break; - } - if (store_idx == MAX_NB_POOL) { - _ODP_ERR("Max number of pool reached (MAX_NB_POOL)"); - return NULL; - } - - if ((min_alloc == 0) || ((max_alloc / min_alloc) > 2)) { - /* alloc variation is not constant enough: we go for a buddy - * allocator. The pool efficiency may go as low as 50% - * so we double the required size to make sure we can satisfy - * the user request */ - real_pool_sz = 2 * size; - return _odp_ishmbud_pool_create(pool_name, store_idx, - real_pool_sz, - BUDDY_MIN_SIZE, flags); - } else { - /* min and max are close enough so we go for constant size - * allocator: - * make sure the pool can fit the required size, even when - * only min_alloc allocation are performed: */ - real_pool_sz = ((size / min_alloc) + - ((size % min_alloc) ? 1 : 0)) - * max_alloc; - return _odp_ishmslab_pool_create(pool_name, store_idx, - real_pool_sz, - max_alloc, flags); - } -} - -/* destroy a pool. everything goes away. no operation on the pool should - * follow. */ -int _odp_ishm_pool_destroy(pool_t *pool) -{ - int store_idx; - - for (store_idx = 0; store_idx < MAX_NB_POOL; store_idx++) { - if (pool_blk_idx[store_idx] == pool->ctrl.ishm_blk_idx) { - pool_blk_idx[store_idx] = -1; - break; - } - } - - return _odp_ishm_free_by_index(pool->ctrl.ishm_blk_idx); -} - -/* allocated a buffer from a pool */ -void *_odp_ishm_pool_alloc(_odp_ishm_pool_t *pool, uint64_t size) -{ - if (!pool->ctrl.element_sz) - return _odp_ishmbud_alloc(pool, size); - else - return _odp_ishmslab_alloc(pool, size); -} - -/* free a previously allocated buffer from a pool */ -int _odp_ishm_pool_free(_odp_ishm_pool_t *pool, void *addr) -{ - if (!pool->ctrl.element_sz) - return _odp_ishmbud_free(pool, addr); - else - return _odp_ishmslab_free(pool, addr); -} - -void _odp_ishm_pool_init(void) -{ - int i; - - for (i = 0; i < MAX_NB_POOL; i++) - pool_blk_idx[i] = -1; -} diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c index 248e7743a7..1fdf724646 100644 --- a/platform/linux-generic/odp_queue_if.c +++ b/platform/linux-generic/odp_queue_if.c @@ -22,9 +22,6 @@ const _odp_queue_api_fn_t *_odp_queue_api; #include -extern const _odp_queue_api_fn_t _odp_queue_scalable_api; -extern const queue_fn_t _odp_queue_scalable_fn; - extern const _odp_queue_api_fn_t _odp_queue_basic_api; extern const queue_fn_t _odp_queue_basic_fn; @@ -127,9 +124,6 @@ int _odp_queue_init_global(void) if (!strcmp(sched, "basic") || !strcmp(sched, "sp")) { _odp_queue_fn = &_odp_queue_basic_fn; _odp_queue_api = &_odp_queue_basic_api; - } else if (!strcmp(sched, "scalable")) { - _odp_queue_fn = &_odp_queue_scalable_fn; - _odp_queue_api = &_odp_queue_scalable_api; } else { _ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c deleted file mode 100644 index f2eb82d23c..0000000000 --- a/platform/linux-generic/odp_queue_scalable.c +++ /dev/null @@ -1,1199 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#define LOCK(a) odp_ticketlock_lock(a) -#define UNLOCK(a) odp_ticketlock_unlock(a) -#define LOCK_INIT(a) odp_ticketlock_init(a) - -extern __thread sched_scalable_thread_state_t *_odp_sched_ts; -extern _odp_queue_inline_offset_t _odp_queue_inline_offset; - -typedef struct queue_table_t { - queue_entry_t queue[CONFIG_MAX_QUEUES]; -} queue_table_t; - -static queue_table_t *queue_tbl; -static _odp_ishm_pool_t *queue_shm_pool; - -static int _queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr); -static _odp_event_hdr_t *_queue_deq(odp_queue_t handle); -static int _queue_enq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], - int num); -static int _queue_deq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], - int num); - -static queue_entry_t *get_qentry(uint32_t queue_id) -{ - return &queue_tbl->queue[queue_id]; -} - -queue_entry_t *_odp_qentry_from_ext(odp_queue_t handle) -{ - return (queue_entry_t *)(uintptr_t)handle; -} - -static int _odp_queue_disable_enq(sched_elem_t *q) -{ - ringidx_t old_read, old_write, new_write; - uint32_t size; - - old_write = q->prod_write; - size = q->prod_mask + 1; - do { - /* Need __atomic_load to avoid compiler reordering */ - old_read = __atomic_load_n(&q->prod_read, __ATOMIC_ACQUIRE); - if (old_write != old_read) { - /* Queue is not empty, cannot claim all elements - * Cannot disable enqueue. - */ - return -1; - } - /* Claim all elements in ring */ - new_write = old_write + size; - } while (!__atomic_compare_exchange_n(&q->prod_write, - &old_write, /* Updated on failure */ - new_write, - true, - __ATOMIC_RELAXED, - __ATOMIC_RELAXED)); - /* All remaining elements claimed, no one else can enqueue */ - return 0; -} - -static int queue_init(queue_entry_t *queue, const char *name, - const odp_queue_param_t *param) -{ - ringidx_t ring_idx; - sched_elem_t *sched_elem; - uint32_t ring_size; - _odp_event_hdr_t **ring; - uint32_t size; - - sched_elem = &queue->sched_elem; - ring_size = param->size > 0 ? - _ODP_ROUNDUP_POWER2_U32(param->size) : CONFIG_SCAL_QUEUE_SIZE; - _odp_strcpy(queue->name, name ? name : "", ODP_QUEUE_NAME_LEN); - memcpy(&queue->param, param, sizeof(odp_queue_param_t)); - - size = ring_size * sizeof(_odp_event_hdr_t *); - ring = (_odp_event_hdr_t **)shm_pool_alloc_align(queue_shm_pool, size); - if (NULL == ring) - return -1; - - for (ring_idx = 0; ring_idx < ring_size; ring_idx++) - ring[ring_idx] = NULL; - - queue->type = queue->param.type; - - if (queue->type == ODP_QUEUE_TYPE_SCHED) - queue->param.deq_mode = ODP_QUEUE_OP_DISABLED; - - odp_atomic_init_u64(&queue->num_timers, 0); - - queue->enqueue = _queue_enq; - queue->dequeue = _queue_deq; - queue->enqueue_multi = _queue_enq_multi; - queue->dequeue_multi = _queue_deq_multi; - queue->orig_dequeue_multi = _queue_deq_multi; - queue->pktin = PKTIN_INVALID; - - sched_elem->node.next = NULL; -#ifdef CONFIG_QSCHST_LOCK - LOCK_INIT(&sched_elem->qschlock); -#endif - sched_elem->qschst.numevts = 0; - sched_elem->qschst.wrr_budget = CONFIG_WRR_WEIGHT; - sched_elem->qschst.cur_ticket = 0; - sched_elem->qschst.nxt_ticket = 0; - sched_elem->pop_deficit = 0; - if (queue->type == ODP_QUEUE_TYPE_SCHED) - sched_elem->qschst_type = queue->param.sched.sync; - else - sched_elem->qschst_type = ODP_NO_SCHED_QUEUE; - /* 2nd cache line - enqueue */ - sched_elem->prod_read = 0; - sched_elem->prod_write = 0; - sched_elem->prod_ring = ring; - sched_elem->prod_mask = ring_size - 1; - /* 3rd cache line - dequeue */ - sched_elem->cons_read = 0; - sched_elem->cons_write = 0; - sched_elem->rwin = NULL; - sched_elem->schedq = NULL; - sched_elem->user_ctx = queue->param.context; -#ifdef CONFIG_SPLIT_PRODCONS - sched_elem->cons_ring = ring; - sched_elem->cons_mask = ring_size - 1; - sched_elem->cons_type = sched_elem->qschst_type; -#endif - - /* Queue initialized successfully, add it to the sched group */ - if (queue->type == ODP_QUEUE_TYPE_SCHED) { - int prio = odp_schedule_max_prio() - param->sched.prio; - - if (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED) { - sched_elem->rwin = - _odp_rwin_alloc(queue_shm_pool, - queue->param.sched.lock_count); - if (sched_elem->rwin == NULL) { - _ODP_ERR("Reorder window not created\n"); - goto rwin_create_failed; - } - } - sched_elem->sched_grp = param->sched.group; - sched_elem->sched_prio = prio; - sched_elem->schedq = - _odp_sched_queue_add(param->sched.group, prio); - _ODP_ASSERT(sched_elem->schedq != NULL); - } - - return 0; - -rwin_create_failed: - _odp_ishm_pool_free(queue_shm_pool, ring); - - return -1; -} - -static int queue_init_global(void) -{ - uint32_t i; - uint64_t pool_size; - uint64_t min_alloc; - uint64_t max_alloc; - - _ODP_DBG("Queue init ... "); - - /* Fill in queue entry field offsets for inline functions */ - memset(&_odp_queue_inline_offset, 0, - sizeof(_odp_queue_inline_offset_t)); - _odp_queue_inline_offset.context = offsetof(queue_entry_t, - param.context); - - /* Create shared memory pool to allocate shared memory for the - * queues. Use the default queue size. - */ - /* Add size of the array holding the queues */ - pool_size = sizeof(queue_table_t); - /* Add storage required for queues */ - pool_size += (CONFIG_SCAL_QUEUE_SIZE * - sizeof(_odp_event_hdr_t *)) * CONFIG_MAX_QUEUES; - - /* Add the reorder window size */ - pool_size += sizeof(reorder_window_t) * CONFIG_MAX_QUEUES; - /* Choose min_alloc and max_alloc such that buddy allocator is selected. */ - min_alloc = 0; - max_alloc = CONFIG_SCAL_QUEUE_SIZE * sizeof(_odp_event_hdr_t *); - queue_shm_pool = _odp_ishm_pool_create("queue_shm_pool", - pool_size, - min_alloc, max_alloc, 0); - if (queue_shm_pool == NULL) { - _ODP_ERR("Failed to allocate shared memory pool for" - " queues\n"); - goto queue_shm_pool_create_failed; - } - - queue_tbl = (queue_table_t *) - shm_pool_alloc_align(queue_shm_pool, - sizeof(queue_table_t)); - if (queue_tbl == NULL) { - _ODP_ERR("Failed to reserve shared memory for queue table\n"); - goto queue_tbl_ishm_alloc_failed; - } - - memset(queue_tbl, 0, sizeof(queue_table_t)); - - for (i = 0; i < CONFIG_MAX_QUEUES; i++) { - /* init locks */ - queue_entry_t *queue; - - queue = get_qentry(i); - LOCK_INIT(&queue->lock); - queue->index = i; - queue->handle = (odp_queue_t)queue; - } - - _ODP_DBG("done\n"); - _ODP_DBG("Queue init global\n"); - _ODP_DBG(" struct queue_entry_s size %zu\n", sizeof(struct queue_entry_s)); - _ODP_DBG(" queue_entry_t size %zu\n", sizeof(queue_entry_t)); - _ODP_DBG("\n"); - - return 0; - -queue_shm_pool_create_failed: - -queue_tbl_ishm_alloc_failed: - _odp_ishm_pool_destroy(queue_shm_pool); - - return -1; -} - -static int queue_term_global(void) -{ - int ret = 0; - int rc = 0; - queue_entry_t *queue; - int i; - - for (i = 0; i < CONFIG_MAX_QUEUES; i++) { - queue = &queue_tbl->queue[i]; - if (__atomic_load_n(&queue->status, - __ATOMIC_RELAXED) != QUEUE_STATUS_FREE) { - _ODP_ERR("Not destroyed queue: %s\n", queue->name); - rc = -1; - } - } - - _odp_ishm_pool_free(queue_shm_pool, queue_tbl); - - ret = _odp_ishm_pool_destroy(queue_shm_pool); - if (ret < 0) { - _ODP_ERR("Failed to destroy shared memory pool for queues\n"); - rc = -1; - } - - return rc; -} - -static int queue_init_local(void) -{ - return 0; -} - -static int queue_term_local(void) -{ - return 0; -} - -static int queue_capability(odp_queue_capability_t *capa) -{ - memset(capa, 0, sizeof(odp_queue_capability_t)); - - /* Reserve some queues for internal use */ - capa->max_queues = CONFIG_MAX_QUEUES - CONFIG_INTERNAL_QUEUES; - - capa->plain.max_num = CONFIG_MAX_PLAIN_QUEUES; - capa->plain.max_size = 0; - - return 0; -} - -static odp_queue_type_t queue_type(odp_queue_t handle) -{ - return _odp_qentry_from_ext(handle)->type; -} - -static odp_schedule_sync_t queue_sched_type(odp_queue_t handle) -{ - return _odp_qentry_from_ext(handle)->param.sched.sync; -} - -static odp_schedule_prio_t queue_sched_prio(odp_queue_t handle) -{ - return _odp_qentry_from_ext(handle)->param.sched.prio; -} - -static odp_schedule_group_t queue_sched_group(odp_queue_t handle) -{ - return _odp_qentry_from_ext(handle)->param.sched.group; -} - -static uint32_t queue_lock_count(odp_queue_t handle) -{ - queue_entry_t *queue = _odp_qentry_from_ext(handle); - - return queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? - queue->param.sched.lock_count : 0; -} - -static odp_queue_t queue_create(const char *name, - const odp_queue_param_t *param) -{ - int queue_idx; - int max_idx; - queue_entry_t *queue; - odp_queue_type_t type; - odp_queue_param_t default_param; - odp_queue_t handle = ODP_QUEUE_INVALID; - - if (param == NULL) { - odp_queue_param_init(&default_param); - param = &default_param; - } - - type = param->type; - - if (type == ODP_QUEUE_TYPE_SCHED) { - if (param->sched.prio < odp_schedule_min_prio() || - param->sched.prio > odp_schedule_max_prio()) { - _ODP_ERR("Bad queue priority: %i\n", param->sched.prio); - return ODP_QUEUE_INVALID; - } - } - - if (type == ODP_QUEUE_TYPE_SCHED) { - /* Start scheduled queue indices from zero to enable direct - * mapping to scheduler implementation indices. */ - queue_idx = 0; - max_idx = CONFIG_MAX_SCHED_QUEUES; - } else { - queue_idx = CONFIG_MAX_SCHED_QUEUES; - /* All internal queues are of type plain */ - max_idx = CONFIG_MAX_QUEUES; - } - - for (; queue_idx < max_idx; queue_idx++) { - queue = &queue_tbl->queue[queue_idx]; - - if (queue->status != QUEUE_STATUS_FREE) - continue; - - LOCK(&queue->lock); - if (queue->status == QUEUE_STATUS_FREE) { - if (queue_init(queue, name, param)) { - UNLOCK(&queue->lock); - return handle; - } - queue->status = QUEUE_STATUS_READY; - handle = queue->handle; - UNLOCK(&queue->lock); - break; - } - UNLOCK(&queue->lock); - } - return handle; -} - -static int queue_create_multi(const char *name[], const odp_queue_param_t param[], - odp_bool_t share_param, odp_queue_t queue[], int num) -{ - int i; - - _ODP_ASSERT(param != NULL); - _ODP_ASSERT(queue != NULL); - _ODP_ASSERT(num > 0); - - for (i = 0; i < num; i++) { - odp_queue_t cur_queue; - const char *cur_name = name != NULL ? name[i] : NULL; - const odp_queue_param_t *cur_param = share_param ? ¶m[0] : ¶m[i]; - - cur_queue = queue_create(cur_name, cur_param); - if (cur_queue == ODP_QUEUE_INVALID) - return (i == 0) ? -1 : i; - - queue[i] = cur_queue; - } - return i; -} - -static int queue_destroy(odp_queue_t handle) -{ - queue_entry_t *queue; - sched_elem_t *q; - - if (handle == ODP_QUEUE_INVALID) - return -1; - - queue = _odp_qentry_from_ext(handle); - LOCK(&queue->lock); - if (queue->status != QUEUE_STATUS_READY) { - UNLOCK(&queue->lock); - return -1; - } - q = &queue->sched_elem; - -#ifdef CONFIG_QSCHST_LOCK - LOCK(&q->qschlock); -#endif - if (_odp_queue_disable_enq(q)) { - /* Producer side not empty */ -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&q->qschlock); -#endif - UNLOCK(&queue->lock); - return -1; - } - /* Enqueue is now disabled */ - if (q->cons_read != q->cons_write) { - /* Consumer side is not empty - * Roll back previous change, enable enqueue again. - */ - uint32_t size; - - size = q->prod_mask + 1; - __atomic_fetch_sub(&q->prod_write, size, __ATOMIC_RELAXED); -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&q->qschlock); -#endif - UNLOCK(&queue->lock); - return -1; - } -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&q->qschlock); -#endif - /* Producer and consumer sides empty, enqueue disabled - * Now wait until schedq state is empty and no outstanding tickets - */ - while (__atomic_load_n(&q->qschst.numevts, __ATOMIC_RELAXED) != 0 || - __atomic_load_n(&q->qschst.cur_ticket, __ATOMIC_RELAXED) != - __atomic_load_n(&q->qschst.nxt_ticket, __ATOMIC_RELAXED)) - _odp_wait_until_eq_u32((uint32_t *)&q->qschst.numevts, 0); - - if (q->schedq != NULL) { - _odp_sched_queue_rem(q->sched_grp, q->sched_prio); - q->schedq = NULL; - } - - _odp_ishm_pool_free(queue_shm_pool, q->prod_ring); - - if (q->rwin != NULL) { - if (_odp_rwin_free(queue_shm_pool, q->rwin) < 0) { - _ODP_ERR("Failed to free reorder window\n"); - UNLOCK(&queue->lock); - return -1; - } - q->rwin = NULL; - } - queue->status = QUEUE_STATUS_FREE; - UNLOCK(&queue->lock); - return 0; -} - -static int queue_destroy_multi(odp_queue_t handle[], int num) -{ - int i; - - _ODP_ASSERT(handle != NULL); - _ODP_ASSERT(num > 0); - - for (i = 0; i < num; i++) { - int ret = queue_destroy(handle[i]); - - if (ret) - return (i == 0) ? ret : i; - } - - return i; -} - -static int queue_context_set(odp_queue_t handle, void *context, - uint32_t len ODP_UNUSED) -{ - odp_mb_full(); - _odp_qentry_from_ext(handle)->param.context = context; - odp_mb_full(); - return 0; -} - -static odp_queue_t queue_lookup(const char *name) -{ - uint32_t i; - - for (i = 0; i < CONFIG_MAX_QUEUES; i++) { - queue_entry_t *queue = &queue_tbl->queue[i]; - - if (queue->status == QUEUE_STATUS_FREE || - queue->status == QUEUE_STATUS_DESTROYED) - continue; - - LOCK(&queue->lock); - if (strcmp(name, queue->name) == 0) { - /* found it */ - UNLOCK(&queue->lock); - return queue->handle; - } - UNLOCK(&queue->lock); - } - - return ODP_QUEUE_INVALID; -} - -#ifndef CONFIG_QSCHST_LOCK -static inline int _odp_queue_enq(sched_elem_t *q, - _odp_event_hdr_t *event_hdr[], - int num) -{ - ringidx_t old_read; - ringidx_t old_write; - ringidx_t new_write; - int actual; - uint32_t mask; - _odp_event_hdr_t **ring; - - mask = q->prod_mask; - ring = q->prod_ring; - - /* Load producer ring state (read & write index) */ - old_write = __atomic_load_n(&q->prod_write, __ATOMIC_RELAXED); - do { - /* Consumer does store-release prod_read, we need - * load-acquire. - */ - old_read = __atomic_load_n(&q->prod_read, __ATOMIC_ACQUIRE); - - actual = _ODP_MIN(num, (int)((mask + 1) - (old_write - old_read))); - if (odp_unlikely(actual <= 0)) - return 0; - - new_write = old_write + actual; - } while (!__atomic_compare_exchange_n(&q->prod_write, - &old_write, /* Updated on failure */ - new_write, - true, - __ATOMIC_RELAXED, - __ATOMIC_RELAXED)); - -#ifdef CONFIG_SPLIT_PRODCONS - __builtin_prefetch(&q->cons_write, 0, 0); -#endif - /* Store our event(s) in the ring */ - do { - ring[old_write & mask] = *event_hdr++; - } while (++old_write != new_write); - old_write -= actual; - -#ifdef CONFIG_SPLIT_PRODCONS - __builtin_prefetch(&q->node, 1, 0); -#endif - /* Wait for our turn to signal consumers */ - if (odp_unlikely(__atomic_load_n(&q->cons_write, __ATOMIC_RELAXED) != old_write)) - _odp_wait_until_eq_u32(&q->cons_write, old_write); - - /* Signal consumers that events are available (release events) - * Enable other producers to continue - */ - /* Wait for writes (to ring slots) to complete */ - atomic_store_release(&q->cons_write, new_write, /*readonly=*/false); - - return actual; -} - -#endif - -int _odp_queue_enq_sp(sched_elem_t *q, - _odp_event_hdr_t *event_hdr[], - int num) -{ - ringidx_t old_read; - ringidx_t old_write; - ringidx_t new_write; - int actual; - uint32_t mask; - _odp_event_hdr_t **ring; - - mask = q->prod_mask; - ring = q->prod_ring; - - /* Load producer ring state (read & write index) */ - old_write = q->prod_write; - /* Consumer does store-release prod_read, we need load-acquire */ - old_read = __atomic_load_n(&q->prod_read, __ATOMIC_ACQUIRE); - actual = _ODP_MIN(num, (int)((mask + 1) - (old_write - old_read))); - if (odp_unlikely(actual <= 0)) - return 0; - - new_write = old_write + actual; - q->prod_write = new_write; - - /* Store our event(s) in the ring */ - do { - ring[old_write & mask] = *event_hdr++; - } while (++old_write != new_write); - old_write -= actual; - -#ifdef CONFIG_SPLIT_PRODCONS - __builtin_prefetch(&q->node, 1, 0); -#endif - - /* Signal consumers that events are available (release events) - * Enable other producers to continue - */ -#ifdef CONFIG_QSCHST_LOCK - q->cons_write = new_write; -#else - atomic_store_release(&q->cons_write, new_write, /*readonly=*/false); -#endif - - return actual; -} - -static int _queue_enq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], - int num) -{ - int actual; - queue_entry_t *queue; - sched_scalable_thread_state_t *ts; - - queue = qentry_from_int(handle); - ts = _odp_sched_ts; - if (ts && odp_unlikely(ts->out_of_order) && - (queue->param.order == ODP_QUEUE_ORDER_KEEP)) { - actual = _odp_rctx_save(queue, event_hdr, num); - return actual; - } - -#ifdef CONFIG_QSCHST_LOCK - LOCK(&queue->sched_elem.qschlock); - actual = _odp_queue_enq_sp(&queue->sched_elem, event_hdr, num); -#else - actual = _odp_queue_enq(&queue->sched_elem, event_hdr, num); -#endif - - if (odp_likely(queue->sched_elem.schedq != NULL && actual != 0)) { - /* Perform scheduler related updates. */ -#ifdef CONFIG_QSCHST_LOCK - _odp_sched_update_enq_sp(&queue->sched_elem, actual); -#else - _odp_sched_update_enq(&queue->sched_elem, actual); -#endif - } - -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&queue->sched_elem.qschlock); -#endif - return actual; -} - -static int _queue_enq(odp_queue_t handle, _odp_event_hdr_t *event_hdr) -{ - return odp_likely(_queue_enq_multi(handle, &event_hdr, 1) == 1) ? 0 : -1; -} - -static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) -{ - _odp_event_hdr_t *event_hdr[QUEUE_MULTI_MAX]; - queue_entry_t *queue; - int i; - - if (num > QUEUE_MULTI_MAX) - num = QUEUE_MULTI_MAX; - - queue = _odp_qentry_from_ext(handle); - - for (i = 0; i < num; i++) - event_hdr[i] = _odp_event_hdr(ev[i]); - - return queue->enqueue_multi(handle, event_hdr, num); -} - -static int queue_enq(odp_queue_t handle, odp_event_t ev) -{ - _odp_event_hdr_t *event_hdr; - queue_entry_t *queue; - - queue = _odp_qentry_from_ext(handle); - event_hdr = _odp_event_hdr(ev); - - return queue->enqueue(handle, event_hdr); -} - -/* Single-consumer dequeue. */ -int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) -{ - int actual; - ringidx_t old_read; - ringidx_t old_write; - ringidx_t new_read; - uint32_t mask; - _odp_event_hdr_t **ring; - - /* Load consumer ring state (read & write index). */ - old_read = q->cons_read; - /* Producer does store-release cons_write, we need load-acquire */ - old_write = __atomic_load_n(&q->cons_write, __ATOMIC_ACQUIRE); - actual = _ODP_MIN(num, (int)(old_write - old_read)); - - if (odp_unlikely(actual <= 0)) - return 0; - -#ifdef CONFIG_SPLIT_PRODCONS - __builtin_prefetch(&q->node, 1, 0); -#endif - new_read = old_read + actual; - q->cons_read = new_read; - - mask = q->cons_mask; - ring = q->cons_ring; - do { - *evp++ = _odp_event_from_hdr(ring[old_read & mask]); - } while (++old_read != new_read); - - /* Signal producers that empty slots are available - * (release ring slots). Enable other consumers to continue. - */ -#ifdef CONFIG_QSCHST_LOCK - q->prod_read = new_read; -#else - /* Wait for loads (from ring slots) to complete. */ - atomic_store_release(&q->prod_read, new_read, /*readonly=*/true); -#endif - return actual; -} - -int _odp_queue_deq(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num) -{ - int actual; - ringidx_t old_read; - ringidx_t old_write; - ringidx_t new_read; - uint32_t mask; - _odp_event_hdr_t **ring; - _odp_event_hdr_t **p_event_hdr; - - mask = q->cons_mask; - ring = q->cons_ring; - - /* Load consumer ring state (read & write index) */ - old_read = __atomic_load_n(&q->cons_read, __ATOMIC_RELAXED); - do { - /* Need __atomic_load to avoid compiler reordering - * Producer does store-release cons_write, we need - * load-acquire. - */ - old_write = __atomic_load_n(&q->cons_write, __ATOMIC_ACQUIRE); - /* Prefetch ring buffer array */ - __builtin_prefetch(&q->cons_ring[old_read & mask], 0, 0); - - actual = _ODP_MIN(num, (int)(old_write - old_read)); - if (odp_unlikely(actual <= 0)) - return 0; - - /* Attempt to free ring slot(s) */ - new_read = old_read + actual; - } while (!__atomic_compare_exchange_n(&q->cons_read, - &old_read, /* Updated on failure */ - new_read, - true, - __ATOMIC_RELAXED, - __ATOMIC_RELAXED)); -#ifdef CONFIG_SPLIT_PRODCONS - __builtin_prefetch(&q->prod_read, 0, 0); -#endif - p_event_hdr = event_hdr; - do { - *p_event_hdr++ = ring[old_read & mask]; - } while (++old_read != new_read); - old_read -= actual; - -#ifdef CONFIG_SPLIT_PRODCONS - __builtin_prefetch(&q->node, 1, 0); -#endif - /* Wait for our turn to signal producers */ - if (odp_unlikely(__atomic_load_n(&q->prod_read, __ATOMIC_RELAXED) != old_read)) - _odp_wait_until_eq_u32(&q->prod_read, old_read); - - /* Signal producers that empty slots are available - * (release ring slots) - * Enable other consumers to continue - */ - /* Wait for loads (from ring slots) to complete */ - atomic_store_release(&q->prod_read, new_read, /*readonly=*/true); - - return actual; -} - -int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) -{ - int ret, evt_idx; - _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; - - if (num > QUEUE_MULTI_MAX) - num = QUEUE_MULTI_MAX; - - ret = _odp_queue_deq(q, hdr_tbl, num); - if (odp_likely(ret != 0)) { - for (evt_idx = 0; evt_idx < num; evt_idx++) - evp[evt_idx] = _odp_event_from_hdr(hdr_tbl[evt_idx]); - } - - return ret; -} - -static int _queue_deq_multi(odp_queue_t handle, _odp_event_hdr_t *event_hdr[], - int num) -{ - sched_elem_t *q; - queue_entry_t *queue; - - queue = qentry_from_int(handle); - q = &queue->sched_elem; - return _odp_queue_deq(q, event_hdr, num); -} - -static _odp_event_hdr_t *_queue_deq(odp_queue_t handle) -{ - sched_elem_t *q; - _odp_event_hdr_t *event_hdr; - queue_entry_t *queue; - - queue = qentry_from_int(handle); - q = &queue->sched_elem; - if (_odp_queue_deq(q, &event_hdr, 1) == 1) - return event_hdr; - else - return NULL; -} - -static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) -{ - queue_entry_t *queue; - int ret; - - if (num > QUEUE_MULTI_MAX) - num = QUEUE_MULTI_MAX; - - queue = _odp_qentry_from_ext(handle); - - ret = queue->dequeue_multi(handle, (_odp_event_hdr_t **)ev, num); - - if (odp_global_rw->inline_timers && - odp_atomic_load_u64(&queue->num_timers)) - timer_run(ret ? 2 : 1); - - return ret; -} - -static odp_event_t queue_deq(odp_queue_t handle) -{ - queue_entry_t *queue = _odp_qentry_from_ext(handle); - odp_event_t ev = (odp_event_t)queue->dequeue(handle); - - if (odp_global_rw->inline_timers && - odp_atomic_load_u64(&queue->num_timers)) - timer_run(ev != ODP_EVENT_INVALID ? 2 : 1); - - return ev; -} - -static void queue_param_init(odp_queue_param_t *params) -{ - memset(params, 0, sizeof(odp_queue_param_t)); - params->type = ODP_QUEUE_TYPE_PLAIN; - params->enq_mode = ODP_QUEUE_OP_MT; - params->deq_mode = ODP_QUEUE_OP_MT; - params->nonblocking = ODP_BLOCKING; - params->sched.prio = odp_schedule_default_prio(); - params->sched.sync = ODP_SCHED_SYNC_PARALLEL; - params->sched.group = ODP_SCHED_GROUP_ALL; - params->order = ODP_QUEUE_ORDER_KEEP; -} - -static int queue_info(odp_queue_t handle, odp_queue_info_t *info) -{ - uint32_t queue_id; - queue_entry_t *queue; - int status; - - if (odp_unlikely(info == NULL)) { - _ODP_ERR("Unable to store info, NULL ptr given\n"); - return -1; - } - - queue_id = queue_to_id(handle); - - if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - _ODP_ERR("Invalid queue handle:%" PRIu64 "\n", odp_queue_to_u64(handle)); - return -1; - } - - queue = get_qentry(queue_id); - - LOCK(&queue->lock); - status = queue->status; - - if (odp_unlikely(status == QUEUE_STATUS_FREE || - status == QUEUE_STATUS_DESTROYED)) { - UNLOCK(&queue->lock); - _ODP_ERR("Invalid queue status:%d\n", status); - return -1; - } - - info->name = queue->name; - info->param = queue->param; - - UNLOCK(&queue->lock); - - return 0; -} - -static void queue_print(odp_queue_t handle) -{ - odp_pktio_info_t pktio_info; - queue_entry_t *queue; - uint32_t queue_id; - int status; - - queue_id = queue_to_id(handle); - - if (odp_unlikely(queue_id >= CONFIG_MAX_QUEUES)) { - _ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); - return; - } - - queue = get_qentry(queue_id); - - LOCK(&queue->lock); - status = queue->status; - - if (odp_unlikely(status == QUEUE_STATUS_FREE || - status == QUEUE_STATUS_DESTROYED)) { - UNLOCK(&queue->lock); - _ODP_ERR("Invalid queue status:%d\n", status); - return; - } - _ODP_PRINT("\nQueue info\n"); - _ODP_PRINT("----------\n"); - _ODP_PRINT(" handle %p\n", (void *)queue->handle); - _ODP_PRINT(" index %" PRIu32 "\n", queue->index); - _ODP_PRINT(" name %s\n", queue->name); - _ODP_PRINT(" enq mode %s\n", - queue->param.enq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.enq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.enq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - _ODP_PRINT(" deq mode %s\n", - queue->param.deq_mode == ODP_QUEUE_OP_MT ? "ODP_QUEUE_OP_MT" : - (queue->param.deq_mode == ODP_QUEUE_OP_MT_UNSAFE ? "ODP_QUEUE_OP_MT_UNSAFE" : - (queue->param.deq_mode == ODP_QUEUE_OP_DISABLED ? "ODP_QUEUE_OP_DISABLED" : - "unknown"))); - _ODP_PRINT(" type %s\n", - queue->type == ODP_QUEUE_TYPE_PLAIN ? "ODP_QUEUE_TYPE_PLAIN" : - (queue->type == ODP_QUEUE_TYPE_SCHED ? "ODP_QUEUE_TYPE_SCHED" : "unknown")); - if (queue->type == ODP_QUEUE_TYPE_SCHED) { - _ODP_PRINT(" sync %s\n", - queue->param.sched.sync == ODP_SCHED_SYNC_PARALLEL ? - "ODP_SCHED_SYNC_PARALLEL" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ATOMIC ? - "ODP_SCHED_SYNC_ATOMIC" : - (queue->param.sched.sync == ODP_SCHED_SYNC_ORDERED ? - "ODP_SCHED_SYNC_ORDERED" : "unknown"))); - _ODP_PRINT(" priority %d\n", queue->param.sched.prio); - _ODP_PRINT(" group %d\n", queue->param.sched.group); - } - if (queue->pktin.pktio != ODP_PKTIO_INVALID) { - if (!odp_pktio_info(queue->pktin.pktio, &pktio_info)) - _ODP_PRINT(" pktin %s\n", pktio_info.name); - } - if (queue->pktout.pktio != ODP_PKTIO_INVALID) { - if (!odp_pktio_info(queue->pktout.pktio, &pktio_info)) - _ODP_PRINT(" pktout %s\n", pktio_info.name); - } - _ODP_PRINT(" timers %" PRIu64 "\n", odp_atomic_load_u64(&queue->num_timers)); - _ODP_PRINT(" param.size %" PRIu32 "\n", queue->param.size); - _ODP_PRINT("\n"); - - UNLOCK(&queue->lock); -} - -static void queue_print_all(void) -{ - uint32_t i, index; - const char *name; - int status; - odp_queue_type_t type; - odp_nonblocking_t blocking; - odp_queue_op_mode_t enq_mode; - odp_queue_op_mode_t deq_mode; - odp_queue_order_t order; - odp_schedule_sync_t sync; - int prio; - const char *bl_str; - char type_c, enq_c, deq_c, order_c, sync_c; - const int col_width = 24; - - _ODP_PRINT("\nList of all queues\n"); - _ODP_PRINT("------------------\n"); - _ODP_PRINT(" idx %-*s type blk enq deq ord sync prio\n", col_width, "name"); - - for (i = 0; i < CONFIG_MAX_QUEUES; i++) { - queue_entry_t *queue = &queue_tbl->queue[i]; - - if (queue->status != QUEUE_STATUS_READY) - continue; - - LOCK(&queue->lock); - - status = queue->status; - index = queue->index; - name = queue->name; - type = queue->type; - blocking = queue->param.nonblocking; - enq_mode = queue->param.enq_mode; - deq_mode = queue->param.deq_mode; - order = queue->param.order; - prio = queue->param.sched.prio; - sync = queue->param.sched.sync; - - UNLOCK(&queue->lock); - - if (status != QUEUE_STATUS_READY) - continue; - - type_c = (type == ODP_QUEUE_TYPE_PLAIN) ? 'P' : 'S'; - - bl_str = (blocking == ODP_BLOCKING) ? "B" : - ((blocking == ODP_NONBLOCKING_LF) ? "LF" : "WF"); - - enq_c = (enq_mode == ODP_QUEUE_OP_MT) ? 'S' : - ((enq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); - - deq_c = (deq_mode == ODP_QUEUE_OP_MT) ? 'S' : - ((deq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); - - order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; - - _ODP_PRINT("%4u %-*s %c %2s", index, col_width, name, type_c, bl_str); - _ODP_PRINT(" %c %c %c", enq_c, deq_c, order_c); - - if (type == ODP_QUEUE_TYPE_SCHED) { - sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : - ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); - _ODP_PRINT(" %c %4i", sync_c, prio); - } - - _ODP_PRINT("\n"); - } - - _ODP_PRINT("\n"); -} - -static uint64_t queue_to_u64(odp_queue_t hdl) -{ - return _odp_pri(hdl); -} - -static odp_pktout_queue_t queue_get_pktout(odp_queue_t handle) -{ - return qentry_from_int(handle)->pktout; -} - -static void queue_set_pktout(odp_queue_t handle, odp_pktio_t pktio, int index) -{ - qentry_from_int(handle)->pktout.pktio = pktio; - qentry_from_int(handle)->pktout.index = index; -} - -static odp_pktin_queue_t queue_get_pktin(odp_queue_t handle) -{ - return qentry_from_int(handle)->pktin; -} - -static void queue_set_pktin(odp_queue_t handle, odp_pktio_t pktio, int index) -{ - qentry_from_int(handle)->pktin.pktio = pktio; - qentry_from_int(handle)->pktin.index = index; -} - -static void queue_set_enq_deq_func(odp_queue_t handle, - queue_enq_fn_t enq, - queue_enq_multi_fn_t enq_multi, - queue_deq_fn_t deq, - queue_deq_multi_fn_t deq_multi) -{ - if (enq) - qentry_from_int(handle)->enqueue = enq; - - if (enq_multi) - qentry_from_int(handle)->enqueue_multi = enq_multi; - - if (deq) - qentry_from_int(handle)->dequeue = deq; - - if (deq_multi) - qentry_from_int(handle)->dequeue_multi = deq_multi; -} - -static int queue_orig_multi(odp_queue_t handle, - _odp_event_hdr_t **event_hdr, int num) -{ - return qentry_from_int(handle)->orig_dequeue_multi(handle, - event_hdr, num); -} - -static void queue_timer_add(odp_queue_t handle) -{ - queue_entry_t *queue = _odp_qentry_from_ext(handle); - - odp_atomic_inc_u64(&queue->num_timers); -} - -static void queue_timer_rem(odp_queue_t handle) -{ - queue_entry_t *queue = _odp_qentry_from_ext(handle); - - odp_atomic_dec_u64(&queue->num_timers); -} - -/* API functions */ -_odp_queue_api_fn_t _odp_queue_scalable_api = { - .queue_create = queue_create, - .queue_create_multi = queue_create_multi, - .queue_destroy = queue_destroy, - .queue_destroy_multi = queue_destroy_multi, - .queue_lookup = queue_lookup, - .queue_capability = queue_capability, - .queue_context_set = queue_context_set, - .queue_enq = queue_enq, - .queue_enq_multi = queue_enq_multi, - .queue_deq = queue_deq, - .queue_deq_multi = queue_deq_multi, - .queue_type = queue_type, - .queue_sched_type = queue_sched_type, - .queue_sched_prio = queue_sched_prio, - .queue_sched_group = queue_sched_group, - .queue_lock_count = queue_lock_count, - .queue_to_u64 = queue_to_u64, - .queue_param_init = queue_param_init, - .queue_info = queue_info, - .queue_print = queue_print, - .queue_print_all = queue_print_all -}; - -/* Functions towards internal components */ -queue_fn_t _odp_queue_scalable_fn = { - .init_global = queue_init_global, - .term_global = queue_term_global, - .init_local = queue_init_local, - .term_local = queue_term_local, - .get_pktout = queue_get_pktout, - .set_pktout = queue_set_pktout, - .get_pktin = queue_get_pktin, - .set_pktin = queue_set_pktin, - .set_enq_deq_fn = queue_set_enq_deq_func, - .orig_deq_multi = queue_orig_multi, - .timer_add = queue_timer_add, - .timer_rem = queue_timer_rem -}; diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 617b92f5d6..f0423fe131 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -30,7 +30,6 @@ int _odp_schedule_configured(void) extern const schedule_fn_t _odp_schedule_sp_fn; extern const schedule_fn_t _odp_schedule_basic_fn; -extern const schedule_fn_t _odp_schedule_scalable_fn; const schedule_fn_t *_odp_sched_fn; int _odp_sched_id; @@ -154,9 +153,6 @@ int _odp_schedule_init_global(void) } else if (!strcmp(sched, "sp")) { _odp_sched_id = _ODP_SCHED_ID_SP; _odp_sched_fn = &_odp_schedule_sp_fn; - } else if (!strcmp(sched, "scalable")) { - _odp_sched_id = _ODP_SCHED_ID_SCALABLE; - _odp_sched_fn = &_odp_schedule_scalable_fn; } else { _ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c deleted file mode 100644 index 5513acb043..0000000000 --- a/platform/linux-generic/odp_schedule_scalable.c +++ /dev/null @@ -1,2210 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#define LOCK(a) odp_ticketlock_lock((a)) -#define UNLOCK(a) odp_ticketlock_unlock((a)) - -#define MAXTHREADS ATOM_BITSET_SIZE - -#define FLAG_PKTIN 0x80 - -ODP_STATIC_ASSERT(_ODP_CHECK_IS_POWER2(CONFIG_MAX_SCHED_QUEUES), - "Number_of_queues_is_not_power_of_two"); - -#define SCHED_GROUP_JOIN 0 -#define SCHED_GROUP_LEAVE 1 -#define NUM_AUTO_GROUPS (ODP_SCHED_GROUP_CONTROL + 1) - -typedef struct { - odp_shm_t shm; - _odp_ishm_pool_t *sched_shm_pool; - /** Currently used scheduler groups */ - sched_group_mask_t sg_free; - sched_group_t *sg_vec[MAX_SCHED_GROUP]; - /** Group lock for MT-safe APIs */ - odp_spinlock_t sched_grp_lock; - /** Initialization lock */ - odp_spinlock_t init_lock; - /** Per thread state */ - sched_scalable_thread_state_t thread_state[MAXTHREADS]; - uint16_t poll_count[CONFIG_PKTIO_ENTRIES]; - /* Scheduler interface config options (not used in fast path) */ - schedule_config_t config_if; -} sched_global_t; - -static sched_global_t *global; - -__thread sched_scalable_thread_state_t *_odp_sched_ts; - -static int thread_state_init(int tidx) -{ - sched_scalable_thread_state_t *ts; - uint32_t i; - - _ODP_ASSERT(tidx < MAXTHREADS); - ts = &global->thread_state[tidx]; - ts->atomq = NULL; - ts->src_schedq = NULL; - ts->rctx = NULL; - ts->pause = false; - ts->out_of_order = false; - ts->tidx = tidx; - ts->dequeued = 0; - ts->ticket = TICKET_INVALID; - ts->priv_rvec_free = 0; - ts->rvec_free = (1ULL << TS_RVEC_SIZE) - 1; - ts->num_schedq = 0; - ts->sg_sem = 1; /* Start with sched group semaphore changed */ - ts->loop_cnt = 0; - memset(ts->sg_actual, 0, sizeof(ts->sg_actual)); - for (i = 0; i < TS_RVEC_SIZE; i++) { - ts->rvec[i].rvec_free = &ts->rvec_free; - ts->rvec[i].idx = i; - } - _odp_sched_ts = ts; - - return 0; -} - -static void insert_schedq_in_list(sched_scalable_thread_state_t *ts, - sched_queue_t *schedq) -{ - /* Find slot for schedq */ - for (uint32_t i = 0; i < ts->num_schedq; i++) { - /* Lower value is higher priority and closer to start of list */ - if (schedq->prio <= ts->schedq_list[i]->prio) { - /* This is the slot! */ - sched_queue_t *tmp; - - tmp = ts->schedq_list[i]; - ts->schedq_list[i] = schedq; - schedq = tmp; - /* Continue the insertion procedure with the - * new schedq. - */ - } - } - if (ts->num_schedq == SCHEDQ_PER_THREAD) - _ODP_ABORT("Too many schedqs\n"); - ts->schedq_list[ts->num_schedq++] = schedq; -} - -static void remove_schedq_from_list(sched_scalable_thread_state_t *ts, - sched_queue_t *schedq) -{ - /* Find schedq */ - for (uint32_t i = 0; i < ts->num_schedq; i++) - if (ts->schedq_list[i] == schedq) { - /* Move remaining schedqs */ - for (uint32_t j = i + 1; j < ts->num_schedq; j++) - ts->schedq_list[j - 1] = ts->schedq_list[j]; - ts->num_schedq--; - return; - } - _ODP_ABORT("Cannot find schedq\n"); -} - -/******************************************************************************* - * Scheduler queues - ******************************************************************************/ -#ifndef odp_container_of -#define odp_container_of(pointer, type, member) \ - ((type *)(void *)(((char *)pointer) - offsetof(type, member))) -#endif - -static inline void schedq_init(sched_queue_t *schedq, uint32_t prio) -{ - llqueue_init(&schedq->llq); - schedq->prio = prio; -} - -static inline sched_elem_t *schedq_peek(sched_queue_t *schedq) -{ - struct llnode *ptr; - - ptr = llq_head(&schedq->llq); - return odp_container_of(ptr, sched_elem_t, node); -} - -static inline odp_bool_t schedq_cond_pop(sched_queue_t *schedq, - sched_elem_t *elem) -{ - return llq_dequeue_cond(&schedq->llq, &elem->node); -} - -static inline void schedq_push(sched_queue_t *schedq, sched_elem_t *elem) -{ - llq_enqueue(&schedq->llq, &elem->node); -} - -static inline odp_bool_t schedq_cond_rotate(sched_queue_t *schedq, - sched_elem_t *elem) -{ - return llq_cond_rotate(&schedq->llq, &elem->node); -} - -static inline bool schedq_elem_on_queue(sched_elem_t *elem) -{ - return llq_on_queue(&elem->node); -} - -/******************************************************************************* - * Shared metadata btwn scheduler and queue - ******************************************************************************/ - -void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual) -{ - qschedstate_t oss, nss; - uint32_t ticket; - - oss = q->qschst; - /* Update event counter, optionally taking a ticket. */ - do { - ticket = TICKET_INVALID; - nss = oss; - nss.numevts += actual; - if (odp_unlikely(oss.numevts <= 0 && nss.numevts > 0)) - /* E -> NE transition */ - if (q->qschst_type != ODP_SCHED_SYNC_ATOMIC || - oss.cur_ticket == oss.nxt_ticket) - /* Parallel or ordered queues: always take - * ticket. - * Atomic queue: only take ticket if one is - * immediately available. - * Otherwise ticket already taken => queue - * processed by some thread. - */ - ticket = nss.nxt_ticket++; - /* Else queue already was non-empty. */ - /* Attempt to update numevts counter and optionally take ticket. */ - } while (!__atomic_compare_exchange(&q->qschst, &oss, &nss, - true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); - - if (odp_unlikely(ticket != TICKET_INVALID)) { - /* Wait for our turn to update schedq. */ - if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) - _odp_wait_until_eq_acq_u8(&q->qschst.cur_ticket, ticket); - - /* Enqueue at end of scheduler queue */ - /* We are here because of empty-to-non-empty transition - * This means queue must be pushed to schedq if possible - * but we can't do that if it already is on the schedq - */ - if (odp_likely(!schedq_elem_on_queue(q) && - q->pop_deficit == 0)) { - /* Queue not already on schedq and no pop deficit means - * we can push queue to schedq */ - schedq_push(q->schedq, q); - } else { - /* Missed push => cancels one missed pop */ - q->pop_deficit--; - } - atomic_store_release(&q->qschst.cur_ticket, ticket + 1, - /*readonly=*/false); - } - /* Else queue was not empty or atomic queue already busy. */ -} - -void _odp_sched_update_enq_sp(sched_elem_t *q, uint32_t actual) -{ - qschedstate_t oss, nss; - uint32_t ticket; - - oss = q->qschst; - /* Update event counter, optionally taking a ticket. */ - ticket = TICKET_INVALID; - nss = oss; - nss.numevts += actual; - if (odp_unlikely(oss.numevts <= 0 && nss.numevts > 0)) { - /* E -> NE transition */ - if (q->qschst_type != ODP_SCHED_SYNC_ATOMIC || - oss.cur_ticket == oss.nxt_ticket) { - /* Parallel or ordered queues: always take - * ticket. - * Atomic queue: only take ticket if one is - * immediately available. Otherwise ticket already - * taken => queue owned/processed by some thread - */ - ticket = nss.nxt_ticket++; - } - } - /* Else queue already was non-empty. */ - /* Attempt to update numevts counter and optionally take ticket. */ - q->qschst = nss; - - if (odp_unlikely(ticket != TICKET_INVALID)) { - /* Enqueue at end of scheduler queue */ - /* We are here because of empty-to-non-empty transition - * This means queue must be pushed to schedq if possible - * but we can't do that if it already is on the schedq - */ - if (odp_likely(!schedq_elem_on_queue(q) && - q->pop_deficit == 0)) { - /* Queue not already on schedq and no pop deficit means - * we can push queue to schedq */ - schedq_push(q->schedq, q); - } else { - /* Missed push => cancels one missed pop */ - q->pop_deficit--; - } - q->qschst.cur_ticket = ticket + 1; - } - /* Else queue was not empty or atomic queue already busy. */ -} - -#ifndef CONFIG_QSCHST_LOCK -/* The scheduler is the only entity that performs the dequeue from a queue. */ -static void -sched_update_deq(sched_elem_t *q, - uint32_t actual, - bool atomic) __attribute__((always_inline)); -static inline void -sched_update_deq(sched_elem_t *q, - uint32_t actual, bool atomic) -{ - qschedstate_t oss, nss; - uint32_t ticket; - - if (atomic) { - bool pushed = false; - - /* We own this atomic queue, only we can dequeue from it and - * thus decrease numevts. Other threads may enqueue and thus - * increase numevts. - * This means that numevts can't unexpectedly become 0 and - * invalidate a push operation already performed - */ - oss = q->qschst; - do { - _ODP_ASSERT(oss.cur_ticket == _odp_sched_ts->ticket); - nss = oss; - nss.numevts -= actual; - if (nss.numevts > 0 && !pushed) { - schedq_push(q->schedq, q); - pushed = true; - } - /* Attempt to release ticket expecting our view of - * numevts to be correct - * Unfortunately nxt_ticket will also be included in - * the CAS operation - */ - nss.cur_ticket = _odp_sched_ts->ticket + 1; - } while (odp_unlikely(!__atomic_compare_exchange(&q->qschst, &oss, &nss, true, - __ATOMIC_RELEASE, - __ATOMIC_RELAXED))); - return; - } - - oss = q->qschst; - do { - ticket = TICKET_INVALID; - nss = oss; - nss.numevts -= actual; - nss.wrr_budget -= actual; - if ((oss.numevts > 0 && nss.numevts <= 0) || - oss.wrr_budget <= actual) { - /* If we have emptied parallel/ordered queue or - * exhausted its WRR budget, we need a ticket - * for a later pop. - */ - ticket = nss.nxt_ticket++; - /* Reset wrr_budget as we might also push the - * queue to the schedq. - */ - nss.wrr_budget = CONFIG_WRR_WEIGHT; - } - /* Attempt to update numevts and optionally take ticket. */ - } while (!__atomic_compare_exchange(&q->qschst, &oss, &nss, - true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); - - if (odp_unlikely(ticket != TICKET_INVALID)) { - _ODP_ASSERT(q->qschst_type != ODP_SCHED_SYNC_ATOMIC); - /* Wait for our turn to update schedq. */ - if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) - _odp_wait_until_eq_acq_u8(&q->qschst.cur_ticket, ticket); - - /* We are here because of non-empty-to-empty transition or - * WRR budget exhausted - * This means the queue must be popped from the schedq, now or - * later - * If there was no NE->E transition but instead the WRR budget - * was exhausted, the queue needs to be moved (popped and - * pushed) to the tail of the schedq - */ - if (oss.numevts > 0 && nss.numevts <= 0) { - /* NE->E transition, need to pop */ - if (!schedq_elem_on_queue(q) || - !schedq_cond_pop(q->schedq, q)) { - /* Queue not at head, failed to dequeue - * Missed a pop. - */ - q->pop_deficit++; - } - } else { - /* WRR budget exhausted - * Need to move queue to tail of schedq if possible - */ - if (odp_likely(schedq_elem_on_queue(q))) { - /* Queue is on schedq, try to move it to - * the tail - */ - (void)schedq_cond_rotate(q->schedq, q); - } - /* Else queue not on schedq or not at head of schedq - * No pop => no push - */ - } - atomic_store_release(&q->qschst.cur_ticket, ticket + 1, - /*readonly=*/false); - } -} -#endif - -#ifdef CONFIG_QSCHST_LOCK -static void -sched_update_deq_sc(sched_elem_t *q, - uint32_t actual, - bool atomic) __attribute__((always_inline)); -static inline void -sched_update_deq_sc(sched_elem_t *q, - uint32_t actual, bool atomic) -{ - qschedstate_t oss, nss; - uint32_t ticket; - - if (atomic) { - _ODP_ASSERT(q->qschst.cur_ticket == _odp_sched_ts->ticket); - _ODP_ASSERT(q->qschst.cur_ticket != q->qschst.nxt_ticket); - q->qschst.numevts -= actual; - q->qschst.cur_ticket = _odp_sched_ts->ticket + 1; - if (q->qschst.numevts > 0) - schedq_push(q->schedq, q); - return; - } - - oss = q->qschst; - ticket = TICKET_INVALID; - nss = oss; - nss.numevts -= actual; - nss.wrr_budget -= actual; - if ((oss.numevts > 0 && nss.numevts <= 0) || oss.wrr_budget <= actual) { - /* If we emptied the queue or - * if we have served the maximum number of events - * then we need a ticket for a later pop. - */ - ticket = nss.nxt_ticket++; - /* Also reset wrr_budget as we might also push the - * queue to the schedq. - */ - nss.wrr_budget = CONFIG_WRR_WEIGHT; - } - q->qschst = nss; - - if (ticket != TICKET_INVALID) { - if (oss.numevts > 0 && nss.numevts <= 0) { - /* NE->E transition, need to pop */ - if (!schedq_elem_on_queue(q) || - !schedq_cond_pop(q->schedq, q)) { - /* Queue not at head, failed to dequeue. - * Missed a pop. - */ - q->pop_deficit++; - } - } else { - /* WRR budget exhausted - * Need to move queue to tail of schedq if possible - */ - if (odp_likely(schedq_elem_on_queue(q))) { - /* Queue is on schedq, try to move it to - * the tail - */ - (void)schedq_cond_rotate(q->schedq, q); - } - /* Else queue not on schedq or not at head of schedq - * No pop => no push - */ - } - q->qschst.cur_ticket = ticket + 1; - } -} -#endif - -static inline void sched_update_popd_sc(sched_elem_t *elem) -{ - if (elem->pop_deficit != 0 && - schedq_elem_on_queue(elem) && - schedq_cond_pop(elem->schedq, elem)) - elem->pop_deficit--; -} - -#ifndef CONFIG_QSCHST_LOCK -static inline void sched_update_popd(sched_elem_t *elem) -{ - uint32_t ticket = __atomic_fetch_add(&elem->qschst.nxt_ticket, - 1, - __ATOMIC_RELAXED); - if (odp_unlikely(__atomic_load_n(&elem->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) - _odp_wait_until_eq_acq_u8(&elem->qschst.cur_ticket, ticket); - - sched_update_popd_sc(elem); - atomic_store_release(&elem->qschst.cur_ticket, ticket + 1, - /*readonly=*/false); -} -#endif - -static void signal_threads_add(sched_group_t *sg, uint32_t sgi, uint32_t prio) -{ - sched_group_mask_t thrds = sg->thr_wanted; - uint32_t thr; - - while (!bitset_is_null(thrds)) { - thr = bitset_ffs(thrds) - 1; - thrds = bitset_clr(thrds, thr); - /* Notify the thread about membership in this - * group/priority. - */ - atom_bitset_set(&global->thread_state[thr].sg_wanted[prio], - sgi, __ATOMIC_RELEASE); - __atomic_store_n(&global->thread_state[thr].sg_sem, 1, - __ATOMIC_RELEASE); - } -} - -sched_queue_t *_odp_sched_queue_add(odp_schedule_group_t grp, uint32_t prio) -{ - uint32_t sgi; - sched_group_t *sg; - uint32_t x; - - _ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - _ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); - _ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); - - sgi = grp; - sg = global->sg_vec[sgi]; - - /* Use xcount to spread queues over the xfactor schedq's - * per priority. - */ - x = __atomic_fetch_add(&sg->xcount[prio], 1, __ATOMIC_RELAXED); - if (x == 0) { - /* First ODP queue for this priority - * Notify all threads in sg->thr_wanted that they - * should join. - */ - signal_threads_add(sg, sgi, prio); - } - return &sg->schedq[prio * sg->xfactor + x % sg->xfactor]; -} - -static uint32_t sched_pktin_add(odp_schedule_group_t grp, uint32_t prio) -{ - uint32_t sgi; - sched_group_t *sg; - - _ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - _ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); - _ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); - - sgi = grp; - sg = global->sg_vec[sgi]; - - (void)_odp_sched_queue_add(grp, ODP_SCHED_PRIO_PKTIN); - return (ODP_SCHED_PRIO_PKTIN - prio) * sg->xfactor; -} - -static void signal_threads_rem(sched_group_t *sg, uint32_t sgi, uint32_t prio) -{ - sched_group_mask_t thrds = sg->thr_wanted; - uint32_t thr; - - while (!bitset_is_null(thrds)) { - thr = bitset_ffs(thrds) - 1; - thrds = bitset_clr(thrds, thr); - /* Notify the thread about membership in this - * group/priority. - */ - atom_bitset_clr(&global->thread_state[thr].sg_wanted[prio], - sgi, __ATOMIC_RELEASE); - __atomic_store_n(&global->thread_state[thr].sg_sem, 1, - __ATOMIC_RELEASE); - } -} - -void _odp_sched_queue_rem(odp_schedule_group_t grp, uint32_t prio) -{ - uint32_t sgi; - sched_group_t *sg; - uint32_t x; - - _ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - _ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); - _ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); - - sgi = grp; - sg = global->sg_vec[sgi]; - - x = __atomic_sub_fetch(&sg->xcount[prio], 1, __ATOMIC_RELAXED); - if (x == 0) { - /* Last ODP queue for this priority - * Notify all threads in sg->thr_wanted that they - * should leave. - */ - signal_threads_rem(sg, sgi, prio); - } -} - -static void sched_pktin_rem(odp_schedule_group_t grp) -{ - _odp_sched_queue_rem(grp, ODP_SCHED_PRIO_PKTIN); -} - -static void update_sg_add(sched_scalable_thread_state_t *ts, - uint32_t p, - sched_group_mask_t sg_wanted) -{ - sched_group_mask_t added; - uint32_t sgi; - sched_group_t *sg; - uint32_t x; - - added = bitset_andn(sg_wanted, ts->sg_actual[p]); - while (!bitset_is_null(added)) { - sgi = bitset_ffs(added) - 1; - sg = global->sg_vec[sgi]; - for (x = 0; x < sg->xfactor; x++) { - /* Include our thread index to shift - * (rotate) the order of schedq's - */ - insert_schedq_in_list(ts, - &sg->schedq[p * sg->xfactor + - (x + ts->tidx) % sg->xfactor]); - } - atom_bitset_set(&sg->thr_actual[p], ts->tidx, __ATOMIC_RELAXED); - added = bitset_clr(added, sgi); - } -} - -static void update_sg_rem(sched_scalable_thread_state_t *ts, - uint32_t p, - sched_group_mask_t sg_wanted) -{ - sched_group_mask_t removed; - uint32_t sgi; - sched_group_t *sg; - uint32_t x; - - removed = bitset_andn(ts->sg_actual[p], sg_wanted); - while (!bitset_is_null(removed)) { - sgi = bitset_ffs(removed) - 1; - sg = global->sg_vec[sgi]; - for (x = 0; x < sg->xfactor; x++) { - remove_schedq_from_list(ts, - &sg->schedq[p * - sg->xfactor + x]); - } - atom_bitset_clr(&sg->thr_actual[p], ts->tidx, __ATOMIC_RELAXED); - removed = bitset_clr(removed, sgi); - } -} - -static void update_sg_membership(sched_scalable_thread_state_t *ts) -{ - uint32_t p; - sched_group_mask_t sg_wanted; - - for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { - sg_wanted = atom_bitset_load(&ts->sg_wanted[p], - __ATOMIC_ACQUIRE); - if (!bitset_is_eql(ts->sg_actual[p], sg_wanted)) { - /* Our sched_group membership has changed */ - update_sg_add(ts, p, sg_wanted); - update_sg_rem(ts, p, sg_wanted); - ts->sg_actual[p] = sg_wanted; - } - } -} - -/******************************************************************************* - * Scheduler - ******************************************************************************/ - -static inline void _schedule_release_atomic(sched_scalable_thread_state_t *ts) -{ -#ifdef CONFIG_QSCHST_LOCK - sched_update_deq_sc(ts->atomq, ts->dequeued, true); - _ODP_ASSERT(ts->atomq->qschst.cur_ticket != ts->ticket); - _ODP_ASSERT(ts->atomq->qschst.cur_ticket == - ts->atomq->qschst.nxt_ticket); -#else - sched_update_deq(ts->atomq, ts->dequeued, true); -#endif - ts->atomq = NULL; - ts->ticket = TICKET_INVALID; -} - -static inline void _schedule_release_ordered(sched_scalable_thread_state_t *ts) -{ - ts->out_of_order = false; - _odp_rctx_release(ts->rctx); - ts->rctx = NULL; -} - -static void pktio_start(int pktio_idx, - int num_in_queue, - int in_queue_idx[], - odp_queue_t odpq[]) -{ - int i, rxq; - queue_entry_t *qentry; - sched_elem_t *elem; - - _ODP_ASSERT(pktio_idx < CONFIG_PKTIO_ENTRIES); - for (i = 0; i < num_in_queue; i++) { - rxq = in_queue_idx[i]; - _ODP_ASSERT(rxq < ODP_PKTIN_MAX_QUEUES); - __atomic_fetch_add(&global->poll_count[pktio_idx], 1, - __ATOMIC_RELAXED); - qentry = _odp_qentry_from_ext(odpq[i]); - elem = &qentry->sched_elem; - elem->cons_type |= FLAG_PKTIN; /* Set pktin queue flag */ - elem->pktio_idx = pktio_idx; - elem->rx_queue = rxq; - elem->xoffset = sched_pktin_add(elem->sched_grp, elem->sched_prio); - _ODP_ASSERT(elem->schedq != NULL); - schedq_push(elem->schedq, elem); - } -} - -static void pktio_stop(sched_elem_t *elem) -{ - sched_pktin_rem(elem->sched_grp); - if (__atomic_sub_fetch(&global->poll_count[elem->pktio_idx], - 1, __ATOMIC_RELAXED) == 0) { - /* Call stop_finalize when all queues - * of the pktio have been removed */ - _odp_sched_cb_pktio_stop_finalize(elem->pktio_idx); - } -} - -static bool have_reorder_ctx(sched_scalable_thread_state_t *ts) -{ - if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { - ts->priv_rvec_free = atom_bitset_xchg(&ts->rvec_free, 0, - __ATOMIC_RELAXED); - if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { - /* No free reorder contexts for this thread */ - return false; - } - } - return true; -} - -static inline bool is_pktin(sched_elem_t *elem) -{ - return (elem->cons_type & FLAG_PKTIN) != 0; -} - -static inline bool is_atomic(sched_elem_t *elem) -{ - return elem->cons_type == (ODP_SCHED_SYNC_ATOMIC | FLAG_PKTIN); -} - -static inline bool is_ordered(sched_elem_t *elem) -{ - return elem->cons_type == (ODP_SCHED_SYNC_ORDERED | FLAG_PKTIN); -} - -static int poll_pktin(sched_elem_t *elem, odp_event_t ev[], int num_evts) -{ - sched_scalable_thread_state_t *ts = _odp_sched_ts; - int num, i; - /* For ordered queues only */ - reorder_context_t *rctx; - reorder_window_t *rwin = NULL; - uint32_t sn = 0; - uint32_t idx; - - if (is_ordered(elem)) { - /* Need reorder context and slot in reorder window */ - rwin = queue_get_rwin((queue_entry_t *)elem); - _ODP_ASSERT(rwin != NULL); - if (odp_unlikely(!have_reorder_ctx(ts) || - !_odp_rwin_reserve_sc(rwin, &sn))) { - /* Put back queue on source schedq */ - schedq_push(ts->src_schedq, elem); - return 0; - } - /* Slot in reorder window reserved! */ - } - - /* Try to dequeue events from the ingress queue itself */ - num = _odp_queue_deq_sc(elem, ev, num_evts); - if (odp_likely(num > 0)) { -events_dequeued: - if (is_atomic(elem)) { - ts->atomq = elem; /* Remember */ - ts->dequeued += num; - /* Don't push atomic queue on schedq */ - } else /* Parallel or ordered */ { - if (is_ordered(elem)) { - /* Find and initialise an unused reorder - * context. */ - idx = bitset_ffs(ts->priv_rvec_free) - 1; - ts->priv_rvec_free = - bitset_clr(ts->priv_rvec_free, idx); - rctx = &ts->rvec[idx]; - _odp_rctx_init(rctx, idx, rwin, sn); - /* Are we in-order or out-of-order? */ - ts->out_of_order = sn != rwin->hc.head; - ts->rctx = rctx; - } - schedq_push(elem->schedq, elem); - } - return num; - } - - /* Ingress queue empty => poll pktio RX queue */ - _odp_event_hdr_t *rx_evts[QUEUE_MULTI_MAX]; - int num_rx = _odp_sched_cb_pktin_poll(elem->pktio_idx, elem->rx_queue, - rx_evts, QUEUE_MULTI_MAX); - - if (odp_likely(num_rx > 0)) { - num = num_rx < num_evts ? num_rx : num_evts; - for (i = 0; i < num; i++) { - /* Return events directly to caller */ - ev[i] = _odp_event_from_hdr(rx_evts[i]); - } - if (num_rx > num) { - /* Events remain, enqueue them */ - i = _odp_queue_enq_sp(elem, &rx_evts[num], num_rx - num); - /* Enqueue must succeed as the queue was empty */ - _ODP_ASSERT(i == num_rx - num); - } - goto events_dequeued; - } - /* No packets received, reset state and undo side effects */ - if (is_atomic(elem)) - ts->atomq = NULL; - else if (is_ordered(elem)) - _odp_rwin_unreserve_sc(rwin, sn); - - if (odp_likely(num_rx == 0)) { - /* RX queue empty, push it to pktin priority schedq */ - sched_queue_t *schedq = ts->src_schedq; - /* Check if queue came from the designated schedq */ - if (schedq == elem->schedq) { - /* Yes, add offset to the pktin priority level - * in order to get alternate schedq */ - schedq += elem->xoffset; - } - /* Else no, queue must have come from alternate schedq */ - schedq_push(schedq, elem); - } else /* num_rx < 0 => pktio stopped or closed */ { - /* Remove queue */ - pktio_stop(elem); - /* Don't push queue to schedq */ - } - - _ODP_ASSERT(ts->atomq == NULL); - _ODP_ASSERT(!ts->out_of_order); - _ODP_ASSERT(ts->rctx == NULL); - return 0; -} - -static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) -{ - sched_scalable_thread_state_t *ts; - sched_elem_t *atomq; - int num; - int cpu_id; - uint32_t i; - - ts = _odp_sched_ts; - atomq = ts->atomq; - - timer_run(1); - - /* Once an atomic queue has been scheduled to a thread, it will stay - * on that thread until empty or 'rotated' by WRR - */ - if (atomq != NULL && is_pktin(atomq)) { - /* Atomic pktin queue */ - if (ts->dequeued < atomq->qschst.wrr_budget) { - _ODP_ASSERT(ts->src_schedq != NULL); - num = poll_pktin(atomq, ev, num_evts); - if (odp_likely(num != 0)) { - if (from) - *from = queue_get_handle((queue_entry_t *)atomq); - return num; - } - } else { - /* WRR budget exhausted, move queue to end of schedq */ - schedq_push(atomq->schedq, atomq); - } - ts->atomq = NULL; - } else if (atomq != NULL) { - _ODP_ASSERT(ts->ticket != TICKET_INVALID); -#ifdef CONFIG_QSCHST_LOCK - LOCK(&atomq->qschlock); -#endif -dequeue_atomic: - _ODP_ASSERT(ts->ticket == atomq->qschst.cur_ticket); - _ODP_ASSERT(ts->ticket != atomq->qschst.nxt_ticket); - /* Atomic queues can be dequeued without lock since this thread - * has the only reference to the atomic queue being processed. - */ - if (ts->dequeued < atomq->qschst.wrr_budget) { - num = _odp_queue_deq_sc(atomq, ev, num_evts); - if (odp_likely(num != 0)) { -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&atomq->qschlock); -#endif - ts->dequeued += num; - /* Allow this thread to continue to 'own' this - * atomic queue until all events have been - * processed and the thread re-invokes the - * scheduler. - */ - if (from) - *from = queue_get_handle((queue_entry_t *)atomq); - return num; - } - } - /* Atomic queue was empty or interrupted by WRR, release it. */ - _schedule_release_atomic(ts); -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&atomq->qschlock); -#endif - } - - /* Check for and perform any scheduler group updates. */ - if (odp_unlikely(__atomic_load_n(&ts->sg_sem, __ATOMIC_RELAXED) != 0)) { - (void)__atomic_load_n(&ts->sg_sem, __ATOMIC_ACQUIRE); - ts->sg_sem = 0; - update_sg_membership(ts); - } - - cpu_id = odp_cpu_id(); - if (odp_unlikely(cpu_id < 0)) - return 0; - - /* Scan our schedq list from beginning to end */ - for (i = 0; i < ts->num_schedq; i++) { - sched_queue_t *schedq = ts->schedq_list[i]; - sched_elem_t *elem; - - ts->loop_cnt++; -restart_same: - elem = schedq_peek(schedq); - if (odp_unlikely(elem == NULL)) { - /* Schedq empty, look at next one. */ - continue; - } - if (is_pktin(elem)) { - /* Pktio ingress queue */ - if (elem->schedq != schedq) { /* Low priority schedq*/ - if (elem->loop_check[cpu_id] != ts->loop_cnt) - elem->loop_check[cpu_id] = ts->loop_cnt; - else /* Wrapped around */ - continue; /* Go to next schedq */ - } - - if (odp_unlikely(!schedq_cond_pop(schedq, elem))) - goto restart_same; - - ts->src_schedq = schedq; /* Remember source schedq */ - num = poll_pktin(elem, ev, num_evts); - if (odp_unlikely(num <= 0)) - goto restart_same; - if (from) - *from = queue_get_handle((queue_entry_t *)elem); - return num; - } else if (elem->cons_type == ODP_SCHED_SYNC_ATOMIC) { - /* Dequeue element only if it is still at head - * of schedq. - */ - if (odp_unlikely(!schedq_cond_pop(schedq, elem))) { - /* Queue not at head of schedq anymore, some - * other thread popped it. - */ - goto restart_same; - } - ts->atomq = elem; - atomq = elem; - ts->dequeued = 0; -#ifdef CONFIG_QSCHST_LOCK - LOCK(&atomq->qschlock); - ts->ticket = atomq->qschst.nxt_ticket++; - _ODP_ASSERT(atomq->qschst.cur_ticket == ts->ticket); -#else - /* Dequeued atomic queue from the schedq, only we - * can process it and any qschst updates are our - * responsibility. - */ - /* The ticket taken below will signal producers */ - ts->ticket = __atomic_fetch_add(&atomq->qschst.nxt_ticket, 1, - __ATOMIC_RELAXED); - while (__atomic_load_n(&atomq->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ts->ticket) { - /* No need to use WFE, spinning here seems - * very infrequent. - */ - odp_cpu_pause(); - } -#endif - goto dequeue_atomic; - } else if (elem->cons_type == ODP_SCHED_SYNC_PARALLEL) { -#ifdef CONFIG_QSCHST_LOCK - LOCK(&elem->qschlock); - num = _odp_queue_deq_sc(elem, ev, num_evts); - if (odp_likely(num != 0)) { - sched_update_deq_sc(elem, num, false); - UNLOCK(&elem->qschlock); - if (from) - *from = - queue_get_handle((queue_entry_t *)elem); - return num; - } - UNLOCK(&elem->qschlock); -#else - num = _odp_queue_deq_mc(elem, ev, num_evts); - if (odp_likely(num != 0)) { - sched_update_deq(elem, num, false); - if (from) - *from = - queue_get_handle((queue_entry_t *)elem); - return num; - } -#endif - } else if (elem->cons_type == ODP_SCHED_SYNC_ORDERED) { - reorder_window_t *rwin; - reorder_context_t *rctx; - uint32_t sn; - uint32_t idx; - - /* The ordered queue has a reorder window so requires - * order restoration. We must use a reorder context to - * collect all outgoing events. Ensure there is at least - * one available reorder context. - */ - if (odp_unlikely(!have_reorder_ctx(ts))) - continue; - - /* _odp_rwin_reserve and odp_queue_deq must be atomic or - * there will be a potential race condition. - * Allocate a slot in the reorder window. - */ - rwin = queue_get_rwin((queue_entry_t *)elem); - _ODP_ASSERT(rwin != NULL); - if (odp_unlikely(!_odp_rwin_reserve(rwin, &sn))) { - /* Reorder window full */ - /* Look at next schedq, find other queue */ - continue; - } - /* Wait for our turn to dequeue */ - if (odp_unlikely(__atomic_load_n(&rwin->turn, __ATOMIC_ACQUIRE) != sn)) - _odp_wait_until_eq_acq_u32(&rwin->turn, sn); -#ifdef CONFIG_QSCHST_LOCK - LOCK(&elem->qschlock); -#endif - num = _odp_queue_deq_sc(elem, ev, num_evts); - /* Wait for prod_read write in _odp_queue_dequeue_sc() - * to complete before we signal the next consumer - */ - atomic_store_release(&rwin->turn, sn + 1, - /*readonly=*/false); - /* Find and initialise an unused reorder context. */ - idx = bitset_ffs(ts->priv_rvec_free) - 1; - ts->priv_rvec_free = - bitset_clr(ts->priv_rvec_free, idx); - rctx = &ts->rvec[idx]; - /* Need to initialise reorder context or we can't - * release it later. - */ - _odp_rctx_init(rctx, idx, rwin, sn); - - /* Was dequeue successful? */ - if (odp_likely(num != 0)) { - /* Perform scheduler related updates */ -#ifdef CONFIG_QSCHST_LOCK - sched_update_deq_sc(elem, num, - /*atomic=*/false); - UNLOCK(&elem->qschlock); -#else - sched_update_deq(elem, num, /*atomic=*/false); -#endif - - /* Are we in-order or out-of-order? */ - ts->out_of_order = sn != rwin->hc.head; - - ts->rctx = rctx; - if (from) - *from = queue_get_handle((queue_entry_t *)elem); - return num; - } -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&elem->qschlock); -#endif - /* Since a slot was reserved in the reorder window, - * the reorder context needs to be released and - * inserted into the reorder window. - */ - _odp_rctx_release(rctx); - _ODP_ASSERT(ts->rctx == NULL); - } - /* Dequeue from parallel/ordered queue failed - * Check if we have a queue at the head of the schedq that needs - * to be popped - */ - if (odp_unlikely(__atomic_load_n(&elem->pop_deficit, - __ATOMIC_RELAXED) != 0)) { -#ifdef CONFIG_QSCHST_LOCK - LOCK(&elem->qschlock); - sched_update_popd_sc(elem); - UNLOCK(&elem->qschlock); -#else - sched_update_popd(elem); -#endif - } - } - - return 0; -} - -/******************************************************************************/ - -static void schedule_order_lock(uint32_t lock_index) -{ - struct reorder_context *rctx = _odp_sched_ts->rctx; - - if (odp_unlikely(rctx == NULL || - rctx->rwin == NULL || - lock_index >= rctx->rwin->lock_count)) { - _ODP_ERR("Invalid call to odp_schedule_order_lock\n"); - return; - } - if (odp_unlikely(__atomic_load_n(&rctx->rwin->olock[lock_index], - __ATOMIC_ACQUIRE) != rctx->sn)) - _odp_wait_until_eq_acq_u32(&rctx->rwin->olock[lock_index], rctx->sn); -} - -static void schedule_order_unlock(uint32_t lock_index) -{ - struct reorder_context *rctx; - - rctx = _odp_sched_ts->rctx; - if (odp_unlikely(rctx == NULL || - rctx->rwin == NULL || - lock_index >= rctx->rwin->lock_count || - rctx->rwin->olock[lock_index] != rctx->sn)) { - _ODP_ERR("Invalid call to odp_schedule_order_unlock\n"); - return; - } - atomic_store_release(&rctx->rwin->olock[lock_index], - rctx->sn + 1, - /*readonly=*/false); - rctx->olock_flags |= 1U << lock_index; -} - -static void schedule_order_unlock_lock(uint32_t unlock_index, - uint32_t lock_index) -{ - schedule_order_unlock(unlock_index); - schedule_order_lock(lock_index); -} - -static void schedule_order_lock_start(uint32_t lock_index) -{ - (void)lock_index; -} - -static void schedule_order_lock_wait(uint32_t lock_index) -{ - schedule_order_lock(lock_index); -} - -static void schedule_release_atomic(void) -{ - sched_scalable_thread_state_t *ts; - - ts = _odp_sched_ts; - if (odp_likely(ts->atomq != NULL)) { -#ifdef CONFIG_QSCHST_LOCK - sched_elem_t *atomq; - - atomq = ts->atomq; - LOCK(&atomq->qschlock); -#endif - _schedule_release_atomic(ts); -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&atomq->qschlock); -#endif - } -} - -static void schedule_release_ordered(void) -{ - sched_scalable_thread_state_t *ts; - - ts = _odp_sched_ts; - if (ts->rctx != NULL) - _schedule_release_ordered(ts); -} - -static int schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t ev[], - int num) -{ - sched_scalable_thread_state_t *ts; - int n; - odp_time_t start; - odp_time_t deadline; - - ts = _odp_sched_ts; - /* Release any previous reorder context. */ - if (ts->rctx != NULL) - _schedule_release_ordered(ts); - - if (odp_unlikely(ts->pause)) { - if (ts->atomq != NULL) { -#ifdef CONFIG_QSCHST_LOCK - sched_elem_t *atomq; - - atomq = ts->atomq; - LOCK(&atomq->qschlock); -#endif - _schedule_release_atomic(ts); -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&atomq->qschlock); -#endif - } - return 0; - } - - if (wait == ODP_SCHED_NO_WAIT) - return _schedule(from, ev, num); - - if (wait == ODP_SCHED_WAIT) { - for (;;) { - n = _schedule(from, ev, num); - if (odp_likely(n > 0)) - return n; - } - } - - start = odp_time_local(); - - n = _schedule(from, ev, num); - if (odp_likely(n > 0)) - return n; - - deadline = odp_time_add_ns(start, wait); - - while (odp_time_cmp(deadline, odp_time_local()) > 0) { - n = _schedule(from, ev, num); - if (odp_likely(n > 0)) - return n; - } - - return 0; -} - -static odp_event_t schedule(odp_queue_t *from, uint64_t wait) -{ - odp_event_t ev = ODP_EVENT_INVALID; - const int num = 1; - sched_scalable_thread_state_t *ts; - int n; - odp_time_t start; - odp_time_t deadline; - - ts = _odp_sched_ts; - /* Release any previous reorder context. */ - if (ts->rctx != NULL) - _schedule_release_ordered(ts); - - if (odp_unlikely(ts->pause)) { - if (ts->atomq != NULL) { -#ifdef CONFIG_QSCHST_LOCK - sched_elem_t *atomq; - - atomq = ts->atomq; - LOCK(&atomq->qschlock); -#endif - _schedule_release_atomic(ts); -#ifdef CONFIG_QSCHST_LOCK - UNLOCK(&atomq->qschlock); -#endif - } - return ev; - } - - if (wait == ODP_SCHED_NO_WAIT) { - (void)_schedule(from, &ev, num); - return ev; - } - - if (wait == ODP_SCHED_WAIT) { - for (;;) { - n = _schedule(from, &ev, num); - if (odp_likely(n > 0)) - return ev; - } - } - - start = odp_time_local(); - - n = _schedule(from, &ev, num); - if (odp_likely(n > 0)) - return ev; - - deadline = odp_time_add_ns(start, wait); - - while (odp_time_cmp(deadline, odp_time_local()) > 0) { - n = _schedule(from, &ev, num); - if (odp_likely(n > 0)) - return ev; - } - - return ev; -} - -static int schedule_multi_wait(odp_queue_t *from, odp_event_t events[], - int max_num) -{ - return schedule_multi(from, ODP_SCHED_WAIT, events, max_num); -} - -static int schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[], - int max_num) -{ - return schedule_multi(from, ODP_SCHED_NO_WAIT, events, max_num); -} - -static void schedule_pause(void) -{ - _odp_sched_ts->pause = true; -} - -static void schedule_resume(void) -{ - _odp_sched_ts->pause = false; -} - -static uint64_t schedule_wait_time(uint64_t ns) -{ - return ns; -} - -static int schedule_num_prio(void) -{ - return ODP_SCHED_PRIO_NUM - 1; /* Discount the pktin priority level */ -} - -static int schedule_min_prio(void) -{ - return 0; -} - -static int schedule_max_prio(void) -{ - return schedule_num_prio() - 1; -} - -static int schedule_default_prio(void) -{ - return schedule_max_prio() / 2; -} - -static int schedule_group_update(sched_group_t *sg, - uint32_t sgi, - const odp_thrmask_t *mask, - int join_leave) -{ - int thr; - uint32_t p; - - /* Internal function, do not validate inputs */ - - /* Notify relevant threads about the change */ - thr = odp_thrmask_first(mask); - while (0 <= thr) { - /* Add thread to scheduler group's wanted thread mask */ - if (join_leave == SCHED_GROUP_JOIN) - atom_bitset_set(&sg->thr_wanted, thr, __ATOMIC_RELAXED); - else - atom_bitset_clr(&sg->thr_wanted, thr, __ATOMIC_RELAXED); - for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { - if (sg->xcount[p] != 0) { - sched_scalable_thread_state_t *state; - - state = &global->thread_state[thr]; - - /* This priority level has ODP queues - * Notify the thread about membership in - * this group/priority - */ - if (join_leave == SCHED_GROUP_JOIN) - atom_bitset_set(&state->sg_wanted[p], - sgi, __ATOMIC_RELEASE); - else - atom_bitset_clr(&state->sg_wanted[p], - sgi, __ATOMIC_RELEASE); - __atomic_store_n(&state->sg_sem, 1, - __ATOMIC_RELEASE); - } - } - thr = odp_thrmask_next(mask, thr); - } - - return 0; -} - -static int _schedule_group_thrmask(sched_group_t *sg, odp_thrmask_t *mask) -{ - bitset_t bs; - uint32_t bit; - - /* Internal function, do not validate inputs */ - - odp_thrmask_zero(mask); - bs = sg->thr_wanted; - while (!bitset_is_null(bs)) { - bit = bitset_ffs(bs) - 1; - bs = bitset_clr(bs, bit); - odp_thrmask_set(mask, bit); - } - - return 0; -} - -static odp_schedule_group_t schedule_group_create(const char *name, - const odp_thrmask_t *mask) -{ - uint32_t sgi; - sched_group_mask_t free; - uint32_t xfactor; - sched_group_t *sg; - uint32_t p; - uint32_t x; - uint32_t size; - - /* Validate inputs */ - if (mask == NULL) - _ODP_ABORT("mask is NULL\n"); - - odp_spinlock_lock(&global->sched_grp_lock); - - /* Allocate a scheduler group */ - free = atom_bitset_load(&global->sg_free, __ATOMIC_RELAXED); - do { - /* All sched_groups in use */ - if (bitset_is_null(free)) - goto no_free_sched_group; - - sgi = bitset_ffs(free) - 1; - /* All sched_groups in use */ - if (sgi >= MAX_SCHED_GROUP) - goto no_free_sched_group; - } while (!atom_bitset_cmpxchg(&global->sg_free, - &free, - bitset_clr(free, sgi), - true, - __ATOMIC_ACQUIRE, - __ATOMIC_ACQUIRE)); - - /* Compute xfactor (spread factor) from the number of threads - * present in the thread mask. Preferable this would be an - * explicit parameter. - */ - xfactor = odp_thrmask_count(mask); - if (xfactor < 1) - xfactor = CONFIG_DEFAULT_XFACTOR; - - size = sizeof(sched_group_t) + - (ODP_SCHED_PRIO_NUM * xfactor - 1) * sizeof(sched_queue_t); - sg = (sched_group_t *)shm_pool_alloc_align(global->sched_shm_pool, - size); - if (sg == NULL) - goto shm_pool_alloc_failed; - - _odp_strcpy(sg->name, name ? name : "", ODP_SCHED_GROUP_NAME_LEN); - global->sg_vec[sgi] = sg; - memset(sg->thr_actual, 0, sizeof(sg->thr_actual)); - sg->thr_wanted = bitset_null(); - sg->xfactor = xfactor; - for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { - sg->xcount[p] = 0; - for (x = 0; x < xfactor; x++) - schedq_init(&sg->schedq[p * xfactor + x], p); - } - if (odp_thrmask_count(mask) != 0) - schedule_group_update(sg, sgi, mask, SCHED_GROUP_JOIN); - - odp_spinlock_unlock(&global->sched_grp_lock); - - return (odp_schedule_group_t)(sgi); - -shm_pool_alloc_failed: - /* Free the allocated group index */ - atom_bitset_set(&global->sg_free, sgi, __ATOMIC_RELAXED); - -no_free_sched_group: - odp_spinlock_unlock(&global->sched_grp_lock); - - return ODP_SCHED_GROUP_INVALID; -} - -static int schedule_group_destroy(odp_schedule_group_t group) -{ - uint32_t sgi; - sched_group_t *sg; - uint32_t p; - int ret = 0; - - /* Validate inputs */ - if (group < 0 || group >= (odp_schedule_group_t)MAX_SCHED_GROUP) { - ret = -1; - goto invalid_group; - } - - if (_odp_sched_ts && - odp_unlikely(__atomic_load_n(&_odp_sched_ts->sg_sem, - __ATOMIC_RELAXED) != 0)) { - (void)__atomic_load_n(&_odp_sched_ts->sg_sem, - __ATOMIC_ACQUIRE); - _odp_sched_ts->sg_sem = 0; - update_sg_membership(_odp_sched_ts); - } - odp_spinlock_lock(&global->sched_grp_lock); - - sgi = (uint32_t)group; - if (bitset_is_set(global->sg_free, sgi)) { - ret = -1; - goto group_not_found; - } - - sg = global->sg_vec[sgi]; - /* First ensure all threads have processed group_join/group_leave - * requests. - */ - for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { - if (sg->xcount[p] != 0) { - bitset_t wanted = atom_bitset_load(&sg->thr_wanted, __ATOMIC_RELAXED); - - _odp_wait_until_eq_bitset(&sg->thr_actual[p], wanted); - } - /* Else ignore because no ODP queues on this prio */ - } - - /* Check if all threads/queues have left the group */ - for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { - if (!bitset_is_null(sg->thr_actual[p])) { - _ODP_ERR("Group has threads\n"); - ret = -1; - goto thrd_q_present_in_group; - } - if (p != ODP_SCHED_PRIO_PKTIN && sg->xcount[p] != 0) { - _ODP_ERR("Group has queues\n"); - ret = -1; - goto thrd_q_present_in_group; - } - } - - _odp_ishm_pool_free(global->sched_shm_pool, sg); - global->sg_vec[sgi] = NULL; - atom_bitset_set(&global->sg_free, sgi, __ATOMIC_RELEASE); - - odp_spinlock_unlock(&global->sched_grp_lock); - - return ret; - -thrd_q_present_in_group: - -group_not_found: - odp_spinlock_unlock(&global->sched_grp_lock); - -invalid_group: - - return ret; -} - -static odp_schedule_group_t schedule_group_lookup(const char *name) -{ - uint32_t sgi; - odp_schedule_group_t group; - - /* Validate inputs */ - if (name == NULL) - _ODP_ABORT("name or mask is NULL\n"); - - group = ODP_SCHED_GROUP_INVALID; - - odp_spinlock_lock(&global->sched_grp_lock); - - /* Scan through the schedule group array */ - for (sgi = 0; sgi < MAX_SCHED_GROUP; sgi++) { - if ((global->sg_vec[sgi] != NULL) && - (strncmp(name, global->sg_vec[sgi]->name, - ODP_SCHED_GROUP_NAME_LEN) == 0)) { - group = (odp_schedule_group_t)sgi; - break; - } - } - - odp_spinlock_unlock(&global->sched_grp_lock); - - return group; -} - -static int schedule_group_join(odp_schedule_group_t group, - const odp_thrmask_t *mask) -{ - uint32_t sgi; - sched_group_t *sg; - int ret; - - /* Validate inputs */ - if (group < 0 || group >= ((odp_schedule_group_t)MAX_SCHED_GROUP)) - return -1; - - if (mask == NULL) - _ODP_ABORT("name or mask is NULL\n"); - - odp_spinlock_lock(&global->sched_grp_lock); - - sgi = (uint32_t)group; - if (bitset_is_set(global->sg_free, sgi)) { - odp_spinlock_unlock(&global->sched_grp_lock); - return -1; - } - - sg = global->sg_vec[sgi]; - ret = schedule_group_update(sg, sgi, mask, SCHED_GROUP_JOIN); - - odp_spinlock_unlock(&global->sched_grp_lock); - - return ret; -} - -static int schedule_group_leave(odp_schedule_group_t group, - const odp_thrmask_t *mask) -{ - uint32_t sgi; - sched_group_t *sg; - int ret = 0; - - /* Validate inputs */ - if (group < 0 || group >= (odp_schedule_group_t)MAX_SCHED_GROUP) { - ret = -1; - goto invalid_group; - } - - if (mask == NULL) - _ODP_ABORT("name or mask is NULL\n"); - - odp_spinlock_lock(&global->sched_grp_lock); - - sgi = (uint32_t)group; - if (bitset_is_set(global->sg_free, sgi)) { - ret = -1; - goto group_not_found; - } - - sg = global->sg_vec[sgi]; - - ret = schedule_group_update(sg, sgi, mask, SCHED_GROUP_LEAVE); - - odp_spinlock_unlock(&global->sched_grp_lock); - - return ret; - -group_not_found: - odp_spinlock_unlock(&global->sched_grp_lock); - -invalid_group: - return ret; -} - -static int schedule_group_thrmask(odp_schedule_group_t group, - odp_thrmask_t *mask) -{ - uint32_t sgi; - sched_group_t *sg; - int ret = 0; - - /* Validate inputs */ - if (group < 0 || group >= ((odp_schedule_group_t)MAX_SCHED_GROUP)) { - ret = -1; - goto invalid_group; - } - - if (mask == NULL) - _ODP_ABORT("name or mask is NULL\n"); - - odp_spinlock_lock(&global->sched_grp_lock); - - sgi = (uint32_t)group; - if (bitset_is_set(global->sg_free, sgi)) { - ret = -1; - goto group_not_found; - } - - sg = global->sg_vec[sgi]; - ret = _schedule_group_thrmask(sg, mask); - - odp_spinlock_unlock(&global->sched_grp_lock); - - return ret; - -group_not_found: - odp_spinlock_unlock(&global->sched_grp_lock); - -invalid_group: - return ret; -} - -static int schedule_group_info(odp_schedule_group_t group, - odp_schedule_group_info_t *info) -{ - uint32_t sgi; - sched_group_t *sg; - int ret = 0; - - /* Validate inputs */ - if (group < 0 || group >= ((odp_schedule_group_t)MAX_SCHED_GROUP)) { - ret = -1; - goto invalid_group; - } - - if (info == NULL) - _ODP_ABORT("name or mask is NULL\n"); - - odp_spinlock_lock(&global->sched_grp_lock); - - sgi = (uint32_t)group; - if (bitset_is_set(global->sg_free, sgi)) { - ret = -1; - goto group_not_found; - } - - sg = global->sg_vec[sgi]; - - ret = _schedule_group_thrmask(sg, &info->thrmask); - - info->name = sg->name; - - odp_spinlock_unlock(&global->sched_grp_lock); - - return ret; - -group_not_found: - odp_spinlock_unlock(&global->sched_grp_lock); - -invalid_group: - return ret; -} - -static int schedule_init_global(void) -{ - odp_thrmask_t mask; - odp_schedule_group_t tmp_all; - odp_schedule_group_t tmp_wrkr; - odp_schedule_group_t tmp_ctrl; - odp_shm_t shm; - _odp_ishm_pool_t *pool; - uint32_t bits; - uint32_t pool_size; - uint64_t min_alloc; - uint64_t max_alloc; - - shm = odp_shm_reserve("_odp_sched_scalable_global", - sizeof(sched_global_t), - ODP_CACHE_LINE_SIZE, 0); - - global = odp_shm_addr(shm); - if (global == NULL) { - _ODP_ERR("Schedule init: Shm reserve failed.\n"); - return -1; - } - - memset(global, 0, sizeof(sched_global_t)); - global->shm = shm; - - /* Add storage required for sched groups. Assume worst case - * xfactor of MAXTHREADS. - */ - pool_size = (sizeof(sched_group_t) + - (ODP_SCHED_PRIO_NUM * MAXTHREADS - 1) * - sizeof(sched_queue_t)) * MAX_SCHED_GROUP; - /* Choose min_alloc and max_alloc such that slab allocator - * is selected. - */ - min_alloc = sizeof(sched_group_t) + - (ODP_SCHED_PRIO_NUM * MAXTHREADS - 1) * - sizeof(sched_queue_t); - max_alloc = min_alloc; - pool = _odp_ishm_pool_create("sched_shm_pool", pool_size, - min_alloc, max_alloc, 0); - if (pool == NULL) { - _ODP_ERR("Failed to allocate shared memory pool " - "for sched\n"); - goto failed_sched_shm_pool_create; - } - global->sched_shm_pool = pool; - - odp_spinlock_init(&global->sched_grp_lock); - odp_spinlock_init(&global->init_lock); - - bits = MAX_SCHED_GROUP; - if (MAX_SCHED_GROUP == sizeof(global->sg_free) * CHAR_BIT) - global->sg_free = ~0; - else - global->sg_free = (1 << bits) - 1; - - for (uint32_t i = 0; i < MAX_SCHED_GROUP; i++) - global->sg_vec[i] = NULL; - for (uint32_t i = 0; i < MAXTHREADS; i++) { - global->thread_state[i].sg_sem = 0; - for (uint32_t j = 0; j < ODP_SCHED_PRIO_NUM; j++) - global->thread_state[i].sg_wanted[j] = bitset_null(); - } - - /* Create sched groups for default GROUP_ALL, GROUP_WORKER and - * GROUP_CONTROL groups. - */ - odp_thrmask_zero(&mask); - tmp_all = schedule_group_create("__group_all", &mask); - if (tmp_all != ODP_SCHED_GROUP_ALL) { - _ODP_ERR("Could not create ODP_SCHED_GROUP_ALL()\n"); - goto failed_create_group_all; - } - - tmp_wrkr = schedule_group_create("__group_worker", &mask); - if (tmp_wrkr != ODP_SCHED_GROUP_WORKER) { - _ODP_ERR("Could not create ODP_SCHED_GROUP_WORKER()\n"); - goto failed_create_group_worker; - } - - tmp_ctrl = schedule_group_create("__group_control", &mask); - if (tmp_ctrl != ODP_SCHED_GROUP_CONTROL) { - _ODP_ERR("Could not create ODP_SCHED_GROUP_CONTROL()\n"); - goto failed_create_group_control; - } - - global->config_if.group_enable.all = 1; - global->config_if.group_enable.control = 1; - global->config_if.group_enable.worker = 1; - - return 0; - -failed_create_group_control: - if (tmp_ctrl != ODP_SCHED_GROUP_INVALID) - schedule_group_destroy(ODP_SCHED_GROUP_CONTROL); - -failed_create_group_worker: - if (tmp_wrkr != ODP_SCHED_GROUP_INVALID) - schedule_group_destroy(ODP_SCHED_GROUP_WORKER); - -failed_create_group_all: - if (tmp_all != ODP_SCHED_GROUP_INVALID) - schedule_group_destroy(ODP_SCHED_GROUP_ALL); - -failed_sched_shm_pool_create: - - return -1; -} - -static int schedule_term_global(void) -{ - /* Destroy enabled sched groups for default GROUP_ALL, GROUP_WORKER and - * GROUP_CONTROL groups. */ - if (global->config_if.group_enable.all) { - if (schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) - _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); - } - if (global->config_if.group_enable.worker) { - if (schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) - _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); - } - if (global->config_if.group_enable.control) { - if (schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) - _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); - } - - _odp_ishm_pool_destroy(global->sched_shm_pool); - - if (odp_shm_free(global->shm)) { - _ODP_ERR("Shm free failed for scalable scheduler"); - return -1; - } - - return 0; -} - -static int schedule_init_local(void) -{ - int thr_id; - odp_thread_type_t thr_type; - odp_thrmask_t mask; - - thr_id = odp_thread_id(); - if (thread_state_init(thr_id)) - goto failed_to_init_ts; - - /* Add this thread to default schedule groups */ - thr_type = odp_thread_type(); - odp_thrmask_zero(&mask); - odp_thrmask_set(&mask, thr_id); - - odp_spinlock_lock(&global->init_lock); - - if (global->config_if.group_enable.all) { - if (schedule_group_join(ODP_SCHED_GROUP_ALL, &mask) != 0) { - _ODP_ERR("Failed to join ODP_SCHED_GROUP_ALL\n"); - goto failed_to_join_grp_all; - } - } - if (global->config_if.group_enable.control && thr_type == ODP_THREAD_CONTROL) { - if (schedule_group_join(ODP_SCHED_GROUP_CONTROL, &mask) != 0) { - _ODP_ERR("Failed to join ODP_SCHED_GROUP_CONTROL\n"); - goto failed_to_join_grp_ctrl; - } - } - if (global->config_if.group_enable.worker && thr_type == ODP_THREAD_WORKER) { - if (schedule_group_join(ODP_SCHED_GROUP_WORKER, &mask) != 0) { - _ODP_ERR("Failed to join ODP_SCHED_GROUP_WORKER\n"); - goto failed_to_join_grp_wrkr; - } - } - - odp_spinlock_unlock(&global->init_lock); - - return 0; - -failed_to_join_grp_wrkr: -failed_to_join_grp_ctrl: - if (global->config_if.group_enable.all) - schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask); - -failed_to_join_grp_all: - odp_spinlock_unlock(&global->init_lock); - -failed_to_init_ts: - return -1; -} - -static int schedule_term_local(void) -{ - int thr_id; - odp_thread_type_t thr_type; - odp_thrmask_t mask; - int rc = 0; - - /* Remove this thread from default schedule groups */ - thr_id = odp_thread_id(); - thr_type = odp_thread_type(); - odp_thrmask_zero(&mask); - odp_thrmask_set(&mask, thr_id); - - if (global->config_if.group_enable.all) { - if (schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask) != 0) - _ODP_ERR("Failed to leave ODP_SCHED_GROUP_ALL\n"); - } - if (global->config_if.group_enable.control && thr_type == ODP_THREAD_CONTROL) { - if (schedule_group_leave(ODP_SCHED_GROUP_CONTROL, &mask) != 0) - _ODP_ERR("Failed to leave ODP_SCHED_GROUP_CONTROL\n"); - } - if (global->config_if.group_enable.worker && thr_type == ODP_THREAD_WORKER) { - if (schedule_group_leave(ODP_SCHED_GROUP_WORKER, &mask) != 0) - _ODP_ERR("Failed to leave ODP_SCHED_GROUP_WORKER\n"); - } - - update_sg_membership(_odp_sched_ts); - - /* Check if the thread is still part of any groups */ - if (_odp_sched_ts->num_schedq != 0) { - _ODP_ERR("Thread %d still part of scheduler group(s)\n", _odp_sched_ts->tidx); - rc = -1; - } - - return rc; -} - -static void schedule_config_init(odp_schedule_config_t *config) -{ - config->num_queues = CONFIG_MAX_SCHED_QUEUES; - config->queue_size = 0; /* FIXME ? */ - config->sched_group.all = true; - config->sched_group.control = true; - config->sched_group.worker = true; -} - -static int schedule_config(const odp_schedule_config_t *config) -{ - odp_spinlock_lock(&global->init_lock); - - global->config_if.group_enable.all = config->sched_group.all; - global->config_if.group_enable.control = config->sched_group.control; - global->config_if.group_enable.worker = config->sched_group.worker; - - /* Destroy disabled predefined scheduling groups. */ - if (!config->sched_group.all) { - if (schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) - _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); - } - if (!config->sched_group.worker) { - if (schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) - _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); - } - - if (!config->sched_group.control) { - if (schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) - _ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); - } - - odp_spinlock_unlock(&global->init_lock); - - return 0; -} - -static int num_grps(void) -{ - return MAX_SCHED_GROUP - NUM_AUTO_GROUPS; -} - -/* - * Stubs for internal scheduler abstraction layer due to absence of NULL - * checking before calling the function pointer. - */ - -static int thr_add(odp_schedule_group_t group, int thr) -{ - /* This function is a schedule_init_local duplicate. */ - (void)group; - (void)thr; - return 0; -} - -static int thr_rem(odp_schedule_group_t group, int thr) -{ - /* This function is a schedule_term_local duplicate. */ - (void)group; - (void)thr; - return 0; -} - -static int create_queue(uint32_t queue_index, - const odp_schedule_param_t *sched_param) -{ - /* Not used in scalable scheduler. */ - (void)queue_index; - (void)sched_param; - return 0; -} - -static void destroy_queue(uint32_t queue_index) -{ - /* Not used in scalable scheduler. */ - (void)queue_index; -} - -static int sched_queue(uint32_t queue_index) -{ - /* Not used in scalable scheduler. */ - (void)queue_index; - return 0; -} - -static int ord_enq_multi(odp_queue_t handle, void *event_hdr[], int num, - int *ret) - -{ - queue_entry_t *queue; - sched_scalable_thread_state_t *ts; - int actual; - - ts = _odp_sched_ts; - queue = qentry_from_int(handle); - if (ts && odp_unlikely(ts->out_of_order) && - (queue->param.order == ODP_QUEUE_ORDER_KEEP)) { - actual = _odp_rctx_save(queue, (_odp_event_hdr_t **)event_hdr, num); - *ret = actual; - return 1; - } - return 0; -} - -static void schedule_prefetch(int num) -{ - (void)num; -} - -/* Wait until we are in-order (when processing an ordered queue) - * Note: this function may be called also when processing other queue types - */ -static void order_lock(void) -{ - sched_scalable_thread_state_t *ts; - reorder_window_t *rwin; - uint32_t sn; - - ts = _odp_sched_ts; - if (odp_unlikely(ts->out_of_order)) { - /* We are processing ordered queue and are currently - * out-of-order. - * We are in-order when our reorder window slot number (sn) - * equals the head of the reorder window. - */ - _ODP_ASSERT(ts->rctx != NULL); - rwin = ts->rctx->rwin; - sn = ts->rctx->sn; - /* Use acquire ordering to be on the safe side even if - * this isn't an acquire/release situation (aka lock). - */ - _odp_wait_until_eq_acq_u32(&rwin->hc.head, sn); - } -} - -/* This function is unnecessary. - * The next thread becomes in-order when we release our reorder context - * (i.e. when odp_schedule() is called again. - */ -static void order_unlock(void) -{ - /* Nothing to do */ -} - -static uint32_t schedule_max_ordered_locks(void) -{ - return CONFIG_QUEUE_MAX_ORD_LOCKS; -} - -static int schedule_capability(odp_schedule_capability_t *capa) -{ - memset(capa, 0, sizeof(odp_schedule_capability_t)); - - capa->max_ordered_locks = schedule_max_ordered_locks(); - capa->max_groups = num_grps(); - capa->max_prios = schedule_num_prio(); - capa->max_queues = CONFIG_MAX_SCHED_QUEUES; - capa->max_queue_size = 0; - capa->order_wait = ODP_SUPPORT_YES; - - return 0; -} - -static void schedule_print(void) -{ - odp_schedule_capability_t capa; - - (void)schedule_capability(&capa); - - _ODP_PRINT("\nScheduler debug info\n"); - _ODP_PRINT("--------------------\n"); - _ODP_PRINT(" scheduler: scalable\n"); - _ODP_PRINT(" max groups: %u\n", capa.max_groups); - _ODP_PRINT(" max priorities: %u\n", capa.max_prios); - _ODP_PRINT("\n"); -} - -const _odp_schedule_api_fn_t _odp_schedule_scalable_api; - -static const _odp_schedule_api_fn_t *sched_api(void) -{ - return &_odp_schedule_scalable_api; -} - -const schedule_fn_t _odp_schedule_scalable_fn = { - .pktio_start = pktio_start, - .thr_add = thr_add, - .thr_rem = thr_rem, - .num_grps = num_grps, - .create_queue = create_queue, - .destroy_queue = destroy_queue, - .sched_queue = sched_queue, - .ord_enq_multi = ord_enq_multi, - .init_global = schedule_init_global, - .term_global = schedule_term_global, - .init_local = schedule_init_local, - .term_local = schedule_term_local, - .order_lock = order_lock, - .order_unlock = order_unlock, - .max_ordered_locks = schedule_max_ordered_locks, - .sched_api = sched_api, -}; - -const _odp_schedule_api_fn_t _odp_schedule_scalable_api = { - .schedule_wait_time = schedule_wait_time, - .schedule_capability = schedule_capability, - .schedule_config_init = schedule_config_init, - .schedule_config = schedule_config, - .schedule = schedule, - .schedule_multi = schedule_multi, - .schedule_multi_wait = schedule_multi_wait, - .schedule_multi_no_wait = schedule_multi_no_wait, - .schedule_pause = schedule_pause, - .schedule_resume = schedule_resume, - .schedule_release_atomic = schedule_release_atomic, - .schedule_release_ordered = schedule_release_ordered, - .schedule_prefetch = schedule_prefetch, - .schedule_min_prio = schedule_min_prio, - .schedule_max_prio = schedule_max_prio, - .schedule_default_prio = schedule_default_prio, - .schedule_num_prio = schedule_num_prio, - .schedule_group_create = schedule_group_create, - .schedule_group_destroy = schedule_group_destroy, - .schedule_group_lookup = schedule_group_lookup, - .schedule_group_join = schedule_group_join, - .schedule_group_leave = schedule_group_leave, - .schedule_group_thrmask = schedule_group_thrmask, - .schedule_group_info = schedule_group_info, - .schedule_order_lock = schedule_order_lock, - .schedule_order_unlock = schedule_order_unlock, - .schedule_order_unlock_lock = schedule_order_unlock_lock, - .schedule_order_lock_start = schedule_order_lock_start, - .schedule_order_lock_wait = schedule_order_lock_wait, - .schedule_order_wait = order_lock, - .schedule_print = schedule_print -}; diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c deleted file mode 100644 index d8e0bcaaac..0000000000 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ /dev/null @@ -1,367 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2017 ARM Limited - * Copyright (c) 2017-2018 Linaro Limited - */ - -#include -#include -#include - -#include -#include -#include -#include - -#include - -extern __thread sched_scalable_thread_state_t *_odp_sched_ts; - -reorder_window_t *_odp_rwin_alloc(_odp_ishm_pool_t *pool, unsigned int lock_count) -{ - reorder_window_t *rwin; - uint32_t i; - - rwin = (reorder_window_t *) - shm_pool_alloc_align(pool, sizeof(reorder_window_t)); - if (rwin == NULL) - return NULL; - - rwin->hc.head = 0; - rwin->hc.chgi = 0; - rwin->winmask = RWIN_SIZE - 1; - rwin->tail = 0; - rwin->turn = 0; - rwin->lock_count = (uint16_t)lock_count; - memset(rwin->olock, 0, sizeof(rwin->olock)); - for (i = 0; i < RWIN_SIZE; i++) - rwin->ring[i] = NULL; - - return rwin; -} - -int _odp_rwin_free(_odp_ishm_pool_t *pool, reorder_window_t *rwin) -{ - return _odp_ishm_pool_free(pool, rwin); -} - -bool _odp_rwin_reserve(reorder_window_t *rwin, uint32_t *sn) -{ - uint32_t head; - uint32_t oldt; - uint32_t newt; - uint32_t winmask; - - /* Read head and tail separately */ - oldt = rwin->tail; - winmask = rwin->winmask; - do { - /* Need __atomic_load to avoid compiler reordering */ - head = __atomic_load_n(&rwin->hc.head, __ATOMIC_RELAXED); - if (odp_unlikely(oldt - head >= winmask)) - return false; - - newt = oldt + 1; - } while (!__atomic_compare_exchange(&rwin->tail, - &oldt, - &newt, - true, - __ATOMIC_RELAXED, - __ATOMIC_RELAXED)); - *sn = oldt; - - return true; -} - -bool _odp_rwin_reserve_sc(reorder_window_t *rwin, uint32_t *sn) -{ - uint32_t head; - uint32_t oldt; - uint32_t newt; - uint32_t winmask; - - /* Read head and tail separately */ - oldt = rwin->tail; - winmask = rwin->winmask; - head = rwin->hc.head; - if (odp_unlikely(oldt - head >= winmask)) - return false; - newt = oldt + 1; - rwin->tail = newt; - *sn = oldt; - - return true; -} - -void _odp_rwin_unreserve_sc(reorder_window_t *rwin, uint32_t sn) -{ - _ODP_ASSERT(rwin->tail == sn + 1); - rwin->tail = sn; -} - -static void rwin_insert(reorder_window_t *rwin, - reorder_context_t *rctx, - uint32_t sn, - void (*callback)(reorder_context_t *)) -{ - /* Initialise to silence scan-build */ - hc_t old = {0, 0}; - hc_t new; - uint32_t winmask; - - __atomic_load(&rwin->hc, &old, __ATOMIC_ACQUIRE); - winmask = rwin->winmask; - if (old.head != sn) { - /* We are out-of-order. Store context in reorder window, - * releasing its content. - */ - _ODP_ASSERT(rwin->ring[sn & winmask] == NULL); - atomic_store_release(&rwin->ring[sn & winmask], - rctx, - /*readonly=*/false); - rctx = NULL; - do { - new.head = old.head; - new.chgi = old.chgi + 1; /* Changed value */ - /* Update head & chgi, fail if any has changed */ - if (__atomic_compare_exchange(&rwin->hc, - /* Updated on fail */ - &old, - &new, - true, - /* Rel our ring update */ - __ATOMIC_RELEASE, - __ATOMIC_ACQUIRE)) - /* CAS succeeded => head same (we are not - * in-order), chgi updated. - */ - return; - /* CAS failed => head and/or chgi changed. - * We might not be out-of-order anymore. - */ - } while (old.head != sn); - } - - /* old.head == sn => we are now in-order! */ - _ODP_ASSERT(old.head == sn); - /* We are in-order so our responsibility to retire contexts */ - new.head = old.head; - new.chgi = old.chgi + 1; - - /* Retire our in-order context (if we still have it) */ - if (rctx != NULL) { - callback(rctx); - new.head++; - } - - /* Retire in-order contexts in the ring - * The first context might actually be ours (if we were originally - * out-of-order) - */ - do { - for (;;) { - rctx = __atomic_load_n(&rwin->ring[new.head & winmask], - __ATOMIC_ACQUIRE); - if (rctx == NULL) - break; - /* We are the only thread that are in-order - * (until head updated) so don't have to use - * atomic load-and-clear (exchange) - */ - rwin->ring[new.head & winmask] = NULL; - callback(rctx); - new.head++; - } - /* Update head&chgi, fail if chgi has changed (head cannot change) */ - } while (!__atomic_compare_exchange(&rwin->hc, - &old, /* Updated on failure */ - &new, - false, /* weak */ - __ATOMIC_RELEASE, /* Release our ring updates */ - __ATOMIC_ACQUIRE)); -} - -void _odp_rctx_init(reorder_context_t *rctx, uint16_t idx, - reorder_window_t *rwin, uint32_t sn) -{ - /* rctx->rvec_free and rctx->idx already initialised in - * thread_state_init function. - */ - _ODP_ASSERT(rctx->idx == idx); - rctx->rwin = rwin; - rctx->sn = sn; - rctx->olock_flags = 0; - /* First => no next reorder context */ - rctx->next_idx = idx; - /* Where to store next event */ - rctx->cur_idx = idx; - rctx->numevts = 0; -} - -static inline void rctx_free(const reorder_context_t *rctx) -{ - const reorder_context_t *const base = &rctx[-(int)rctx->idx]; - const uint32_t first = rctx->idx; - uint32_t next_idx; - - next_idx = rctx->next_idx; - - _ODP_ASSERT(rctx->rwin != NULL); - /* Set free bit */ - if (rctx->rvec_free == &_odp_sched_ts->rvec_free) - /* Since it is our own reorder context, we can instead - * perform a non-atomic and relaxed update on our private - * rvec_free. - */ - _odp_sched_ts->priv_rvec_free = - bitset_set(_odp_sched_ts->priv_rvec_free, rctx->idx); - else - atom_bitset_set(rctx->rvec_free, rctx->idx, __ATOMIC_RELEASE); - - /* Can't dereference rctx after the corresponding free bit is set */ - while (next_idx != first) { - rctx = &base[next_idx]; - next_idx = rctx->next_idx; - /* Set free bit */ - if (rctx->rvec_free == &_odp_sched_ts->rvec_free) - _odp_sched_ts->priv_rvec_free = - bitset_set(_odp_sched_ts->priv_rvec_free, rctx->idx); - else - atom_bitset_set(rctx->rvec_free, rctx->idx, - __ATOMIC_RELEASE); - } -} - -static inline void olock_unlock(const reorder_context_t *rctx, - reorder_window_t *rwin, - uint32_t lock_index) -{ - if ((rctx->olock_flags & (1U << lock_index)) == 0) { - /* Use relaxed ordering, we are not releasing any updates */ - rwin->olock[lock_index] = rctx->sn + 1; - } -} - -static void olock_release(const reorder_context_t *rctx) -{ - reorder_window_t *rwin; - uint32_t i; - - rwin = rctx->rwin; - - for (i = 0; i < rwin->lock_count; i++) - olock_unlock(rctx, rwin, i); -} - -static void blocking_enqueue(queue_entry_t *q, _odp_event_hdr_t **evts, int num) -{ - int actual; - - /* Iterate until all events have been successfully enqueued */ - for (;;) { - /* Attempt to enqueue remaining events */ - actual = q->enqueue_multi(qentry_to_int(q), evts, num); - if (odp_unlikely(actual < 0)) - _ODP_ERR("Failed to enqueue deferred events\n"); - /* Update for potential partial success */ - evts += actual; - num -= actual; - if (num == 0) - break; - /* Back-off to decrease load on the system */ - odp_cpu_pause(); - } -} - -static void rctx_retire(reorder_context_t *first) -{ - reorder_context_t *rctx; - queue_entry_t *q; - uint32_t i; - uint32_t j; - uint32_t num; - - rctx = first; - do { - /* Process all events in this reorder context */ - for (i = 0; i < rctx->numevts;) { - q = rctx->destq[i]; - /* Find index of next different destq */ - j = i + 1; - while (j < rctx->numevts && rctx->destq[j] == q) - j++; - num = j - i; - /* Blocking enqueue of events to this destq */ - blocking_enqueue(q, &rctx->events[i], num); - i += num; - } - /* Update rctx pointer to point to 'next_idx' element */ - rctx += (int)rctx->next_idx - (int)rctx->idx; - } while (rctx != first); - olock_release(first); - rctx_free(first); -} - -void _odp_rctx_release(reorder_context_t *rctx) -{ - /* Insert reorder context into reorder window, potentially calling the - * rctx_retire function for all pending reorder_contexts. - */ - rwin_insert(rctx->rwin, rctx, rctx->sn, rctx_retire); -} - -/* Save destination queue and events in the reorder context for deferred - * enqueue. - */ -int _odp_rctx_save(queue_entry_t *queue, _odp_event_hdr_t *event_hdr[], int num) -{ - int i; - sched_scalable_thread_state_t *ts; - reorder_context_t *first; - reorder_context_t *cur; - bitset_t next_idx; - - ts = _odp_sched_ts; - first = ts->rctx; - _ODP_ASSERT(ts->rctx != NULL); - cur = &first[(int)first->cur_idx - (int)first->idx]; - for (i = 0; i < num; i++) { - if (odp_unlikely(cur->numevts == RC_EVT_SIZE)) { - /* No more space in current reorder context - * Try to allocate another. - */ - if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { - ts->priv_rvec_free = atom_bitset_xchg(&ts->rvec_free, 0, - __ATOMIC_RELAXED); - - if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) - /* Out of reorder contexts. - * Return the number of events - * stored so far. - */ - return i; - } - next_idx = bitset_ffs(ts->priv_rvec_free) - 1; - ts->priv_rvec_free = - bitset_clr(ts->priv_rvec_free, - next_idx); - /* Link current to next (for eventual - * retiring) - */ - cur->next_idx = next_idx; - /* Link first to next (for next call to - * queue_enq_multi()) - */ - first->cur_idx = next_idx; - /* Update current to next */ - cur = &ts->rvec[next_idx]; - _odp_rctx_init(cur, next_idx, NULL, 0); - /* The last rctx (so far) */ - cur->next_idx = first->idx; - } - cur->events[cur->numevts] = event_hdr[i]; - cur->destq[cur->numevts] = queue; - cur->numevts++; - } - /* All events stored. */ - return num; -} diff --git a/scripts/ci/check_inline_timer.sh b/scripts/ci/check_inline_timer.sh index 48800be354..45f1ae062e 100755 --- a/scripts/ci/check_inline_timer.sh +++ b/scripts/ci/check_inline_timer.sh @@ -11,6 +11,5 @@ cd "$(dirname "$0")"/../.. ODP_SCHEDULER=basic ./test/validation/api/timer/timer_main ODP_SCHEDULER=sp ./test/validation/api/timer/timer_main -ODP_SCHEDULER=scalable ./test/validation/api/timer/timer_main umount /mnt/huge diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh index c59a704e15..caa3fe8b58 100755 --- a/scripts/ci/coverage.sh +++ b/scripts/ci/coverage.sh @@ -23,9 +23,8 @@ export CI="true" ODP_SCHEDULER=basic make check -# Run only validation tests for scalable and sp schedulers +# Run only validation tests for SP scheduler pushd ./test/validation/api/ -ODP_SCHEDULER=scalable CI_SKIP=pktio_test_pktin_event_sched make check ODP_SCHEDULER=sp make check popd