diff --git a/doc/swss-schema.md b/doc/swss-schema.md index 74bfd687b8..631594b345 100644 --- a/doc/swss-schema.md +++ b/doc/swss-schema.md @@ -27,6 +27,8 @@ Stores information for physical switch ports managed by the switch chip. Ports t preemphasis = 1*8HEXDIG *( "," 1*8HEXDIG) ; list of hex values, one per lane idriver = 1*8HEXDIG *( "," 1*8HEXDIG) ; list of hex values, one per lane ipredriver = 1*8HEXDIG *( "," 1*8HEXDIG) ; list of hex values, one per lane + pt_interface_id = 1*4DIGIT ; Path Tracing Interface ID (1-4095) + pt_timestamp_template = "template1" / "template2" / "template3" / "template4" ; Path Tracing Timestamp Template ;QOS Mappings map_dscp_to_tc = ref_hash_key_reference @@ -1021,6 +1023,8 @@ Stores information for physical switch ports managed by the switch chip. Ports t mtu = 1*4DIGIT ; port MTU fec = 1*64VCHAR ; port fec mode autoneg = BIT ; auto-negotiation mode + pt_interface_id = 1*4DIGIT ; Path Tracing Interface ID (1-4095) + pt_timestamp_template = "template1" / "template2" / "template3" / "template4" ; Path Tracing Timestamp Template ### MGMT_PORT_TABLE ;Configuration for management port, including at least one key diff --git a/orchagent/port.h b/orchagent/port.h index dc8241ce3a..d153b20318 100644 --- a/orchagent/port.h +++ b/orchagent/port.h @@ -208,6 +208,10 @@ class Port int m_cap_an = -1; /* Capability - AutoNeg, -1 means not set */ int m_cap_lt = -1; /* Capability - LinkTraining, -1 means not set */ + + /* Path Tracing */ + uint16_t m_pt_intf_id = 0; + sai_port_path_tracing_timestamp_type_t m_pt_timestamp_template = SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23; }; } diff --git a/orchagent/port/portcnt.h b/orchagent/port/portcnt.h index 2ff66383f9..9e3e63f9b7 100644 --- a/orchagent/port/portcnt.h +++ b/orchagent/port/portcnt.h @@ -207,6 +207,16 @@ class PortConfig final bool is_set = false; } subport; // Port subport + struct { + std::uint16_t value; + bool is_set = false; + } pt_intf_id; // Port interface ID for Path Tracing + + struct { + sai_port_path_tracing_timestamp_type_t value; + bool is_set = false; + } pt_timestamp_template; // Port timestamp template for Path Tracing + std::string key; std::string op; diff --git a/orchagent/port/porthlpr.cpp b/orchagent/port/porthlpr.cpp index 80029cb569..7ac9c15c52 100644 --- a/orchagent/port/porthlpr.cpp +++ b/orchagent/port/porthlpr.cpp @@ -118,6 +118,14 @@ static const std::unordered_map portRoleMap = { PORT_ROLE_DPC, Port::Role::Dpc } }; +static const std::unordered_map portPtTimestampTemplateMap = +{ + { PORT_PT_TIMESTAMP_TEMPLATE_1, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_8_15 }, + { PORT_PT_TIMESTAMP_TEMPLATE_2, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_12_19 }, + { PORT_PT_TIMESTAMP_TEMPLATE_3, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23 }, + { PORT_PT_TIMESTAMP_TEMPLATE_4, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_20_27 } +}; + // functions ---------------------------------------------------------------------------------------------------------- template @@ -233,6 +241,11 @@ std::string PortHelper::getAdminStatusStr(const PortConfig &port) const return this->getFieldValueStr(port, PORT_ADMIN_STATUS); } +std::string PortHelper::getPtTimestampTemplateStr(const PortConfig &port) const +{ + return this->getFieldValueStr(port, PORT_PT_TIMESTAMP_TEMPLATE); +} + bool PortHelper::parsePortAlias(PortConfig &port, const std::string &field, const std::string &value) const { SWSS_LOG_ENTER(); @@ -773,6 +786,73 @@ bool PortHelper::parsePortSubport(PortConfig &port, const std::string &field, co return true; } +bool PortHelper::parsePortPtIntfId(PortConfig &port, const std::string &field, const std::string &value) const +{ + SWSS_LOG_ENTER(); + + uint16_t pt_intf_id; + try + { + if (value != "None") + { + pt_intf_id = to_uint(value); + if (pt_intf_id < 1 || pt_intf_id > 4095) + { + throw std::invalid_argument("Out of range Path Tracing Interface ID: " + value); + } + + port.pt_intf_id.value = pt_intf_id; + } + else + { + /* + * In SAI, Path Tracing Interface ID 0 means Path Tracing disabled. + * When Path Tracing Interface ID is not set (i.e., value is None), + * we set the Interface ID to 0 in ASIC DB in order to disable + * Path Tracing on the port. + */ + port.pt_intf_id.value = 0; + } + port.pt_intf_id.is_set = true; + } + catch (const std::exception &e) + { + SWSS_LOG_ERROR("Failed to parse field(%s): %s", field.c_str(), e.what()); + return false; + } + + return true; +} + +bool PortHelper::parsePortPtTimestampTemplate(PortConfig &port, const std::string &field, const std::string &value) const +{ + SWSS_LOG_ENTER(); + std::unordered_map::const_iterator cit; + + if (value != "None") + { + cit = portPtTimestampTemplateMap.find(value); + } + else + { + /* + * When Path Tracing Timestamp Template is not specified (i.e., value is None), + * we use Template3 (which is the default template in SAI). + */ + cit = portPtTimestampTemplateMap.find("template3"); + } + if (cit == portPtTimestampTemplateMap.cend()) + { + SWSS_LOG_ERROR("Failed to parse field(%s): invalid value(%s)", field.c_str(), value.c_str()); + return false; + } + + port.pt_timestamp_template.value = cit->second; + port.pt_timestamp_template.is_set = true; + + return true; +} + bool PortHelper::parsePortConfig(PortConfig &port) const { SWSS_LOG_ENTER(); @@ -1027,6 +1107,20 @@ bool PortHelper::parsePortConfig(PortConfig &port) const return false; } } + else if (field == PORT_PT_INTF_ID) + { + if (!this->parsePortPtIntfId(port, field, value)) + { + return false; + } + } + else if (field == PORT_PT_TIMESTAMP_TEMPLATE) + { + if (!this->parsePortPtTimestampTemplate(port, field, value)) + { + return false; + } + } else { SWSS_LOG_WARN("Unknown field(%s): skipping ...", field.c_str()); diff --git a/orchagent/port/porthlpr.h b/orchagent/port/porthlpr.h index 04ddfa5231..3852759975 100644 --- a/orchagent/port/porthlpr.h +++ b/orchagent/port/porthlpr.h @@ -26,6 +26,7 @@ class PortHelper final std::string getLearnModeStr(const PortConfig &port) const; std::string getLinkTrainingStr(const PortConfig &port) const; std::string getAdminStatusStr(const PortConfig &port) const; + std::string getPtTimestampTemplateStr(const PortConfig &port) const; bool parsePortConfig(PortConfig &port) const; bool validatePortConfig(PortConfig &port) const; @@ -54,4 +55,6 @@ class PortHelper final bool parsePortAdminStatus(PortConfig &port, const std::string &field, const std::string &value) const; bool parsePortDescription(PortConfig &port, const std::string &field, const std::string &value) const; bool parsePortSubport(PortConfig &port, const std::string &field, const std::string &value) const; + bool parsePortPtIntfId(PortConfig &port, const std::string &field, const std::string &value) const; + bool parsePortPtTimestampTemplate(PortConfig &port, const std::string &field, const std::string &value) const; }; diff --git a/orchagent/port/portschema.h b/orchagent/port/portschema.h index ca3d859cae..c9a3274913 100644 --- a/orchagent/port/portschema.h +++ b/orchagent/port/portschema.h @@ -53,38 +53,45 @@ #define PORT_ROLE_REC "Rec" #define PORT_ROLE_DPC "Dpc" -#define PORT_ALIAS "alias" -#define PORT_INDEX "index" -#define PORT_LANES "lanes" -#define PORT_SPEED "speed" -#define PORT_AUTONEG "autoneg" -#define PORT_ADV_SPEEDS "adv_speeds" -#define PORT_INTERFACE_TYPE "interface_type" -#define PORT_ADV_INTERFACE_TYPES "adv_interface_types" -#define PORT_FEC "fec" -#define PORT_MTU "mtu" -#define PORT_TPID "tpid" -#define PORT_PFC_ASYM "pfc_asym" -#define PORT_LEARN_MODE "learn_mode" -#define PORT_LINK_TRAINING "link_training" -#define PORT_PREEMPHASIS "preemphasis" -#define PORT_IDRIVER "idriver" -#define PORT_IPREDRIVER "ipredriver" -#define PORT_PRE1 "pre1" -#define PORT_PRE2 "pre2" -#define PORT_PRE3 "pre3" -#define PORT_MAIN "main" -#define PORT_POST1 "post1" -#define PORT_POST2 "post2" -#define PORT_POST3 "post3" -#define PORT_ATTN "attn" -#define PORT_OB_M2LP "ob_m2lp" -#define PORT_OB_ALEV_OUT "ob_alev_out" -#define PORT_OBPLEV "obplev" -#define PORT_OBNLEV "obnlev" -#define PORT_REGN_BFM1P "regn_bfm1p" -#define PORT_REGN_BFM1N "regn_bfm1n" -#define PORT_ROLE "role" -#define PORT_ADMIN_STATUS "admin_status" -#define PORT_DESCRIPTION "description" -#define PORT_SUBPORT "subport" +#define PORT_PT_TIMESTAMP_TEMPLATE_1 "template1" +#define PORT_PT_TIMESTAMP_TEMPLATE_2 "template2" +#define PORT_PT_TIMESTAMP_TEMPLATE_3 "template3" +#define PORT_PT_TIMESTAMP_TEMPLATE_4 "template4" + +#define PORT_ALIAS "alias" +#define PORT_INDEX "index" +#define PORT_LANES "lanes" +#define PORT_SPEED "speed" +#define PORT_AUTONEG "autoneg" +#define PORT_ADV_SPEEDS "adv_speeds" +#define PORT_INTERFACE_TYPE "interface_type" +#define PORT_ADV_INTERFACE_TYPES "adv_interface_types" +#define PORT_FEC "fec" +#define PORT_MTU "mtu" +#define PORT_TPID "tpid" +#define PORT_PFC_ASYM "pfc_asym" +#define PORT_LEARN_MODE "learn_mode" +#define PORT_LINK_TRAINING "link_training" +#define PORT_PREEMPHASIS "preemphasis" +#define PORT_IDRIVER "idriver" +#define PORT_IPREDRIVER "ipredriver" +#define PORT_PRE1 "pre1" +#define PORT_PRE2 "pre2" +#define PORT_PRE3 "pre3" +#define PORT_MAIN "main" +#define PORT_POST1 "post1" +#define PORT_POST2 "post2" +#define PORT_POST3 "post3" +#define PORT_ATTN "attn" +#define PORT_OB_M2LP "ob_m2lp" +#define PORT_OB_ALEV_OUT "ob_alev_out" +#define PORT_OBPLEV "obplev" +#define PORT_OBNLEV "obnlev" +#define PORT_REGN_BFM1P "regn_bfm1p" +#define PORT_REGN_BFM1N "regn_bfm1n" +#define PORT_ROLE "role" +#define PORT_ADMIN_STATUS "admin_status" +#define PORT_DESCRIPTION "description" +#define PORT_SUBPORT "subport" +#define PORT_PT_INTF_ID "pt_interface_id" +#define PORT_PT_TIMESTAMP_TEMPLATE "pt_timestamp_template" diff --git a/orchagent/portsorch.cpp b/orchagent/portsorch.cpp old mode 100755 new mode 100644 index 09e80ec651..bac498fa3d --- a/orchagent/portsorch.cpp +++ b/orchagent/portsorch.cpp @@ -34,6 +34,8 @@ #include "stringutility.h" #include "subscriberstatetable.h" +#include "saitam.h" + extern sai_switch_api_t *sai_switch_api; extern sai_bridge_api_t *sai_bridge_api; extern sai_port_api_t *sai_port_api; @@ -44,6 +46,7 @@ extern sai_acl_api_t* sai_acl_api; extern sai_queue_api_t *sai_queue_api; extern sai_object_id_t gSwitchId; extern sai_fdb_api_t *sai_fdb_api; +extern sai_tam_api_t *sai_tam_api; extern sai_l2mc_group_api_t *sai_l2mc_group_api; extern sai_buffer_api_t *sai_buffer_api; extern IntfsOrch *gIntfsOrch; @@ -147,6 +150,15 @@ static map interface_type_map = { "kr8", SAI_PORT_INTERFACE_TYPE_KR8 } }; +// Timestamp Template map used for Path Tracing +static map pt_timestamp_template_map = +{ + { "template1", SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_8_15 }, + { "template2", SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_12_19 }, + { "template3", SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23 }, + { "template4", SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_20_27 } +}; + const vector port_stat_ids = { SAI_PORT_STAT_IF_IN_OCTETS, @@ -388,6 +400,83 @@ static void getPortSerdesAttr(PortSerdesAttrMap_t &map, const PortConfig &port) } +static bool isPathTracingSupported() +{ + /* + * Path Tracing is supported when four conditions are met: + * + * 1. The switch supports SAI_OBJECT_TYPE_TAM + * 2. SAI_OBJECT_TYPE_PORT supports SAI_PORT_ATTR_PATH_TRACING_INTF attribute + * 3. SAI_OBJECT_TYPE_PORT supports SAI_PORT_ATTR_PATH_TRACING_TIMESTAMP_TYPE attribute + * 4. SAI_OBJECT_TYPE_PORT supports SAI_PORT_ATTR_TAM_OBJECT attribute + */ + + /* First, query switch capabilities */ + sai_attribute_t attr; + std::vector switchCapabilities(SAI_OBJECT_TYPE_MAX); + attr.id = SAI_SWITCH_ATTR_SUPPORTED_OBJECT_TYPE_LIST; + attr.value.s32list.count = static_cast(switchCapabilities.size()); + attr.value.s32list.list = switchCapabilities.data(); + + bool is_tam_supported = false; + auto status = sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr); + if (status == SAI_STATUS_SUCCESS) + { + for (std::uint32_t i = 0; i < attr.value.s32list.count; i++) + { + switch(static_cast(attr.value.s32list.list[i])) + { + case SAI_OBJECT_TYPE_TAM: + is_tam_supported = true; + break; + default: + /* Received an attribute in which we are not interested, ignoring it */ + break; + } + } + } + else + { + SWSS_LOG_ERROR( + "Failed to get a list of supported switch capabilities. Error=%d", status + ); + return false; + } + + /* Then verify if the four conditions are met */ + if (!is_tam_supported || + !gSwitchOrch->querySwitchCapability(SAI_OBJECT_TYPE_PORT, SAI_PORT_ATTR_PATH_TRACING_INTF) || + !gSwitchOrch->querySwitchCapability(SAI_OBJECT_TYPE_PORT, SAI_PORT_ATTR_PATH_TRACING_TIMESTAMP_TYPE) || + !gSwitchOrch->querySwitchCapability(SAI_OBJECT_TYPE_PORT, SAI_PORT_ATTR_TAM_OBJECT)) + { + return false; + } + + return true; +} + +bool PortsOrch::checkPathTracingCapability() +{ + vector fvVector; + if (isPathTracingSupported()) + { + SWSS_LOG_INFO("Path Tracing is supported"); + /* Set PATH_TRACING_CAPABLE = true in STATE DB */ + fvVector.emplace_back(SWITCH_CAPABILITY_TABLE_PATH_TRACING_CAPABLE, "true"); + m_isPathTracingSupported = true; + } + else + { + SWSS_LOG_INFO("Path Tracing is not supported"); + /* Set PATH_TRACING_CAPABLE = false in STATE DB */ + fvVector.emplace_back(SWITCH_CAPABILITY_TABLE_PATH_TRACING_CAPABLE, "false"); + m_isPathTracingSupported = false; + } + gSwitchOrch->set_switch_capability(fvVector); + + return m_isPathTracingSupported; +} + // Port OA ------------------------------------------------------------------------------------------------------------ /* @@ -674,6 +763,9 @@ PortsOrch::PortsOrch(DBConnector *db, DBConnector *stateDb, vector (new LagIdAllocator(chassisAppDb)); } + /* Query Path Tracing capability */ + checkPathTracingCapability(); + auto executor = new ExecutableTimer(m_port_state_poller, this, "PORT_STATE_POLLER"); Orch::addExecutor(executor); } @@ -861,6 +953,75 @@ bool PortsOrch::addPortBulk(const std::vector &portList) attr.value.booldata = false; attrList.push_back(attr); } + + if (cit.pt_intf_id.is_set) + { + if (!m_isPathTracingSupported) + { + SWSS_LOG_WARN( + "Failed to set Path Tracing Interface ID: Path Tracing is not supported by the switch" + ); + continue; + } + + /* + * First, let's check the Path Tracing Interface ID configured for the port. + * + * Path Tracing Interface ID > 0 -> Path Tracing ENABLED on the port + * Path Tracing Interface ID == 0 -> Path Tracing DISABLED on the port + */ + if (cit.pt_intf_id.value != 0) + { + /* Path Tracing ENABLED case */ + + /* + * The port does not have a TAM object assigned to it. + * + * Let's create a new TAM object (if we don't already have one) + * and assign it to the port. + */ + if (m_ptTam == SAI_NULL_OBJECT_ID) + { + if (!createPtTam()) + { + SWSS_LOG_ERROR( + "Failed to create TAM object for Path Tracing" + ); + } + } + + if (m_ptTam != SAI_NULL_OBJECT_ID) + { + vector tam_objects_list; + tam_objects_list.push_back(m_ptTam); + attr.id = SAI_PORT_ATTR_TAM_OBJECT; + attr.value.objlist.count = (uint32_t)tam_objects_list.size(); + attr.value.objlist.list = tam_objects_list.data(); + + m_ptTamRefCount++; + m_portPtTam[cit.key] = m_ptTam; + } + } + + attr.id = SAI_PORT_ATTR_PATH_TRACING_INTF; + attr.value.u16 = cit.pt_intf_id.value; + attrList.push_back(attr); + } + + if (cit.pt_timestamp_template.is_set) + { + if (!m_isPathTracingSupported) + { + SWSS_LOG_WARN( + "Failed to set Path Tracing Timestamp Template: Path Tracing is not supported by the switch" + ); + continue; + } + + attr.id = SAI_PORT_ATTR_PATH_TRACING_TIMESTAMP_TYPE; + attr.value.u16 = cit.pt_timestamp_template.value; + attrList.push_back(attr); + } attrDataList.push_back(attrList); attrCountList.push_back(static_cast(attrDataList.back().size())); @@ -941,6 +1102,22 @@ bool PortsOrch::removePortBulk(const std::vector &portList) // Remove port serdes (if exists) before removing port since this reference is dependency removePortSerdesAttribute(cit); + + /* + * Decrease TAM object ref count before removing the port, if the port + * has a TAM object assigned + */ + if (m_portPtTam.find(p.m_alias) != m_portPtTam.end()) + { + m_ptTamRefCount--; + if (m_ptTamRefCount == 0) + { + if (!removePtTam(m_ptTam)) + { + throw runtime_error("Remove port TAM object for Path Tracing failed"); + } + } + } } auto portCount = static_cast(portList.size()); @@ -4329,6 +4506,110 @@ void PortsOrch::doPortTask(Consumer &consumer) ); } } + + if (pCfg.pt_intf_id.is_set) + { + if (!m_isPathTracingSupported) + { + SWSS_LOG_WARN( + "Failed to set Path Tracing Interface ID: Path Tracing is not supported by the switch" + ); + it = taskMap.erase(it); + continue; + } + + if (p.m_pt_intf_id != pCfg.pt_intf_id.value) + { + /* + * First, let's check the Path Tracing Interface ID configured for the port. + * + * Path Tracing Interface ID > 0 -> Path Tracing ENABLED on the port + * Path Tracing Interface ID == 0 -> Path Tracing DISABLED on the port + */ + if (pCfg.pt_intf_id.value != 0) + { + /* Path Tracing ENABLED case */ + + /* Create and set port TAM object */ + if (!createAndSetPortPtTam(p)) + { + SWSS_LOG_ERROR( + "Failed to create and set port %s TAM object for Path Tracing", + p.m_alias.c_str() + ); + it++; + continue; + } + } + else + { + /* Path Tracing DISABLED case */ + + /* Unset port TAM object */ + if (!unsetPortPtTam(p)) + { + SWSS_LOG_ERROR( + "Failed to unset port %s TAM object for Path Tracing", + p.m_alias.c_str() + ); + it++; + continue; + } + } + + /* Set Path Tracing Interface ID */ + if (!setPortPtIntfId(p, pCfg.pt_intf_id.value)) + { + SWSS_LOG_ERROR( + "Failed to set port %s Intf ID to %u", + p.m_alias.c_str(), pCfg.pt_intf_id.value + ); + it++; + continue; + } + + p.m_pt_intf_id = pCfg.pt_intf_id.value; + m_portList[p.m_alias] = p; + + SWSS_LOG_NOTICE( + "Set port %s Intf ID to %u", + p.m_alias.c_str(), pCfg.pt_intf_id.value + ); + } + } + + if (pCfg.pt_timestamp_template.is_set) + { + if (!m_isPathTracingSupported) + { + SWSS_LOG_WARN( + "Failed to set Path Tracing Timestamp Template: Path Tracing is not supported by the switch" + ); + it = taskMap.erase(it); + continue; + } + + if (p.m_pt_timestamp_template != pCfg.pt_timestamp_template.value) + { + if (!setPortPtTimestampTemplate(p, pCfg.pt_timestamp_template.value)) + { + SWSS_LOG_ERROR( + "Failed to set port %s Timestamp Template to %s", + p.m_alias.c_str(), m_portHlpr.getPtTimestampTemplateStr(pCfg).c_str() + ); + it++; + continue; + } + + p.m_pt_timestamp_template = pCfg.pt_timestamp_template.value; + m_portList[p.m_alias] = p; + + SWSS_LOG_NOTICE( + "Set port %s Timestamp Template to %s", + p.m_alias.c_str(), m_portHlpr.getPtTimestampTemplateStr(pCfg).c_str() + ); + } + } } } else if (op == DEL_COMMAND) @@ -4385,6 +4666,18 @@ void PortsOrch::doPortTask(Consumer &consumer) } } + /* + * Unset port Path Tracing TAM object and decrease TAM object refcount before + * removing the port (if the port has a TAM object associated) + */ + if (!unsetPortPtTam(p)) + { + SWSS_LOG_ERROR( + "Failed to unset port %s TAM object for Path Tracing", + p.m_alias.c_str() + ); + } + sai_status_t status = removePort(port_id); if (SAI_STATUS_SUCCESS != status) { @@ -9106,6 +9399,311 @@ void PortsOrch::updatePortStatePoll(const Port &port, port_state_poll_t type, bo } } +bool PortsOrch::createAndSetPortPtTam(const Port &p) +{ + /* + * First, let's check if a TAM object is already assigned to the port. + */ + + /* If the port has already a TAM object, nothing to do */ + if (m_portPtTam.find(p.m_alias) != m_portPtTam.end()) + { + SWSS_LOG_DEBUG( + "Port %s has already a TAM object", p.m_alias.c_str() + ); + return true; + } + + /* + * The port does not have a TAM object assigned to it. + * + * Let's create a new TAM object (if we don't already have one) + * and assign it to the port. + */ + if (m_ptTam == SAI_NULL_OBJECT_ID) + { + if (!createPtTam()) + { + SWSS_LOG_ERROR( + "Failed to create TAM object for Path Tracing" + ); + return false; + } + } + + if (!setPortPtTam(p, m_ptTam)) + { + SWSS_LOG_ERROR( + "Failed to set port %s TAM object for Path Tracing", + p.m_alias.c_str() + ); + return false; + } + + m_ptTamRefCount++; + m_portPtTam[p.m_alias] = m_ptTam; + + return true; +} + +bool PortsOrch::unsetPortPtTam(const Port &p) +{ + /* + * Let's unassign the TAM object from the port and decrease ref counter + */ + if (m_portPtTam.find(p.m_alias) != m_portPtTam.end()) + { + if (!setPortPtTam(p, SAI_NULL_OBJECT_ID)) + { + SWSS_LOG_ERROR( + "Failed to unset port %s TAM object for Path Tracing", + p.m_alias.c_str() + ); + return false; + } + m_ptTamRefCount--; + m_portPtTam.erase(p.m_alias); + + /* + * If the TAM object is no longer used, we can safely remove it. + */ + if (m_ptTamRefCount == 0) + { + if (!removePtTam(m_ptTam)) + { + SWSS_LOG_ERROR( + "Failed to remove TAM object for Path Tracing" + ); + return false; + } + } + } + + return true; +} + +bool PortsOrch::setPortPtIntfId(const Port& port, sai_uint16_t intf_id) +{ + sai_attribute_t attr; + attr.id = SAI_PORT_ATTR_PATH_TRACING_INTF; + attr.value.u16 = intf_id; + + sai_status_t status = sai_port_api->set_port_attribute(port.m_port_id, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + task_process_status handle_status = handleSaiSetStatus(SAI_API_PORT, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + + return true; +} + +bool PortsOrch::setPortPtTimestampTemplate(const Port& port, sai_port_path_tracing_timestamp_type_t ts_type) +{ + sai_attribute_t attr; + attr.id = SAI_PORT_ATTR_PATH_TRACING_TIMESTAMP_TYPE; + attr.value.s32 = ts_type; + + sai_status_t status = sai_port_api->set_port_attribute(port.m_port_id, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + task_process_status handle_status = handleSaiSetStatus(SAI_API_PORT, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + + return true; +} + +bool PortsOrch::setPortPtTam(const Port& port, sai_object_id_t tam_id) +{ + sai_attribute_t attr; + + attr.id = SAI_PORT_ATTR_TAM_OBJECT; + + if (tam_id != SAI_NULL_OBJECT_ID) + { + attr.value.objlist.count = 1; + attr.value.objlist.list = &tam_id; + } + else + { + attr.value.objlist.count = 0; + } + + sai_status_t status = sai_port_api->set_port_attribute(port.m_port_id, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + task_process_status handle_status = handleSaiSetStatus(SAI_API_PORT, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + + return true; +} + +bool PortsOrch::createPtTam() +{ + SWSS_LOG_ENTER(); + + sai_attribute_t attr; + vector attrs; + sai_status_t status; + + /* First, create a TAM report */ + if (m_ptTamReport == SAI_NULL_OBJECT_ID) + { + sai_object_id_t tam_report_id; + + attr.id = SAI_TAM_REPORT_ATTR_TYPE; + attr.value.s32 = SAI_TAM_REPORT_TYPE_VENDOR_EXTN; + attrs.push_back(attr); + + status = sai_tam_api->create_tam_report(&tam_report_id, gSwitchId, static_cast(attrs.size()), attrs.data()); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create TAM Report object for Path Tracing, rv:%d", status); + task_process_status handle_status = handleSaiCreateStatus(SAI_API_TAM, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + + m_ptTamReport = tam_report_id; + SWSS_LOG_NOTICE("Created TAM Report object %" PRIx64 " for Path Tracing", tam_report_id); + } + + /* Second, create a TAM INT object */ + if (m_ptTamInt == SAI_NULL_OBJECT_ID) + { + sai_object_id_t tam_int_id; + + attrs.clear(); + + attr.id = SAI_TAM_INT_ATTR_TYPE; + attr.value.s32 = SAI_TAM_INT_TYPE_PATH_TRACING; + attrs.push_back(attr); + + attr.id = SAI_TAM_INT_ATTR_DEVICE_ID; + attr.value.u32 = 0; + attrs.push_back(attr); + + attr.id = SAI_TAM_INT_ATTR_INT_PRESENCE_TYPE; + attr.value.u32 = SAI_TAM_INT_PRESENCE_TYPE_UNDEFINED; + attrs.push_back(attr); + + attr.id = SAI_TAM_INT_ATTR_INLINE; + attr.value.u32 = false; + attrs.push_back(attr); + + attr.id = SAI_TAM_INT_ATTR_REPORT_ID; + attr.value.oid = m_ptTamReport; + attrs.push_back(attr); + + status = sai_tam_api->create_tam_int(&tam_int_id, gSwitchId, static_cast(attrs.size()), attrs.data()); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create TAM INT object for Path Tracing, rv:%d", status); + task_process_status handle_status = handleSaiCreateStatus(SAI_API_TAM, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + + m_ptTamInt = tam_int_id; + SWSS_LOG_NOTICE("Created TAM INT object %" PRIx64 " for Path Tracing", tam_int_id); + } + + /* Finally, create a TAM object */ + if (m_ptTam == SAI_NULL_OBJECT_ID) + { + sai_object_id_t tam_id; + + attrs.clear(); + + attr.id = SAI_TAM_ATTR_INT_OBJECTS_LIST; + attr.value.objlist.count = 1; + attr.value.objlist.list = &m_ptTamInt; + attrs.push_back(attr); + + status = sai_tam_api->create_tam(&tam_id, gSwitchId, static_cast(attrs.size()), attrs.data()); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create TAM object for Path Tracing, rv:%d", status); + task_process_status handle_status = handleSaiCreateStatus(SAI_API_TAM, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + + m_ptTam = tam_id; + SWSS_LOG_NOTICE("Created TAM object %" PRIx64 " for Path Tracing", tam_id); + } + + return true; +} + +bool PortsOrch::removePtTam(sai_object_id_t tam_id) +{ + SWSS_LOG_ENTER(); + + sai_status_t status; + + if (m_ptTam != SAI_NULL_OBJECT_ID) + { + status = sai_tam_api->remove_tam(m_ptTam); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove TAM object for Path Tracing, rv:%d", status); + return false; + } + + SWSS_LOG_NOTICE("Removed TAM %" PRIx64, m_ptTam); + m_ptTam = SAI_NULL_OBJECT_ID; + } + + if (m_ptTamInt != SAI_NULL_OBJECT_ID) + { + status = sai_tam_api->remove_tam_int(m_ptTamInt); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove TAM INT object for Path Tracing, rv:%d", status); + return false; + } + + SWSS_LOG_NOTICE("Removed TAM INT %" PRIx64, m_ptTamInt); + m_ptTamInt = SAI_NULL_OBJECT_ID; + } + + if (m_ptTamReport != SAI_NULL_OBJECT_ID) + { + status = sai_tam_api->remove_tam_report(m_ptTamReport); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove TAM Report for Path Tracing, rv:%d", status); + return false; + } + + SWSS_LOG_NOTICE("Removed TAM Report %" PRIx64, m_ptTamReport); + m_ptTamReport = SAI_NULL_OBJECT_ID; + } + + return true; +} + void PortsOrch::doTask(swss::SelectableTimer &timer) { Port port; diff --git a/orchagent/portsorch.h b/orchagent/portsorch.h old mode 100755 new mode 100644 index 689d2c0dda..70f6248fda --- a/orchagent/portsorch.h +++ b/orchagent/portsorch.h @@ -243,6 +243,9 @@ class PortsOrch : public Orch, public Subject bool isMACsecPort(sai_object_id_t port_id) const; vector getPortVoQIds(Port& port); + bool setPortPtIntfId(const Port& port, sai_uint16_t intf_id); + bool setPortPtTimestampTemplate(const Port& port, sai_port_path_tracing_timestamp_type_t ts_type); + private: unique_ptr m_counterTable; unique_ptr
m_counterSysPortTable; @@ -514,6 +517,9 @@ class PortsOrch : public Orch, public Subject std::unordered_set generateCounterStats(const string& type, bool gearbox = false); map m_queueInfo; + /* Protoypes for Path tracing */ + bool setPortPtTam(const Port& port, sai_object_id_t tam_id); + private: void initializeCpuPort(); void initializePorts(); @@ -524,6 +530,20 @@ class PortsOrch : public Orch, public Subject bool addPortBulk(const std::vector &portList); bool removePortBulk(const std::vector &portList); + /* Prototypes for Path Tracing */ + bool checkPathTracingCapability(); + bool createPtTam(); + bool removePtTam(sai_object_id_t tam_id); + bool createAndSetPortPtTam(const Port &p); + bool unsetPortPtTam(const Port &p); + sai_object_id_t m_ptTamReport = SAI_NULL_OBJECT_ID; + sai_object_id_t m_ptTamInt = SAI_NULL_OBJECT_ID; + sai_object_id_t m_ptTam = SAI_NULL_OBJECT_ID; + uint32_t m_ptTamRefCount = 0; + map m_portPtTam; + // Define whether the switch supports or not Path Tracing + bool m_isPathTracingSupported = false; + private: // Port config aggregator std::unordered_map> m_portConfigMap; diff --git a/orchagent/saihelper.cpp b/orchagent/saihelper.cpp index d8d35bd62b..1265364d97 100644 --- a/orchagent/saihelper.cpp +++ b/orchagent/saihelper.cpp @@ -84,6 +84,7 @@ sai_dash_eni_api_t* sai_dash_eni_api; sai_dash_vip_api_t* sai_dash_vip_api; sai_dash_direction_lookup_api_t* sai_dash_direction_lookup_api; sai_twamp_api_t* sai_twamp_api; +sai_tam_api_t* sai_tam_api; extern sai_object_id_t gSwitchId; extern bool gTraditionalFlexCounter; @@ -230,6 +231,7 @@ void initSaiApi() sai_api_query((sai_api_t)SAI_API_DASH_VIP, (void**)&sai_dash_vip_api); sai_api_query((sai_api_t)SAI_API_DASH_DIRECTION_LOOKUP, (void**)&sai_dash_direction_lookup_api); sai_api_query(SAI_API_TWAMP, (void **)&sai_twamp_api); + sai_api_query(SAI_API_TAM, (void **)&sai_tam_api); sai_log_set(SAI_API_SWITCH, SAI_LOG_LEVEL_NOTICE); sai_log_set(SAI_API_BRIDGE, SAI_LOG_LEVEL_NOTICE); @@ -270,6 +272,7 @@ void initSaiApi() sai_log_set(SAI_API_MY_MAC, SAI_LOG_LEVEL_NOTICE); sai_log_set(SAI_API_GENERIC_PROGRAMMABLE, SAI_LOG_LEVEL_NOTICE); sai_log_set(SAI_API_TWAMP, SAI_LOG_LEVEL_NOTICE); + sai_log_set(SAI_API_TAM, SAI_LOG_LEVEL_NOTICE); } void initFlexCounterTables() diff --git a/orchagent/switchorch.h b/orchagent/switchorch.h index 5c347e0887..95cd04dbdf 100644 --- a/orchagent/switchorch.h +++ b/orchagent/switchorch.h @@ -15,6 +15,7 @@ #define SWITCH_CAPABILITY_TABLE_ORDERED_ECMP_CAPABLE "ORDERED_ECMP_CAPABLE" #define SWITCH_CAPABILITY_TABLE_PFC_DLR_INIT_CAPABLE "PFC_DLR_INIT_CAPABLE" #define SWITCH_CAPABILITY_TABLE_PORT_EGRESS_SAMPLE_CAPABLE "PORT_EGRESS_SAMPLE_CAPABLE" +#define SWITCH_CAPABILITY_TABLE_PATH_TRACING_CAPABLE "PATH_TRACING_CAPABLE" #define ASIC_SDK_HEALTH_EVENT_ELIMINATE_INTERVAL 3600 #define SWITCH_CAPABILITY_TABLE_ASIC_SDK_HEALTH_EVENT_CAPABLE "ASIC_SDK_HEALTH_EVENT" diff --git a/tests/mock_tests/fdborch/flush_syncd_notif_ut.cpp b/tests/mock_tests/fdborch/flush_syncd_notif_ut.cpp index e6bd8bea1c..eab62450f4 100644 --- a/tests/mock_tests/fdborch/flush_syncd_notif_ut.cpp +++ b/tests/mock_tests/fdborch/flush_syncd_notif_ut.cpp @@ -77,7 +77,20 @@ namespace fdb_syncd_flush_test m_asic_db = std::make_shared("ASIC_DB", 0); // Construct dependencies - // 1) Portsorch + // 1) SwitchOrch + TableConnector stateDbSwitchTable(m_state_db.get(), "SWITCH_CAPABILITY"); + TableConnector app_switch_table(m_app_db.get(), APP_SWITCH_TABLE_NAME); + TableConnector conf_asic_sensors(m_config_db.get(), CFG_ASIC_SENSORS_TABLE_NAME); + + vector switch_tables = { + conf_asic_sensors, + app_switch_table + }; + + ASSERT_EQ(gSwitchOrch, nullptr); + gSwitchOrch = new SwitchOrch(m_app_db.get(), switch_tables, stateDbSwitchTable); + + // 2) Portsorch const int portsorch_base_pri = 40; vector ports_tables = { @@ -90,7 +103,7 @@ namespace fdb_syncd_flush_test m_portsOrch = std::make_shared(m_app_db.get(), m_state_db.get(), ports_tables, m_chassis_app_db.get()); - // 2) Crmorch + // 3) Crmorch ASSERT_EQ(gCrmOrch, nullptr); gCrmOrch = new CrmOrch(m_config_db.get(), CFG_CRM_TABLE_NAME); VxlanTunnelOrch *vxlan_tunnel_orch_1 = new VxlanTunnelOrch(m_state_db.get(), m_app_db.get(), APP_VXLAN_TUNNEL_TABLE_NAME); @@ -114,6 +127,8 @@ namespace fdb_syncd_flush_test } virtual void TearDown() override { + delete gSwitchOrch; + gSwitchOrch = nullptr; delete gCrmOrch; gCrmOrch = nullptr; gDirectory.m_values.clear(); diff --git a/tests/mock_tests/mock_orch_test.h b/tests/mock_tests/mock_orch_test.h index 9cf34975d2..f0e022a7bc 100644 --- a/tests/mock_tests/mock_orch_test.h +++ b/tests/mock_tests/mock_orch_test.h @@ -119,6 +119,19 @@ namespace mock_orch_test { APP_LAG_MEMBER_TABLE_NAME, portsorch_base_pri } }; + TableConnector stateDbSwitchTable(m_state_db.get(), STATE_SWITCH_CAPABILITY_TABLE_NAME); + TableConnector app_switch_table(m_app_db.get(), APP_SWITCH_TABLE_NAME); + TableConnector conf_asic_sensors(m_config_db.get(), CFG_ASIC_SENSORS_TABLE_NAME); + + vector switch_tables = { + conf_asic_sensors, + app_switch_table + }; + + gSwitchOrch = new SwitchOrch(m_app_db.get(), switch_tables, stateDbSwitchTable); + gDirectory.set(gSwitchOrch); + ut_orch_list.push_back((Orch **)&gSwitchOrch); + vector flex_counter_tables = { CFG_FLEX_COUNTER_TABLE_NAME }; @@ -199,14 +212,6 @@ namespace mock_orch_test gBufferOrch = new BufferOrch(m_app_db.get(), m_config_db.get(), m_state_db.get(), buffer_tables); ut_orch_list.push_back((Orch **)&gBufferOrch); - TableConnector stateDbSwitchTable(m_state_db.get(), STATE_SWITCH_CAPABILITY_TABLE_NAME); - TableConnector app_switch_table(m_app_db.get(), APP_SWITCH_TABLE_NAME); - TableConnector conf_asic_sensors(m_config_db.get(), CFG_ASIC_SENSORS_TABLE_NAME); - - vector switch_tables = { - conf_asic_sensors, - app_switch_table - }; vector policer_tables = { TableConnector(m_config_db.get(), CFG_POLICER_TABLE_NAME), TableConnector(m_config_db.get(), CFG_PORT_STORM_CONTROL_TABLE_NAME) @@ -217,10 +222,6 @@ namespace mock_orch_test gDirectory.set(gPolicerOrch); ut_orch_list.push_back((Orch **)&gPolicerOrch); - gSwitchOrch = new SwitchOrch(m_app_db.get(), switch_tables, stateDbSwitchTable); - gDirectory.set(gSwitchOrch); - ut_orch_list.push_back((Orch **)&gSwitchOrch); - gNhgOrch = new NhgOrch(m_app_db.get(), APP_NEXTHOP_GROUP_TABLE_NAME); gDirectory.set(gNhgOrch); ut_orch_list.push_back((Orch **)&gNhgOrch); diff --git a/tests/mock_tests/mock_orchagent_main.h b/tests/mock_tests/mock_orchagent_main.h index 850bcb7ed2..e44d8e1612 100644 --- a/tests/mock_tests/mock_orchagent_main.h +++ b/tests/mock_tests/mock_orchagent_main.h @@ -88,3 +88,4 @@ extern sai_counter_api_t* sai_counter_api; extern sai_samplepacket_api_t *sai_samplepacket_api; extern sai_fdb_api_t* sai_fdb_api; extern sai_twamp_api_t* sai_twamp_api; +extern sai_tam_api_t* sai_tam_api; diff --git a/tests/mock_tests/portmgr_ut.cpp b/tests/mock_tests/portmgr_ut.cpp index 27dc61e03e..b7b83590bd 100644 --- a/tests/mock_tests/portmgr_ut.cpp +++ b/tests/mock_tests/portmgr_ut.cpp @@ -123,4 +123,82 @@ namespace portmgr_ut ASSERT_EQ("/sbin/ip link set dev \"Ethernet0\" mtu \"1518\"", mockCallArgs[0]); ASSERT_EQ("/sbin/ip link set dev \"Ethernet0\" up", mockCallArgs[1]); } + + TEST_F(PortMgrTest, ConfigurePortPTDefaultTimestampTemplate) + { + Table state_port_table(m_state_db.get(), STATE_PORT_TABLE_NAME); + Table app_port_table(m_app_db.get(), APP_PORT_TABLE_NAME); + Table cfg_port_table(m_config_db.get(), CFG_PORT_TABLE_NAME); + + // Port is not ready, verify that doTask does not handle port configuration + + cfg_port_table.set("Ethernet0", { + {"speed", "100000"}, + {"index", "1"}, + {"pt_interface_id", "129"} + }); + mockCallArgs.clear(); + m_portMgr->addExistingData(&cfg_port_table); + m_portMgr->doTask(); + ASSERT_TRUE(mockCallArgs.empty()); + std::vector values; + app_port_table.get("Ethernet0", values); + auto value_opt = swss::fvsGetValue(values, "mtu", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ(DEFAULT_MTU_STR, value_opt.get()); + value_opt = swss::fvsGetValue(values, "admin_status", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ(DEFAULT_ADMIN_STATUS_STR, value_opt.get()); + value_opt = swss::fvsGetValue(values, "speed", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ("100000", value_opt.get()); + value_opt = swss::fvsGetValue(values, "index", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ("1", value_opt.get()); + value_opt = swss::fvsGetValue(values, "pt_interface_id", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ("129", value_opt.get()); + value_opt = swss::fvsGetValue(values, "pt_timestamp_template", true); + ASSERT_FALSE(value_opt); + } + + TEST_F(PortMgrTest, ConfigurePortPTNonDefaultTimestampTemplate) + { + Table state_port_table(m_state_db.get(), STATE_PORT_TABLE_NAME); + Table app_port_table(m_app_db.get(), APP_PORT_TABLE_NAME); + Table cfg_port_table(m_config_db.get(), CFG_PORT_TABLE_NAME); + + // Port is not ready, verify that doTask does not handle port configuration + + cfg_port_table.set("Ethernet0", { + {"speed", "100000"}, + {"index", "1"}, + {"pt_interface_id", "129"}, + {"pt_timestamp_template", "template2"} + }); + mockCallArgs.clear(); + m_portMgr->addExistingData(&cfg_port_table); + m_portMgr->doTask(); + ASSERT_TRUE(mockCallArgs.empty()); + std::vector values; + app_port_table.get("Ethernet0", values); + auto value_opt = swss::fvsGetValue(values, "mtu", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ(DEFAULT_MTU_STR, value_opt.get()); + value_opt = swss::fvsGetValue(values, "admin_status", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ(DEFAULT_ADMIN_STATUS_STR, value_opt.get()); + value_opt = swss::fvsGetValue(values, "speed", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ("100000", value_opt.get()); + value_opt = swss::fvsGetValue(values, "index", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ("1", value_opt.get()); + value_opt = swss::fvsGetValue(values, "pt_interface_id", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ("129", value_opt.get()); + value_opt = swss::fvsGetValue(values, "pt_timestamp_template", true); + ASSERT_TRUE(value_opt); + ASSERT_EQ("template2", value_opt.get()); + } } diff --git a/tests/mock_tests/portsorch_ut.cpp b/tests/mock_tests/portsorch_ut.cpp index 968a578d44..afa26dc439 100644 --- a/tests/mock_tests/portsorch_ut.cpp +++ b/tests/mock_tests/portsorch_ut.cpp @@ -83,6 +83,15 @@ namespace portsorch_test uint32_t _sai_set_pfc_mode_count; uint32_t _sai_set_admin_state_up_count; uint32_t _sai_set_admin_state_down_count; + bool set_pt_interface_id_fail = false; + bool set_pt_timestamp_template_fail = false; + bool set_port_tam_fail = false; + uint32_t set_pt_interface_id_count = false; + uint32_t set_pt_timestamp_template_count = false; + uint32_t set_port_tam_count = false; + uint32_t set_pt_interface_id_failures; + uint32_t set_pt_timestamp_template_failures; + uint32_t set_port_tam_failures; sai_status_t _ut_stub_sai_set_port_attribute( _In_ sai_object_id_t port_id, _In_ const sai_attribute_t *attr) @@ -109,9 +118,74 @@ namespace portsorch_test _sai_set_admin_state_down_count++; } } + else if (attr[0].id == SAI_PORT_ATTR_PATH_TRACING_INTF) + { + set_pt_interface_id_count++; + /* Simulating failure case */ + if (set_pt_interface_id_fail) + { + set_pt_interface_id_failures++; + return SAI_STATUS_INVALID_ATTR_VALUE_0; + } + } + else if (attr[0].id == SAI_PORT_ATTR_PATH_TRACING_TIMESTAMP_TYPE) + { + set_pt_timestamp_template_count++; + /* Simulating failure case */ + if (set_pt_timestamp_template_fail) + { + set_pt_timestamp_template_failures++; + return SAI_STATUS_INVALID_ATTR_VALUE_0; + } + } + else if (attr[0].id == SAI_PORT_ATTR_TAM_OBJECT) + { + set_port_tam_count++; + /* Simulating failure case */ + if (set_port_tam_fail) + { + set_port_tam_failures++; + return SAI_STATUS_INVALID_ATTR_VALUE_0; + } + } return pold_sai_port_api->set_port_attribute(port_id, attr); } + vector supported_sai_objects = { + SAI_OBJECT_TYPE_PORT, + SAI_OBJECT_TYPE_LAG, + SAI_OBJECT_TYPE_TAM, + SAI_OBJECT_TYPE_TAM_INT, + SAI_OBJECT_TYPE_TAM_COLLECTOR, + SAI_OBJECT_TYPE_TAM_REPORT, + SAI_OBJECT_TYPE_TAM_TRANSPORT, + SAI_OBJECT_TYPE_TAM_TELEMETRY, + SAI_OBJECT_TYPE_TAM_EVENT_THRESHOLD + }; + + sai_status_t _ut_stub_sai_get_switch_attribute( + _In_ sai_object_id_t switch_id, + _In_ uint32_t attr_count, + _Inout_ sai_attribute_t *attr_list) + { + sai_status_t status; + if (attr_count == 1 && attr_list[0].id == SAI_SWITCH_ATTR_SUPPORTED_OBJECT_TYPE_LIST) + { + uint32_t i; + for (i = 0; i < attr_list[0].value.s32list.count && i < supported_sai_objects.size(); i++) + { + attr_list[0].value.s32list.list[i] = supported_sai_objects[i]; + } + attr_list[0].value.s32list.count = i; + status = SAI_STATUS_SUCCESS; + } + else + { + status = pold_sai_switch_api->get_switch_attribute(switch_id, attr_count, attr_list); + } + return status; + } + uint32_t *_sai_syncd_notifications_count; int32_t *_sai_syncd_notification_event; uint32_t _sai_switch_dlr_packet_action_count; @@ -152,6 +226,7 @@ namespace portsorch_test ut_sai_switch_api = *sai_switch_api; pold_sai_switch_api = sai_switch_api; ut_sai_switch_api.set_switch_attribute = _ut_stub_sai_set_switch_attribute; + ut_sai_switch_api.get_switch_attribute = _ut_stub_sai_get_switch_attribute; sai_switch_api = &ut_sai_switch_api; } @@ -1054,6 +1129,506 @@ namespace portsorch_test _unhook_sai_queue_api(); } + TEST_F(PortsOrchTest, PortPTConfigDefaultTimestampTemplate) + { + auto portTable = Table(m_app_db.get(), APP_PORT_TABLE_NAME); + Port p; + std::deque kfvList; + auto consumer = dynamic_cast(gPortsOrch->getExecutor(APP_PORT_TABLE_NAME)); + + // Get SAI default ports to populate DB + auto &ports = defaultPortList; + ASSERT_TRUE(!ports.empty()); + + // Generate port config + for (const auto &cit : ports) + { + portTable.set(cit.first, cit.second); + } + + // Set PortConfigDone + portTable.set("PortConfigDone", { { "count", std::to_string(ports.size()) } }); + + // Refill consumer + gPortsOrch->addExistingData(&portTable); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + + // Port count: 32 Data + 1 CPU + ASSERT_EQ(gPortsOrch->getAllPorts().size(), ports.size() + 1); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet8", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23); + + // Enable Path Tracing on Ethernet8 with Interface ID 128 and default Timestamp Template + kfvList = {{ + "Ethernet8", + SET_COMMAND, { + { "pt_interface_id", "128" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + kfvList.clear(); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet8", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 128); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23); + + // Disable Path Tracing on Ethernet8 + kfvList = {{ + "Ethernet8", + SET_COMMAND, { + { "pt_interface_id", "None" }, + { "pt_timestamp_template", "None" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + kfvList.clear(); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet8", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23); + + // Dump pending tasks + std::vector taskList; + gPortsOrch->dumpPendingTasks(taskList); + ASSERT_TRUE(taskList.empty()); + } + + TEST_F(PortsOrchTest, PortPTConfigNonDefaultTimestampTemplate) + { + auto portTable = Table(m_app_db.get(), APP_PORT_TABLE_NAME); + Port p; + std::deque kfvList; + auto consumer = dynamic_cast(gPortsOrch->getExecutor(APP_PORT_TABLE_NAME)); + + // Get SAI default ports + auto &ports = defaultPortList; + ASSERT_TRUE(!ports.empty()); + + // Generate port config + for (const auto &cit : ports) + { + portTable.set(cit.first, cit.second); + } + + // Set PortConfigDone + portTable.set("PortConfigDone", { { "count", std::to_string(ports.size()) } }); + + // Refill consumer + gPortsOrch->addExistingData(&portTable); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + + // Port count: 32 Data + 1 CPU + ASSERT_EQ(gPortsOrch->getAllPorts().size(), ports.size() + 1); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet9", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23); + + // Enable Path Tracing on Ethernet9 with Interface ID 129 and Timestamp Template template2 + kfvList = {{ + "Ethernet9", + SET_COMMAND, { + { "pt_interface_id", "129" }, + { "pt_timestamp_template", "template2" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + kfvList.clear(); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet9", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 129); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_12_19); + + // Disable Path Tracing on Ethernet9 + kfvList = {{ + "Ethernet9", + SET_COMMAND, { + { "pt_interface_id", "None" }, + { "pt_timestamp_template", "None" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + kfvList.clear(); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet9", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23); + + // Dump pending tasks + std::vector taskList; + gPortsOrch->dumpPendingTasks(taskList); + ASSERT_TRUE(taskList.empty()); + } + + TEST_F(PortsOrchTest, PortPTConfigInvalidInterfaceID) + { + auto portTable = Table(m_app_db.get(), APP_PORT_TABLE_NAME); + Port p; + std::deque kfvList; + auto consumer = dynamic_cast(gPortsOrch->getExecutor(APP_PORT_TABLE_NAME)); + + // Get SAI default ports + auto &ports = defaultPortList; + ASSERT_TRUE(!ports.empty()); + + // Generate port config + for (const auto &cit : ports) + { + portTable.set(cit.first, cit.second); + } + + // Set PortConfigDone + portTable.set("PortConfigDone", { { "count", std::to_string(ports.size()) } }); + + // Refill consumer + gPortsOrch->addExistingData(&portTable); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + + // Port count: 32 Data + 1 CPU + ASSERT_EQ(gPortsOrch->getAllPorts().size(), ports.size() + 1); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet9", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23); + + // Enable Path Tracing on Ethernet9 with Interface ID 4096 (INVALID) and Timestamp Template template2 + kfvList = {{ + "Ethernet9", + SET_COMMAND, { + { "pt_interface_id", "4096" }, + { "pt_timestamp_template", "template2" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + kfvList.clear(); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet9", p)); + + // Verify PT Interface ID + // We provided an invalid Path Tracing Interface ID, therefore we expect PortsOrch rejects the port + // configuration and Path Tracing remains disabled (i.e., Tracing Interface ID should be 0) + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Dump pending tasks + std::vector taskList; + gPortsOrch->dumpPendingTasks(taskList); + ASSERT_TRUE(taskList.empty()); + } + + TEST_F(PortsOrchTest, PortPTConfigInvalidInterfaceTimestampTemplate) + { + auto portTable = Table(m_app_db.get(), APP_PORT_TABLE_NAME); + Port p; + std::deque kfvList; + auto consumer = dynamic_cast(gPortsOrch->getExecutor(APP_PORT_TABLE_NAME)); + + // Get SAI default ports + auto &ports = defaultPortList; + ASSERT_TRUE(!ports.empty()); + + // Generate port config + for (const auto &cit : ports) + { + portTable.set(cit.first, cit.second); + } + + // Set PortConfigDone + portTable.set("PortConfigDone", { { "count", std::to_string(ports.size()) } }); + + // Refill consumer + gPortsOrch->addExistingData(&portTable); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + + // Port count: 32 Data + 1 CPU + ASSERT_EQ(gPortsOrch->getAllPorts().size(), ports.size() + 1); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet9", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23); + + // Enable Path Tracing on Ethernet9 with Interface ID 129 and Timestamp Template template5 (INVALID) + kfvList = {{ + "Ethernet9", + SET_COMMAND, { + { "pt_interface_id", "129" }, + { "pt_timestamp_template", "template5" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + kfvList.clear(); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet9", p)); + + // Verify PT Interface ID + // We provided an invalid Timestamp Template, therefore we expect PortsOrch rejects the port + // configuration and Path Tracing remains disabled (i.e., Tracing Interface ID should be 0) + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Dump pending tasks + std::vector taskList; + gPortsOrch->dumpPendingTasks(taskList); + ASSERT_TRUE(taskList.empty()); + } + + TEST_F(PortsOrchTest, PortPTSAIFailureHandling) + { + _hook_sai_port_api(); + _hook_sai_switch_api(); + + auto portTable = Table(m_app_db.get(), APP_PORT_TABLE_NAME); + Port p; + std::deque kfvList; + auto consumer = dynamic_cast(gPortsOrch->getExecutor(APP_PORT_TABLE_NAME)); + + // Get SAI default ports + auto &ports = defaultPortList; + ASSERT_TRUE(!ports.empty()); + + // Generate port config + for (const auto &cit : ports) + { + portTable.set(cit.first, cit.second); + } + + // Set PortConfigDone + portTable.set("PortConfigDone", { { "count", std::to_string(ports.size()) } }); + + // Refill consumer + gPortsOrch->addExistingData(&portTable); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + + // Port count: 32 Data + 1 CPU + ASSERT_EQ(gPortsOrch->getAllPorts().size(), ports.size() + 1); + + // Get port + ASSERT_TRUE(gPortsOrch->getPort("Ethernet9", p)); + + // Verify PT Interface ID + ASSERT_EQ(p.m_pt_intf_id, 0); + + // Verify PT Timestamp Template + ASSERT_EQ(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23); + + // Simulate failure when PortsOrch attempts to set the Path Tracing Interface ID + set_pt_interface_id_fail = true; + + // Enable Path Tracing on Ethernet9 with Interface ID 129 and Timestamp Template template2 + kfvList = {{ + "Ethernet9", + SET_COMMAND, { + { "pt_interface_id", "129" }, + { "pt_timestamp_template", "template2" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + static_cast(gPortsOrch)->doTask(); + + ASSERT_EQ(set_pt_interface_id_fail, 1); + + set_pt_interface_id_fail = false; + + + // Simulate failure when PortsOrch attempts to set the Path Tracing Timestamp Template + set_pt_timestamp_template_fail = true; + + // Enable Path Tracing on Ethernet10 with Interface ID 129 and Timestamp Template template2 + kfvList = {{ + "Ethernet10", + SET_COMMAND, { + { "pt_interface_id", "129" }, + { "pt_timestamp_template", "template2" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + static_cast(gPortsOrch)->doTask(); + + ASSERT_EQ(set_pt_timestamp_template_failures, 1); + + set_pt_timestamp_template_fail = false; + + + // Simulate failure when PortsOrch attempts to set the port TAM object + set_port_tam_fail = true; + + // Enable Path Tracing on Ethernet11 with Interface ID 129 and Timestamp Template template2 + kfvList = {{ + "Ethernet11", + SET_COMMAND, { + { "pt_interface_id", "129" }, + { "pt_timestamp_template", "template2" } + } + }}; + + // Refill consumer + consumer->addToSync(kfvList); + + static_cast(gPortsOrch)->doTask(); + + ASSERT_EQ(set_port_tam_fail, 1); + + set_port_tam_fail = false; + + _unhook_sai_switch_api(); + _unhook_sai_port_api(); + } + + TEST_F(PortsOrchTest, PortPTCapabilityUnsupported) + { + _hook_sai_port_api(); + _hook_sai_switch_api(); + + auto portTable = Table(m_app_db.get(), APP_PORT_TABLE_NAME); + Port p; + std::deque kfvList; + auto consumer = dynamic_cast(gPortsOrch->getExecutor(APP_PORT_TABLE_NAME)); + + // Get SAI default ports + auto &ports = defaultPortList; + ASSERT_TRUE(!ports.empty()); + + // Generate port config + for (const auto &cit : ports) + { + portTable.set(cit.first, cit.second); + } + + // Set PortConfigDone + portTable.set("PortConfigDone", { { "count", std::to_string(ports.size()) } }); + + // Refill consumer + gPortsOrch->addExistingData(&portTable); + + // Apply configuration + static_cast(gPortsOrch)->doTask(); + + // Port count: 32 Data + 1 CPU + ASSERT_EQ(gPortsOrch->getAllPorts().size(), ports.size() + 1); + + // Scenario 1: Path Tracing supported + ASSERT_TRUE(gPortsOrch->checkPathTracingCapability()); + + // Scenario 2: Path Tracing is not supported + supported_sai_objects.erase(std::remove(supported_sai_objects.begin(), supported_sai_objects.end(), SAI_OBJECT_TYPE_TAM), supported_sai_objects.end()); + ASSERT_FALSE(gPortsOrch->checkPathTracingCapability()); + + kfvList = {{ + "Ethernet10", + SET_COMMAND, { + { "pt_interface_id", "129"} + } + }}; + consumer->addToSync(kfvList); + static_cast(gPortsOrch)->doTask(); + ASSERT_TRUE(gPortsOrch->getPort("Ethernet10", p)); + // Expect Path Tracing Interface ID is ignored because Path Tracing is not supported on the switch + ASSERT_EQ(p.m_pt_intf_id, 0); + + kfvList = {{ + "Ethernet10", + SET_COMMAND, { + { "pt_timestamp_template", "template2"} + } + }}; + consumer->addToSync(kfvList); + static_cast(gPortsOrch)->doTask(); + ASSERT_TRUE(gPortsOrch->getPort("Ethernet10", p)); + // Expect Path Tracing template is ignored because Path Tracing is not supported on the switch + ASSERT_NE(p.m_pt_timestamp_template, SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_12_19); + + _unhook_sai_switch_api(); + _unhook_sai_port_api(); + } + /** * Test case: PortsOrch::addBridgePort() does not add router port to .1Q bridge */ diff --git a/tests/mock_tests/ut_saihelper.cpp b/tests/mock_tests/ut_saihelper.cpp index c9bed67691..f2b7c54ad5 100644 --- a/tests/mock_tests/ut_saihelper.cpp +++ b/tests/mock_tests/ut_saihelper.cpp @@ -90,6 +90,7 @@ namespace ut_helper sai_api_query(SAI_API_COUNTER, (void**)&sai_counter_api); sai_api_query(SAI_API_FDB, (void**)&sai_fdb_api); sai_api_query(SAI_API_TWAMP, (void**)&sai_twamp_api); + sai_api_query(SAI_API_TAM, (void**)&sai_tam_api); return SAI_STATUS_SUCCESS; } @@ -120,6 +121,7 @@ namespace ut_helper sai_queue_api = nullptr; sai_counter_api = nullptr; sai_twamp_api = nullptr; + sai_tam_api = nullptr; return SAI_STATUS_SUCCESS; } diff --git a/tests/test_port.py b/tests/test_port.py index 3853a61ffe..d7bf62d2d7 100644 --- a/tests/test_port.py +++ b/tests/test_port.py @@ -305,6 +305,133 @@ def test_PortHostTxSignalSet(self, dvs, testlog): expected_fields = {"SAI_PORT_ATTR_HOST_TX_SIGNAL_ENABLE":"false"} adb.wait_for_field_match("ASIC_STATE:SAI_OBJECT_TYPE_PORT", port_oid, expected_fields) + def test_PortPathTracing(self, dvs, testlog): + pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0) + adb = swsscommon.DBConnector(1, dvs.redis_sock, 0) + cdb = swsscommon.DBConnector(4, dvs.redis_sock, 0) + + ctbl = swsscommon.Table(cdb, "PORT") + ptbl = swsscommon.Table(pdb, "PORT_TABLE") + atbl = swsscommon.Table(adb, "ASIC_STATE:SAI_OBJECT_TYPE_PORT") + + # get the number of ports before removal + num_of_ports = len(atbl.getKeys()) + + initial_entries = set(atbl.getKeys()) + + # read port info and save it + (status, ports_info) = ctbl.get("Ethernet124") + assert status + + # remove buffer pg cfg for the port (record the buffer pgs before removing them) + pgs = dvs.get_config_db().get_keys('BUFFER_PG') + buffer_pgs = {} + for key in pgs: + if "Ethernet124" in key: + buffer_pgs[key] = dvs.get_config_db().get_entry('BUFFER_PG', key) + dvs.get_config_db().delete_entry('BUFFER_PG', key) + dvs.get_app_db().wait_for_deleted_entry("BUFFER_PG_TABLE", key) + + # remove buffer queue cfg for the port + queues = dvs.get_config_db().get_keys('BUFFER_QUEUE') + buffer_queues = {} + for key in queues: + if "Ethernet124" in key: + buffer_queues[key] = dvs.get_config_db().get_entry('BUFFER_QUEUE', key) + dvs.get_config_db().delete_entry('BUFFER_QUEUE', key) + dvs.get_app_db().wait_for_deleted_entry('BUFFER_QUEUE_TABLE', key) + + # shutdown port + dvs.port_admin_set("Ethernet124", 'down') + + # remove this port + ctbl.delete("Ethernet124") + + # verify that the port has been removed + num = dvs.get_asic_db().wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_PORT", num_of_ports - 1) + assert len(num) == num_of_ports - 1 + + # re-add the port with Path Tracing enabled + fvs = swsscommon.FieldValuePairs(ports_info + (("pt_interface_id", "129"), ("pt_timestamp_template", "template2"))) + ctbl.set("Ethernet124", fvs) + + # check application database + dvs.get_app_db().wait_for_entry('PORT_TABLE', "Ethernet124") + (status, fvs) = ptbl.get("Ethernet124") + assert status + for fv in fvs: + if fv[0] == "pt_interface_id": + assert fv[1] == "129" + if fv[0] == "pt_timestamp_template": + assert fv[1] == "template2" + + # verify that the port has been re-added + num = dvs.get_asic_db().wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_PORT", num_of_ports) + assert len(num) == num_of_ports + + # check ASIC DB + atbl = swsscommon.Table(adb, "ASIC_STATE:SAI_OBJECT_TYPE_PORT") + # get PT Interface ID and validate it to be 129 + entries = set(atbl.getKeys()) + new_entries = list(entries - initial_entries) + assert len(new_entries) == 1, "Wrong number of created entries." + + (status, fvs) = atbl.get(new_entries[0]) + assert status + + for fv in fvs: + if fv[0] == "SAI_PORT_ATTR_PATH_TRACING_INTF": + assert fv[1] == "129" + if fv[0] == "SAI_PORT_ATTR_PATH_TRACING_TIMESTAMP_TYPE": + assert fv[1] == "SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_12_19" + + # change Path Tracing Interface ID and Timestamp Template on the port + fvs = swsscommon.FieldValuePairs([("pt_interface_id", "130"), ("pt_timestamp_template", "template3")]) + ctbl.set("Ethernet124", fvs) + time.sleep(5) + + # check application database + (status, fvs) = ptbl.get("Ethernet124") + assert status + for fv in fvs: + if fv[0] == "pt_interface_id": + assert fv[1] == "130" + if fv[0] == "pt_timestamp_template": + assert fv[1] == "template3" + + time.sleep(5) + + # check ASIC DB + # get PT Interface ID and validate it to be 130 + (status, fvs) = atbl.get(new_entries[0]) + assert status + + for fv in fvs: + if fv[0] == "SAI_PORT_ATTR_PATH_TRACING_INTF": + assert fv[1] == "130" + if fv[0] == "SAI_PORT_ATTR_PATH_TRACING_TIMESTAMP_TYPE": + assert fv[1] == "SAI_PORT_PATH_TRACING_TIMESTAMP_TYPE_16_23" + + # shutdown port + dvs.port_admin_set("Ethernet124", 'down') + + # remove the port + ctbl.delete("Ethernet124") + + # re-add the port with the original configuration + ctbl.set("Ethernet124", ports_info) + + # verify that the port has been re-added + num = dvs.get_asic_db().wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_PORT", num_of_ports) + assert len(num) == num_of_ports + + # re-add buffer pg and queue cfg to the port + for key, pg in buffer_pgs.items(): + dvs.get_config_db().update_entry("BUFFER_PG", key, pg) + + for key, queue in buffer_queues.items(): + dvs.get_config_db().update_entry("BUFFER_QUEUE", key, queue) + # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying