From 3a000ebed32f1659ee2552e46075bfb8ef8586f3 Mon Sep 17 00:00:00 2001 From: Doug Cook <45909143+idigdoug@users.noreply.github.com> Date: Fri, 5 Jan 2024 12:23:07 -0700 Subject: [PATCH] dwc_eqos - vlan, flow, INF config options (#32) - Change checksum offload INF options from granular to combined. - Add INF options for FlowControl, PriorityVlanTag. - Respect INF option for FlowControl at MAC level. - Set DisableReceiveOwn on MAC options. - Correct Rx VLAN tag behavior: always strip tag, always copy tag to OOB. - Correct Tx VLAN tag behavior: add tag if present in OOB. - Stop tracking Tx-Own statistics. They don't seem useful. The Tx VLAN tag behavior required significant refactoring of TxQueueAdvance. Previously we depended on one-to-one mapping between descriptor and fragment, but VLAN tags require context descriptors so there can now be descriptors without a matching fragment, so our descriptor drain code needs to deal with that. --- drivers/net/dwc_eqos/descriptors.h | 1 + drivers/net/dwc_eqos/device.cpp | 75 +++++++- drivers/net/dwc_eqos/device.h | 3 +- drivers/net/dwc_eqos/driver.cpp | 9 +- drivers/net/dwc_eqos/dwc_eqos.inf | 100 +++++----- drivers/net/dwc_eqos/dwc_eqos_perf.man | 13 +- drivers/net/dwc_eqos/dwc_eqos_perf_data.h | 1 - drivers/net/dwc_eqos/precomp.h | 1 + drivers/net/dwc_eqos/registers.h | 39 +++- drivers/net/dwc_eqos/rxqueue.cpp | 17 ++ drivers/net/dwc_eqos/txqueue.cpp | 214 +++++++++++++--------- 11 files changed, 306 insertions(+), 167 deletions(-) diff --git a/drivers/net/dwc_eqos/descriptors.h b/drivers/net/dwc_eqos/descriptors.h index d8776dc..fca7785 100644 --- a/drivers/net/dwc_eqos/descriptors.h +++ b/drivers/net/dwc_eqos/descriptors.h @@ -168,6 +168,7 @@ struct TxDescriptorContext UINT8 OneStepEnable : 1; // OSTC UINT8 Reserved28 : 2; UINT8 ContextType : 1; // CTXT = 1 + UINT8 Own : 1; // OWN #if DBG UINT32 FragmentIndex; diff --git a/drivers/net/dwc_eqos/device.cpp b/drivers/net/dwc_eqos/device.cpp index cf9f40c..2abc963 100644 --- a/drivers/net/dwc_eqos/device.cpp +++ b/drivers/net/dwc_eqos/device.cpp @@ -90,7 +90,6 @@ struct DeviceContext UINT32 dpcFatalBusError; // Updated only in DPC. UINT32 rxOwnDescriptors; // Updated only during RxQueueAdvance. UINT32 rxDoneFragments; // Updated only during RxQueueAdvance. - UINT32 txOwnDescriptors; // Updated only during TxQueueAdvance. UINT32 txDoneFragments; // Updated only during TxQueueAdvance. }; WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(DeviceContext, DeviceGetContext) @@ -593,7 +592,7 @@ DeviceD0Entry( : txFifoSize / QueuesSupported; MacTxFlowCtrl_t txFlowCtrl = {}; - txFlowCtrl.TransmitFlowControlEnable = true; + txFlowCtrl.TransmitFlowControlEnable = context->config.txFlowControl; txFlowCtrl.PauseTime = 0xFFFF; Write32(&context->regs->Mac_Tx_Flow_Ctrl, txFlowCtrl); // TxFlow control, pause time. @@ -617,7 +616,7 @@ DeviceD0Entry( UINT32 const rxFlowControlDeactivate = rxFlowControlActivate + rxFlowControlActivate / 2u; // Unpause when 3/4 or 48KB remains. - Write32(&context->regs->Mac_Rx_Flow_Ctrl, 0x3); // Rx flow control, pause packet detect. + Write32(&context->regs->Mac_Rx_Flow_Ctrl, context->config.rxFlowControl ? 0x3 : 0x0); // Rx flow control, pause packet detect. Write32(&context->regs->Mac_RxQ_Ctrl0, 0x2); // RxQ0 enabled for DCB/generic. MtlRxOperationMode_t rxOperationMode = {}; @@ -626,7 +625,7 @@ DeviceD0Entry( rxOperationMode.ForwardErrorPackets = false; rxOperationMode.ForwardUndersizedGoodPackets = true; rxOperationMode.QueueSize = rxQueueSize / 256u - 1; - rxOperationMode.HardwareFlowControl = rxQueueSize >= 2048; + rxOperationMode.HardwareFlowControl = context->config.rxFlowControl && rxQueueSize >= 2048; rxOperationMode.FlowControlActivate = (rxFlowControlActivate / 512u) - 2u; rxOperationMode.FlowControlDeactivate = (rxFlowControlDeactivate / 512u) - 2u; Write32(&context->regs->Mtl_Q[0].Rx_Operation_Mode, rxOperationMode); @@ -634,13 +633,21 @@ DeviceD0Entry( // MAC configuration. MacConfiguration_t macConfig = {}; - macConfig.DisableCarrierSenseDuringTransmit = true; - macConfig.PacketBurstEnable = true; macConfig.ReceiverEnable = true; macConfig.TransmitterEnable = true; + macConfig.DisableCarrierSenseDuringTransmit = true; + macConfig.DisableReceiveOwn = true; + macConfig.PacketBurstEnable = true; + //macConfig.PadOrCrcStripEnable = true; // Why doesn't this work? + //macConfig.CrcStripEnableForType = true; // Why doesn't this work? macConfig.ChecksumOffloadEnable = context->config.txCoeSel || context->config.rxCoeSel; Write32(&context->regs->Mac_Configuration, macConfig); + Mac_Vlan_Tag_Ctrl_t vlanTagCtrl = {}; + vlanTagCtrl.StripOnReceive = VlanTagStripOnReceive_Always; + vlanTagCtrl.RxStatusEnable = true; + Write32(&context->regs->Mac_Vlan_Tag_Ctrl, vlanTagCtrl); + // Clear any pending interrupts, then unmask them. NT_ASSERT(ReadNoFence8(&context->updateLinkStateBusy) == 0); @@ -707,6 +714,7 @@ DevicePrepareHardware( PHYSICAL_ADDRESS maxPhysicalAddress; auto const context = DeviceGetContext(device); bool configHasMacAddress = false; + ULONG flowControlConfiguration; // Read configuration @@ -746,6 +754,15 @@ DevicePrepareHardware( memcpy(context->currentMacAddress, configAddress.Address, sizeof(context->currentMacAddress)); configHasMacAddress = true; } + + DECLARE_CONST_UNICODE_STRING(FlowControl_Name, L"FlowControl"); + status = NetConfigurationQueryUlong(configuration, NET_CONFIGURATION_QUERY_ULONG_NO_FLAGS, &FlowControl_Name, &flowControlConfiguration); + if (!NT_SUCCESS(status)) + { + TraceWrite("QueryFlowControl-not-found", LEVEL_VERBOSE, + TraceLoggingNTStatus(status)); + flowControlConfiguration = 3; // Default = TxRxEnabled + } } // Configure resources @@ -908,6 +925,14 @@ DevicePrepareHardware( status = STATUS_DEVICE_CONFIGURATION_ERROR; goto Done; } + + if (!context->feature0.SaVlanIns) + { + // Could adapt at runtime if needed, but assume it's present for now. + TraceWrite("DevicePrepareHardware-SaVlanIns-required", LEVEL_ERROR); + status = STATUS_DEVICE_CONFIGURATION_ERROR; + goto Done; + } } // Device Config @@ -925,6 +950,33 @@ DevicePrepareHardware( context->config.rd_osr_lmt = 8; context->config.blen = 0x7; // 0x7 = 4, 8, 16 + switch (flowControlConfiguration) + { + case 0: // Disabled + context->config.txFlowControl = false; + context->config.rxFlowControl = false; + break; + case 1: // TxEnabled + context->config.txFlowControl = true; + context->config.rxFlowControl = false; + break; + case 2: // RxEnabled + context->config.txFlowControl = false; + context->config.rxFlowControl = true; + break; + case 3: // TxRxEnabled + case 4: // AutoNegotiate + context->config.txFlowControl = true; + context->config.rxFlowControl = true; + break; + default: + TraceWrite("QueryFlowControl-bad-value", LEVEL_WARNING, + TraceLoggingUInt32(flowControlConfiguration, "value")); + context->config.txFlowControl = true; + context->config.rxFlowControl = true; + break; + } + auto const deviceObject = WdfDeviceWdmGetPhysicalDevice(device); PACPI_EVAL_OUTPUT_BUFFER outputBuffer = nullptr; ACPI_METHOD_ARGUMENT const UNALIGNED* properties = nullptr; @@ -1139,7 +1191,7 @@ DevicePrepareHardware( NET_ADAPTER_TX_CAPABILITIES txCaps; NET_ADAPTER_TX_CAPABILITIES_INIT_FOR_DMA(&txCaps, &dmaCaps, QueuesSupported); - txCaps.MaximumNumberOfFragments = QueueDescriptorMinCount - 1; + txCaps.MaximumNumberOfFragments = QueueDescriptorMinCount - 2; // = 1 hole in the ring + 1 context descriptor. NET_ADAPTER_RX_CAPABILITIES rxCaps; // TODO: Might use less memory if driver-managed. NET_ADAPTER_RX_CAPABILITIES_INIT_SYSTEM_MANAGED_DMA(&rxCaps, &dmaCaps, RxBufferSize, QueuesSupported); // TODO: Jumbo packets. @@ -1181,6 +1233,12 @@ DevicePrepareHardware( NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES_INIT(&rxChecksumCaps, AdapterOffloadSetRxChecksum); NetAdapterOffloadSetRxChecksumCapabilities(context->adapter, &rxChecksumCaps); + + NET_ADAPTER_OFFLOAD_IEEE8021Q_TAG_CAPABILITIES ieee8021qCaps; + NET_ADAPTER_OFFLOAD_IEEE8021Q_TAG_CAPABILITIES_INIT(&ieee8021qCaps, + NetAdapterOffloadIeee8021PriorityTaggingFlag | + NetAdapterOffloadIeee8021VlanTaggingFlag); + NetAdapterOffloadSetIeee8021qTagCapabilities(context->adapter, &ieee8021qCaps); } // Initialize adapter. @@ -1325,11 +1383,9 @@ _IRQL_requires_max_(DISPATCH_LEVEL) void DeviceAddStatisticsTxQueue( _Inout_ DeviceContext* context, - UINT32 ownDescriptors, UINT32 doneFragments) { // DISPATCH_LEVEL - context->txOwnDescriptors += ownDescriptors; context->txDoneFragments += doneFragments; } @@ -1544,7 +1600,6 @@ PerfDataInit( data->DpcFatalBusError = context->dpcFatalBusError; data->RxOwnDescriptors = context->rxOwnDescriptors; data->RxDoneFragments = context->rxDoneFragments; - data->TxOwnDescriptors = context->txOwnDescriptors; data->TxDoneFragments = context->txDoneFragments; } diff --git a/drivers/net/dwc_eqos/device.h b/drivers/net/dwc_eqos/device.h index 298da46..bcc1739 100644 --- a/drivers/net/dwc_eqos/device.h +++ b/drivers/net/dwc_eqos/device.h @@ -19,6 +19,8 @@ struct DeviceConfig UINT8 wr_osr_lmt; // AXIC\snps,wr_osr_lmt (default = 4). UINT8 rd_osr_lmt; // AXIC\snps,rd_osr_lmt (default = 8). UINT8 blen : 7; // AXIC\snps,blen bitmask of 7 booleans 4..256 (default = 4, 8, 16). + bool txFlowControl; // Adapter configuration (Ndi\params\*FlowControl). + bool rxFlowControl; // Adapter configuration (Ndi\params\*FlowControl). }; // Referenced in driver.cpp DriverEntry. @@ -66,5 +68,4 @@ _IRQL_requires_max_(DISPATCH_LEVEL) void DeviceAddStatisticsTxQueue( _Inout_ DeviceContext* context, - UINT32 ownDescriptors, UINT32 doneFragments); diff --git a/drivers/net/dwc_eqos/driver.cpp b/drivers/net/dwc_eqos/driver.cpp index 379c529..12af233 100644 --- a/drivers/net/dwc_eqos/driver.cpp +++ b/drivers/net/dwc_eqos/driver.cpp @@ -1,15 +1,14 @@ #include "precomp.h" -#include "device.h" // DeviceAdd +#include "device.h" #include "trace.h" /* Possible areas for improvement: -- Run against network test suites and fix any issues. - Jumbo frames. -- Configuration in registry (e.g. flow control, speed, duplex). - Tx segmentation offload. -- Wake-on-LAN. -- ARP offload. +- Run against network test suites and fix any issues. +- Power control, wake-on-LAN, ARP offload. +- Configure speed, duplex in Ndi\params. - Multi-queue support? */ diff --git a/drivers/net/dwc_eqos/dwc_eqos.inf b/drivers/net/dwc_eqos/dwc_eqos.inf index 99fea27..24e7475 100644 --- a/drivers/net/dwc_eqos/dwc_eqos.inf +++ b/drivers/net/dwc_eqos/dwc_eqos.inf @@ -77,52 +77,44 @@ HKR, Ndi\Interfaces, LowerRange, 0, "ethernet" [DWCEQOS_AddReg_Ndi_params] -HKR, Ndi\params\NetworkAddress, ParamDesc, 0, %NetworkAddress% -HKR, Ndi\params\NetworkAddress, type, 0, "edit" -HKR, Ndi\params\NetworkAddress, default, 0, "" -HKR, Ndi\params\NetworkAddress, LimitText, 0, "12" -HKR, Ndi\params\NetworkAddress, UpperCase, 0, "1" -HKR, Ndi\params\NetworkAddress, Optional, 0, "1" - -HKR, Ndi\params\*IPChecksumOffloadIPv4, ParamDesc, 0, %IPChksumOffV4% -HKR, Ndi\params\*IPChecksumOffloadIPv4, default, 0, "3" -HKR, Ndi\params\*IPChecksumOffloadIPv4, type, 0, "enum" -HKR, Ndi\params\*IPChecksumOffloadIPv4\enum, "0", 0, %Disabled% -HKR, Ndi\params\*IPChecksumOffloadIPv4\enum, "1", 0, %TxEnabled% -HKR, Ndi\params\*IPChecksumOffloadIPv4\enum, "2", 0, %RxEnabled% -HKR, Ndi\params\*IPChecksumOffloadIPv4\enum, "3", 0, %RxTxEnabled% - -HKR, Ndi\params\*TCPChecksumOffloadIPv4, ParamDesc, 0, %TCPChksumOffV4% -HKR, Ndi\params\*TCPChecksumOffloadIPv4, default, 0, "3" -HKR, Ndi\params\*TCPChecksumOffloadIPv4, type, 0, "enum" -HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum, "0", 0, %Disabled% -HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum, "1", 0, %TxEnabled% -HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum, "2", 0, %RxEnabled% -HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum, "3", 0, %RxTxEnabled% - -HKR, Ndi\params\*UDPChecksumOffloadIPv4, ParamDesc, 0, %UDPChksumOffV4% -HKR, Ndi\params\*UDPChecksumOffloadIPv4, default, 0, "3" -HKR, Ndi\params\*UDPChecksumOffloadIPv4, type, 0, "enum" -HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum, "0", 0, %Disabled% -HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum, "1", 0, %TxEnabled% -HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum, "2", 0, %RxEnabled% -HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum, "3", 0, %RxTxEnabled% - -HKR, Ndi\params\*TCPChecksumOffloadIPv6, ParamDesc, 0, %TCPChksumOffV6% -HKR, Ndi\params\*TCPChecksumOffloadIPv6, default, 0, "3" -HKR, Ndi\params\*TCPChecksumOffloadIPv6, type, 0, "enum" -HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum, "0", 0, %Disabled% -HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum, "1", 0, %TxEnabled% -HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum, "2", 0, %RxEnabled% -HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum, "3", 0, %RxTxEnabled% - -HKR, Ndi\params\*UDPChecksumOffloadIPv6, ParamDesc, 0, %UDPChksumOffV6% -HKR, Ndi\params\*UDPChecksumOffloadIPv6, default, 0, "3" -HKR, Ndi\params\*UDPChecksumOffloadIPv6, type, 0, "enum" -HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum, "0", 0, %Disabled% -HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum, "1", 0, %TxEnabled% -HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum, "2", 0, %RxEnabled% -HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum, "3", 0, %RxTxEnabled% +HKR, Ndi\params\NetworkAddress, ParamDesc, 0, %NetworkAddress% +HKR, Ndi\params\NetworkAddress, type, 0, "edit" +HKR, Ndi\params\NetworkAddress, default, 0, "" +HKR, Ndi\params\NetworkAddress, LimitText, 0, "12" +HKR, Ndi\params\NetworkAddress, UpperCase, 0, "1" +HKR, Ndi\params\NetworkAddress, Optional, 0, "1" + +HKR, Ndi\params\*FlowControl, ParamDesc, 0, %FlowControl% +HKR, Ndi\params\*FlowControl, default, 0, "3" +HKR, Ndi\params\*FlowControl, type, 0, "enum" +HKR, Ndi\params\*FlowControl\enum, "0", 0, %Disabled% +HKR, Ndi\params\*FlowControl\enum, "1", 0, %TxEnabled% +HKR, Ndi\params\*FlowControl\enum, "2", 0, %RxEnabled% +HKR, Ndi\params\*FlowControl\enum, "3", 0, %TxRxEnabled% + +HKR, Ndi\Params\*PriorityVLANTag, ParamDesc, 0, %PriorityVlanTag% +HKR, Ndi\Params\*PriorityVLANTag, Default, 0, "3" +HKR, Ndi\Params\*PriorityVLANTag, Type, 0, "enum" +HKR, Ndi\Params\*PriorityVLANTag\enum, "0", 0, %Disabled% +HKR, Ndi\Params\*PriorityVLANTag\enum, "1", 0, %PriorityEnabled% +HKR, Ndi\Params\*PriorityVLANTag\enum, "2", 0, %VlanEnabled% +HKR, Ndi\Params\*PriorityVLANTag\enum, "3", 0, %PriorityVlanEnabled% + +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv4, ParamDesc, 0, %TCPUDPChecksumOffloadIPv4% +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv4, default, 0, "3" +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv4, type, 0, "enum" +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv4\enum, "0", 0, %Disabled% +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv4\enum, "1", 0, %TxEnabled% +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv4\enum, "2", 0, %RxEnabled% +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv4\enum, "3", 0, %TxRxEnabled% + +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv6, ParamDesc, 0, %TCPUDPChecksumOffloadIPv6% +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv6, default, 0, "3" +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv6, type, 0, "enum" +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv6\enum, "0", 0, %Disabled% +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv6\enum, "1", 0, %TxEnabled% +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv6\enum, "2", 0, %RxEnabled% +HKR, Ndi\params\*TCPUDPChecksumOffloadIPv6\enum, "3", 0, %TxRxEnabled% [DWCEQOS_Device.NT.Services] AddService = %ServiceName%, 2, DWCEQOS_AddService, DWCEQOS_AddService_EventLog @@ -153,19 +145,21 @@ KmdfLibraryVersion = $KMDFVERSION$ [Strings] ProviderName = "Open Source" -NetworkAddress = "Network Address" RKCP = "Rockchip" DWCEQOS.DeviceDesc = "Synopsys DesignWare Ethernet Quality of Service (GMAC)" DWCEQOS.ServiceDesc = "DesignWare Ethernet" -IPChksumOffV4 = "IPv4 Checksum Offload" -TCPChksumOffV4 = "TCP Checksum Offload (IPv4)" -UDPChksumOffV4 = "UDP Checksum Offload (IPv4)" -TCPChksumOffV6 = "TCP Checksum Offload (IPv6)" -UDPChksumOffV6 = "UDP Checksum Offload (IPv6)" +NetworkAddress = "Network Address" +FlowControl = "Flow Control" +PriorityVlanTag = "Packet Priority & VLAN" +TCPUDPChecksumOffloadIPv4 = "TCP/UDP Checksum Offload (IPv4)" +TCPUDPChecksumOffloadIPv6 = "TCP/UDP Checksum Offload (IPv6)" Disabled = "Disabled" TxEnabled = "Tx Enabled" RxEnabled = "Rx Enabled" -RxTxEnabled = "Rx & Tx Enabled" +TxRxEnabled = "Tx and Rx Enabled" +PriorityEnabled = "Packet Priority Enabled" +VlanEnabled = "VLAN Enabled" +PriorityVlanEnabled = "Packet Priority & VLAN Enabled" ; Not localized ServiceName = "dwc_eqos" diff --git a/drivers/net/dwc_eqos/dwc_eqos_perf.man b/drivers/net/dwc_eqos/dwc_eqos_perf.man index 832a077..4a31a2b 100644 --- a/drivers/net/dwc_eqos/dwc_eqos_perf.man +++ b/drivers/net/dwc_eqos/dwc_eqos_perf.man @@ -452,21 +452,12 @@ type="perf_counter_rawcount" uri="uri:opensource/dwc_eqos/perf/debug/RxDoneFragments" /> - diff --git a/drivers/net/dwc_eqos/dwc_eqos_perf_data.h b/drivers/net/dwc_eqos/dwc_eqos_perf_data.h index c562940..1bd05c1 100644 --- a/drivers/net/dwc_eqos/dwc_eqos_perf_data.h +++ b/drivers/net/dwc_eqos/dwc_eqos_perf_data.h @@ -30,7 +30,6 @@ struct PERF_DEBUG_DATA UINT32 DpcFatalBusError; UINT32 RxOwnDescriptors; UINT32 RxDoneFragments; - UINT32 TxOwnDescriptors; UINT32 TxDoneFragments; }; diff --git a/drivers/net/dwc_eqos/precomp.h b/drivers/net/dwc_eqos/precomp.h index bf51ec8..f555bce 100644 --- a/drivers/net/dwc_eqos/precomp.h +++ b/drivers/net/dwc_eqos/precomp.h @@ -8,6 +8,7 @@ #pragma warning(pop) #include +#include #include #include #include diff --git a/drivers/net/dwc_eqos/registers.h b/drivers/net/dwc_eqos/registers.h index 17323da..1c7500b 100644 --- a/drivers/net/dwc_eqos/registers.h +++ b/drivers/net/dwc_eqos/registers.h @@ -639,6 +639,43 @@ union MacPacketFilter_t }; }; +enum VlanTagStripOnReceive : UINT8 +{ + VlanTagStripOnReceive_Never, + VlanTagStripOnReceive_IfFilterPasses, + VlanTagStripOnReceive_IfFilterFails, + VlanTagStripOnReceive_Always, +}; + +union Mac_Vlan_Tag_Ctrl_t +{ + UINT32 Value32; + struct + { + UINT8 OperationBusy : 1; // OB + UINT8 CommandType : 1; // CT, 0 = write, 1 = read. + UINT8 Offset : 5; // OFS + UINT8 Reserved7 : 1; + + UINT8 Reserved8; + + UINT8 Reserved16 : 1; + UINT8 InverseMatchEnable : 1; // VTIM + UINT8 SvlanEnable : 1; // ESVL + UINT8 Reserved19 : 2; + VlanTagStripOnReceive StripOnReceive : 2; // EVLS + UINT8 Reserved23 : 1; + + UINT8 RxStatusEnable : 1; // EVLRXS + UINT8 HashTableMatchEnable : 1; // VTHM + UINT8 DoubleVlanEnable : 1; // EDVLP + UINT8 InnerEnable : 1; // ERIVLT + VlanTagStripOnReceive InnerStripOnReceive : 2; // EIVLS + UINT8 Reserved30 : 1; + UINT8 InnerRxStatusEnable : 1; // EIVLRXS + }; +}; + union MacInterruptStatus_t { UINT32 Value32; @@ -924,7 +961,7 @@ struct MacRegisters // This register is the redefined format of the MAC VLAN Tag Register. It is used // for indirect addressing. It contains the address offset, command type and Busy // Bit for CSR access of the Per VLAN Tag registers. - ULONG Mac_Vlan_Tag_Ctrl; + Mac_Vlan_Tag_Ctrl_t Mac_Vlan_Tag_Ctrl; // MAC_VLAN_Tag_Data @ 0x0054 = 0x0: // This register holds the read/write data for Indirect Access of the Per VLAN Tag diff --git a/drivers/net/dwc_eqos/rxqueue.cpp b/drivers/net/dwc_eqos/rxqueue.cpp index 5a98be7..b0a21d3 100644 --- a/drivers/net/dwc_eqos/rxqueue.cpp +++ b/drivers/net/dwc_eqos/rxqueue.cpp @@ -18,6 +18,7 @@ struct RxQueueContext WDFCOMMONBUFFER descBuffer; RxDescriptor* descVirtual; PHYSICAL_ADDRESS descPhysical; + NET_EXTENSION packetIeee8021Q; NET_EXTENSION packetChecksum; NET_EXTENSION fragmentLogical; UINT32 descCount; // A power of 2 between QueueDescriptorMinCount and QueueDescriptorMaxCount. @@ -171,6 +172,16 @@ RxQueueAdvance(_In_ NETPACKETQUEUE queue) NT_ASSERT(descWrite.PacketLength >= 4); // PacketLength includes CRC frag->ValidLength = descWrite.PacketLength - 4; + if (descWrite.Rdes0Valid && descWrite.OuterVlanTag != 0) + { + NET_PACKET_IEEE8021Q tag = {}; + tag.PriorityCodePoint = descWrite.OuterVlanTag >> 13; + tag.VlanIdentifier = descWrite.OuterVlanTag; + + auto const ieee8021Q = NetExtensionGetPacketIeee8021Q(&context->packetIeee8021Q, pktIndex); + *ieee8021Q = tag; + } + // If checksum offload is disabled by hardware then no IP headers will be // detected. If checksum offload is disabled by software then NetAdapterCx // will ignore our evaluation. @@ -416,6 +427,12 @@ RxQueueCreate( NET_EXTENSION_QUERY query; + NET_EXTENSION_QUERY_INIT(&query, + NET_PACKET_EXTENSION_IEEE8021Q_NAME, + NET_PACKET_EXTENSION_IEEE8021Q_VERSION_1, + NetExtensionTypePacket); + NetRxQueueGetExtension(queue, &query, &context->packetIeee8021Q); + NET_EXTENSION_QUERY_INIT(&query, NET_PACKET_EXTENSION_CHECKSUM_NAME, NET_PACKET_EXTENSION_CHECKSUM_VERSION_1, diff --git a/drivers/net/dwc_eqos/txqueue.cpp b/drivers/net/dwc_eqos/txqueue.cpp index f29ecd6..a15a2bc 100644 --- a/drivers/net/dwc_eqos/txqueue.cpp +++ b/drivers/net/dwc_eqos/txqueue.cpp @@ -18,17 +18,21 @@ struct TxQueueContext WDFCOMMONBUFFER descBuffer; TxDescriptor* descVirtual; PHYSICAL_ADDRESS descPhysical; + NET_EXTENSION packetIeee8021Q; NET_EXTENSION packetChecksum; NET_EXTENSION fragmentLogical; UINT32 descCount; // A power of 2 between QueueDescriptorMinCount and QueueDescriptorMaxCount. UINT8 txPbl; bool txChecksumOffload; + UINT16 lastVlanTag; UINT32 descBegin; // Start of the TRANSMIT region. UINT32 descEnd; // End of the TRANSMIT region, start of the EMPTY region. }; WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(TxQueueContext, TxQueueGetContext) +#if DBG + // Gets channelRegs->Current_App_TxDesc, converted to an index. static UINT32 GetDescNext(_In_ TxQueueContext* context) @@ -38,6 +42,8 @@ GetDescNext(_In_ TxQueueContext* context) return QueueDescriptorAddressToIndex(current, context->descPhysical, context->descCount); } +#endif // DBG + // Sets channelRegs->TxDesc_Tail_Pointer to the physical address of the descriptor at the given index. static void SetDescEnd( @@ -58,6 +64,7 @@ TxQueueStart(_In_ NETPACKETQUEUE queue) // PASSIVE_LEVEL, nonpaged (resume path) auto const context = TxQueueGetContext(queue); + context->lastVlanTag = 0; // Make no assumptions about the device's current vlan tag. context->descBegin = 0; context->descEnd = 0; @@ -80,123 +87,121 @@ TxQueueStart(_In_ NETPACKETQUEUE queue) TraceEntryExit(TxQueueStart, LEVEL_INFO); } +// Starting at pktIndex, scan forward until we reach packetRing->NextIndex or a +// non-ignored packet. If we reach a non-ignored packet, sets *pktLastFrag to the last +// fragment of the found packet and returns the packet's index. Otherwise, sets +// *pktLastFrag to 0xFFFFFFFF and returns packetRing->NextIndex. +static UINT32 +SkipIgnorePackets(UINT32 pktIndex, _In_ TxQueueContext const* context, _Out_ UINT32* pktLastFrag) +{ + // DISPATCH_LEVEL + auto const pktNext = context->packetRing->NextIndex; + for (; pktIndex != pktNext; pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex)) + { + auto const pkt = NetRingGetPacketAtIndex(context->packetRing, pktIndex); + if (!pkt->Ignore) + { + NT_ASSERT(pkt->FragmentCount != 0); + *pktLastFrag = NetRingAdvanceIndex(context->fragmentRing, pkt->FragmentIndex, pkt->FragmentCount - 1u); + return pktIndex; + } + } + + *pktLastFrag = 0xFFFFFFFF; + return pktNext; +} + static EVT_PACKET_QUEUE_ADVANCE TxQueueAdvance; static void TxQueueAdvance(_In_ NETPACKETQUEUE queue) { // DISPATCH_LEVEL auto const context = TxQueueGetContext(queue); - auto const pktBegin = context->packetRing->BeginIndex; - auto const pktNext = context->packetRing->NextIndex; - auto const pktEnd = context->packetRing->EndIndex; auto const descMask = context->descCount - 1u; - UINT32 descIndex, pktIndex, fragIndex; - UINT32 ownDescriptors = 0, doneFrags = 0, queuedFrags = 0; + auto const descEnd = context->descEnd; + UINT32 descIndex, pktIndex, pktLastFrag, fragIndex; + UINT32 doneFrags = 0, queuedFrags = 0; /* Packet indexes: [pktBegin] old [pktNext] new [pktEnd] owned by NetAdapterCx [pktBegin] Descriptor indexes: - [descBegin] TRANSMITTED [descNext] TRANSMITTING [descEnd] EMPTY [descBegin-1] + [descBegin] TRANSMITTED TRANSMITTING [descEnd] EMPTY [descBegin-1] */ // Indicate transmitted packets. - pktIndex = pktBegin; + // Process any descriptors that the adapter has handed back to us. + // Release the corresponding fragments and packets. + pktIndex = SkipIgnorePackets(context->packetRing->BeginIndex, context, &pktLastFrag); fragIndex = context->fragmentRing->BeginIndex; - - auto const descNext = GetDescNext(context); - descIndex = context->descBegin; - auto descReady = (descNext - descIndex) & descMask; // Number of descriptors ready to be indicated. - while (descIndex != descNext) + for (descIndex = context->descBegin; descIndex != descEnd; descIndex = (descIndex + 1) & descMask) { - NT_ASSERT(pktIndex != pktNext); + NT_ASSERT(pktIndex != context->packetRing->NextIndex); + NT_ASSERT(pktLastFrag != 0xFFFFFFFF); - auto const pkt = NetRingGetPacketAtIndex(context->packetRing, pktIndex); - auto const fragmentCount = pkt->FragmentCount; - if (!pkt->Ignore) + auto const& desc = context->descVirtual[descIndex]; + auto const descWrite = desc.Write; + + if (descWrite.Own) { - NT_ASSERT(fragmentCount != 0); + // Descriptor is still owned by the DMA engine. + break; + } - if (fragmentCount > descReady) - { - break; - } + NT_ASSERT(descWrite.PacketIndex == pktIndex); + NT_ASSERT(descWrite.FragmentIndex == fragIndex); - for (unsigned i = 0; i != fragmentCount; i += 1) - { - auto const descIndex2 = (descIndex + i) & descMask; - NT_ASSERT(descIndex2 != descNext); + if (descWrite.ContextType) + { + // Ignore context descriptors. + continue; + } - auto const& desc = context->descVirtual[descIndex2]; - auto const descWrite = desc.Write; - NT_ASSERT(descWrite.PacketIndex == pktIndex); - NT_ASSERT(descWrite.FragmentIndex == NetRingAdvanceIndex(context->fragmentRing, pkt->FragmentIndex, i)); + if (descWrite.ErrorSummary || + descWrite.ContextType || + descWrite.DescriptorError || + (descWrite.LastDescriptor != 0) != (fragIndex == pktLastFrag)) + { + TraceWrite("TxQueueAdvance-error", LEVEL_ERROR, + TraceLoggingUInt32(descIndex), + TraceLoggingHexInt32(reinterpret_cast(&descWrite)[3], "TDES3"), + TraceLoggingUInt32(fragIndex), + TraceLoggingUInt32(pktLastFrag)); + } - // Descriptor is still owned by the DMA engine? - if (descWrite.Own) - { - /* - There's a race condition where we see the update to Current_App_TxDesc - before we see the update to the descriptor. We don't want to write to - the descriptor's memory while the update is in flight, but this is - otherwise harmless (we only use the descriptor for assertions and - logging), and a fix for this would likely affect performance, so just - stop when we see a descriptor with the Own bit still set. We'll - process it the next time we're called. - - This doesn't seem to happen in the Rx path. - */ - TraceWrite("TxQueueAdvance-own", LEVEL_VERBOSE, - TraceLoggingUInt32(descIndex2, "descIndex"), - TraceLoggingHexInt32(reinterpret_cast(&descWrite)[3], "TDES3"), - TraceLoggingUInt32(i, "fragment"), - TraceLoggingUInt32(fragmentCount)); - ownDescriptors = 1; - goto DoneIndicating; - } - else if ( - descWrite.ErrorSummary || - descWrite.ContextType || - descWrite.DescriptorError || - (descWrite.LastDescriptor != 0) != (i == fragmentCount - 1u)) - { - TraceWrite("TxQueueAdvance-error", LEVEL_ERROR, - TraceLoggingUInt32(descIndex2, "descIndex"), - TraceLoggingHexInt32(reinterpret_cast(&descWrite)[3], "TDES3"), - TraceLoggingUInt32(i, "fragment"), - TraceLoggingUInt32(fragmentCount)); - } - } + // Non-context descriptors map one-to-one with fragments. - doneFrags += fragmentCount; - descReady -= fragmentCount; - descIndex = (descIndex + fragmentCount) & descMask; + if (fragIndex == pktLastFrag) + { + // Packet is complete. Move to the next one. + pktIndex = SkipIgnorePackets(NetRingIncrementIndex(context->packetRing, pktIndex), context, &pktLastFrag); } - pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex); - fragIndex = NetRingAdvanceIndex(context->fragmentRing, pkt->FragmentIndex, fragmentCount); + // Fragment is complete. Move to the next one. + fragIndex = NetRingIncrementIndex(context->fragmentRing, fragIndex); } -DoneIndicating: - + // Return the completed packets and fragments to NetAdapterCx. context->packetRing->BeginIndex = pktIndex; context->fragmentRing->BeginIndex = fragIndex; - context->descBegin = descIndex; + + auto const descBegin = descIndex; + context->descBegin = descBegin; // Fill more descriptors. - pktIndex = pktNext; + pktIndex = context->packetRing->NextIndex; // Number of EMPTY is (descBegin-1) - descEnd (wrapping around if necessary). + auto const pktEnd = context->packetRing->EndIndex; auto const txChecksumOffload = context->txChecksumOffload; - auto const descEnd = context->descEnd; - auto descEmpty = ((context->descBegin - 1) - descEnd) & descMask; + auto descEmpty = (descBegin - 1 - descEnd) & descMask; descIndex = descEnd; while (descEmpty != 0) { - NT_ASSERT(descIndex != ((context->descBegin - 1) & descMask)); + NT_ASSERT(descIndex != ((descBegin - 1) & descMask)); if (pktIndex == pktEnd) { @@ -220,11 +225,46 @@ TxQueueAdvance(_In_ NETPACKETQUEUE queue) : checksum.Layer3 ? TxChecksumInsertionEnabledHeaderOnly : TxChecksumInsertionDisabled; - auto const fragmentCount = pkt->FragmentCount; + UINT32 const fragmentCount = pkt->FragmentCount; NT_ASSERT(fragmentCount != 0); - if (fragmentCount > descEmpty) + + auto const ieee8021Q = NetExtensionGetPacketIeee8021Q(&context->packetIeee8021Q, pktIndex); + if (ieee8021Q->TxTagging != 0) { - break; + UINT16 const newTag = (ieee8021Q->PriorityCodePoint << 13) | ieee8021Q->VlanIdentifier; + NT_ASSERT(newTag != 0); + if (newTag == context->lastVlanTag) + { + goto NoContextPacket; + } + + if (fragmentCount + 1 > descEmpty) + { + break; + } + + TxDescriptorContext descCtx = {}; + descCtx.VlanTag = newTag; + descCtx.VlanTagValid = true; + descCtx.ContextType = true; + descCtx.Own = true; +#if DBG + descCtx.PacketIndex = pktIndex; + descCtx.FragmentIndex = fragIndex; +#endif + + context->descVirtual[descIndex].Context = descCtx; + descIndex = (descIndex + 1) & descMask; + descEmpty -= 1; + } + else + { + NoContextPacket: + + if (fragmentCount > descEmpty) + { + break; + } } UINT32 frameLength = 0; @@ -263,24 +303,22 @@ TxQueueAdvance(_In_ NETPACKETQUEUE queue) fragIndex = NetRingIncrementIndex(context->fragmentRing, fragIndex); queuedFrags += 1; } + + descEmpty -= fragmentCount; } pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex); } - // In some error cases, the device may stall until we write to the tail pointer - // again. Write to the tail pointer if there are pending descriptors, even if we - // didn't fill any new ones. - if (descIndex != descNext) + if (descIndex != descEnd) { SetDescEnd(context, descIndex); context->packetRing->NextIndex = pktIndex; } - DeviceAddStatisticsTxQueue(context->deviceContext, ownDescriptors, doneFrags); + DeviceAddStatisticsTxQueue(context->deviceContext, doneFrags); TraceEntryExit(TxQueueAdvance, LEVEL_VERBOSE, - TraceLoggingUInt32(ownDescriptors), TraceLoggingUInt32(doneFrags), TraceLoggingUInt32(queuedFrags)); } @@ -491,6 +529,12 @@ TxQueueCreate( NET_EXTENSION_QUERY query; + NET_EXTENSION_QUERY_INIT(&query, + NET_PACKET_EXTENSION_IEEE8021Q_NAME, + NET_PACKET_EXTENSION_IEEE8021Q_VERSION_1, + NetExtensionTypePacket); + NetTxQueueGetExtension(queue, &query, &context->packetIeee8021Q); + if (context->txChecksumOffload) { NET_EXTENSION_QUERY_INIT(&query,