diff --git a/drivers/net/dwc_eqos/device.cpp b/drivers/net/dwc_eqos/device.cpp index 5df8f97..f77a2fe 100644 --- a/drivers/net/dwc_eqos/device.cpp +++ b/drivers/net/dwc_eqos/device.cpp @@ -424,7 +424,8 @@ AdapterCreateTxQueue( queueInit, context->dma, &context->regs->Dma_Ch[0], - &context->regs->Mtl_Q[0]); + &context->regs->Mtl_Q[0], + context->feature0.TxChecksumOffload != 0); } static EVT_NET_ADAPTER_CREATE_RXQUEUE AdapterCreateRxQueue; @@ -450,7 +451,6 @@ AdapterSetReceiveFilter( _In_ NETRECEIVEFILTER receiveFilter) { // PASSIVE_LEVEL, nonpaged (resume path) - TraceEntry(AdapterSetReceiveFilter, LEVEL_INFO); auto const context = DeviceGetContext(AdapterGetContext(adapter)->device); auto const flags = NetReceiveFilterGetPacketFilter(receiveFilter); @@ -487,11 +487,45 @@ AdapterSetReceiveFilter( Write32(&context->regs->Mac_Packet_Filter, filter); - TraceExit(AdapterSetReceiveFilter, LEVEL_INFO, + TraceEntryExit(AdapterSetReceiveFilter, LEVEL_INFO, TraceLoggingHexInt32(flags), TraceLoggingUIntPtr(mcastCount)); } +static EVT_NET_ADAPTER_OFFLOAD_SET_TX_CHECKSUM AdapterOffloadSetTxChecksum; +static void +AdapterOffloadSetTxChecksum( + _In_ NETADAPTER adapter, + _In_ NETOFFLOAD offload) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + UNREFERENCED_PARAMETER(adapter); + auto const IPv4 = NetOffloadIsTxChecksumIPv4Enabled(offload); + auto const Tcp = NetOffloadIsTxChecksumTcpEnabled(offload); + auto const Udp = NetOffloadIsTxChecksumUdpEnabled(offload); + TraceEntryExit(AdapterOffloadSetTxChecksum, LEVEL_INFO, + TraceLoggingBoolean(IPv4), + TraceLoggingBoolean(Tcp), + TraceLoggingBoolean(Udp)); +} + +static EVT_NET_ADAPTER_OFFLOAD_SET_RX_CHECKSUM AdapterOffloadSetRxChecksum; +static void +AdapterOffloadSetRxChecksum( + _In_ NETADAPTER adapter, + _In_ NETOFFLOAD offload) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + UNREFERENCED_PARAMETER(adapter); + auto const IPv4 = NetOffloadIsRxChecksumIPv4Enabled(offload); + auto const Tcp = NetOffloadIsRxChecksumTcpEnabled(offload); + auto const Udp = NetOffloadIsRxChecksumUdpEnabled(offload); + TraceEntryExit(AdapterOffloadSetRxChecksum, LEVEL_INFO, + TraceLoggingBoolean(IPv4), + TraceLoggingBoolean(Tcp), + TraceLoggingBoolean(Udp)); +} + static EVT_WDF_DEVICE_D0_ENTRY DeviceD0Entry; static NTSTATUS DeviceD0Entry( @@ -541,7 +575,8 @@ DeviceD0Entry( MtlRxOperationMode_t rxOperationMode = {}; rxOperationMode.StoreAndForward = true; - rxOperationMode.ForwardErrorPackets = true; + rxOperationMode.DisableDropTcpChecksumError = true; + rxOperationMode.ForwardErrorPackets = false; rxOperationMode.ForwardUndersizedGoodPackets = true; rxOperationMode.QueueSize = rxQueueSize / 256u - 1; rxOperationMode.HardwareFlowControl = rxQueueSize >= 2048; @@ -556,6 +591,7 @@ DeviceD0Entry( macConfig.PacketBurstEnable = true; macConfig.ReceiverEnable = true; macConfig.TransmitterEnable = true; + macConfig.ChecksumOffloadEnable = context->feature0.TxChecksumOffload | context->feature0.RxChecksumOffload; Write32(&context->regs->Mac_Configuration, macConfig); // Clear and then enable interrupts. @@ -906,6 +942,27 @@ DevicePrepareHardware( NetPacketFilterFlagBroadcast | NetPacketFilterFlagPromiscuous; NetAdapterSetReceiveFilterCapabilities(context->adapter, &rxFilterCaps); + + if (context->feature0.TxChecksumOffload) + { + NET_ADAPTER_OFFLOAD_TX_CHECKSUM_CAPABILITIES txChecksumCaps; + NET_ADAPTER_OFFLOAD_TX_CHECKSUM_CAPABILITIES_INIT(&txChecksumCaps, {}, AdapterOffloadSetTxChecksum); + txChecksumCaps.Layer3Flags = + NetAdapterOffloadLayer3FlagIPv4NoOptions | + NetAdapterOffloadLayer3FlagIPv4WithOptions | + NetAdapterOffloadLayer3FlagIPv6NoExtensions | + NetAdapterOffloadLayer3FlagIPv6WithExtensions; + txChecksumCaps.Layer4Flags = + NetAdapterOffloadLayer4FlagTcpNoOptions | + NetAdapterOffloadLayer4FlagTcpWithOptions | + NetAdapterOffloadLayer4FlagUdp; + NetAdapterOffloadSetTxChecksumCapabilities(context->adapter, &txChecksumCaps); + } + + NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES rxChecksumCaps; + NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES_INIT(&rxChecksumCaps, + AdapterOffloadSetRxChecksum); + NetAdapterOffloadSetRxChecksumCapabilities(context->adapter, &rxChecksumCaps); } // Initialize adapter. @@ -983,34 +1040,6 @@ DeviceReleaseHardware( auto const context = DeviceGetContext(device); if (context->regs != nullptr) { -#define CtxStat(x) TraceLoggingUInt32(context->x, #x) -#define RegStat(x) TraceLoggingUInt32(Read32(&context->regs->x), #x) - - TraceWrite("DeviceReleaseHardware-MacStats", LEVEL_INFO, - CtxStat(isrHandled), - CtxStat(isrIgnored), - CtxStat(dpcLinkState), - CtxStat(dpcRx), - CtxStat(dpcTx), - CtxStat(dpcAbnormalStatus), - CtxStat(dpcFatalBusError)); - -#if 0 // MMC frozen for now - TraceWrite("DeviceReleaseHardware-TxStats", LEVEL_INFO, - RegStat(TxPacketCountGoodBad), - RegStat(TxUnderflowErrorPackets), - RegStat(TxCarrierErrorPackets), - RegStat(TxPacketCountGood), - RegStat(TxPausePackets)); - TraceWrite("DeviceReleaseHardware-RxStats", LEVEL_INFO, - RegStat(RxPacketCountGoodBad), - RegStat(RxCrcErrorPackets), - RegStat(RxLengthErrorPackets), - RegStat(RxPausePackets), - RegStat(RxFifoOverflowPackets), - RegStat(RxWatchdogErrorPackets)); -#endif - DeviceReset(context->regs, context->permanentMacAddress); MmUnmapIoSpace(context->regs, sizeof(*context->regs)); context->regs = nullptr; diff --git a/drivers/net/dwc_eqos/driver.cpp b/drivers/net/dwc_eqos/driver.cpp index 810c6cf..3623c2b 100644 --- a/drivers/net/dwc_eqos/driver.cpp +++ b/drivers/net/dwc_eqos/driver.cpp @@ -9,7 +9,6 @@ TODO list: - Jumbo frames. - Receive queue memory optimization? - Configuration in registry (e.g. flow control). -- Checksum offload. - Tx segmentation offload. - Rx segmentation offload. - Wake-on-LAN. diff --git a/drivers/net/dwc_eqos/dwc_eqos.inf b/drivers/net/dwc_eqos/dwc_eqos.inf index 7ce273f..99fea27 100644 --- a/drivers/net/dwc_eqos/dwc_eqos.inf +++ b/drivers/net/dwc_eqos/dwc_eqos.inf @@ -76,6 +76,7 @@ HKR, Ndi\Interfaces, UpperRange, 0, "ndis5" HKR, Ndi\Interfaces, LowerRange, 0, "ethernet" [DWCEQOS_AddReg_Ndi_params] + HKR, Ndi\params\NetworkAddress, ParamDesc, 0, %NetworkAddress% HKR, Ndi\params\NetworkAddress, type, 0, "edit" HKR, Ndi\params\NetworkAddress, default, 0, "" @@ -83,6 +84,46 @@ HKR, Ndi\params\NetworkAddress, LimitText, 0, "12" HKR, Ndi\params\NetworkAddress, UpperCase, 0, "1" HKR, Ndi\params\NetworkAddress, Optional, 0, "1" +HKR, Ndi\params\*IPChecksumOffloadIPv4, ParamDesc, 0, %IPChksumOffV4% +HKR, Ndi\params\*IPChecksumOffloadIPv4, default, 0, "3" +HKR, Ndi\params\*IPChecksumOffloadIPv4, type, 0, "enum" +HKR, Ndi\params\*IPChecksumOffloadIPv4\enum, "0", 0, %Disabled% +HKR, Ndi\params\*IPChecksumOffloadIPv4\enum, "1", 0, %TxEnabled% +HKR, Ndi\params\*IPChecksumOffloadIPv4\enum, "2", 0, %RxEnabled% +HKR, Ndi\params\*IPChecksumOffloadIPv4\enum, "3", 0, %RxTxEnabled% + +HKR, Ndi\params\*TCPChecksumOffloadIPv4, ParamDesc, 0, %TCPChksumOffV4% +HKR, Ndi\params\*TCPChecksumOffloadIPv4, default, 0, "3" +HKR, Ndi\params\*TCPChecksumOffloadIPv4, type, 0, "enum" +HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum, "0", 0, %Disabled% +HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum, "1", 0, %TxEnabled% +HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum, "2", 0, %RxEnabled% +HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum, "3", 0, %RxTxEnabled% + +HKR, Ndi\params\*UDPChecksumOffloadIPv4, ParamDesc, 0, %UDPChksumOffV4% +HKR, Ndi\params\*UDPChecksumOffloadIPv4, default, 0, "3" +HKR, Ndi\params\*UDPChecksumOffloadIPv4, type, 0, "enum" +HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum, "0", 0, %Disabled% +HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum, "1", 0, %TxEnabled% +HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum, "2", 0, %RxEnabled% +HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum, "3", 0, %RxTxEnabled% + +HKR, Ndi\params\*TCPChecksumOffloadIPv6, ParamDesc, 0, %TCPChksumOffV6% +HKR, Ndi\params\*TCPChecksumOffloadIPv6, default, 0, "3" +HKR, Ndi\params\*TCPChecksumOffloadIPv6, type, 0, "enum" +HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum, "0", 0, %Disabled% +HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum, "1", 0, %TxEnabled% +HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum, "2", 0, %RxEnabled% +HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum, "3", 0, %RxTxEnabled% + +HKR, Ndi\params\*UDPChecksumOffloadIPv6, ParamDesc, 0, %UDPChksumOffV6% +HKR, Ndi\params\*UDPChecksumOffloadIPv6, default, 0, "3" +HKR, Ndi\params\*UDPChecksumOffloadIPv6, type, 0, "enum" +HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum, "0", 0, %Disabled% +HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum, "1", 0, %TxEnabled% +HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum, "2", 0, %RxEnabled% +HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum, "3", 0, %RxTxEnabled% + [DWCEQOS_Device.NT.Services] AddService = %ServiceName%, 2, DWCEQOS_AddService, DWCEQOS_AddService_EventLog @@ -111,11 +152,20 @@ KmdfService = %ServiceName%, DWCEQOS_KmdfService KmdfLibraryVersion = $KMDFVERSION$ [Strings] -ProviderName = "Open Source" -NetworkAddress = "Network Address" -RKCP = "Rockchip" -DWCEQOS.DeviceDesc = "Synopsys DesignWare Ethernet Quality of Service (GMAC)" +ProviderName = "Open Source" +NetworkAddress = "Network Address" +RKCP = "Rockchip" +DWCEQOS.DeviceDesc = "Synopsys DesignWare Ethernet Quality of Service (GMAC)" DWCEQOS.ServiceDesc = "DesignWare Ethernet" +IPChksumOffV4 = "IPv4 Checksum Offload" +TCPChksumOffV4 = "TCP Checksum Offload (IPv4)" +UDPChksumOffV4 = "UDP Checksum Offload (IPv4)" +TCPChksumOffV6 = "TCP Checksum Offload (IPv6)" +UDPChksumOffV6 = "UDP Checksum Offload (IPv6)" +Disabled = "Disabled" +TxEnabled = "Tx Enabled" +RxEnabled = "Rx Enabled" +RxTxEnabled = "Rx & Tx Enabled" ; Not localized ServiceName = "dwc_eqos" diff --git a/drivers/net/dwc_eqos/precomp.h b/drivers/net/dwc_eqos/precomp.h index 2960475..bf51ec8 100644 --- a/drivers/net/dwc_eqos/precomp.h +++ b/drivers/net/dwc_eqos/precomp.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/dwc_eqos/registers.h b/drivers/net/dwc_eqos/registers.h index ea4dd6b..743e881 100644 --- a/drivers/net/dwc_eqos/registers.h +++ b/drivers/net/dwc_eqos/registers.h @@ -1735,7 +1735,6 @@ constexpr unsigned DESCRIPTOR_ALIGN = 64u; #define RX_DESCRIPTOR_SIZE 16 #endif - enum TxChecksumInsertion : UINT16 { TxChecksumInsertionDisabled = 0, @@ -1937,6 +1936,18 @@ struct RxDescriptorRead }; static_assert(sizeof(RxDescriptorRead) == RX_DESCRIPTOR_SIZE); +enum RxPayloadType : UINT8 +{ + RxPayloadTypeUnknown = 0, + RxPayloadTypeUdp, + RxPayloadTypeTcp, + RxPayloadTypeIcmp, + RxPayloadTypeIgmp, + RxPayloadTypeAvUntaggedControl, + RxPayloadTypeAvTaggedData, + RxPayloadTypeAvTaggedControl, +}; + struct RxDescriptorWrite { // RDES0 @@ -1946,7 +1957,7 @@ struct RxDescriptorWrite // RDES1 - UINT8 PayloadType : 3; // PT + RxPayloadType PayloadType : 3; // PT UINT8 IPHeaderError : 1; // IPHE UINT8 IPv4HeaderPresent : 1; // IPV4 UINT8 IPv6HeaderPresent : 1; // IPV6 diff --git a/drivers/net/dwc_eqos/rxqueue.cpp b/drivers/net/dwc_eqos/rxqueue.cpp index d43d4e3..632a1c4 100644 --- a/drivers/net/dwc_eqos/rxqueue.cpp +++ b/drivers/net/dwc_eqos/rxqueue.cpp @@ -17,6 +17,7 @@ struct RxQueueContext WDFCOMMONBUFFER descBuffer; RxDescriptor* descVirtual; PHYSICAL_ADDRESS descPhysical; + NET_EXTENSION packetChecksum; NET_EXTENSION fragmentLogical; UINT32 descCount; // A power of 2 between QueueDescriptorMinCount and QueueDescriptorMaxCount. bool running; @@ -167,6 +168,39 @@ RxQueueAdvance(_In_ NETPACKETQUEUE queue) { NT_ASSERT(descWrite.PacketLength >= 4); // PacketLength includes CRC frag->ValidLength = descWrite.PacketLength - 4; + + // If checksum offload is disabled by hardware then no IP headers will be + // detected. If checksum offload is disabled by software then NetAdapterCx + // will ignore our evaluation. + if (descWrite.IPv4HeaderPresent | descWrite.IPv6HeaderPresent) + { + auto const checksum = NetExtensionGetPacketChecksum(&context->packetChecksum, pktIndex); + checksum->Layer2 = NetPacketRxChecksumEvaluationValid; + pkt->Layout.Layer2Type = NetPacketLayer2TypeEthernet; + + checksum->Layer3 = descWrite.IPHeaderError + ? NetPacketRxChecksumEvaluationInvalid + : NetPacketRxChecksumEvaluationValid; + pkt->Layout.Layer3Type = descWrite.IPv4HeaderPresent + ? NetPacketLayer3TypeIPv4UnspecifiedOptions + : NetPacketLayer3TypeIPv6UnspecifiedExtensions; + + checksum->Layer4 = descWrite.IPPayloadError + ? NetPacketRxChecksumEvaluationInvalid + : NetPacketRxChecksumEvaluationValid; + switch (descWrite.PayloadType) + { + case RxPayloadTypeUdp: + pkt->Layout.Layer4Type = NetPacketLayer4TypeUdp; + break; + case RxPayloadTypeTcp: + pkt->Layout.Layer4Type = NetPacketLayer4TypeTcp; + break; + default: + pkt->Layout.Layer4Type = NetPacketLayer4TypeUnspecified; + break; + } + } } pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex); @@ -377,6 +411,13 @@ RxQueueCreate( TraceLoggingPointer(context->descVirtual, "virtual")); NET_EXTENSION_QUERY query; + + NET_EXTENSION_QUERY_INIT(&query, + NET_PACKET_EXTENSION_CHECKSUM_NAME, + NET_PACKET_EXTENSION_CHECKSUM_VERSION_1, + NetExtensionTypePacket); + NetRxQueueGetExtension(queue, &query, &context->packetChecksum); + NET_EXTENSION_QUERY_INIT(&query, NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_NAME, NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_VERSION_1, diff --git a/drivers/net/dwc_eqos/txqueue.cpp b/drivers/net/dwc_eqos/txqueue.cpp index 5e5b0f9..15ac44b 100644 --- a/drivers/net/dwc_eqos/txqueue.cpp +++ b/drivers/net/dwc_eqos/txqueue.cpp @@ -17,8 +17,10 @@ struct TxQueueContext WDFCOMMONBUFFER descBuffer; TxDescriptor* descVirtual; PHYSICAL_ADDRESS descPhysical; + NET_EXTENSION packetChecksum; NET_EXTENSION fragmentLogical; UINT32 descCount; // A power of 2 between QueueDescriptorMinCount and QueueDescriptorMaxCount. + bool checksumOffloadEnabled; UINT32 descBegin; // Start of the TRANSMIT region. UINT32 descEnd; // End of the TRANSMIT region, start of the EMPTY region. @@ -202,6 +204,18 @@ TxQueueAdvance(_In_ NETPACKETQUEUE queue) { fragIndex = pkt->FragmentIndex; + // If checksum offload is disabled by hardware then we can't call + // NetExtensionGetPacketChecksum because device.cpp didn't call + // NetAdapterOffloadSetTxChecksumCapabilities. + // If offload is disabled by software then the extension will be zeroed. + auto const checksum = context->checksumOffloadEnabled + ? *NetExtensionGetPacketChecksum(&context->packetChecksum, pktIndex) + : NET_PACKET_CHECKSUM{}; // Disabled by hardware. + auto const checksumInsertion = + checksum.Layer4 ? TxChecksumInsertionEnabledIncludingPseudo + : checksum.Layer3 ? TxChecksumInsertionEnabledHeaderOnly + : TxChecksumInsertionDisabled; + auto const fragmentCount = pkt->FragmentCount; NT_ASSERT(fragmentCount != 0); if (fragmentCount > descEmpty) @@ -229,8 +243,9 @@ TxQueueAdvance(_In_ NETPACKETQUEUE queue) descRead.Buf2Ap = static_cast(fragLogicalAddress >> 32); NT_ASSERT(frag->ValidLength <= 0x3FFF); // 14 bits descRead.Buf1Length = frag->ValidLength & 0x3FFF; - descRead.InterruptOnCompletion = true; + descRead.InterruptOnCompletion = i == fragmentCount - 1u; descRead.FrameLength = static_cast(frameLength); + descRead.ChecksumInsertion = checksumInsertion; descRead.LastDescriptor = i == fragmentCount - 1u; descRead.FirstDescriptor = i == 0; descRead.Own = true; @@ -398,7 +413,8 @@ TxQueueCreate( NETTXQUEUE_INIT* queueInit, WDFDMAENABLER dma, ChannelRegisters* channelRegs, - MtlQueueRegisters* mtlRegs) + MtlQueueRegisters* mtlRegs, + bool checksumOffloadEnabled) { // PASSIVE_LEVEL, nonpaged (resume path) NTSTATUS status; @@ -437,6 +453,7 @@ TxQueueCreate( context->packetRing = NetRingCollectionGetPacketRing(rings); context->fragmentRing = NetRingCollectionGetFragmentRing(rings); context->descCount = QueueDescriptorCount(context->fragmentRing->NumberOfElements); + context->checksumOffloadEnabled = checksumOffloadEnabled; TraceWrite("TxQueueCreate-size", LEVEL_VERBOSE, TraceLoggingHexInt32(context->packetRing->NumberOfElements, "packets"), @@ -468,6 +485,16 @@ TxQueueCreate( TraceLoggingPointer(context->descVirtual, "virtual")); NET_EXTENSION_QUERY query; + + if (context->checksumOffloadEnabled) + { + NET_EXTENSION_QUERY_INIT(&query, + NET_PACKET_EXTENSION_CHECKSUM_NAME, + NET_PACKET_EXTENSION_CHECKSUM_VERSION_1, + NetExtensionTypePacket); + NetTxQueueGetExtension(queue, &query, &context->packetChecksum); + } + NET_EXTENSION_QUERY_INIT(&query, NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_NAME, NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_VERSION_1, diff --git a/drivers/net/dwc_eqos/txqueue.h b/drivers/net/dwc_eqos/txqueue.h index 6236b69..bf2e348 100644 --- a/drivers/net/dwc_eqos/txqueue.h +++ b/drivers/net/dwc_eqos/txqueue.h @@ -16,4 +16,5 @@ TxQueueCreate( _Inout_ NETTXQUEUE_INIT* queueInit, _In_ WDFDMAENABLER dma, _Inout_ ChannelRegisters* channelRegs, - _Inout_ MtlQueueRegisters* mtlRegs); + _Inout_ MtlQueueRegisters* mtlRegs, + bool checksumOffloadEnabled);