Merge branch 'main' into psu_presence_1201

build/fbcode_builder/getdeps.py

@@ -333,6 +333,12 @@ def run_project_cmd(self, args, loader, manifest):
 
         cache = cache_module.create_cache()
         for m in projects:
+            fetcher = loader.create_fetcher(m)
+            if isinstance(fetcher, SystemPackageFetcher):
+                # We are guaranteed that if the fetcher is set to
+                # SystemPackageFetcher then this item is completely
+                # satisfied by the appropriate system packages
+                continue
             cached_project = CachedProject(cache, loader, m)
             if cached_project.download():
                 continue
@@ -348,7 +354,6 @@ def run_project_cmd(self, args, loader, manifest):
                     continue
 
             # We need to fetch the sources
-            fetcher = loader.create_fetcher(m)
             fetcher.update()
 
 

build/fbcode_builder/getdeps/manifest.py

@@ -430,23 +430,27 @@ def _create_fetcher(self, build_options, ctx):
             # We can use the code from fbsource
             return ShipitTransformerFetcher(build_options, self.shipit_project)
 
+        # If both of these are None, the package can only be coming from
+        # preinstalled toolchain or system packages
+        repo_url = self.get_repo_url(ctx)
+        url = self.get("download", "url", ctx=ctx)
+
         # Can we satisfy this dep with system packages?
-        if build_options.allow_system_packages:
+        if (repo_url is None and url is None) or build_options.allow_system_packages:
             if self._is_satisfied_by_preinstalled_environment(ctx):
                 return PreinstalledNopFetcher()
 
-            packages = self.get_required_system_packages(ctx)
-            package_fetcher = SystemPackageFetcher(build_options, packages)
-            if package_fetcher.packages_are_installed():
-                return package_fetcher
+            if build_options.host_type.get_package_manager():
+                packages = self.get_required_system_packages(ctx)
+                package_fetcher = SystemPackageFetcher(build_options, packages)
+                if package_fetcher.packages_are_installed():
+                    return package_fetcher
 
-        repo_url = self.get_repo_url(ctx)
         if repo_url:
             rev = self.get("git", "rev")
             depth = self.get("git", "depth")
             return GitFetcher(build_options, self, repo_url, rev, depth)
 
-        url = self.get("download", "url", ctx=ctx)
         if url:
             # We need to defer this import until now to avoid triggering
             # a cycle when the facebook/__init__.py is loaded.
@@ -464,7 +468,8 @@ def _create_fetcher(self, build_options, ctx):
                 )
 
         raise KeyError(
-            "project %s has no fetcher configuration matching %s" % (self.name, ctx)
+            "project %s has no fetcher configuration or system packages matching %s"
+            % (self.name, ctx)
         )
 
     def create_fetcher(self, build_options, loader, ctx):

build/fbcode_builder/manifests/openssl

@@ -5,7 +5,7 @@ name = openssl
 libssl-dev
 
 [homebrew]
-openssl@1.1
+openssl
 # on homebrew need the matching curl and ca-
 
 [rpms]
@@ -16,17 +16,19 @@ openssl-libs
 [pps]
 openssl
 
-[download]
-url = https://www.openssl.org/source/openssl-1.1.1l.tar.gz
-sha256 = 0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
+# no need to download on the systems where we always use the system libs
+[download.not(any(os=linux, os=freebsd))]
+# match the openssl version packages in ubuntu LTS folly current supports
+url = https://www.openssl.org/source/openssl-3.0.15.tar.gz
+sha256 = 23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533
 
 # We use the system openssl on these platforms even without --allow-system-packages
 [build.any(os=linux, os=freebsd)]
 builder = nop
 
 [build.not(any(os=linux, os=freebsd))]
 builder = openssl
-subdir = openssl-1.1.1l
+subdir = openssl-3.0.15
 
 [dependencies.os=windows]
 perl

cmake/AgentPacket.cmake

@@ -60,4 +60,5 @@ target_link_libraries(packet_factory
   ctrl_cpp2
   switch_config_cpp2
   Folly::folly
+  sflow_structs
 )

cmake/AgentTestAgentHwTests.cmake

@@ -49,6 +49,7 @@ add_library(agent_hw_test_src
   fboss/agent/test/agent_hw_tests/AgentMmuTuningTests.cpp
   fboss/agent/test/agent_hw_tests/AgentSflowMirrorTest.cpp
   fboss/agent/test/agent_hw_tests/AgentAclPriorityTests.cpp
+  fboss/agent/test/agent_hw_tests/AgentTrafficPauseTests.cpp
   fboss/agent/test/agent_hw_tests/AgentTrunkLoadBalancerTests.cpp
   fboss/agent/test/agent_hw_tests/AgentTrunkTests.cpp
   fboss/agent/test/agent_hw_tests/AgentRxReasonTests.cpp

fboss/agent/packet/BUCK

@@ -85,6 +85,7 @@ cpp_library(
         "//fboss/agent:fboss-types",
         "//fboss/agent:hw_switch",
         "//fboss/agent:packet",
+        "//fboss/agent/packet:sflow_structs",
         "//folly:network_address",
         "//folly/io:iobuf",
     ],

fboss/agent/packet/PktFactory.cpp

@@ -789,4 +789,264 @@ std::unique_ptr<TxPacket> makeTCPTxPacket(
       255,
       payload);
 }
+
+template <typename IPHDR>
+std::unique_ptr<TxPacket> makeSflowV5Packet(
+    const AllocatePktFn& allocatePacket,
+    const EthHdr& ethHdr,
+    const IPHDR& ipHdr,
+    const UDPHeader& udpHdr,
+    bool computeChecksum,
+    folly::IPAddress agentIp,
+    uint32_t ingressInterface,
+    uint32_t egressInterface,
+    uint32_t samplingRate,
+    const std::vector<uint8_t>& payload) {
+  auto txPacket = allocatePacket(
+      ethHdr.size() + ipHdr.size() + udpHdr.size() + 104 + payload.size());
+
+  folly::io::RWPrivateCursor rwCursor(txPacket->buf());
+  // Write EthHdr
+  writeEthHeader(
+      txPacket,
+      &rwCursor,
+      ethHdr.getDstMac(),
+      ethHdr.getSrcMac(),
+      ethHdr.getVlanTags(),
+      ethHdr.getEtherType());
+  ipHdr.serialize(&rwCursor);
+
+  // write UDP header, payload and compute checksum
+  rwCursor.writeBE<uint16_t>(udpHdr.srcPort);
+  rwCursor.writeBE<uint16_t>(udpHdr.dstPort);
+  rwCursor.writeBE<uint16_t>(udpHdr.length);
+  // Skip 2 bytes and compuete checksum later
+  rwCursor.skip(2);
+
+  sflow::SampleDatagram datagram;
+  sflow::SampleDatagramV5 datagramV5;
+  sflow::SampleRecord record;
+  sflow::FlowSample fsample;
+  sflow::FlowRecord frecord;
+  sflow::SampledHeader hdr;
+
+  int bufSize = 1024;
+
+  hdr.protocol = sflow::HeaderProtocol::ETHERNET_ISO88023;
+  hdr.frameLength = 0;
+  hdr.stripped = 0;
+  hdr.header = payload.data();
+  hdr.headerLength = payload.size();
+  auto hdrSize = hdr.size();
+  if (hdrSize % sflow::XDR_BASIC_BLOCK_SIZE > 0) {
+    hdrSize = (hdrSize / sflow::XDR_BASIC_BLOCK_SIZE + 1) *
+        sflow::XDR_BASIC_BLOCK_SIZE;
+  }
+
+  std::vector<uint8_t> hb(bufSize);
+  auto hbuf = folly::IOBuf::wrapBuffer(hb.data(), bufSize);
+  auto hc = std::make_shared<folly::io::RWPrivateCursor>(hbuf.get());
+  hdr.serialize(hc.get());
+
+  frecord.flowFormat = 1; // single flow sample
+  frecord.flowDataLen = hdrSize;
+  frecord.flowData = hb.data();
+
+  fsample.sequenceNumber = 0;
+  fsample.sourceID = 0;
+  fsample.samplingRate = samplingRate;
+  fsample.samplePool = 0;
+  fsample.drops = 0;
+  fsample.input = ingressInterface;
+  fsample.output = egressInterface;
+  fsample.flowRecordsCnt = 1;
+  fsample.flowRecords = &frecord;
+
+  std::vector<uint8_t> fsb(bufSize);
+  auto fbuf = folly::IOBuf::wrapBuffer(fsb.data(), bufSize);
+  auto fc = std::make_shared<folly::io::RWPrivateCursor>(fbuf.get());
+  fsample.serialize(fc.get());
+  size_t fsampleSize = bufSize - fc->length();
+
+  record.sampleType = 1; // raw header
+  record.sampleDataLen = fsampleSize;
+  record.sampleData = fsb.data();
+
+  datagramV5.agentAddress = agentIp;
+  datagramV5.subAgentID = 0; // no sub agent
+  datagramV5.sequenceNumber = 0; // not used
+  datagramV5.uptime = 0; // not used
+  datagramV5.samplesCnt = 1; // So far only 1 sample encapsuled
+  datagramV5.samples = &record;
+
+  datagram.datagramV5 = datagramV5;
+  datagram.serialize(&rwCursor);
+
+  if (computeChecksum) {
+    // Need to revisit when we compute the checksum of UDP payload
+    // folly::io::Cursor payloadStart(rwCursor);
+    // uint16_t csum = udpHdr.computeChecksum(ipHdr, payloadStart);
+    // csumCursor.writeBE<uint16_t>(csum);
+  }
+  return txPacket;
+}
+
+std::unique_ptr<facebook::fboss::TxPacket> makeSflowV5Packet(
+    const AllocatePktFn& allocator,
+    std::optional<VlanID> vlan,
+    folly::MacAddress srcMac,
+    folly::MacAddress dstMac,
+    const folly::IPAddressV4& srcIp,
+    const folly::IPAddressV4& dstIp,
+    uint16_t srcPort,
+    uint16_t dstPort,
+    uint8_t dscp,
+    uint8_t ttl,
+    uint32_t ingressInterface,
+    uint32_t egressInterface,
+    uint32_t samplingRate,
+    bool computeChecksum,
+    std::optional<std::vector<uint8_t>> payload) {
+  if (!payload) {
+    payload = kDefaultPayload;
+  }
+  const auto& payloadBytes = payload.value();
+  // EthHdr
+  auto ethHdr = makeEthHdr(srcMac, dstMac, vlan, ETHERTYPE::ETHERTYPE_IPV4);
+  // TODO: This assumes the Sflow V5 packet contains one sample header
+  // and one sample record. Need to be computed dynamically.
+  auto sampleHdrSize = 104;
+
+  // IPv4Hdr - total_length field includes the payload + UDP hdr + ip hdr
+  IPv4Hdr ipHdr(
+      srcIp,
+      dstIp,
+      static_cast<uint8_t>(IP_PROTO::IP_PROTO_UDP),
+      payloadBytes.size() + UDPHeader::size() + sampleHdrSize);
+  ipHdr.dscp = dscp;
+  ipHdr.ttl = ttl;
+  ipHdr.computeChecksum();
+  // UDPHeader
+  UDPHeader udpHdr(
+      srcPort,
+      dstPort,
+      UDPHeader::size() + payloadBytes.size() + sampleHdrSize);
+
+  return makeSflowV5Packet(
+      allocator,
+      ethHdr,
+      ipHdr,
+      udpHdr,
+      computeChecksum,
+      folly::IPAddress(srcIp),
+      ingressInterface,
+      egressInterface,
+      samplingRate,
+      payloadBytes);
+}
+
+std::unique_ptr<facebook::fboss::TxPacket> makeSflowV5Packet(
+    const AllocatePktFn& allocator,
+    std::optional<VlanID> vlan,
+    folly::MacAddress srcMac,
+    folly::MacAddress dstMac,
+    const folly::IPAddressV6& srcIp,
+    const folly::IPAddressV6& dstIp,
+    uint16_t srcPort,
+    uint16_t dstPort,
+    uint8_t trafficClass,
+    uint8_t hopLimit,
+    uint32_t ingressInterface,
+    uint32_t egressInterface,
+    uint32_t samplingRate,
+    bool computeChecksum,
+    std::optional<std::vector<uint8_t>> payload) {
+  if (!payload) {
+    payload = kDefaultPayload;
+  }
+  const auto& payloadBytes = payload.value();
+  // EthHdr
+  auto ethHdr = makeEthHdr(srcMac, dstMac, vlan, ETHERTYPE::ETHERTYPE_IPV6);
+  // TODO: This assumes the Sflow V5 packet contains one sample header
+  // and one sample record. Need to be computed dynamically.
+  auto sampleHdrSize = 104;
+
+  // IPv6Hdr
+  IPv6Hdr ipHdr(srcIp, dstIp);
+  ipHdr.nextHeader = static_cast<uint8_t>(IP_PROTO::IP_PROTO_UDP);
+  ipHdr.trafficClass = trafficClass;
+  ipHdr.payloadLength = UDPHeader::size() + payloadBytes.size() + sampleHdrSize;
+  ipHdr.hopLimit = hopLimit;
+  // UDPHeader
+  UDPHeader udpHdr(
+      srcPort,
+      dstPort,
+      UDPHeader::size() + payloadBytes.size() + sampleHdrSize);
+
+  return makeSflowV5Packet(
+      allocator,
+      ethHdr,
+      ipHdr,
+      udpHdr,
+      computeChecksum,
+      folly::IPAddress(srcIp),
+      ingressInterface,
+      egressInterface,
+      samplingRate,
+      payloadBytes);
+}
+
+std::unique_ptr<facebook::fboss::TxPacket> makeSflowV5Packet(
+    const AllocatePktFn& allocator,
+    std::optional<VlanID> vlan,
+    folly::MacAddress srcMac,
+    folly::MacAddress dstMac,
+    const folly::IPAddress& srcIp,
+    const folly::IPAddress& dstIp,
+    uint16_t srcPort,
+    uint16_t dstPort,
+    uint8_t trafficClass,
+    uint8_t hopLimit,
+    uint32_t ingressInterface,
+    uint32_t egressInterface,
+    uint32_t samplingRate,
+    bool computeChecksum,
+    std::optional<std::vector<uint8_t>> payload) {
+  CHECK_EQ(srcIp.isV6(), dstIp.isV6());
+  if (srcIp.isV6()) {
+    return makeSflowV5Packet(
+        allocator,
+        vlan,
+        srcMac,
+        dstMac,
+        srcIp.asV6(),
+        dstIp.asV6(),
+        srcPort,
+        dstPort,
+        trafficClass,
+        hopLimit,
+        ingressInterface,
+        egressInterface,
+        samplingRate,
+        computeChecksum,
+        payload);
+  }
+  return makeSflowV5Packet(
+      allocator,
+      vlan,
+      srcMac,
+      dstMac,
+      srcIp.asV4(),
+      dstIp.asV4(),
+      srcPort,
+      dstPort,
+      trafficClass,
+      hopLimit,
+      ingressInterface,
+      egressInterface,
+      samplingRate,
+      computeChecksum,
+      payload);
+}
+
 } // namespace facebook::fboss::utility