Revert "Merge branch 'master' into nacho/SendUserAgentHeaderAsStringT…

…oTheWAF"

This reverts commit 4091dea, reversing
changes made to 7ebd3aa.

profiler/src/ProfilerEngine/Datadog.Profiler.Native/StackSamplerLoopManager.cpp

@@ -91,8 +91,8 @@ const char* StackSamplerLoopManager::GetName()
 
 bool StackSamplerLoopManager::StartImpl()
 {
- InitializeSampler();
- RunWatcherAndSampler();
+ RunStackSampling();
+ RunWatcher();
 
  return true;
 }
@@ -106,7 +106,7 @@ bool StackSamplerLoopManager::StopImpl()
  return true;
 }
 
-void StackSamplerLoopManager::InitializeSampler()
+void StackSamplerLoopManager::RunStackSampling()
 {
  _pStackSamplerLoop = std::make_unique<StackSamplerLoop>(
  _pCorProfilerInfo,
@@ -119,9 +119,11 @@ void StackSamplerLoopManager::InitializeSampler()
  _pWallTimeCollector,
  _pCpuTimeCollector,
  _metricsRegistry);
+
+ _pStackSamplerLoop->Start();
 }
 
-void StackSamplerLoopManager::RunWatcherAndSampler()
+void StackSamplerLoopManager::RunWatcher()
 {
  _pWatcherThread = std::make_unique<std::thread>([this]
  {
@@ -145,9 +147,6 @@ void StackSamplerLoopManager::WatcherLoop()
  Log::Info("StackSamplerLoopManager::WatcherLoop started.");
  _pThreadsCpuManager->Map(OpSysTools::GetThreadId(), WatcherThreadName);
 
- // Start the sampler loop only when the watcher is ready
- _pStackSamplerLoop->Start();
-
  while (false == _isWatcherShutdownRequested)
  {
  try

profiler/src/ProfilerEngine/Datadog.Profiler.Native/StackSamplerLoopManager.h

@@ -105,9 +105,9 @@ class StackSamplerLoopManager
  inline bool GetUpdateIsThreadSafeForStackSampleCollection(ManagedThreadInfo* pThreadInfo, bool* pIsStatusChanged);
  static inline bool ShouldCollectThread(std::uint64_t threadAggPeriodDeadlockCount, std::uint64_t globalAggPeriodDeadlockCount) ;
 
- void InitializeSampler();
+ void RunStackSampling();
 
- void RunWatcherAndSampler();
+ void RunWatcher();
  void ShutdownWatcher();
 
  void WatcherLoop();

tracer/missing-nullability-files.csv

@@ -270,6 +270,7 @@ src/Datadog.Trace/Agent/Transports/MimeTypes.cs
 src/Datadog.Trace/Agent/Transports/SocketHandlerRequestFactory.cs
 src/Datadog.Trace/AppSec/Concurrency/ReaderWriterLock.Core.cs
 src/Datadog.Trace/AppSec/Concurrency/ReaderWriterLock.Framework.cs
+src/Datadog.Trace/AppSec/Waf/IWaf.cs
 src/Datadog.Trace/AppSec/Waf/WafConstants.cs
 src/Datadog.Trace/AppSec/Waf/WafReturnCode.cs
 src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs

tracer/src/Datadog.Trace/AppSec/Rcm/AsmDdProduct.cs

@@ -40,9 +40,18 @@ public void ProcessUpdates(ConfigurationStatus configurationStatus, List<RemoteC
 
  public void ProcessRemovals(ConfigurationStatus configurationStatus, List<RemoteConfigurationPath> removedConfigsForThisProduct)
  {
+ var oneRemoved = false;
  foreach (var removedConfig in removedConfigsForThisProduct)
  {
- configurationStatus.RulesByFile.Remove(removedConfig.Path);
+ oneRemoved |= configurationStatus.RulesByFile.Remove(removedConfig.Path);
+ }
+
+ if (configurationStatus.RulesByFile.Count == 0)
+ {
+ configurationStatus.IncomingUpdateState.FallbackToEmbeddedRuleset();
+ }
+ else if (oneRemoved)
+ {
  configurationStatus.IncomingUpdateState.WafKeysToApply.Add(ConfigurationStatus.WafRulesKey);
  }
  }

tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs

@@ -13,6 +13,7 @@
 using Datadog.Trace.AppSec.Rcm.Models.AsmDd;
 using Datadog.Trace.AppSec.Rcm.Models.AsmFeatures;
 using Datadog.Trace.AppSec.Waf.Initialization;
+using Datadog.Trace.ExtensionMethods;
 using Datadog.Trace.Logging;
 using Datadog.Trace.RemoteConfigurationManagement;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;
@@ -47,6 +48,8 @@ internal record ConfigurationStatus
 
  public ConfigurationStatus(string? embeddedRulesPath) => _embeddedRulesPath = embeddedRulesPath;
 
+ internal RuleSet? FallbackEmbeddedRuleSet { get; set; }
+
  internal bool? EnableAsm { get; set; } = null;
 
  internal string? AutoUserInstrumMode { get; set; } = null;
@@ -101,73 +104,67 @@ internal static List<RuleData> MergeRuleData(IEnumerable<RuleData> res)
  return finalRuleData;
  }
 
- internal object? BuildDictionaryForWafAccordingToIncomingUpdate(string? embeddedRulesetPath)
+ internal Dictionary<string, object> BuildDictionaryForWafAccordingToIncomingUpdate()
  {
- var configuration = new Dictionary<string, object>();
+ var dictionary = new Dictionary<string, object>();
 
  if (IncomingUpdateState.WafKeysToApply.Contains(WafExclusionsKey))
  {
  var exclusions = ExclusionsByFile.SelectMany(x => x.Value).ToList();
- configuration.Add(WafExclusionsKey, new JArray(exclusions));
+ dictionary.Add(WafExclusionsKey, new JArray(exclusions));
  }
 
  if (IncomingUpdateState.WafKeysToApply.Contains(WafRulesOverridesKey))
  {
  var overrides = RulesOverridesByFile.SelectMany(x => x.Value).ToList();
- configuration.Add(WafRulesOverridesKey, overrides.Select(r => r.ToKeyValuePair()).ToArray());
+ dictionary.Add(WafRulesOverridesKey, overrides.Select(r => r.ToKeyValuePair()).ToArray());
  }
 
  if (IncomingUpdateState.WafKeysToApply.Contains(WafRulesDataKey))
  {
  var rulesData = MergeRuleData(RulesDataByFile.SelectMany(x => x.Value));
- configuration.Add(WafRulesDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray());
+ dictionary.Add(WafRulesDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray());
  }
 
  if (IncomingUpdateState.WafKeysToApply.Contains(WafExclusionsDataKey))
  {
  var rulesData = MergeRuleData(ExclusionsDataByFile.SelectMany(x => x.Value));
- configuration.Add(WafExclusionsDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray());
+ dictionary.Add(WafExclusionsDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray());
  }
 
  if (IncomingUpdateState.WafKeysToApply.Contains(WafActionsKey))
  {
  var actions = ActionsByFile.SelectMany(x => x.Value).ToList();
- configuration.Add(WafActionsKey, actions.Select(r => r.ToKeyValuePair()).ToArray());
+ dictionary.Add(WafActionsKey, actions.Select(r => r.ToKeyValuePair()).ToArray());
  }
 
  if (IncomingUpdateState.WafKeysToApply.Contains(WafCustomRulesKey))
  {
  var customRules = CustomRulesByFile.SelectMany(x => x.Value).ToList();
  var mergedCustomRules = new JArray(customRules);
- configuration.Add(WafCustomRulesKey, mergedCustomRules);
+ dictionary.Add(WafCustomRulesKey, mergedCustomRules);
  }
 
- // if there's incoming rules or empty rules, or if asm is to be activated, we also want the rules key in waf arguments
- if (IncomingUpdateState.WafKeysToApply.Contains(WafRulesKey) || (IncomingUpdateState.SecurityStateChange && (EnableAsm ?? false)))
+ if (IncomingUpdateState.FallbackToEmbeddedRulesetAtNextUpdate)
  {
- var rulesetFromRcm = RulesByFile.Values.FirstOrDefault();
- // should deserialize from LocalRuleFile
- if (rulesetFromRcm is null)
+ if (FallbackEmbeddedRuleSet == null)
  {
- var deserializedFromLocalRules = WafConfigurator.DeserializeEmbeddedOrStaticRules(embeddedRulesetPath);
- if (deserializedFromLocalRules is not null)
+ var result = WafConfigurator.DeserializeEmbeddedOrStaticRules(_embeddedRulesPath);
+ if (result != null)
  {
- if (configuration.Count == 0)
- {
- return deserializedFromLocalRules;
- }
-
- var ruleSet = RuleSet.From(deserializedFromLocalRules);
- ruleSet.AddToDictionaryAtRoot(configuration);
+ FallbackEmbeddedRuleSet = RuleSet.From(result);
  }
  }
- else
- {
- rulesetFromRcm?.AddToDictionaryAtRoot(configuration);
- }
+
+ FallbackEmbeddedRuleSet?.AddToDictionaryAtRoot(dictionary);
+ }
+ else if (IncomingUpdateState.WafKeysToApply.Contains(WafRulesKey))
+ {
+ var rulesetFromRcm = RulesByFile.Values.FirstOrDefault();
+ rulesetFromRcm?.AddToDictionaryAtRoot(dictionary);
  }
 
- return configuration.Count > 0 ? configuration : null;
+ return dictionary;
  }
 
  /// <summary>
@@ -250,7 +247,7 @@ public bool StoreLastConfigState(Dictionary<string, List<RemoteConfiguration>> c
  }
  }
 
- // only deserialize and apply asm_features as it will decide if asm gets toggled on and if we deserialize all the others
+ // only treat asm_features as it will decide if asm gets toggled on and if we deserialize all the others
  // (the enable of auto user instrumentation as added to asm_features)
  _asmFeatureProduct.ProcessUpdates(this, asmFeaturesToUpdate);
  _asmFeatureProduct.ProcessRemovals(this, asmFeaturesToRemove);
@@ -285,14 +282,19 @@ internal record IncomingUpdateStatus
  {
  internal HashSet<string> WafKeysToApply { get; } = new();
 
+ internal bool FallbackToEmbeddedRulesetAtNextUpdate { get; private set; }
+
  internal bool SecurityStateChange { get; set; }
 
  public void Reset()
  {
+ FallbackToEmbeddedRulesetAtNextUpdate = false;
  WafKeysToApply.Clear();
  SecurityStateChange = false;
  }
 
+ public void FallbackToEmbeddedRuleset() => FallbackToEmbeddedRulesetAtNextUpdate = true;
+
  public void SignalSecurityStateChange() => SecurityStateChange = true;
  }
 }

tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmDd/RuleSet.cs

@@ -1,4 +1,4 @@
-// <copyright file="RuleSet.cs" company="Datadog">
+// <copyright file="RuleSet.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
@@ -22,32 +22,21 @@ internal class RuleSet
  [JsonProperty("processors")]
  internal JToken? Processors { get; set; }
 
- [JsonProperty("actions")]
- internal JToken? Actions { get; set; }
-
  [JsonProperty("scanners")]
  internal JToken? Scanners { get; set; }
 
- [JsonProperty("exclusions")]
- internal JToken? Exclusions { get; set; }
-
- [JsonProperty("custom_rules")]
- internal JToken? CustomRules { get; set; }
+ public JToken? All { get; set; }
 
  public static RuleSet From(JToken result)
  {
- // can rules from rc contains exclusions and custom rules?
-
  var ruleset = new RuleSet
  {
  Version = result["version"]?.ToString(),
  Metadata = result["metadata"],
  Rules = result["rules"],
  Processors = result["processors"],
  Scanners = result["scanners"],
- Actions = result["actions"],
- Exclusions = result["exclusions"],
- CustomRules = result["custom_rules"]
+ All = result
  };
  return ruleset;
  }
@@ -60,37 +49,27 @@ public void AddToDictionaryAtRoot(Dictionary<string, object> dictionary)
  {
  if (Rules != null)
  {
- dictionary["rules"] = Rules;
+ dictionary.Add("rules", Rules);
  }
 
  if (Metadata != null)
  {
- dictionary["metadata"] = Metadata;
+ dictionary.Add("metadata", Metadata);
  }
 
  if (Version != null)
  {
- dictionary["version"] = Version;
+ dictionary.Add("version", Version);
  }
 
  if (Processors != null)
  {
- dictionary["processors"] = Processors;
+ dictionary.Add("processors", Processors);
  }
 
  if (Scanners != null)
  {
- dictionary["scanners"] = Scanners;
- }
-
- if (Exclusions is not null)
- {
- dictionary["exclusions"] = Exclusions;
- }
-
- if (CustomRules is not null)
- {
- dictionary["custom_rules"] = CustomRules;
+ dictionary.Add("scanners", Scanners);
  }
  }
 }

tracer/src/Datadog.Trace/AppSec/Security.cs

@@ -208,6 +208,7 @@ private ApplyDetails[] UpdateFromRcm(Dictionary<string, List<RemoteConfiguration
  // store the last config state, clearing any previous state, without deserializing any payloads yet.
  var anyChange = _configurationStatus.StoreLastConfigState(configsByProduct, removedConfigs);
  var securityStateChange = Enabled != _configurationStatus.EnableAsm;
+
  // normally CanBeToggled should not need a check as asm_features capacity is only sent if AppSec env var is null, but still guards it in case
  if (securityStateChange && _settings.CanBeToggled)
  {
@@ -231,7 +232,7 @@ private ApplyDetails[] UpdateFromRcm(Dictionary<string, List<RemoteConfiguration
  else if (Enabled && anyChange)
  {
  _configurationStatus.ApplyStoredFiles();
- updateResult = _waf?.UpdateWafFromConfigurationStatus(_configurationStatus, _settings.Rules);
+ updateResult = _waf?.UpdateWafFromConfigurationStatus(_configurationStatus);
  if (updateResult?.Success ?? false)
  {
  if (!string.IsNullOrEmpty(updateResult.RuleFileVersion))
@@ -538,7 +539,7 @@ private void InitWafAndInstrumentations(bool configurationFromRcm = false)
  _wafLibraryInvoker!,
  _settings.ObfuscationParameterKeyRegex,
  _settings.ObfuscationParameterValueRegex,
- embeddedRulesetPath: _settings.Rules,
+ _settings.Rules,
  configurationFromRcm ? _configurationStatus : null,
  _settings.UseUnsafeEncoder,
  GlobalSettings.Instance.DebugEnabledInternal && _settings.WafDebugEnabled);
@@ -556,6 +557,10 @@ private void InitWafAndInstrumentations(bool configurationFromRcm = false)
  Enabled = true;
  InitializationError = null;
  Log.Information("AppSec is now Enabled, _settings.Enabled is {EnabledValue}, coming from remote config: {EnableFromRemoteConfig}", _settings.Enabled, configurationFromRcm);
+ if (_wafInitResult.EmbeddedRules != null)
+ {
+ _configurationStatus.FallbackEmbeddedRuleSet ??= RuleSet.From(_wafInitResult.EmbeddedRules);
+ }
 
  if (!configurationFromRcm)
  {
@@ -585,7 +590,7 @@ private void RemoveInstrumentationsAndProducts(bool fromRemoteConfig)
  {
  if (_rcmSubscription != null)
  {
- var newKeys = _rcmSubscription.ProductKeys.Except([RcmProducts.AsmData, RcmProducts.Asm]).ToArray();
+ var newKeys = _rcmSubscription.ProductKeys.Except(new[] { RcmProducts.AsmData, RcmProducts.Asm }).ToArray();
  if (newKeys.Length > 0)
  {
  var newSubscription = new Subscription(UpdateFromRcm, newKeys);

tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs

@@ -2,7 +2,7 @@
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
-#nullable enable
+
 using System;
 using Datadog.Trace.AppSec.Rcm;
 using Datadog.Trace.AppSec.Waf.NativeBindings;
@@ -14,11 +14,11 @@ internal interface IWaf : IDisposable
  {
  public string Version { get; }
 
- public IContext? CreateContext();
+ public IContext CreateContext();
 
  internal unsafe WafReturnCode Run(IntPtr contextHandle, DdwafObjectStruct* rawPersistentData, DdwafObjectStruct* rawEphemeralData, ref DdwafResultStruct retNative, ulong timeoutMicroSeconds);
 
- UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configurationStatus, string? staticRulesFilePath = null);
+ UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configurationStatus);
 
  public string[] GetKnownAddresses();
 

tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs

@@ -82,13 +82,6 @@ private static void LogRuleDetailsIfDebugEnabled(JToken root)
  return File.OpenRead(rulesFile);
  }
 
- /// <summary>
- /// Deserialize rules for the waf as Jtoken
- /// If null is passed, will deserialize embedded rule file in the app
- /// If a path is given but file isn't found, it won't fallback on the embedded rule file
- /// </summary>
- /// <param name="rulesFilePath">if null, will fallback on embedded rules file</param>
- /// <returns>the rules, might be null if file not found</returns>
  internal static JToken? DeserializeEmbeddedOrStaticRules(string? rulesFilePath)
  {
  JToken root;