refactor: Move CDClient Database away from constructor queries (#1164)

* Move away from constructor queries

Fix up other large tables to have proper backup lookups

Revert "idk im just dumb ig"

This reverts commit 5d5be5df53b8959b42b291613d7db749a65a3585.

idk im just dumb ig

* Fix slow components registry lookup

* add define for cdclient cache all

* Update CDBehaviorParameterTable.cpp

CMakeVariables.txt

@@ -18,3 +18,5 @@ __maria_db_connector_compile_jobs__=1
 __enable_testing__=1
 # The path to OpenSSL.  Change this if your OpenSSL install path is different than the default.
 OPENSSL_ROOT_DIR=/usr/local/opt/openssl@3/
+# Uncomment the below line to cache the entire CDClient into memory
+# CDCLIENT_CACHE_ALL=1

dDatabase/CDClientDatabase.h

@@ -13,15 +13,7 @@
 #include <sstream>
 #include <iostream>
 
- // Enable this to cache all entries in each table for fast access, comes with more memory cost
- //#define CDCLIENT_CACHE_ALL
-
-/*!
-  \file CDClientDatabase.hpp
-  \brief An interface between the CDClient.sqlite file and the server
- */
-
- //! The CDClient Database namespace
+//! The CDClient Database namespace
 namespace CDClientDatabase {
 
 	//! Opens a connection with the CDClient

dDatabase/CDClientManager.cpp

@@ -38,43 +38,53 @@
 #include "CDFeatureGatingTable.h"
 #include "CDRailActivatorComponent.h"
 
+// Uncomment this to cache the full cdclient database into memory. This will make the server load faster, but will use more memory.
+// A vanilla CDClient takes about 46MB of memory + the regular world data.
+// #define CDCLIENT_CACHE_ALL
+
+#ifdef CDCLIENT_CACHE_ALL
+	#define CDCLIENT_DONT_CACHE_TABLE(x) x
+#else
+	#define CDCLIENT_DONT_CACHE_TABLE(x)
+#endif
+
 CDClientManager::CDClientManager() {
-	CDActivityRewardsTable::Instance();
-	CDAnimationsTable::Instance();
-	CDBehaviorParameterTable::Instance();
-	CDBehaviorTemplateTable::Instance();
-	CDComponentsRegistryTable::Instance();
-	CDCurrencyTableTable::Instance();
-	CDDestructibleComponentTable::Instance();
-	CDEmoteTableTable::Instance();
-	CDInventoryComponentTable::Instance();
-	CDItemComponentTable::Instance();
-	CDItemSetsTable::Instance();
-	CDItemSetSkillsTable::Instance();
-	CDLevelProgressionLookupTable::Instance();
-	CDLootMatrixTable::Instance();
-	CDLootTableTable::Instance();
-	CDMissionNPCComponentTable::Instance();
-	CDMissionTasksTable::Instance();
-	CDMissionsTable::Instance();
-	CDObjectSkillsTable::Instance();
-	CDObjectsTable::Instance();
-	CDPhysicsComponentTable::Instance();
-	CDRebuildComponentTable::Instance();
-	CDScriptComponentTable::Instance();
-	CDSkillBehaviorTable::Instance();
-	CDZoneTableTable::Instance();
-	CDVendorComponentTable::Instance();
-	CDActivitiesTable::Instance();
-	CDPackageComponentTable::Instance();
-	CDProximityMonitorComponentTable::Instance();
-	CDMovementAIComponentTable::Instance();
-	CDBrickIDTableTable::Instance();
-	CDRarityTableTable::Instance();
-	CDMissionEmailTable::Instance();
-	CDRewardsTable::Instance();
-	CDPropertyEntranceComponentTable::Instance();
-	CDPropertyTemplateTable::Instance();
-	CDFeatureGatingTable::Instance();
-	CDRailActivatorComponentTable::Instance();
+	CDActivityRewardsTable::Instance().LoadValuesFromDatabase();
+	CDActivitiesTable::Instance().LoadValuesFromDatabase();
+	CDCLIENT_DONT_CACHE_TABLE(CDAnimationsTable::Instance().LoadValuesFromDatabase());
+	CDBehaviorParameterTable::Instance().LoadValuesFromDatabase();
+	CDBehaviorTemplateTable::Instance().LoadValuesFromDatabase();
+	CDBrickIDTableTable::Instance().LoadValuesFromDatabase();
+	CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
+	CDCurrencyTableTable::Instance().LoadValuesFromDatabase();
+	CDDestructibleComponentTable::Instance().LoadValuesFromDatabase();
+	CDEmoteTableTable::Instance().LoadValuesFromDatabase();
+	CDFeatureGatingTable::Instance().LoadValuesFromDatabase();
+	CDInventoryComponentTable::Instance().LoadValuesFromDatabase();
+	CDCLIENT_DONT_CACHE_TABLE(CDItemComponentTable::Instance().LoadValuesFromDatabase());
+	CDItemSetSkillsTable::Instance().LoadValuesFromDatabase();
+	CDItemSetsTable::Instance().LoadValuesFromDatabase();
+	CDLevelProgressionLookupTable::Instance().LoadValuesFromDatabase();
+	CDLootMatrixTable::Instance().LoadValuesFromDatabase();
+	CDLootTableTable::Instance().LoadValuesFromDatabase();
+	CDMissionEmailTable::Instance().LoadValuesFromDatabase();
+	CDMissionNPCComponentTable::Instance().LoadValuesFromDatabase();
+	CDMissionTasksTable::Instance().LoadValuesFromDatabase();
+	CDMissionsTable::Instance().LoadValuesFromDatabase();
+	CDMovementAIComponentTable::Instance().LoadValuesFromDatabase();
+	CDObjectSkillsTable::Instance().LoadValuesFromDatabase();
+	CDCLIENT_DONT_CACHE_TABLE(CDObjectsTable::Instance().LoadValuesFromDatabase());
+	CDPhysicsComponentTable::Instance().LoadValuesFromDatabase();
+	CDPackageComponentTable::Instance().LoadValuesFromDatabase();
+	CDProximityMonitorComponentTable::Instance().LoadValuesFromDatabase();
+	CDPropertyEntranceComponentTable::Instance().LoadValuesFromDatabase();
+	CDPropertyTemplateTable::Instance().LoadValuesFromDatabase();
+	CDRailActivatorComponentTable::Instance().LoadValuesFromDatabase();
+	CDRarityTableTable::Instance().LoadValuesFromDatabase();
+	CDRebuildComponentTable::Instance().LoadValuesFromDatabase();
+	CDRewardsTable::Instance().LoadValuesFromDatabase();
+	CDScriptComponentTable::Instance().LoadValuesFromDatabase();
+	CDSkillBehaviorTable::Instance().LoadValuesFromDatabase();
+	CDVendorComponentTable::Instance().LoadValuesFromDatabase();
+	CDZoneTableTable::Instance().LoadValuesFromDatabase();
 }

dDatabase/Tables/CDActivitiesTable.cpp

@@ -1,7 +1,6 @@
 #include "CDActivitiesTable.h"
 
-CDActivitiesTable::CDActivitiesTable(void) {
-
+void CDActivitiesTable::LoadValuesFromDatabase() {
 	// First, get the size of the table
 	unsigned int size = 0;
 	auto tableSize = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM Activities");
@@ -55,8 +54,3 @@ std::vector<CDActivities> CDActivitiesTable::Query(std::function<bool(CDActiviti
 
 	return data;
 }
-
-std::vector<CDActivities> CDActivitiesTable::GetEntries(void) const {
-	return this->entries;
-}
-

dDatabase/Tables/CDActivitiesTable.h

@@ -30,9 +30,10 @@ class CDActivitiesTable : public CDTable<CDActivitiesTable> {
 	std::vector<CDActivities> entries;
 
 public:
-	CDActivitiesTable();
+	void LoadValuesFromDatabase();
+
 	// Queries the table with a custom "where" clause
 	std::vector<CDActivities> Query(std::function<bool(CDActivities)> predicate);
 
-	std::vector<CDActivities> GetEntries(void) const;
+	const std::vector<CDActivities>& GetEntries() const { return this->entries; }
 };

dDatabase/Tables/CDActivityRewardsTable.cpp

@@ -1,6 +1,6 @@
 #include "CDActivityRewardsTable.h"
 
-CDActivityRewardsTable::CDActivityRewardsTable(void) {
+void CDActivityRewardsTable::LoadValuesFromDatabase() {
 
 	// First, get the size of the table
 	unsigned int size = 0;
@@ -43,8 +43,3 @@ std::vector<CDActivityRewards> CDActivityRewardsTable::Query(std::function<bool(
 
 	return data;
 }
-
-std::vector<CDActivityRewards> CDActivityRewardsTable::GetEntries(void) const {
-	return this->entries;
-}
-

dDatabase/Tables/CDActivityRewardsTable.h

@@ -18,10 +18,10 @@ class CDActivityRewardsTable : public CDTable<CDActivityRewardsTable> {
 	std::vector<CDActivityRewards> entries;
 
 public:
-	CDActivityRewardsTable();
+	void LoadValuesFromDatabase();
 	// Queries the table with a custom "where" clause
 	std::vector<CDActivityRewards> Query(std::function<bool(CDActivityRewards)> predicate);
 
-	std::vector<CDActivityRewards> GetEntries(void) const;
+	std::vector<CDActivityRewards> GetEntries() const;
 
 };

dDatabase/Tables/CDAnimationsTable.cpp

@@ -2,6 +2,35 @@
 #include "GeneralUtils.h"
 #include "Game.h"
 
+
+void CDAnimationsTable::LoadValuesFromDatabase() {
+	auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM Animations");
+	while (!tableData.eof()) {
+		std::string animation_type = tableData.getStringField("animation_type", "");
+		DluAssert(!animation_type.empty());
+		AnimationGroupID animationGroupID = tableData.getIntField("animationGroupID", -1);
+		DluAssert(animationGroupID != -1);
+
+		CDAnimation entry;
+		entry.animation_name = tableData.getStringField("animation_name", "");
+		entry.chance_to_play = tableData.getFloatField("chance_to_play", 1.0f);
+		UNUSED_COLUMN(entry.min_loops = tableData.getIntField("min_loops", 0);)
+		UNUSED_COLUMN(entry.max_loops = tableData.getIntField("max_loops", 0);)
+		entry.animation_length = tableData.getFloatField("animation_length", 0.0f);
+		UNUSED_COLUMN(entry.hideEquip = tableData.getIntField("hideEquip", 0) == 1;)
+		UNUSED_COLUMN(entry.ignoreUpperBody = tableData.getIntField("ignoreUpperBody", 0) == 1;)
+		UNUSED_COLUMN(entry.restartable = tableData.getIntField("restartable", 0) == 1;)
+		UNUSED_COLUMN(entry.face_animation_name = tableData.getStringField("face_animation_name", "");)
+		UNUSED_COLUMN(entry.priority = tableData.getFloatField("priority", 0.0f);)
+		UNUSED_COLUMN(entry.blendTime = tableData.getFloatField("blendTime", 0.0f);)
+
+		this->animations[CDAnimationKey(animation_type, animationGroupID)].push_back(entry);
+		tableData.nextRow();
+	}
+
+	tableData.finalize();
+}
+
 bool CDAnimationsTable::CacheData(CppSQLite3Statement& queryToCache) {
 	auto tableData = queryToCache.execQuery();
 	// If we received a bad lookup, cache it anyways so we do not run the query again.

dDatabase/Tables/CDAnimationsTable.h

@@ -27,6 +27,7 @@ class CDAnimationsTable : public CDTable<CDAnimationsTable> {
 	typedef std::string AnimationID;
 	typedef std::pair<std::string, AnimationGroupID> CDAnimationKey;
 public:
+	void LoadValuesFromDatabase();
 	/**
 	 * Given an animationType and the previousAnimationName played, return the next animationType to play.
 	 * If there are more than 1 animationTypes that can be played, one is selected at random but also does not allow

dDatabase/Tables/CDBehaviorParameterTable.cpp

@@ -1,15 +1,15 @@
 #include "CDBehaviorParameterTable.h"
 #include "GeneralUtils.h"
 
-uint64_t GetHash(const uint32_t behaviorID, const uint32_t parameterID) {
-	uint64_t hash = behaviorID;
-	hash <<= 31U;
-	hash |= parameterID;
+uint64_t GetKey(const uint32_t behaviorID, const uint32_t parameterID) {
+	uint64_t key = behaviorID;
+	key <<= 31U;
+	key |= parameterID;
 
-	return hash;
+	return key;
 }
 
-CDBehaviorParameterTable::CDBehaviorParameterTable() {
+void CDBehaviorParameterTable::LoadValuesFromDatabase() {
 	auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM BehaviorParameter");
 	while (!tableData.eof()) {
 		uint32_t behaviorID = tableData.getIntField("behaviorID", -1);
@@ -21,7 +21,7 @@ CDBehaviorParameterTable::CDBehaviorParameterTable() {
 		} else {
 			parameterId = m_ParametersList.insert(std::make_pair(candidateStringToAdd, m_ParametersList.size())).first->second;
 		}
-		uint64_t hash = GetHash(behaviorID, parameterId);
+		uint64_t hash = GetKey(behaviorID, parameterId);
 		float value = tableData.getFloatField("value", -1.0f);
 
 		m_Entries.insert(std::make_pair(hash, value));
@@ -34,7 +34,7 @@ CDBehaviorParameterTable::CDBehaviorParameterTable() {
 float CDBehaviorParameterTable::GetValue(const uint32_t behaviorID, const std::string& name, const float defaultValue) {
 	auto parameterID = this->m_ParametersList.find(name);
 	if (parameterID == this->m_ParametersList.end()) return defaultValue;
-	auto hash = GetHash(behaviorID, parameterID->second);
+	auto hash = GetKey(behaviorID, parameterID->second);
 
 	// Search for specific parameter
 	auto it = m_Entries.find(hash);
@@ -45,7 +45,7 @@ std::map<std::string, float> CDBehaviorParameterTable::GetParametersByBehaviorID
 	uint64_t hashBase = behaviorID;
 	std::map<std::string, float> returnInfo;
 	for (auto& [parameterString, parameterId] : m_ParametersList) {
-		uint64_t hash = GetHash(hashBase, parameterId);
+		uint64_t hash = GetKey(hashBase, parameterId);
 		auto infoCandidate = m_Entries.find(hash);
 		if (infoCandidate != m_Entries.end()) {
 			returnInfo.insert(std::make_pair(parameterString, infoCandidate->second));

dDatabase/Tables/CDBehaviorParameterTable.h

@@ -12,7 +12,7 @@ class CDBehaviorParameterTable : public CDTable<CDBehaviorParameterTable> {
 	std::unordered_map<BehaviorParameterHash, BehaviorParameterValue> m_Entries;
 	std::unordered_map<std::string, uint32_t> m_ParametersList;
 public:
-	CDBehaviorParameterTable();
+	void LoadValuesFromDatabase();
 
 	float GetValue(const uint32_t behaviorID, const std::string& name, const float defaultValue = 0);
 

dDatabase/Tables/CDBehaviorTemplateTable.cpp

@@ -1,6 +1,6 @@
 #include "CDBehaviorTemplateTable.h"
 
-CDBehaviorTemplateTable::CDBehaviorTemplateTable(void) {
+void CDBehaviorTemplateTable::LoadValuesFromDatabase() {
 
 	// First, get the size of the table
 	unsigned int size = 0;
@@ -48,7 +48,7 @@ std::vector<CDBehaviorTemplate> CDBehaviorTemplateTable::Query(std::function<boo
 	return data;
 }
 
-std::vector<CDBehaviorTemplate> CDBehaviorTemplateTable::GetEntries(void) const {
+const std::vector<CDBehaviorTemplate>& CDBehaviorTemplateTable::GetEntries() const {
 	return this->entries;
 }
 
@@ -64,4 +64,3 @@ const CDBehaviorTemplate CDBehaviorTemplateTable::GetByBehaviorID(uint32_t behav
 		return entry->second;
 	}
 }
-

dDatabase/Tables/CDBehaviorTemplateTable.h

@@ -19,11 +19,12 @@ class CDBehaviorTemplateTable : public CDTable<CDBehaviorTemplateTable> {
 	std::unordered_map<uint32_t, CDBehaviorTemplate> entriesMappedByBehaviorID;
 	std::unordered_set<std::string> m_EffectHandles;
 public:
-	CDBehaviorTemplateTable();
+	void LoadValuesFromDatabase();
+
 	// Queries the table with a custom "where" clause
 	std::vector<CDBehaviorTemplate> Query(std::function<bool(CDBehaviorTemplate)> predicate);
 
-	std::vector<CDBehaviorTemplate> GetEntries(void) const;
+	const std::vector<CDBehaviorTemplate>& GetEntries(void) const;
 
 	const CDBehaviorTemplate GetByBehaviorID(uint32_t behaviorID);
 };

dDatabase/Tables/CDBrickIDTableTable.cpp

@@ -1,6 +1,6 @@
 #include "CDBrickIDTableTable.h"
 
-CDBrickIDTableTable::CDBrickIDTableTable(void) {
+void CDBrickIDTableTable::LoadValuesFromDatabase() {
 
 	// First, get the size of the table
 	unsigned int size = 0;
@@ -39,7 +39,7 @@ std::vector<CDBrickIDTable> CDBrickIDTableTable::Query(std::function<bool(CDBric
 	return data;
 }
 
-std::vector<CDBrickIDTable> CDBrickIDTableTable::GetEntries(void) const {
+const std::vector<CDBrickIDTable>& CDBrickIDTableTable::GetEntries() const {
 	return this->entries;
 }
 

dDatabase/Tables/CDBrickIDTableTable.h

@@ -21,9 +21,9 @@ class CDBrickIDTableTable : public CDTable<CDBrickIDTableTable> {
 	std::vector<CDBrickIDTable> entries;
 
 public:
-	CDBrickIDTableTable();
+	void LoadValuesFromDatabase();
 	// Queries the table with a custom "where" clause
 	std::vector<CDBrickIDTable> Query(std::function<bool(CDBrickIDTable)> predicate);
 
-	std::vector<CDBrickIDTable> GetEntries(void) const;
+	const std::vector<CDBrickIDTable>& GetEntries() const;
 };