Merge branch 'search'

exch/exmdb_provider/common_util.cpp

@@ -48,6 +48,7 @@
 #include <gromox/textmaps.hpp>
 #include <gromox/usercvt.hpp>
 #include <gromox/util.hpp>
+#include "db_engine.h"
 #define S2A(x) reinterpret_cast<const char *>(x)
 
 using XUI = unsigned int;
@@ -69,18 +70,13 @@ class fhash {
 	std::string cid;
 };
 
-struct prepared_statements {
-	xstmt msg_norm, msg_str, rcpt_norm, rcpt_str;
-};
-
 }
 
 char g_exmdb_org_name[256];
 static unsigned int g_max_msg, g_cid_use_xxhash = 1;
 thread_local unsigned int g_inside_flush_instance;
 thread_local sqlite3 *g_sqlite_for_oxcmail;
 static thread_local prepared_statements *g_opt_key;
-static thread_local const char *g_opt_key_src;
 unsigned int g_max_rule_num, g_max_extrule_num;
 unsigned int g_cid_compression = 0; /* disabled(0), specific_level(n) */
 static std::atomic<unsigned int> g_sequence_id;
@@ -192,8 +188,6 @@ void common_util_build_tls()
 {
 	g_inside_flush_instance = false;
 	g_sqlite_for_oxcmail = nullptr;
-	g_opt_key = nullptr;
-	g_opt_key_src = nullptr;
 }
 
 unsigned int common_util_sequence_ID()
@@ -469,49 +463,49 @@ BOOL common_util_allocate_cid(sqlite3 *psqlite, uint64_t *pcid)
 	return TRUE;
 }
 
-BOOL common_util_begin_message_optimize(sqlite3 *psqlite, const char *src)
+bool prepared_statements::begin(sqlite3 *psqlite)
 {
-	if (g_opt_key != nullptr) {
-		mlog(LV_ERR, "E-1229: cannot satisfy nested common_util_begin_message_optimize call (previous: %s, new: %s)",
-			znul(g_opt_key_src), znul(src));
-		return TRUE;
-	}
-	std::unique_ptr<prepared_statements> op(new(std::nothrow) prepared_statements);
-	if (op == nullptr)
-		return FALSE;
-	op->msg_norm = gx_sql_prep(psqlite, "SELECT propval"
+	msg_norm = gx_sql_prep(psqlite, "SELECT propval"
 	               " FROM message_properties WHERE "
 	               "message_id=? AND proptag=?");
-	if (op->msg_norm == nullptr)
+	if (msg_norm == nullptr)
 		return FALSE;
-	op->msg_str = gx_sql_prep(psqlite, "SELECT proptag, "
+	msg_str = gx_sql_prep(psqlite, "SELECT proptag, "
 	              "propval FROM message_properties WHERE "
 	              "message_id=? AND proptag IN (?,?)");
-	if (op->msg_str == nullptr)
+	if (msg_str == nullptr)
 		return FALSE;
-	op->rcpt_norm = gx_sql_prep(psqlite, "SELECT propval "
+	rcpt_norm = gx_sql_prep(psqlite, "SELECT propval "
 	                "FROM recipients_properties WHERE "
 	                "recipient_id=? AND proptag=?");
-	if (op->rcpt_norm == nullptr)
+	if (rcpt_norm == nullptr)
 		return FALSE;
-	op->rcpt_str = gx_sql_prep(psqlite, "SELECT proptag, propval"
+	rcpt_str = gx_sql_prep(psqlite, "SELECT proptag, propval"
 	               " FROM recipients_properties WHERE recipient_id=?"
 	               " AND proptag IN (?,?)");
-	if (op->rcpt_str == nullptr)
+	if (rcpt_str == nullptr)
 		return FALSE;
-	g_opt_key = op.release();
-	g_opt_key_src = src;
-	return TRUE;
+	return true;
 }
 
-void common_util_end_message_optimize()
+prepared_statements::~prepared_statements()
 {
-	auto op = g_opt_key;
-	if (op == nullptr)
-		return;
-	g_opt_key = nullptr;
-	g_opt_key_src = nullptr;
-	delete op;
+	if (g_opt_key == this)
+		g_opt_key = nullptr;
+}
+
+std::unique_ptr<prepared_statements> DB_ITEM::begin_optim() try
+{
+	auto op = std::make_unique<prepared_statements>();
+	if (!op->begin(psqlite))
+		return nullptr;
+	if (g_opt_key != nullptr)
+		mlog(LV_ERR, "E-2359: overlapping optimize_statements");
+	g_opt_key = op.get();
+	return op;
+} catch (const std::bad_alloc &) {
+	mlog(LV_ERR, "E-2358: ENOMEM");
+	return nullptr;
 }
 
 static sqlite3_stmt *

exch/exmdb_provider/db_engine.cpp

@@ -211,9 +211,7 @@ db_item_ptr db_engine_get_db(const char *path)
 		++pdb->reference;
 		hhold.unlock();
 		if (!pdb->giant_lock.try_lock_for(DB_LOCK_TIMEOUT)) {
-			hhold.lock();
 			--pdb->reference;
-			hhold.unlock();
 			mlog(LV_DEBUG, "D-2207: rejecting access to %s because of DB contention", path);
 			return NULL;
 		}
@@ -236,9 +234,7 @@ db_item_ptr db_engine_get_db(const char *path)
 	pdb->reference ++;
 	hhold.unlock();
 	if (!pdb->giant_lock.try_lock_for(DB_LOCK_TIMEOUT)) {
-		hhold.lock();
 		pdb->reference --;
-		hhold.unlock();
 		return NULL;
 	}
 	pdb->tables.last_id = 0;
@@ -267,7 +263,6 @@ void db_item_deleter::operator()(DB_ITEM *pdb) const
 {
 	pdb->last_time = time(nullptr);
 	pdb->giant_lock.unlock();
-	std::lock_guard hhold(g_hash_lock);
 	pdb->reference --;
 }
 
@@ -480,6 +475,7 @@ static BOOL db_engine_search_folder(const char *dir, cpid_t cpid,
 			if (!db_reload(pdb, dir))
 				return false;
 			count = 0;
+			t_start = tp_now();
 		}
 		if (!cu_eval_msg_restriction(pdb->psqlite,
 		    cpid, pmessage_ids->pids[i], prestriction))
@@ -496,8 +492,17 @@ static BOOL db_engine_search_folder(const char *dir, cpid_t cpid,
 			break;
 		else if (ret != SQLITE_OK)
 			continue;
+		/*
+		 * Update other search folders (seems like it is allowed to
+		 * have a search folder have a scope containing another search
+		 * folder; exmdb_provider only does a descendant check).
+		 */
 		db_engine_proc_dynamic_event(pdb, cpid, dynamic_event::new_msg,
 			search_fid, pmessage_ids->pids[i], 0);
+		/*
+		 * Regular notifications
+		 */
+		db_engine_notify_link_creation(pdb, search_fid, pmessage_ids->pids[i]);
 	}
 	return TRUE;
 }
@@ -660,6 +665,7 @@ static void *mdpeng_thrwork(void *param)
 		auto pdb = db_engine_get_db(psearch->dir.c_str());
 		if (pdb == nullptr || pdb->psqlite == nullptr)
 			goto NEXT_SEARCH;
+		/* Stop animation (does nothing else in OL really) */
 		db_engine_notify_search_completion(
 			pdb, psearch->folder_id);
 		db_engine_notify_folder_modification(pdb,
@@ -677,6 +683,10 @@ static void *mdpeng_thrwork(void *param)
 			    psearch->folder_id == t.folder_id)
 				table_ids.push_back(t.table_id);
 		pdb.reset();
+		/*
+		 * reload_ct triggers a table_change notification, and the
+		 * client eventually learns of the new message count.
+		 */
 		while (table_ids.size() > 0) {
 			exmdb_server::reload_content_table(psearch->dir.c_str(), table_ids.back());
 			table_ids.pop_back();
@@ -999,6 +1009,17 @@ static void dbeng_dynevt_2(db_item_ptr &pdb, cpid_t cpid, dynamic_event event_ty
 	}
 }
 
+/**
+ * This is the entry function called by most everything else to notify *search
+ * folders* of events that happened elsewhere.
+ *
+ * @id1:        event source folder
+ * @id2:        message involved in the event
+ *
+ * Caveat: id1 may be a regular folder like Inbox, but it also be a search
+ * folder itself (population/depopulation as a result of search criteria
+ * change).
+ */
 void db_engine_proc_dynamic_event(db_item_ptr &pdb, cpid_t cpid,
     dynamic_event event_type, uint64_t id1, uint64_t id2, uint64_t id3)
 {
@@ -1009,8 +1030,16 @@ void db_engine_proc_dynamic_event(db_item_ptr &pdb, cpid_t cpid,
 		mlog(LV_DEBUG, "db_engine: fatal error in %s", __PRETTY_FUNCTION__);
 		return;
 	}
+	/* Iterate over all search folders (event sinks)... */
 	for (auto &dn : pdb->dynamic_list) {
 		auto pdynamic = &dn;
+		/*
+		 * Iterate over source folders (a.k.a. search scope; MS-OXCFOLD
+		 * v23.2 §1.1).
+		 *
+		 * [In conjunction with dynevt_1/2] if id1 is within the scope,
+		 * pdynamic gets the event.
+		 */
 		for (size_t i = 0; i < pdynamic->folder_ids.count; ++i) {
 			if (dynamic_event::move_folder == event_type) {
 				dbeng_dynevt_1(pdb, cpid, id1, id2, id3, folder_type, pdynamic, i);
@@ -1254,11 +1283,7 @@ static void db_engine_notify_content_table_add_row(db_item_ptr &pdb,
 	if (!cu_get_property(MAPI_MESSAGE, message_id, CP_ACP,
 	    pdb->psqlite, PR_ASSOCIATED, &pvalue0))
 		return;	
-	bool b_optimize = false;
-	auto cl_0 = make_scope_exit([&]() {
-		if (b_optimize)
-			common_util_end_message_optimize();
-	});
+	std::unique_ptr<prepared_statements> optim;
 	BOOL b_fai = pvb_enabled(pvalue0) ? TRUE : false;
 	for (auto &tnode : pdb->tables.table_list) {
 		auto ptable = &tnode;
@@ -1288,9 +1313,9 @@ static void db_engine_notify_content_table_add_row(db_item_ptr &pdb,
 			padded_row1->row_folder_id = folder_id;
 			padded_row1->row_instance = 0;
 			datagram1.db_notify.pdata = padded_row1;
-			if (!common_util_begin_message_optimize(pdb->psqlite, __func__))
+			optim = pdb->begin_optim();
+			if (optim == nullptr)
 				return;
-			b_optimize = true;
 		}
 		datagram.id_array = {1, &ptable->table_id};
 		datagram1.id_array = datagram.id_array;
@@ -1941,18 +1966,18 @@ void db_engine_notify_message_creation(db_item_ptr &pdb, uint64_t folder_id,
 	mlog(LV_ERR, "E-2121: ENOMEM");
 }
 
-void db_engine_notify_link_creation(db_item_ptr &pdb, uint64_t parent_id,
+void db_engine_notify_link_creation(db_item_ptr &pdb, uint64_t srch_fld,
     uint64_t message_id) try
 {
-	uint64_t folder_id;
+	uint64_t anchor_fld;
 	DB_NOTIFY_DATAGRAM datagram;
 
-	if (!common_util_get_message_parent_folder(pdb->psqlite, message_id, &folder_id))
+	if (!common_util_get_message_parent_folder(pdb->psqlite, message_id, &anchor_fld))
 		return;
 
 	auto dir = exmdb_server::get_dir();
 	auto parrays = db_engine_classify_id_array(pdb,
-	               NF_OBJECT_CREATED, folder_id, 0);
+	               NF_OBJECT_CREATED, anchor_fld, 0);
 	if (!parrays.has_value())
 		return;
 	if (parrays->count > 0) {
@@ -1962,17 +1987,17 @@ void db_engine_notify_link_creation(db_item_ptr &pdb, uint64_t parent_id,
 		if (plinked_mail == nullptr)
 			return;
 		datagram.db_notify.pdata = plinked_mail;
-		plinked_mail->folder_id = folder_id;
+		plinked_mail->folder_id = anchor_fld;
 		plinked_mail->message_id = message_id;
-		plinked_mail->parent_id = parent_id;
+		plinked_mail->parent_id = srch_fld;
 		plinked_mail->proptags.count = 0;
 		dg_notify(std::move(datagram), std::move(*parrays));
 	}
 	db_engine_notify_content_table_add_row(
-		pdb, parent_id, message_id);
+		pdb, srch_fld, message_id);
 	db_engine_notify_folder_modification(
 		pdb, common_util_get_folder_parent_fid(
-		pdb->psqlite, parent_id), parent_id);
+		pdb->psqlite, srch_fld), srch_fld);
 } catch (const std::bad_alloc &) {
 	mlog(LV_ERR, "E-2122: ENOMEM");
 }

exch/exmdb_provider/db_engine.h

@@ -6,6 +6,7 @@
 #include <mutex>
 #include <sqlite3.h>
 #include <string>
+#include <gromox/database.h>
 #include <gromox/element_data.hpp>
 #include <gromox/mapi_types.hpp>
 #define CONTENT_ROW_HEADER						1
@@ -27,10 +28,10 @@ struct dynamic_node {
 	~dynamic_node();
 	dynamic_node &operator=(dynamic_node &&) noexcept;
 
-	uint64_t folder_id = 0;
+	uint64_t folder_id = 0; /* search folder ID */
 	uint32_t search_flags = 0;
 	RESTRICTION *prestriction = nullptr;
-	LONGLONG_ARRAY folder_ids{};
+	LONGLONG_ARRAY folder_ids{}; /* source folder IDs */
 };
 using DYNAMIC_NODE = dynamic_node;
 
@@ -94,10 +95,18 @@ struct instance_node {
 };
 using INSTANCE_NODE = instance_node;
 
+struct prepared_statements {
+	~prepared_statements();
+	bool begin(sqlite3 *);
+
+	gromox::xstmt msg_norm, msg_str, rcpt_norm, rcpt_str;
+};
+
 struct DB_ITEM {
 	DB_ITEM() = default;
 	~DB_ITEM();
 	NOMOVE(DB_ITEM);
+	std::unique_ptr<prepared_statements> begin_optim();
 
 	/* client reference count, item can be flushed into file system only count is 0 */
 	std::atomic<int> reference{0};

exch/exmdb_provider/instance.cpp

@@ -354,11 +354,11 @@ BOOL exmdb_server::load_message_instance(const char *dir, const char *username,
 		exmdb_server::set_public_username(username);
 	auto cl_0 = make_scope_exit([]() { exmdb_server::set_public_username(nullptr); });
 	auto sql_transact = gx_sql_begin_trans(pdb->psqlite);
-	if (!common_util_begin_message_optimize(pdb->psqlite, __func__))
+	auto optim = pdb->begin_optim();
+	if (optim == nullptr)
 		return FALSE;
 	auto ret = instance_load_message(pdb->psqlite, mid_val, &pinstance->last_id,
 	           reinterpret_cast<MESSAGE_CONTENT **>(&pinstance->pcontent));
-	common_util_end_message_optimize();
 	if (!ret)
 		return FALSE;
 	if (sql_transact.commit() != 0)

exch/exmdb_provider/main.cpp

@@ -46,7 +46,7 @@ static constexpr cfg_directive exmdb_cfg_defaults[] = {
 	{"exmdb_schema_upgrades", "auto"},
 	{"exmdb_search_nice", "0"},
 	{"exmdb_search_pacing", "250", CFG_SIZE},
-	{"exmdb_search_pacing_time", "2s", CFG_TIME_NS},
+	{"exmdb_search_pacing_time", "0.5s", CFG_TIME_NS},
 	{"exmdb_search_yield", "0", CFG_BOOL},
 	{"exrpc_debug", "0"},
 	{"listen_ip", "::1"},

exch/exmdb_provider/message.cpp

@@ -3838,12 +3838,13 @@ BOOL exmdb_server::read_message(const char *dir, const char *username,
 	auto cl_0 = make_scope_exit([]() { exmdb_server::set_public_username(nullptr); });
 	mid_val = rop_util_get_gc_value(message_id);
 	auto sql_transact = gx_sql_begin_trans(pdb->psqlite);
-	if (!common_util_begin_message_optimize(pdb->psqlite, __func__))
+	auto optim = pdb->begin_optim();
+	if (optim == nullptr)
 		return FALSE;
 	auto ret = message_read_message(pdb->psqlite, cpid, mid_val, ppmsgctnt);
-	common_util_end_message_optimize();
 	if (!ret)
 		return FALSE;
+	optim.reset();
 	return sql_transact.commit() == 0 ? TRUE : false;
 }