Merge branch 'prime'

.gitignore

@@ -45,6 +45,7 @@ Makefile.in
 /gromox-kdb2mt
 /gromox-mailq
 /gromox-mbop
+/gromox-mbck
 /gromox-mkmidb
 /gromox-mkprivate
 /gromox-mkpublic

Makefile.am

@@ -60,7 +60,7 @@ svc_plugins = \
 	libgxs_ldap_adaptor.la \
 	libgxs_mysql_adaptor.la libgxs_ruleproc.la \
 	libgxs_user_filter.la
-sbin_PROGRAMS = gromox-abktconv gromox-compress gromox-dbop gromox-dscli gromox-e2ghelper gromox-eml2mbox gromox-eml2mt gromox-exm2eml gromox-mailq gromox-mbop gromox-mkmidb gromox-mkprivate gromox-mkpublic gromox-kdb2mt gromox-mt2exm
+sbin_PROGRAMS = gromox-abktconv gromox-compress gromox-dbop gromox-dscli gromox-e2ghelper gromox-eml2mbox gromox-eml2mt gromox-exm2eml gromox-mailq gromox-mbck gromox-mbop gromox-mkmidb gromox-mkprivate gromox-mkpublic gromox-kdb2mt gromox-mt2exm
 if HAVE_ESEDB
 sbin_PROGRAMS += gromox-edb2mt
 endif
@@ -230,6 +230,8 @@ gromox_mailq_SOURCES = tools/mailq.cpp
 gromox_mailq_LDADD = libgromox_common.la
 gromox_mbop_SOURCES = tools/genimport.cpp tools/genimport.hpp tools/mbop_main.cpp
 gromox_mbop_LDADD = ${libHX_LIBS} ${mysql_LIBS} libgromox_common.la libgromox_exrpc.la libgromox_mapi.la libgxs_mysql_adaptor.la
+gromox_mbck_SOURCES = tools/mbck.cpp
+gromox_mbck_LDADD = ${libHX_LIBS} ${fmt_LIBS} ${sqlite_LIBS} libgromox_common.la
 gromox_mkmidb_SOURCES = tools/mkmidb.cpp tools/mkshared.cpp tools/mkshared.hpp
 gromox_mkmidb_LDADD = ${fmt_LIBS} ${libHX_LIBS} ${mysql_LIBS} ${libssl_LIBS} ${sqlite_LIBS} libgromox_common.la libgromox_dbop.la libgromox_mapi.la
 gromox_mkprivate_SOURCES = tools/mkprivate.cpp tools/mkshared.cpp tools/mkshared.hpp

doc/exmdb_provider.4gx

@@ -14,6 +14,12 @@ a process-local function API, a network client, and a network server.
 The following directives are recognized when they appear in
 /etc/gromox/gromox.cfg.
 .TP
+\fBexmdb_force_write_txn\fP
+(Developer option.) Perform all SQLite transactions as write transactions (i.e.
+with exclusive locking).
+.br
+Default: \fIno\fP
+.TP
 \fBexmdb_ics_log_file\fP
 Log ICS/synchronization requests (and their results) to this file.
 .br

doc/gromox-mbck.8

@@ -0,0 +1,16 @@
+.TH gromox\-mbck 8 "" "Gromox" "Gromox admin reference"
+.SH Name
+\fBgromox\-mbck\fP \(em Mailbox check and repair utility
+.SH Synopsis
+\fBgromox\-mbck\fP [\fB\-p\fP] x.sqlite
+.SH Description
+mbck can be used to check one or more mailboxes for problems, and optionally
+repairing them. Since Gromox 2.30, the utility can be run while the mailbox is
+in active use by gromox\-http.
+.SH Options
+.TP
+\fB\-p\fP
+Perform repairs / write operations. (Default: just readonly checks)
+.TP
+\fB\-?\fP
+Display option summary.

doc/gromox.7

@@ -103,6 +103,8 @@ gromox\-dscli(8) \(em Autodiscover command line utility
 .IP \(bu 4
 gromox\-mailq(8) \(em SMTP queue lister
 .IP \(bu 4
+gromox\-mbck(8gx) \(em Mailbox check and repair utility
+.IP \(bu 4
 gromox\-mbop(8) \(em Mailbox operations utility
 .IP \(bu 4
 gromox\-mkmidb(8) \(em Tool for creating a blank message index database

exch/dnsbl_filter.cpp

@@ -7,8 +7,9 @@
 #include <cerrno>
 #include <cstring>
 #include <string>
-#	include <arpa/inet.h>
+#include <arpa/inet.h>
 #include <libHX/string.h>
+#include <sys/socket.h>
 #include <netinet/in.h>
 #include <gromox/config_file.hpp>
 #include <gromox/scope.hpp>

exch/ews/ews.cpp

@@ -238,6 +238,7 @@ const std::unordered_map<std::string, EWSPlugin::Handler> EWSPlugin::requestMap
 	{"EmptyFolder", process<Structures::mEmptyFolderRequest>},
 	{"FindFolder", process<Structures::mFindFolderRequest>},
 	{"FindItem", process<Structures::mFindItemRequest>},
+	{"GetAppManifests", process<Structures::mGetAppManifestsRequest>},
 	{"GetAttachment", process<Structures::mGetAttachmentRequest>},
 	{"GetEvents", process<Structures::mGetEventsRequest>},
 	{"GetFolder", process<Structures::mGetFolderRequest>},
@@ -703,7 +704,7 @@ int EWSContext::notify()
 	return flush();
 }
 
-int EWSPlugin::retr(int ctx_id)
+int EWSPlugin::retr(int ctx_id) try
 {
 	if(ctx_id < 0 || static_cast<size_t>(ctx_id) >= contexts.size() || !contexts[ctx_id])
 		return HPM_RETRIEVE_DONE;
@@ -730,6 +731,9 @@ int EWSPlugin::retr(int ctx_id)
 		return context.notify();
 	}
 	return HPM_RETRIEVE_DONE;
+} catch (const std::exception &err) {
+	mlog(LV_ERR, "[ews#%d]%s Retrieve error: %s", ctx_id, timestamp().c_str(), err.what());
+	return HPM_RETRIEVE_ERROR;
 }
 
 void EWSPlugin::term(int ctx)

exch/ews/requests.cpp

@@ -575,6 +575,26 @@ void process(mFindItemRequest&& request, XMLElement* response, const EWSContext&
 	data.serialize(response);
 }
 
+/**
+ * @brief      Process GetAppManifests
+ *
+ * Provides a stub that returns an empty Manifests node.
+ *
+ * @todo       This function lacks most of its functionality and is practically worthless.
+ *
+ * @param      request   Request data
+ * @param      response  XMLElement to store response in
+ * @param      ctx       Request context
+ */
+void process(mGetAppManifestsRequest&&, XMLElement* response, const EWSContext&)
+{
+	response->SetName("m:GetAppManifestsResponse");
+
+	mGetAppManifestsResponse data;
+	data.success();
+	data.serialize(response);
+}
+
 /**
  * @brief      Process GetAttachment
  *

exch/ews/requests.hpp

@@ -24,6 +24,7 @@ EWSFUNC(mDeleteItemRequest);
 EWSFUNC(mEmptyFolderRequest);
 EWSFUNC(mFindFolderRequest);
 EWSFUNC(mFindItemRequest);
+EWSFUNC(mGetAppManifestsRequest);
 EWSFUNC(mGetAttachmentRequest);
 EWSFUNC(mGetEventsRequest);
 EWSFUNC(mGetFolderRequest);

exch/ews/serialization.cpp

@@ -1575,6 +1575,12 @@ void mFindItemResponseMessage::serialize(tinyxml2::XMLElement* xml) const
 void mFindItemResponse::serialize(tinyxml2::XMLElement* xml) const
 {XMLDUMPM(ResponseMessages);}
 
+void mGetAppManifestsResponse::serialize(tinyxml2::XMLElement* xml) const
+{
+	mResponseMessageType::serialize(xml);
+	xml->InsertNewChildElement("m:Manifests");
+}
+
 mGetAttachmentRequest::mGetAttachmentRequest(const XMLElement* xml) :
 	XMLINIT(AttachmentIds)
 {}

exch/ews/structures.hpp

@@ -3629,6 +3629,38 @@ struct mFreeBusyResponse : public NS_EWS_Messages
 	std::optional<mResponseMessageType> ResponseMessage;
 };
 
+/**
+ * Messages.xsd:3486
+ */
+struct mGetAppManifestsRequest
+{
+	explicit inline mGetAppManifestsRequest(const tinyxml2::XMLElement*) {} // nothing to do here for now
+
+	//<xs:element name="ApiVersionSupported" type="xs:string" minOccurs="0" maxOccurs="1" />
+	//<xs:element name="SchemaVersionSupported" type="xs:string" minOccurs="0" maxOccurs="1" />
+	//<xs:element name="IncludeAllInstalledAddIns" type="xs:boolean" minOccurs="0" maxOccurs="1" />
+	//<xs:element name="IncludeEntitlementData" type="xs:boolean" minOccurs="0" maxOccurs="1" />
+	//<xs:element name="IncludeManifestData" type="xs:boolean" minOccurs="0" maxOccurs="1" />
+	//<xs:element name="IncludeCustomAppsData" type="xs:boolean" minOccurs="0" maxOccurs="1" />
+	//<xs:element name="ExtensionIds" type="m:ListOfExtensionIdsType" minOccurs="0" maxOccurs="1" />
+	//<xs:element name="AddIns" type="m:ArrayOfPrivateCatalogAddInsType" minOccurs="0" maxOccurs="1"/>
+	//<xs:element name="IncludeExtensionMetaData" type="xs:boolean" minOccurs="0" maxOccurs="1" />
+};
+
+/**
+ * Messages.xsd:3511
+ */
+struct mGetAppManifestsResponse : mResponseMessageType
+{
+	using mResponseMessageType::mResponseMessageType;
+
+	void serialize(tinyxml2::XMLElement*) const;
+	//<xs:choice>
+	//	<xs:element name="Apps" type="t:ArrayOfAppsType" maxOccurs="1"/>
+	//	<xs:element name="Manifests" type="m:ArrayOfAppManifestsType" maxOccurs="1"/>
+	//</xs:choice>
+};
+
 /**
  * Messages.xsd:2204
  */

exch/exmdb/main.cpp

@@ -36,6 +36,7 @@ using namespace exmdb;
 static std::shared_ptr<CONFIG_FILE> g_config_during_init;
 
 static constexpr cfg_directive exmdb_gromox_cfg_defaults[] = {
+	{"exmdb_force_write_txn", "0", CFG_BOOL},
 	{"exmdb_ics_log_file", ""},
 	CFG_TABLE_END,
 };
@@ -110,6 +111,7 @@ static bool exmdb_provider_reload(std::shared_ptr<config_file> gxcfg = nullptr,
 	g_exmdb_search_pacing_time = pconfig->get_ll("exmdb_search_pacing_time");
 	g_exmdb_max_sqlite_spares = pconfig->get_ll("exmdb_max_sqlite_spares");
 	g_sqlite_busy_timeout_ns = pconfig->get_ll("sqlite_busy_timeout");
+	gx_force_write_txn = gxcfg->get_ll("exmdb_force_write_txn");
 	auto s = gxcfg->get_value("exmdb_ics_log_file");
 	if (s != nullptr)
 		g_exmdb_ics_log_file = s;

exch/http/main.cpp

@@ -158,10 +158,9 @@ int main(int argc, char **argv)
 	const char *dns_name, *dns_domain, *netbios_name;
 
 	setvbuf(stdout, nullptr, _IOLBF, 0);
-	if (HX_getopt5(g_options_table, argv, &argc, &argv,
+	if (HX_getopt5(g_options_table, argv, nullptr, nullptr,
 	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS)
 		return EXIT_FAILURE;
-	auto cl_0 = make_scope_exit([=]() { HX_zvecfree(argv); });
 
 	startup_banner("gromox-http");
 	setup_sigalrm();

exch/midb/mail_engine.cpp

@@ -1422,19 +1422,25 @@ static void mail_engine_insert_message(sqlite3_stmt *pstmt, uint32_t *puidnext,
 		mid_string = mid_string1;
 		sprintf(temp_path, "%s/ext/%s", dir, mid_string1);
 		wrapfd fd = open(temp_path, O_CREAT | O_TRUNC | O_WRONLY, FMODE_PRIVATE);
-		if (fd.get() < 0)
+		if (fd.get() < 0) {
+			mlog(LV_ERR, "E-1770: open %s for write: %s", temp_path, strerror(errno));
 			return;
+		}
 		if (HXio_fullwrite(fd.get(), djson.c_str(), djson.size()) < 0 ||
 		    fd.close_wr() != 0) {
 			mlog(LV_ERR, "E-1134: write %s: %s", temp_path, strerror(errno));
 			return;
 		}
 		sprintf(temp_path1, "%s/eml/%s", dir, mid_string1);
 		fd = open(temp_path1, O_CREAT | O_TRUNC | O_WRONLY, FMODE_PRIVATE);
-		if (fd.get() < 0)
+		if (fd.get() < 0) {
+			mlog(LV_ERR, "E-1771: open %s for write: %s", temp_path, strerror(errno));
 			return;
-		if (!imail.to_file(fd.get()))
+		}
+		if (!imail.to_file(fd.get())) {
+			mlog(LV_ERR, "E-1772: to_file %s failed", temp_path);
 			return;
+		}
 	}
 	(*puidnext) ++;
 	auto b_unsent = !!(message_flags & MSGFLAG_UNSENT);

exch/midb/main.cpp

@@ -308,10 +308,9 @@ int main(int argc, char **argv)
 	exmdb_rpc_alloc = common_util_alloc;
 	exmdb_rpc_free = [](void *) {};
 	setvbuf(stdout, nullptr, _IOLBF, 0);
-	if (HX_getopt5(g_options_table, argv, &argc, &argv,
+	if (HX_getopt5(g_options_table, argv, nullptr, nullptr,
 	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS)
 		return EXIT_FAILURE;
-	auto cl_0a = make_scope_exit([=]() { HX_zvecfree(argv); });
 
 	startup_banner("gromox-midb");
 	if (opt_show_version)

exch/zcore/main.cpp

@@ -308,10 +308,9 @@ int main(int argc, char **argv)
 	exmdb_rpc_alloc = common_util_alloc;
 	exmdb_rpc_free = [](void *) {};
 	setvbuf(stdout, nullptr, _IOLBF, 0);
-	if (HX_getopt5(g_options_table, argv, &argc, &argv,
+	if (HX_getopt5(g_options_table, argv, nullptr, nullptr,
 	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS)
 		return EXIT_FAILURE;
-	auto cl_0a = make_scope_exit([=]() { HX_zvecfree(argv); });
 
 	startup_banner("gromox-zcore");
 	if (opt_show_version)

include/gromox/database.h

@@ -104,6 +104,6 @@ static inline uint64_t gx_sql_col_uint64(sqlite3_stmt *s, int c)
 	return x >= 0 ? x : 0;
 }
 
-extern GX_EXPORT unsigned int gx_sqlite_debug;
+extern GX_EXPORT unsigned int gx_sqlite_debug, gx_force_write_txn;
 
 }

lib/dbhelper.cpp

@@ -12,7 +12,7 @@
 
 namespace gromox {
 
-unsigned int gx_sqlite_debug;
+unsigned int gx_sqlite_debug, gx_force_write_txn;
 
 static bool write_statement(const char *q)
 {
@@ -99,6 +99,8 @@ int xtransaction::commit()
 
 xtransaction gx_sql_begin3(const std::string &pos, sqlite3 *db, txn_mode mode)
 {
+	if (gx_force_write_txn)
+		mode = txn_mode::write;
 	auto ret = gx_sql_exec(db, mode == txn_mode::write ? "BEGIN IMMEDIATE" : "BEGIN");
 	if (ret == SQLITE_OK) {
 		auto fn = sqlite3_db_filename(db, nullptr);

lib/dbop_sqlite.cpp

@@ -14,6 +14,8 @@
 /*
  * INDEX names should be suffixed by the schema number, to facilitate
  * dbop_sqlite_upgrade (old table+index exists simultaneously with new table).
+ *
+ * If you add a new index name, add it to mbck.cpp:ck_indices_present too.
  */
 #define TABLE_END {}
 

lib/mail_func.cpp

@@ -312,7 +312,7 @@ size_t parse_mime_field(const char *in_buff, size_t buff_len,
 	auto tmp_ptr = in_buff;
 
 	size_t i = 0, value_length = 0;
-	while (*tmp_ptr != ':' && i < buff_len &&
+	while (i < buff_len && *tmp_ptr != ':' &&
 		i <= MIME_NAME_LEN && *tmp_ptr != '\r'
 		&& *tmp_ptr != '\n') {
 		tmp_ptr ++; 

mda/delivery_app/main.cpp

@@ -111,10 +111,9 @@ int main(int argc, char **argv)
 	char temp_buff[256];
 
 	setvbuf(stdout, nullptr, _IOLBF, 0);
-	if (HX_getopt5(g_options_table, argv, &argc, &argv,
+	if (HX_getopt5(g_options_table, argv, nullptr, nullptr,
 	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS)
 		return EXIT_FAILURE;
-	auto cl_0 = make_scope_exit([=]() { HX_zvecfree(argv); });
 
 	startup_banner("gromox-delivery");
 	setup_sigalrm();

mda/smtp/main.cpp

@@ -338,10 +338,9 @@ int main(int argc, char **argv)
 	smtp_param scfg;
 
 	setvbuf(stdout, nullptr, _IOLBF, 0);
-	if (HX_getopt5(g_options_table, argv, &argc, &argv,
+	if (HX_getopt5(g_options_table, argv, nullptr, nullptr,
 	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS)
 		return EXIT_FAILURE;
-	auto cl_0 = make_scope_exit([=]() { HX_zvecfree(argv); });
 
 	startup_banner("gromox-delivery-queue");
 	setup_sigalrm();

mra/imap/main.cpp

@@ -485,10 +485,9 @@ int main(int argc, char **argv)
 	char temp_buff[256];
 
 	setvbuf(stdout, nullptr, _IOLBF, 0);
-	if (HX_getopt5(g_options_table, argv, &argc, &argv,
+	if (HX_getopt5(g_options_table, argv, nullptr, nullptr,
 	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS)
 		return EXIT_FAILURE;
-	auto cl_0a = make_scope_exit([=]() { HX_zvecfree(argv); });
 
 	startup_banner("gromox-imap");
 	setup_sigalrm();

mra/pop3/main.cpp

@@ -412,10 +412,9 @@ int main(int argc, char **argv)
 	char temp_buff[256];
 
 	setvbuf(stdout, nullptr, _IOLBF, 0);
-	if (HX_getopt5(g_options_table, argv, &argc, &argv,
+	if (HX_getopt5(g_options_table, argv, nullptr, nullptr,
 	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS)
 		return EXIT_FAILURE;
-	auto cl_0a = make_scope_exit([=]() { HX_zvecfree(argv); });
 
 	startup_banner("gromox-pop3");
 	setup_sigalrm();

tools/event.cpp

@@ -195,10 +195,9 @@ void qsock::sk_close()
 int main(int argc, char **argv)
 {
 	setvbuf(stdout, nullptr, _IOLBF, 0);
-	if (HX_getopt5(g_options_table, argv, &argc, &argv,
+	if (HX_getopt5(g_options_table, argv, nullptr, nullptr,
 	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS)
 		return EXIT_FAILURE;
-	auto cl_0 = make_scope_exit([=]() { HX_zvecfree(argv); });
 
 	startup_banner("gromox-event");
 	if (opt_show_version)