feat!: Add lockfile to account manager (#4310)

Opening the same account (context) from multiple processes is dangerous, can result in duplicate
downloads of the same message etc. Same for account manager, attempts to modify the same
accounts.toml even if done atomically with may result in corrupted files as atomic replacement
procedure does not expect that multiple processes may write to the same temporary file.

accounts.toml cannot be used as a lockfile because it is replaced during atomic update. Therefore, a
new file next to accounts.toml is needed to prevent starting second account manager in the same
directory.

But iOS needs to be able to open accounts from multiple processes at the same time. This is required
as the "share-to-DC extension" is a separate process by iOS design -- this process may or may not be
started while the main app is running. Accounts are not altered however by this extension, so let's
add to the `Accounts::new()` constructor an `rdwr` parameter which allows to read the accounts
config w/o locking the lockfile.

Cargo.toml

@@ -48,6 +48,7 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
 fast-socks5 = "0.8"
+fd-lock = "3.0.11"
 futures = "0.3"
 futures-lite = "1.13.0"
 hex = "0.4.0"

benches/create_account.rs

@@ -8,7 +8,8 @@ async fn create_accounts(n: u32) {
  let dir = tempdir().unwrap();
  let p: PathBuf = dir.path().join("accounts");
 
- let mut accounts = Accounts::new(p.clone()).await.unwrap();
+ let writable = true;
+ let mut accounts = Accounts::new(p.clone(), writable).await.unwrap();
 
  for expected_id in 2..n {
  let id = accounts.add_account().await.unwrap();

deltachat-ffi/deltachat.h

@@ -2915,12 +2915,15 @@ int dc_receive_backup (dc_context_t* context, const char* qr);
  * @param dir The directory to create the context-databases in.
  * If the directory does not exist,
  * dc_accounts_new() will try to create it.
+ * @param writable Whether the returned account manager is writable, i.e. calling these functions on
+ * it is possible: dc_accounts_add_account(), dc_accounts_add_closed_account(),
+ * dc_accounts_migrate_account(), dc_accounts_remove_account(), dc_accounts_select_account().
  * @return An account manager object.
  * The object must be passed to the other account manager functions
  * and must be freed using dc_accounts_unref() after usage.
  * On errors, NULL is returned.
  */
-dc_accounts_t* dc_accounts_new (const char* os_name, const char* dir);
+dc_accounts_t* dc_accounts_new (const char* dir, int writable);
 
 
 /**

deltachat-ffi/src/lib.rs

@@ -4676,17 +4676,17 @@ pub type dc_accounts_t = AccountsWrapper;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_new(
- _os_name: *const libc::c_char,
- dbfile: *const libc::c_char,
+ dir: *const libc::c_char,
+ writable: libc::c_int,
 ) -> *mut dc_accounts_t {
  setup_panic!();
 
- if dbfile.is_null() {
+ if dir.is_null() {
  eprintln!("ignoring careless call to dc_accounts_new()");
  return ptr::null_mut();
  }
 
- let accs = block_on(Accounts::new(as_path(dbfile).into()));
+ let accs = block_on(Accounts::new(as_path(dir).into(), writable != 0));
 
  match accs {
  Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),

deltachat-jsonrpc/src/lib.rs

@@ -13,7 +13,8 @@ mod tests {
  #[tokio::test(flavor = "multi_thread")]
  async fn basic_json_rpc_functionality() -> anyhow::Result<()> {
  let tmp_dir = TempDir::new().unwrap().path().into();
- let accounts = Accounts::new(tmp_dir).await?;
+ let writable = true;
+ let accounts = Accounts::new(tmp_dir, writable).await?;
  let api = CommandApi::new(accounts);
 
  let (sender, mut receiver) = unbounded::<String>();
@@ -54,7 +55,8 @@ mod tests {
  #[tokio::test(flavor = "multi_thread")]
  async fn test_batch_set_config() -> anyhow::Result<()> {
  let tmp_dir = TempDir::new().unwrap().path().into();
- let accounts = Accounts::new(tmp_dir).await?;
+ let writable = true;
+ let accounts = Accounts::new(tmp_dir, writable).await?;
  let api = CommandApi::new(accounts);
 
  let (sender, mut receiver) = unbounded::<String>();

deltachat-jsonrpc/src/webserver.rs

@@ -19,7 +19,8 @@ async fn main() -> Result<(), std::io::Error> {
  .map(|port| port.parse::<u16>().expect("DC_PORT must be a number"))
  .unwrap_or(DEFAULT_PORT);
  log::info!("Starting with accounts directory `{path}`.");
- let accounts = Accounts::new(PathBuf::from(&path)).await.unwrap();
+ let writable = true;
+ let accounts = Accounts::new(PathBuf::from(&path), writable).await.unwrap();
  let state = CommandApi::new(accounts);
 
  let app = Router::new()

deltachat-rpc-server/src/main.rs

@@ -56,7 +56,8 @@ async fn main_impl() -> Result<()> {
 
  let path = std::env::var("DC_ACCOUNTS_PATH").unwrap_or_else(|_| "accounts".to_string());
  log::info!("Starting with accounts directory `{}`.", path);
- let accounts = Accounts::new(PathBuf::from(&path)).await?;
+ let writable = true;
+ let accounts = Accounts::new(PathBuf::from(&path), writable).await?;
 
  log::info!("Creating JSON-RPC API.");
  let accounts = Arc::new(RwLock::new(accounts));

node/lib/deltachat.ts

@@ -21,12 +21,15 @@ export class AccountManager extends EventEmitter {
  accountDir: string
  jsonRpcStarted = false
 
- constructor(cwd: string, os = 'deltachat-node') {
+ constructor(cwd: string, writable = true) {
  super()
  debug('DeltaChat constructor')
 
  this.accountDir = cwd
- this.dcn_accounts = binding.dcn_accounts_new(os, this.accountDir)
+ this.dcn_accounts = binding.dcn_accounts_new(
+ this.accountDir,
+ writable ? 1 : 0
+ )
  }
 
  getAllAccountIds() {

node/src/module.c

@@ -2903,8 +2903,8 @@ NAPI_METHOD(dcn_msg_get_webxdc_blob){
 
 NAPI_METHOD(dcn_accounts_new) {
  NAPI_ARGV(2);
- NAPI_ARGV_UTF8_MALLOC(os_name, 0);
- NAPI_ARGV_UTF8_MALLOC(dir, 1);
+ NAPI_ARGV_UTF8_MALLOC(dir, 0);
+ NAPI_ARGV_INT32(writable, 1);
  TRACE("calling..");
 
  dcn_accounts_t* dcn_accounts = calloc(1, sizeof(dcn_accounts_t));
@@ -2913,7 +2913,7 @@ NAPI_METHOD(dcn_accounts_new) {
  }
 
 
- dcn_accounts->dc_accounts = dc_accounts_new(os_name, dir);
+ dcn_accounts->dc_accounts = dc_accounts_new(dir, writable);
 
  napi_value result;
  NAPI_STATUS_THROWS(napi_create_external(env, dcn_accounts,

python/tests/test_4_lowlevel.py

@@ -221,8 +221,9 @@ def ac_process_ffi_event(ffi_event):
 
 def test_jsonrpc_blocking_call(tmp_path):
  accounts_fname = tmp_path / "accounts"
+ writable = True
  accounts = ffi.gc(
- lib.dc_accounts_new(ffi.NULL, str(accounts_fname).encode("ascii")),
+ lib.dc_accounts_new(str(accounts_fname).encode("ascii"), writable),
  lib.dc_accounts_unref,
  )
  jsonrpc = ffi.gc(lib.dc_jsonrpc_init(accounts), lib.dc_jsonrpc_unref)