From 3dbcd67e15edce684dbf9724f5f4672d7652cbd2 Mon Sep 17 00:00:00 2001 From: Dakota Brink Date: Fri, 20 Dec 2024 17:14:36 -0500 Subject: [PATCH] fix migration headaches --- xmtp_mls/diesel.toml | 2 +- .../2024-12-18-170645_add_dm_id/up.sql | 26 --- .../down.sql | 2 +- .../up.sql | 6 +- .../down.sql | 0 .../2024-12-20-214747_add_dm_id/up.sql | 76 +++++++++ .../encrypted_store/conversation_list.rs | 4 +- xmtp_mls/src/storage/encrypted_store/mod.rs | 1 + .../src/storage/encrypted_store/schema.rs | 152 +----------------- .../src/storage/encrypted_store/schema_gen.rs | 146 +++++++++++++++++ 10 files changed, 233 insertions(+), 182 deletions(-) delete mode 100644 xmtp_mls/migrations/2024-12-18-170645_add_dm_id/up.sql rename xmtp_mls/migrations/{2024-12-18-170645_add_dm_id => 2024-12-20-214747_add_dm_id}/down.sql (100%) create mode 100644 xmtp_mls/migrations/2024-12-20-214747_add_dm_id/up.sql create mode 100644 xmtp_mls/src/storage/encrypted_store/schema_gen.rs diff --git a/xmtp_mls/diesel.toml b/xmtp_mls/diesel.toml index 3c3d01e02..148125b34 100644 --- a/xmtp_mls/diesel.toml +++ b/xmtp_mls/diesel.toml @@ -2,7 +2,7 @@ # see https://diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "src/storage/encrypted_store/schema.rs" +file = "src/storage/encrypted_store/schema_gen.rs" [migrations_directory] dir = "migrations" diff --git a/xmtp_mls/migrations/2024-12-18-170645_add_dm_id/up.sql b/xmtp_mls/migrations/2024-12-18-170645_add_dm_id/up.sql deleted file mode 100644 index 0adda6092..000000000 --- a/xmtp_mls/migrations/2024-12-18-170645_add_dm_id/up.sql +++ /dev/null @@ -1,26 +0,0 @@ -ALTER TABLE groups ADD COLUMN dm_id TEXT; -ALTER TABLE groups ADD COLUMN last_message_ns BIGINT; - --- Fill the dm_id column -UPDATE groups -SET dm_id = 'dm:' || - LOWER( - CASE - WHEN LOWER((SELECT inbox_id FROM identity)) < LOWER(dm_inbox_id) - THEN (SELECT inbox_id FROM identity) || ':' || dm_inbox_id - ELSE dm_inbox_id || ':' || (SELECT inbox_id FROM identity) - END - ) -WHERE dm_inbox_id IS NOT NULL; - -DROP INDEX IF EXISTS idx_dm_target; -ALTER TABLE groups DROP COLUMN dm_inbox_id; - --- Create a trigger to auto-update group table on insert -CREATE TRIGGER msg_inserted -AFTER INSERT ON group_messages -BEGIN - UPDATE groups - SET last_message_ns = (strftime('%s', 'now') * 1000000000) + (strftime('%f', 'now') * 1000000) - WHERE id = NEW.group_id; -END; diff --git a/xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/down.sql b/xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/down.sql index 64a220cd8..cfae20537 100644 --- a/xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/down.sql +++ b/xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/down.sql @@ -1 +1 @@ -DROP VIEW IF EXISTS conversation_list; \ No newline at end of file +DROP VIEW IF EXISTS conversation_list; diff --git a/xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/up.sql b/xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/up.sql index 01d5433ab..d7a50ecea 100644 --- a/xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/up.sql +++ b/xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/up.sql @@ -42,6 +42,6 @@ SELECT rm.authority_id FROM groups g - LEFT JOIN ranked_messages rm - ON g.id = rm.group_id AND rm.row_num = 1 -ORDER BY COALESCE(rm.sent_at_ns, g.created_at_ns) DESC; \ No newline at end of file + LEFT JOIN ranked_messages rm + ON g.id = rm.group_id AND rm.row_num = 1 +ORDER BY COALESCE(rm.sent_at_ns, g.created_at_ns) DESC; diff --git a/xmtp_mls/migrations/2024-12-18-170645_add_dm_id/down.sql b/xmtp_mls/migrations/2024-12-20-214747_add_dm_id/down.sql similarity index 100% rename from xmtp_mls/migrations/2024-12-18-170645_add_dm_id/down.sql rename to xmtp_mls/migrations/2024-12-20-214747_add_dm_id/down.sql diff --git a/xmtp_mls/migrations/2024-12-20-214747_add_dm_id/up.sql b/xmtp_mls/migrations/2024-12-20-214747_add_dm_id/up.sql new file mode 100644 index 000000000..d563c3216 --- /dev/null +++ b/xmtp_mls/migrations/2024-12-20-214747_add_dm_id/up.sql @@ -0,0 +1,76 @@ +DROP VIEW IF EXISTS conversation_list; +ALTER TABLE groups ADD COLUMN dm_id TEXT; +ALTER TABLE groups ADD COLUMN last_message_ns BIGINT; + +-- Fill the dm_id column +UPDATE groups +SET dm_id = 'dm:' || + LOWER( + CASE + WHEN LOWER((SELECT inbox_id FROM identity)) < LOWER(dm_inbox_id) + THEN (SELECT inbox_id FROM identity) || ':' || dm_inbox_id + ELSE dm_inbox_id || ':' || (SELECT inbox_id FROM identity) + END + ) +WHERE dm_inbox_id IS NOT NULL; + +DROP INDEX IF EXISTS idx_dm_target; +ALTER TABLE groups DROP COLUMN dm_inbox_id; + +-- Create a trigger to auto-update group table on insert +CREATE TRIGGER msg_inserted +AFTER INSERT ON group_messages +BEGIN + UPDATE groups + SET last_message_ns = (strftime('%s', 'now') * 1000000000) + (strftime('%f', 'now') * 1000000) + WHERE id = NEW.group_id; +END; + + +CREATE VIEW conversation_list AS +WITH ranked_messages AS ( + SELECT + gm.group_id, + gm.id AS message_id, + gm.decrypted_message_bytes, + gm.sent_at_ns, + gm.kind AS message_kind, + gm.sender_installation_id, + gm.sender_inbox_id, + gm.delivery_status, + gm.content_type, + gm.version_major, + gm.version_minor, + gm.authority_id, + ROW_NUMBER() OVER (PARTITION BY gm.group_id ORDER BY gm.sent_at_ns DESC) AS row_num + FROM + group_messages gm + WHERE + gm.kind = 1 +) +SELECT + g.id AS id, + g.created_at_ns, + g.membership_state, + g.installations_last_checked, + g.added_by_inbox_id, + g.welcome_id, + g.dm_id, + g.rotated_at_ns, + g.conversation_type, + rm.message_id, + rm.decrypted_message_bytes, + rm.sent_at_ns, + rm.message_kind, + rm.sender_installation_id, + rm.sender_inbox_id, + rm.delivery_status, + rm.content_type, + rm.version_major, + rm.version_minor, + rm.authority_id +FROM + groups g + LEFT JOIN ranked_messages rm + ON g.id = rm.group_id AND rm.row_num = 1 +ORDER BY COALESCE(rm.sent_at_ns, g.created_at_ns) DESC; diff --git a/xmtp_mls/src/storage/encrypted_store/conversation_list.rs b/xmtp_mls/src/storage/encrypted_store/conversation_list.rs index e82d2fd66..58f387a7c 100644 --- a/xmtp_mls/src/storage/encrypted_store/conversation_list.rs +++ b/xmtp_mls/src/storage/encrypted_store/conversation_list.rs @@ -1,6 +1,6 @@ +use super::schema::conversation_list::dsl::conversation_list; use crate::storage::group::{ConversationType, GroupMembershipState}; use crate::storage::group_message::{ContentType, DeliveryStatus, GroupMessageKind}; -use crate::storage::schema::conversation_list::dsl::conversation_list; use crate::storage::{DbConnection, StorageError}; use diesel::{QueryDsl, Queryable, RunQueryDsl, Table}; use serde::{Deserialize, Serialize}; @@ -23,7 +23,7 @@ pub struct ConversationListItem { /// The sequence id of the welcome message pub welcome_id: Option, /// The inbox_id of the DM target - pub dm_inbox_id: Option, + pub dm_id: Option, /// The last time the leaf node encryption key was rotated pub rotated_at_ns: i64, /// Enum, [`ConversationType`] signifies the group conversation type which extends to who can access it. diff --git a/xmtp_mls/src/storage/encrypted_store/mod.rs b/xmtp_mls/src/storage/encrypted_store/mod.rs index 14bb596b8..6dfe637c2 100644 --- a/xmtp_mls/src/storage/encrypted_store/mod.rs +++ b/xmtp_mls/src/storage/encrypted_store/mod.rs @@ -25,6 +25,7 @@ pub mod key_store_entry; mod native; pub mod refresh_state; pub mod schema; +mod schema_gen; #[cfg(not(target_arch = "wasm32"))] mod sqlcipher_connection; pub mod user_preferences; diff --git a/xmtp_mls/src/storage/encrypted_store/schema.rs b/xmtp_mls/src/storage/encrypted_store/schema.rs index 679729bd2..5d2f68c30 100644 --- a/xmtp_mls/src/storage/encrypted_store/schema.rs +++ b/xmtp_mls/src/storage/encrypted_store/schema.rs @@ -1,133 +1,7 @@ -// @generated automatically by Diesel CLI. +pub use super::schema_gen::*; diesel::table! { - association_state (inbox_id, sequence_id) { - inbox_id -> Text, - sequence_id -> BigInt, - state -> Binary, - } -} - -diesel::table! { - consent_records (entity_type, entity) { - entity_type -> Integer, - state -> Integer, - entity -> Text, - } -} - -diesel::table! { - group_intents (id) { - id -> Integer, - kind -> Integer, - group_id -> Binary, - data -> Binary, - state -> Integer, - payload_hash -> Nullable, - post_commit_data -> Nullable, - publish_attempts -> Integer, - staged_commit -> Nullable, - published_in_epoch -> Nullable, - } -} - -diesel::table! { - group_messages (id) { - id -> Binary, - group_id -> Binary, - decrypted_message_bytes -> Binary, - sent_at_ns -> BigInt, - kind -> Integer, - sender_installation_id -> Binary, - sender_inbox_id -> Text, - delivery_status -> Integer, - content_type -> Integer, - version_minor -> Integer, - version_major -> Integer, - authority_id -> Text, - } -} - -diesel::table! { - groups (id) { - id -> Binary, - created_at_ns -> BigInt, - membership_state -> Integer, - installations_last_checked -> BigInt, - added_by_inbox_id -> Text, - welcome_id -> Nullable, - rotated_at_ns -> BigInt, - conversation_type -> Integer, - dm_id -> Nullable, - last_message_ns -> Nullable, - } -} - -diesel::table! { - identity (rowid) { - inbox_id -> Text, - installation_keys -> Binary, - credential_bytes -> Binary, - rowid -> Nullable, - } -} - -diesel::table! { - identity_updates (inbox_id, sequence_id) { - inbox_id -> Text, - sequence_id -> BigInt, - server_timestamp_ns -> BigInt, - payload -> Binary, - } -} - -diesel::table! { - key_package_history (id) { - id -> Integer, - key_package_hash_ref -> Binary, - created_at_ns -> BigInt, - } -} - -diesel::table! { - openmls_key_store (key_bytes) { - key_bytes -> Binary, - value_bytes -> Binary, - } -} - -diesel::table! { - openmls_key_value (version, key_bytes) { - version -> Integer, - key_bytes -> Binary, - value_bytes -> Binary, - } -} - -diesel::table! { - refresh_state (entity_id, entity_kind) { - entity_id -> Binary, - entity_kind -> Integer, - cursor -> BigInt, - } -} - -diesel::table! { - user_preferences (id) { - id -> Integer, - hmac_key -> Nullable, - } -} - -diesel::table! { - wallet_addresses (wallet_address) { - inbox_id -> Text, - wallet_address -> Text, - } -} - -diesel::table! { - conversation_list (id) { + conversation_list (id) { id -> Binary, created_at_ns -> BigInt, membership_state -> Integer, @@ -148,25 +22,5 @@ diesel::table! { version_major -> Nullable, version_minor -> Nullable, authority_id -> Nullable - } + } } - -diesel::joinable!(group_intents -> groups (group_id)); -diesel::joinable!(group_messages -> groups (group_id)); - -diesel::allow_tables_to_appear_in_same_query!( - association_state, - consent_records, - group_intents, - group_messages, - groups, - identity, - identity_updates, - key_package_history, - openmls_key_store, - openmls_key_value, - refresh_state, - user_preferences, - wallet_addresses, - conversation_list -); diff --git a/xmtp_mls/src/storage/encrypted_store/schema_gen.rs b/xmtp_mls/src/storage/encrypted_store/schema_gen.rs new file mode 100644 index 000000000..82a76b76a --- /dev/null +++ b/xmtp_mls/src/storage/encrypted_store/schema_gen.rs @@ -0,0 +1,146 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + association_state (inbox_id, sequence_id) { + inbox_id -> Text, + sequence_id -> BigInt, + state -> Binary, + } +} + +diesel::table! { + consent_records (entity_type, entity) { + entity_type -> Integer, + state -> Integer, + entity -> Text, + } +} + +diesel::table! { + group_intents (id) { + id -> Integer, + kind -> Integer, + group_id -> Binary, + data -> Binary, + state -> Integer, + payload_hash -> Nullable, + post_commit_data -> Nullable, + publish_attempts -> Integer, + staged_commit -> Nullable, + published_in_epoch -> Nullable, + } +} + +diesel::table! { + group_messages (id) { + id -> Binary, + group_id -> Binary, + decrypted_message_bytes -> Binary, + sent_at_ns -> BigInt, + kind -> Integer, + sender_installation_id -> Binary, + sender_inbox_id -> Text, + delivery_status -> Integer, + content_type -> Integer, + version_minor -> Integer, + version_major -> Integer, + authority_id -> Text, + } +} + +diesel::table! { + groups (id) { + id -> Binary, + created_at_ns -> BigInt, + membership_state -> Integer, + installations_last_checked -> BigInt, + added_by_inbox_id -> Text, + welcome_id -> Nullable, + rotated_at_ns -> BigInt, + conversation_type -> Integer, + dm_id -> Nullable, + last_message_ns -> Nullable, + } +} + +diesel::table! { + identity (rowid) { + inbox_id -> Text, + installation_keys -> Binary, + credential_bytes -> Binary, + rowid -> Nullable, + } +} + +diesel::table! { + identity_updates (inbox_id, sequence_id) { + inbox_id -> Text, + sequence_id -> BigInt, + server_timestamp_ns -> BigInt, + payload -> Binary, + } +} + +diesel::table! { + key_package_history (id) { + id -> Integer, + key_package_hash_ref -> Binary, + created_at_ns -> BigInt, + } +} + +diesel::table! { + openmls_key_store (key_bytes) { + key_bytes -> Binary, + value_bytes -> Binary, + } +} + +diesel::table! { + openmls_key_value (version, key_bytes) { + version -> Integer, + key_bytes -> Binary, + value_bytes -> Binary, + } +} + +diesel::table! { + refresh_state (entity_id, entity_kind) { + entity_id -> Binary, + entity_kind -> Integer, + cursor -> BigInt, + } +} + +diesel::table! { + user_preferences (id) { + id -> Integer, + hmac_key -> Nullable, + } +} + +diesel::table! { + wallet_addresses (wallet_address) { + inbox_id -> Text, + wallet_address -> Text, + } +} + +diesel::joinable!(group_intents -> groups (group_id)); +diesel::joinable!(group_messages -> groups (group_id)); + +diesel::allow_tables_to_appear_in_same_query!( + association_state, + consent_records, + group_intents, + group_messages, + groups, + identity, + identity_updates, + key_package_history, + openmls_key_store, + openmls_key_value, + refresh_state, + user_preferences, + wallet_addresses, +);