From da9aae4ddd6c372a27a2bbea86871b4e3fbc59fd Mon Sep 17 00:00:00 2001
From: Richard Hua <rich@rkhua.com>
Date: Wed, 31 Jul 2024 12:33:29 -0700
Subject: [PATCH] create-db-schema

---
 README.md                                 | 18 ++++++++
 pkg/migrations/00001_init-schema.down.sql |  4 ++
 pkg/migrations/00001_init-schema.up.sql   | 51 ++++++++++++++++++++++-
 pkg/server/server.go                      |  1 -
 4 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index fe6f926f..304ed885 100644
--- a/README.md
+++ b/README.md
@@ -78,3 +78,21 @@ The `xmtpd` node build provides two options for monitoring your node.
     ```
 
     To learn how to visualize node data in Grafana, see [Prometheus Histograms with Grafana Heatmaps](https://towardsdatascience.com/prometheus-histograms-with-grafana-heatmaps-d556c28612c7) and [How to visualize Prometheus histograms in Grafana](https://grafana.com/blog/2020/06/23/how-to-visualize-prometheus-histograms-in-grafana/).
+
+## Modifying the protobuf schema
+
+Submit and land a PR to https://github.com/xmtp/proto. Then run:
+
+  ```sh
+  dev/generate
+  ```
+
+## Modifying the database schema
+
+Create a new migration by running:
+
+  ```sh
+  dev/gen-migration
+  ```
+
+If you are unfamiliar with migrations, you may follow [this guide](https://github.com/golang-migrate/migrate/blob/master/MIGRATIONS.md). The database is PostgreSQL and the driver is PGX.
diff --git a/pkg/migrations/00001_init-schema.down.sql b/pkg/migrations/00001_init-schema.down.sql
index e69de29b..617dbd3f 100644
--- a/pkg/migrations/00001_init-schema.down.sql
+++ b/pkg/migrations/00001_init-schema.down.sql
@@ -0,0 +1,4 @@
+DROP TABLE node_info;
+DROP TABLE all_envelopes;
+DROP TABLE staged_originated_envelopes;
+DROP TABLE address_log;
diff --git a/pkg/migrations/00001_init-schema.up.sql b/pkg/migrations/00001_init-schema.up.sql
index 8dcfd8e8..0c2b1ced 100644
--- a/pkg/migrations/00001_init-schema.up.sql
+++ b/pkg/migrations/00001_init-schema.up.sql
@@ -1,3 +1,50 @@
-SELECT
-	1;
+-- Ensures that if the command-line node configuration mutates,
+-- the existing data in the DB is invalid
+CREATE TABLE node_info(
+	node_id INTEGER NOT NULL,
+	public_key BYTEA NOT NULL,
 
+	singleton_id SMALLINT PRIMARY KEY DEFAULT 1,
+	CONSTRAINT is_singleton CHECK (singleton_id = 1)
+);
+
+CREATE TABLE all_envelopes(
+	-- used to construct gateway_sid
+	id BIGSERIAL PRIMARY KEY,
+	originator_sid BIGINT NOT NULL,
+	topic BYTEA NOT NULL,
+	originator_envelope BYTEA NOT NULL
+);
+-- Client queries
+CREATE INDEX idx_all_envelopes_topic ON all_envelopes(topic);
+-- Node queries
+CREATE UNIQUE INDEX idx_all_envelopes_originator_sid ON all_envelopes(originator_sid);
+
+
+-- Process for originating envelopes:
+-- 1. Perform any necessary validation
+-- 2. Insert into originated_envelopes
+-- 3. Singleton background task will continuously query (or subscribe to)
+--    staged_originated_envelopes, and for each envelope in order of ID:
+--     2.1. Construct and sign OriginatorEnvelope proto
+--     2.2. Atomically insert into all_envelopes and delete from originated_envelopes,
+--	        ignoring unique index violations on originator_sid
+-- This preserves total ordering, while avoiding gaps in sequence ID's.
+CREATE TABLE staged_originated_envelopes(
+	-- used to construct originator_sid
+	id BIGSERIAL PRIMARY KEY,
+	originator_ns TIMESTAMP NOT NULL DEFAULT now(),
+	payer_envelope BYTEA NOT NULL
+);
+
+-- A cached view for looking up the inbox_id that an address belongs to.
+-- Relies on a total ordering of updates across all inbox_ids, from which this
+-- view can be deterministically generated.
+CREATE TABLE address_log(
+	address TEXT NOT NULL,
+	inbox_id BYTEA NOT NULL,
+	association_sequence_id BIGINT,
+	revocation_sequence_id BIGINT,
+
+	PRIMARY KEY (address, inbox_id)
+);
diff --git a/pkg/server/server.go b/pkg/server/server.go
index 2fbfbc69..6c8bf4a7 100644
--- a/pkg/server/server.go
+++ b/pkg/server/server.go
@@ -40,7 +40,6 @@ func NewReplicationServer(ctx context.Context, log *zap.Logger, options Options,
 	if err != nil {
 		return nil, err
 	}
-	// Commenting out the DB stuff until I get the new migrations in
 	s.writerDb, err = db.NewDB(ctx, options.DB.WriterConnectionString, options.DB.WaitForDB, options.DB.ReadTimeout)
 	if err != nil {
 		return nil, err