From 483fbaf0fe3056a3dd046a4dd863aa1cfe62dc02 Mon Sep 17 00:00:00 2001 From: Alextopher Date: Sat, 8 Jul 2023 12:05:19 -0400 Subject: [PATCH] Replace sqlite database with append only file, add resyncing on websocket connection fix bugs found in testing deleting a talk can now be undone :) add talk syncing on websocket reconnect improve syncing --amend --- .gitignore | 4 +- README.md | 3 +- backup.go | 37 ------- client.go | 103 +++++++++++++++---- db.go | 266 +++++++++++++++++++++++++++++++++---------------- db_test.go | 165 ++++++++++++++++++++++++++++++ go.mod | 9 +- go.sum | 22 ++-- handlers.go | 14 +-- hub.go | 42 ++++---- main.go | 29 ++++-- migrate.py | 70 +++++++++++++ models.go | 192 ++++++++++++++++++++++++++++++++--- static/site.js | 112 +++++++++++++++------ utils.go | 2 + utils_test.go | 13 +++ 16 files changed, 841 insertions(+), 242 deletions(-) delete mode 100644 backup.go create mode 100644 db_test.go create mode 100644 migrate.py create mode 100644 utils_test.go diff --git a/.gitignore b/.gitignore index 1157a50..dbacd8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ .DS_Store .env go-talks -talks.db main gin-bin backups -config.toml \ No newline at end of file +config.toml +*.db* \ No newline at end of file diff --git a/README.md b/README.md index d1d9719..55fc2a1 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ go-talks (Or more commonly known as just Talks) is an app to manage talks at COS - /posts directory supports rendering pre-written markdown files - Image caching/proxying - Create future talks and view historic talks -- Backed by SQLITE database +- Backed by an append only log database ## Endpoints @@ -27,3 +27,4 @@ go-talks (Or more commonly known as just Talks) is an app to manage talks at COS | GET | /img/{id} | Image proxy | | GET | /health | Indicates how many active connections there are | | GET | /ws | Websocket endpoint | + diff --git a/backup.go b/backup.go deleted file mode 100644 index 9ac4a2d..0000000 --- a/backup.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os/exec" - "time" -) - -// Saves weekly backups of the database going back 4 weeks -func backup() { - dbLock.Lock() - defer dbLock.Unlock() - - // Get the current date - t := time.Now().In(tz) - date := t.Format("2006-01-02") - - // File name for the backup - fileName := fmt.Sprintf("backups/backup-%s.db", date) - - // Copy the database to the backup file - cmd := fmt.Sprintf("cp talks.db %s", fileName) - err := exec.Command("sh", "-c", cmd).Run() - - if err != nil { - log.Println("[ERROR] Failed to backup database:", err) - } - - // Remove backups older than 4 weeks - cmd = "find backups -type f -mtime +28 -exec rm {} \\;" - err = exec.Command("sh", "-c", cmd).Run() - - if err != nil { - log.Println("[ERROR] Failed to remove old backups:", err) - } -} diff --git a/client.go b/client.go index d08f6d3..cfe9227 100644 --- a/client.go +++ b/client.go @@ -32,27 +32,58 @@ const ( DELETE // AUTH communicates authentication request/response AUTH + // SYNC requests a sync of the talks + SYNC ) // Message is the format of messages sent between the client and server // Since go doesn't have the strongest type system we pack every message // into a single struct type Message struct { - Type MessageType `json:"type"` - ID uint32 `json:"id,omitempty"` - Password string `json:"password,omitempty"` - Name string `json:"name,omitempty"` - Talktype *TalkType `json:"talktype,omitempty"` - Description string `json:"description,omitempty"` - Week string `json:"week,omitempty"` + Type MessageType `json:"type"` + New *NewMessage `json:"new,omitempty"` + Hide *HideMessage `json:"hide,omitempty"` + Del *DeleteMessage `json:"delete,omitempty"` + Auth *AuthMessage `json:"auth,omitempty"` + Sync *SyncMessage `json:"sync,omitempty"` +} + +// NewMessage gives the client all the information needed to add a new talk to +// the page +type NewMessage struct { + ID uint32 `json:"id"` + Name string `json:"name"` + Talktype TalkType `json:"talktype"` + Description string `json:"description"` + Week string `json:"week"` +} + +// HideMessage gives the client the ID of the talk to hide +type HideMessage struct { + ID uint32 `json:"id"` +} + +// DeleteMessage gives the client the ID of the talk to delete +type DeleteMessage struct { + ID uint32 `json:"id"` +} + +// AuthMessage gives the client the password to authenticate +type AuthMessage struct { + Password string `json:"password"` +} + +// SyncMessage starts and caps off a sync +type SyncMessage struct { + Week string `json:"week"` } func authenticatedMessage(b bool) []byte { if b { - return []byte("{\"type\": 3, \"status\": true}") + return []byte("{\"type\": 3, \"auth\": {\"status\": true}}") } - return []byte("{\"type\": 3, \"status\": false}") + return []byte("{\"type\": 3, \"auth\": {\"status\": false}}") } func (c *Client) read() { @@ -73,21 +104,54 @@ func (c *Client) read() { var message Message err = json.Unmarshal(raw, &message) if err != nil { + // Print the message and continue + log.Printf("[WARN] %v", err) continue } - // Handle authentication without consulting the hub - if message.Type == AUTH { + switch message.Type { + case NEW, HIDE, DELETE: + // NEW, HIDE, and DELETE need to be serialized through the hub + hub.broadcast <- message + case AUTH: + // AUTH is handled without having to contact the hub log.Printf("[INFO] Client %v is trying to authenticate", c.conn.RemoteAddr()) - - c.auth = message.Password == config.Password - c.send <- authenticatedMessage(c.auth) - - continue + c.send <- authenticatedMessage(message.Auth.Password == config.Password) + case SYNC: + // SYNC messages don't need to be serialized, go straight to the db + log.Printf("[INFO] Client %v is requesting a sync", c.conn.RemoteAddr()) + for _, talk := range talks.AllTalks(message.Sync.Week) { + var msg Message + if talk.Hidden { + // Send a hide message + msg = Message{ + Type: HIDE, + Hide: &HideMessage{ + ID: talk.ID, + }, + } + } else { + // Send a create message + msg = Message{ + Type: NEW, + New: &NewMessage{ + ID: talk.ID, + Name: talk.Name, + Talktype: talk.Type, + Description: talk.Description, + Week: talk.Week, + }, + } + } + // Send the message + raw, _ := json.Marshal(msg) + c.send <- raw + } + raw, _ := json.Marshal(message) + c.send <- raw + default: + log.Printf("[WARN] Client %v sent an invalid message type", c.conn.RemoteAddr()) } - - // Forward all other message to be processed and broadcasted to other client - hub.broadcast <- message } } @@ -103,6 +167,7 @@ func (c *Client) write() { select { case message, ok := <-c.send: if !ok { + log.Println("[INFO] Closing connection") return } diff --git a/db.go b/db.go index 7d9d188..b4bc95e 100644 --- a/db.go +++ b/db.go @@ -1,137 +1,231 @@ package main import ( + "encoding/json" + "fmt" + "io" "log" "sync" - - "gorm.io/driver/sqlite" - "gorm.io/gorm" ) -// db is a global db connection to be shared -var db *gorm.DB -var dbLock sync.Mutex +// Global talks object +var talks *Talks + +// Talks stores all it's data in an append-only log +type Talks struct { + // Logs are protected by a RWMutex + sync.RWMutex + // The file we're writing to (opened in append mode) + encoder *json.Encoder + // Maps talk IDs to talks + talks map[uint32]*Talk -// ConnectDB sets up the initial connection to the database along with retrying attempts -func ConnectDB(config *Config) error { - dbLock.Lock() - defer dbLock.Unlock() + // An index of talks by week + weeks map[string][]*Talk - var err error - db, err = gorm.Open(sqlite.Open(config.Database), &gorm.Config{}) - return err + // The current ID counter + id uint32 } -// MakeDB sets up the db -func MakeDB() { - dbLock.Lock() - defer dbLock.Unlock() +// NewTalks creates a new talks object, loading the talks from the given reader +// and writing new events to the given writer +func NewTalks(r io.Reader, w io.Writer) (*Talks, error) { + encoder := json.NewEncoder(w) + + // Create the talks object + t := &Talks{ + RWMutex: sync.RWMutex{}, + encoder: encoder, + talks: make(map[uint32]*Talk), + weeks: make(map[string][]*Talk), + id: 0, + } + + // Load the talks from the file + if err := t.load(r); err != nil { + return nil, err + } - // Create all regular tables - db.AutoMigrate( - &Talk{}, - ) + return t, nil } -// DropTables drops everything in the db -func DropTables() { - dbLock.Lock() - defer dbLock.Unlock() +func (t *Talks) load(r io.Reader) error { + // json decode the file + dec := json.NewDecoder(r) + for { + var event TalkEvent + if err := dec.Decode(&event); err == io.EOF { + break + } else if err != nil { + return err + } + + if err := t.event(event); err != nil { + return err + } + } - // Drop tables in an order that won't invoke errors from foreign key constraints - db.Migrator().DropTable(&Talk{}) + return nil } -// VisibleTalks returns all visible talks for a given week -// If week is empty, it will default to this week -func VisibleTalks(week string) []Talk { - dbLock.Lock() - defer dbLock.Unlock() +// Writes an event to the log +func (t *Talks) write(event TalkEvent) error { + // Write the event to the file + return t.encoder.Encode(event) +} - if week == "" { - week = nextWednesday() +// Applies an event to the in-memory state +func (t *Talks) event(event TalkEvent) error { + // switch on the event type + switch event.Type { + case Create: + t.create(event.Create) + case Hide: + t.hide(event.Hide) + case Delete: + t.delete(event.Delete) + default: + return fmt.Errorf("unknown event type: %v", event.Type) } - var talks []Talk - result := db.Where("is_hidden = false").Where("week = ?", week).Order("type").Find(&talks) + return nil +} + +// Write and apply an event and handle any errors +func (t *Talks) writeAndApply(event TalkEvent) { + log.Println("[INFO] Applying event:", event.Type, "[", event, "]") - if result.Error != nil { - log.Println("[WARN] could not get visible talks:", result) + if err := t.write(event); err != nil { + log.Println("[ERROR] Failed to write event:", err) } - return talks + if err := t.event(event); err != nil { + log.Println("[ERROR] Failed to apply event:", err) + } } -// AllTalks returns all talks for a given week -// If week is empty, it will default to this week -func AllTalks(week string) []Talk { - dbLock.Lock() - defer dbLock.Unlock() +// Applies a create event to the in-memory state +func (t *Talks) create(c *CreateTalkEvent) { + if c.ID > t.id { + t.id = c.ID + } - if week == "" { - week = nextWednesday() + t.talks[c.ID] = &Talk{ + ID: c.ID, + Name: c.Name, + Type: c.Type, + Description: c.Description, + Week: c.Week, + Hidden: false, } - var talks []Talk - result := db.Where("week = ?", week).Order("type").Find(&talks) + // Add the talk to the week index + if _, ok := t.weeks[c.Week]; !ok { + t.weeks[c.Week] = make([]*Talk, 0) + } + t.weeks[c.Week] = append(t.weeks[c.Week], t.talks[c.ID]) +} - if result.Error != nil { - log.Println("[WARN] could not get all talks:", result) +// Create creates a new talk +func (t *Talks) Create(name string, talkType TalkType, description string, week string) { + t.Lock() + + // Increment the ID counter + t.id++ + event := TalkEvent{ + Time: Now(), + Type: Create, + Create: &CreateTalkEvent{ + ID: t.id, + Name: name, + Type: talkType, + Description: description, + Week: week, + }, } + t.writeAndApply(event) - return talks + t.Unlock() } -// CreateTalk inserts a new talk into the db -func CreateTalk(talk *Talk) uint32 { - dbLock.Lock() - defer dbLock.Unlock() +// Applies a hide event to the in-memory state +func (t *Talks) hide(h *HideTalkEvent) { + if _, ok := t.talks[h.ID]; !ok { + return + } + + t.talks[h.ID].Hidden = true +} - result := db.Create(talk) +// Hide hides a talk +func (t *Talks) Hide(id uint32) { + t.Lock() - if result.Error != nil { - log.Println("[WARN] could not create talk:", result) + event := TalkEvent{ + Time: Now(), + Type: Hide, + Hide: &HideTalkEvent{ + ID: id, + }, } + t.writeAndApply(event) - log.Println("[INFO] Created talk {", talk.Name, talk.Description, talk.Type, talk.Week, talk.ID, "}") - return talk.ID + t.Unlock() } -// HideTalk updates a talk, setting its isHidden field to true -func HideTalk(id uint32) { - dbLock.Lock() - defer dbLock.Unlock() - - talk := Talk{} - result := db.First(&talk, id) +// Applies a delete event to the in-memory state +func (t *Talks) delete(d *DeleteTalkEvent) { + if _, ok := t.talks[d.ID]; !ok { + return + } - if result.Error != nil { - log.Println("[WARN] could not find talk:", result) + // Remove the talk from the week index + week := t.talks[d.ID].Week + for i, talk := range t.weeks[week] { + if talk.ID == d.ID { + // Swap remove + t.weeks[week][i] = t.weeks[week][len(t.weeks[week])-1] + t.weeks[week] = t.weeks[week][:len(t.weeks[week])-1] + } } + delete(t.talks, d.ID) +} - talk.IsHidden = true - result = db.Save(&talk) +// Delete deletes a talk +func (t *Talks) Delete(id uint32) { + t.Lock() - if result.Error != nil { - log.Println("[WARN] could not hide talk:", result) + event := TalkEvent{ + Time: Now(), + Type: Delete, + Delete: &DeleteTalkEvent{ + ID: id, + }, } + t.writeAndApply(event) + + t.Unlock() } -// DeleteTalk deletes a talk from the db -func DeleteTalk(id uint32) { - dbLock.Lock() - defer dbLock.Unlock() +// AllTalks returns all talks for a given week +func (t *Talks) AllTalks(week string) []*Talk { + t.RLock() + talks := t.weeks[week] + t.RUnlock() - talk := Talk{} - result := db.First(&talk, id) + return talks +} - if result.Error != nil { - log.Println("[WARN] could not find talk:", result) +// VisibleTalks returns all visible talks for a given week +func (t *Talks) VisibleTalks(week string) []*Talk { + t.RLock() + talks := make([]*Talk, 0) + for _, talk := range t.weeks[week] { + if !talk.Hidden { + talks = append(talks, talk) + } } + t.RUnlock() - result = db.Delete(&talk) - - if result.Error != nil { - log.Println("[WARN] could not delete talk:", result) - } + return talks } diff --git a/db_test.go b/db_test.go new file mode 100644 index 0000000..42e91e4 --- /dev/null +++ b/db_test.go @@ -0,0 +1,165 @@ +package main + +import ( + "bytes" + "testing" +) + +// Creates a memory-backed DB for testing +func CreateDB(events []TalkEvent) *Talks { + // Create a writer to save the events + buf := bytes.NewBuffer(nil) + + // Create the DB + db, err := NewTalks(bytes.NewReader(nil), buf) + if err != nil { + panic(err) + } + + // Apply the events + for _, event := range events { + if err := db.event(event); err != nil { + panic(err) + } + + // Write the event to the buffer + if err := db.write(event); err != nil { + panic(err) + } + } + + return db +} + +func TestDB(t *testing.T) { + events := []TalkEvent{ + {Type: Create, Create: &CreateTalkEvent{ + ID: 0, + Name: "", + Type: 0, + Description: "", + Week: "20230705", + }}, + {Type: Create, Create: &CreateTalkEvent{ + ID: 1, + Name: "Test Talk", + Type: 0, + Description: "This is a test talk", + Week: "20230705", + }}, + {Type: Hide, Hide: &HideTalkEvent{ + ID: 1, + }}, + {Type: Delete, Delete: &DeleteTalkEvent{ + ID: 0, + }}, + {Type: Create, Create: &CreateTalkEvent{ + ID: 3, + Name: "Test Talk 2", + Type: 0, + Description: "This is a test talk", + Week: "20230705", + }}, + } + + db := CreateDB(events) + + // Check the state + if len(db.talks) != 2 { + t.Fatal("Expected 2 talks") + } + + // Verify that the talk was hidden + if !db.talks[1].Hidden { + t.Fatal("Expected talk to be hidden") + } + + // Verify that the talk was deleted + if _, ok := db.talks[0]; ok { + t.Fatal("Expected talk to be deleted") + } + + // Verify that talk 2 doesn't exist + if _, ok := db.talks[2]; ok { + t.Fatal("Expected talk to be missing") + } +} + +// Verify that deleting a talk that doesn't exist doesn't cause a panic +func TestDeleteNonExistentTalk(t *testing.T) { + events := []TalkEvent{ + {Type: Delete, Delete: &DeleteTalkEvent{ + ID: 0, + }}, + } + + db := CreateDB(events) + + // Check the state + if len(db.talks) != 0 { + t.Fatal("Expected 0 talks") + } +} + +// Verify that hiding a talk that doesn't exist doesn't cause a panic +func TestHideNonExistentTalk(t *testing.T) { + events := []TalkEvent{ + {Type: Hide, Hide: &HideTalkEvent{ + ID: 0, + }}, + } + + db := CreateDB(events) + + // Check the state + if len(db.talks) != 0 { + t.Fatal("Expected 0 talks") + } +} + +// Verify that leaving holes in the ID sequence doesn't cause a panic +func TestHolesInIDSequence(t *testing.T) { + events := []TalkEvent{ + {Type: Create, Create: &CreateTalkEvent{ + ID: 0, + Name: "", + Type: 0, + Description: "", + Week: "20230705", + }}, + {Type: Create, Create: &CreateTalkEvent{ + ID: 2, + Name: "Test Talk", + Type: 0, + Description: "This is a test talk", + Week: "20230705", + }}, + } + + db := CreateDB(events) + + // Check the state + if len(db.talks) != 2 { + t.Fatal("Expected 2 talks") + } + + // assert that talk 0 has the correct info + if db.talks[0] == nil { + t.Fatal("Expected talk 0 to exist") + } + + if db.talks[1] != nil { + t.Fatal("Expected talk 1 to not exist") + } + + if db.talks[2] == nil { + t.Fatal("Expected talk 2 to exist") + } + + // Create a new event and expect it to have ID 3 + db.Create("Test Talk 2", ForumTopic, "This is a test talk", "20230705") + + if db.talks[3] == nil { + t.Fatal("Expected talk 3 to exist") + } +} diff --git a/go.mod b/go.mod index 2518848..e4cbd28 100644 --- a/go.mod +++ b/go.mod @@ -5,22 +5,19 @@ go 1.20 require ( github.com/BurntSushi/toml v1.3.2 github.com/go-co-op/gocron v1.30.1 + github.com/gofrs/flock v0.8.1 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/microcosm-cc/bluemonday v1.0.24 github.com/russross/blackfriday/v2 v2.1.0 - golang.org/x/net v0.11.0 - gorm.io/driver/sqlite v1.5.2 - gorm.io/gorm v1.25.2 + golang.org/x/net v0.12.0 ) require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jinzhu/now v1.1.5 // indirect - github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/stretchr/testify v1.8.4 // indirect go.uber.org/atomic v1.11.0 // indirect + golang.org/x/sys v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index e8e936f..1976628 100644 --- a/go.sum +++ b/go.sum @@ -8,24 +8,22 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-co-op/gocron v1.30.1 h1:tjWUvJl5KrcwpkEkSXFSQFr4F9h5SfV/m4+RX0cV2fs= github.com/go-co-op/gocron v1.30.1/go.mod h1:39f6KNSGVOU1LO/ZOoZfcSxwlsJDQOKSu8erN0SH48Y= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= -github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -34,6 +32,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -49,16 +48,15 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= -gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= -gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= -gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= diff --git a/handlers.go b/handlers.go index 1afff34..17c25cb 100644 --- a/handlers.go +++ b/handlers.go @@ -18,7 +18,7 @@ var upgrader = websocket.Upgrader{} // TemplateResponse contains the information needed to render the past and future templates type TemplateResponse struct { - Talks []Talk + Talks []*Talk HumanWeek string Week string NextWeek string @@ -32,7 +32,7 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { human, _ := weekForHumans(week) // Prepare response - talks := VisibleTalks(week) + talks := talks.VisibleTalks(week) res := TemplateResponse{Talks: talks, Week: week, HumanWeek: human, NextWeek: addWeek(week), PrevWeek: subtractWeek(week)} // Render the template @@ -60,10 +60,10 @@ func weekHandler(w http.ResponseWriter, r *http.Request) { // Render the template if isPast(nextWednesday(), week) { - res.Talks = AllTalks(week) + res.Talks = talks.AllTalks(week) err = tmpls.ExecuteTemplate(w, "past.gohtml", res) } else { - res.Talks = VisibleTalks(week) + res.Talks = talks.VisibleTalks(week) err = tmpls.ExecuteTemplate(w, "future.gohtml", res) } @@ -83,7 +83,7 @@ func indexTalksHandler(w http.ResponseWriter, r *http.Request) { return } - talks := AllTalks(week) + talks := talks.AllTalks(week) // Parse talks as JSON err = json.NewEncoder(w).Encode(talks) @@ -105,7 +105,7 @@ func talksHandler(w http.ResponseWriter, r *http.Request) { return } - talks := AllTalks(week) + talks := talks.AllTalks(week) // Parse talks as JSON err = json.NewEncoder(w).Encode(talks) @@ -178,6 +178,8 @@ func socketHandler(w http.ResponseWriter, r *http.Request) { if trustedNetworks.Contains(ip) { authenticated = true } + } else { + authenticated = true } } log.Printf("[INFO] New connection from %s (authenticated: %t)", ip, authenticated) diff --git a/hub.go b/hub.go index f99e41c..d440595 100644 --- a/hub.go +++ b/hub.go @@ -29,53 +29,45 @@ func processMessage(message *Message) bool { case NEW: // You can not create a talk for a previous meeting wednesday := nextWednesday() - if message.Week == "" { - message.Week = wednesday - } else if isPast(wednesday, message.Week) { + if message.New.Week == "" { + message.New.Week = wednesday + } else if isPast(wednesday, message.New.Week) { return false } // Validate talk type - talk := &Talk{} - if *message.Talktype > 4 { + if message.New.Talktype > 4 { return false } - talk.Type = *message.Talktype // Validate talk description - if message.Description == "" { + if message.New.Description == "" { return false } - talk.Description = message.Description - - // Update the message's description to be parsed as markdown - message.Description = string(markDownerSafe(message.Description)) - // Validate talk name - if message.Name == "" { + if message.New.Name == "" { return false } - talk.Name = message.Name - talk.Week = message.Week - // TODO: Talk order - talk.Order = 0 + talks.Create(message.New.Name, message.New.Talktype, message.New.Description, message.New.Week) + + // Update the message's description to be parsed as markdown + message.New.Description = string(markDownerSafe(message.New.Description)) - message.ID = CreateTalk(talk) return true case HIDE: // During meetings we hide talks instead of deleting them if duringMeeting() { - log.Println("[INFO] Hide talk {", message.ID, "}") - HideTalk(message.ID) + log.Println("[INFO] Hide talk {", message.Hide.ID, "}") + talks.Hide(message.Hide.ID) } else { - log.Println("[INFO] Delete talk {", message.ID, "}") - DeleteTalk(message.ID) + log.Println("[INFO] Delete talk {", message.Hide.ID, "}") + talks.Delete(message.Hide.ID) } return true case DELETE: - log.Println("[INFO] Delete talk {", message.ID, "}") - DeleteTalk(message.ID) + log.Println("[INFO] Delete talk {", message.Hide.ID, "}") + talks.Delete(message.Hide.ID) return true default: return false @@ -93,6 +85,8 @@ func (hub *Hub) run() { delete(hub.clients, client) close(client.send) case message := <-hub.broadcast: + log.Println("[INFO] Broadcast message:", message) + // broadcasts the message to all clients (including the one that sent the message) if !processMessage(&message) { log.Println("[WARN] Invalid message") diff --git a/main.go b/main.go index f2492b0..03f6fb4 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "bytes" "fmt" "html/template" @@ -11,6 +12,7 @@ import ( "time" "github.com/go-co-op/gocron" + "github.com/gofrs/flock" "github.com/gorilla/mux" "github.com/microcosm-cc/bluemonday" "github.com/russross/blackfriday/v2" @@ -50,14 +52,30 @@ func main() { config.Validate() trustedNetworks = config.Network() - // Connect to the database - err = ConnectDB(&config) + // Use flock to ensure only one instance of go-talks is running + flock := flock.New(config.Database + ".lock") + locked, err := flock.TryLock() + if err != nil { + log.Fatalln("[ERROR] Failed to lock database", err) + } + if !locked { + log.Fatalln("[ERROR] Database is already locked by another instance of go-talks") + } + + // Open the database with r/w and create if it doesn't exist + db, err := os.OpenFile(config.Database, os.O_RDWR|os.O_CREATE, 0644) if err != nil { - log.Fatalln("[ERROR] Failed to connect to the database", err) + log.Fatalln("[ERROR] Failed to open database", err) } + // Split db into a buffered reader and a non-buffered writer + dbReader := bufio.NewReader(db) - // Set up all tables - MakeDB() + // Connect to the database + talks, err = NewTalks(dbReader, db) + if err != nil { + log.Fatalln("[ERROR] Failed to open database", err) + } + log.Println("[INFO] Loaded", len(talks.talks), "talks") // Load templates and add markdown function tmpls = template.Must(template.New("").Funcs(template.FuncMap{"safe_markdown": markDownerSafe, "unsafe_markdown": markDownerUnsafe}).ParseGlob("templates/*.gohtml")) @@ -102,7 +120,6 @@ func main() { // Schedule backup tasks s := gocron.NewScheduler(tz) - s.Wednesday().At("23:59").Do(backup) s.Every(1).Day().At("00:00").Do(invalidateCache) // Start servers diff --git a/migrate.py b/migrate.py new file mode 100644 index 0000000..df44388 --- /dev/null +++ b/migrate.py @@ -0,0 +1,70 @@ +# Python script that migrates our data from the old database to the new +# append only logfile backed database. + +import sqlite3 +import sys +import os +import json + +# Takes as argument the name of the old database +arg = sys.argv[1] + +# Call out to the shell to backup the old database +if os.system('cp ' + arg + ' ' + arg + '.bak') != 0: + print('Error backing up the old database') + sys.exit(1) +# delete the old database +if os.system('rm ' + arg) != 0: + print('Error deleting the old database') + sys.exit(1) + +# Open the old database +old_db = sqlite3.connect(arg + '.bak') + +# Open the new logfile backed database +log = open(arg, 'w') + +# Select all talks from the old database +talks = old_db.execute('SELECT * FROM talks') + +# map talk type to string +talk_type_map = { + 0: 'forum topic', + 1: 'lightning talk', + 2: 'project update', + 3: 'announcement', + 4: 'after meeting slot' +} + +# Write all talks to the new database +for talk in talks: + print(talk) + + # Create the talk event + talk_event = { + 'time': talk[7], + 'type': 'create', + 'create': { + 'id': talk[0], + 'name': talk[1], + 'type': talk_type_map[talk[2]], + 'description': talk[3], + 'week': talk[5] + } + } + + # Write the talk using the JSON format (without extra whitespace) + text = json.dumps(talk_event, separators=(',', ':')) + log.write(text + '\n') + + # If the talk is hidden, write the hide event + if talk[4] == 1: + hide_event = { + 'time': talk[7], + 'type': 'hide', + 'hide': { + 'id': talk[0] + } + } + text = json.dumps(hide_event, separators=(',', ':')) + log.write(text + '\n') diff --git a/models.go b/models.go index b237d5b..5c8cb31 100644 --- a/models.go +++ b/models.go @@ -1,9 +1,61 @@ package main -import "time" +import ( + "encoding/json" + "time" +) + +// TalkEventType is an enum of the different types of events +type TalkEventType int + +const ( + // UnknownEvent is for when we can't parse the event type + UnknownEvent TalkEventType = iota + // Create is the talk creation event {type: "create", create: {id: 1, name: "foo", ...} + Create + // Hide is the talk hiding event {type: "hide", hide: {id: 1}} + Hide + // Delete is the talk deletion event {type: "delete", delete: {id: 1}} + Delete +) + +// MarshalJSON implements the json.Marshaler interface +func (t TalkEventType) MarshalJSON() ([]byte, error) { + var s string + switch t { + case Create: + s = "create" + case Hide: + s = "hide" + case Delete: + s = "delete" + default: + s = "unknown" + } + return json.Marshal(s) +} + +// UnmarshalJSON implements the json.Unmarshaler interface +func (t *TalkEventType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + switch s { + case "create": + *t = Create + case "hide": + *t = Hide + case "delete": + *t = Delete + default: + *t = UnknownEvent + } + return nil +} // TalkType is the type of talk -type TalkType uint32 +type TalkType int const ( // ForumTopic are for general lab discussion @@ -18,21 +70,133 @@ const ( AfterMeetingSlot ) -func (tt TalkType) String() string { - if tt > 4 { +// MarshalJSON implements the json.Marshaler interface +func (t TalkType) MarshalJSON() ([]byte, error) { + var s string + switch t { + case ForumTopic: + s = "forum topic" + case LightningTalk: + s = "lightning talk" + case ProjectUpdate: + s = "project update" + case Announcement: + s = "announcement" + case AfterMeetingSlot: + s = "after meeting slot" + default: + s = "unknown" + } + return json.Marshal(s) +} + +// UnmarshalJSON implements the json.Unmarshaler interface +func (t *TalkType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + switch s { + case "forum topic": + *t = ForumTopic + case "lightning talk": + *t = LightningTalk + case "project update": + *t = ProjectUpdate + case "announcement": + *t = Announcement + case "after meeting slot": + *t = AfterMeetingSlot + } + return nil +} + +// String implements the fmt.Stringer interface. This is used when templating +func (t TalkType) String() string { + switch t { + case ForumTopic: + return "forum topic" + case LightningTalk: + return "lightning talk" + case ProjectUpdate: + return "project update" + case Announcement: + return "announcement" + case AfterMeetingSlot: + return "after meeting slot" + default: return "unknown" } - return []string{"forum topic", "lightning talk", "project update", "announcement", "after meeting slot"}[tt] } -// Talk is used to represent a talk in the database +// JSONTime is a Wrapper for time.Time struct with custom JSON behavior +// matching sqlite's datetime format +type JSONTime struct { + time.Time +} + +var format = "2006-01-02 15:04:05.999999-07:00" + +// MarshalJSON implements the json.Marshaler interface +func (t JSONTime) MarshalJSON() ([]byte, error) { + return json.Marshal(t.Format(format)) +} + +// UnmarshalJSON implements the json.Unmarshaler interface +func (t *JSONTime) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + tt, err := time.Parse(format, s) + if err != nil { + return err + } + t.Time = tt + return nil +} + +// Now returns a JSONTime with the time from time.Now() +func Now() JSONTime { + return JSONTime{time.Now()} +} + +// TalkEvent is stored in the database. Since go doesn't have a good way to +// represent a union type, we'll just use a large struct with a type field +// and a field for each event type +type TalkEvent struct { + Time JSONTime `json:"time"` + Type TalkEventType `json:"type"` + Create *CreateTalkEvent `json:"create,omitempty"` + Hide *HideTalkEvent `json:"hide,omitempty"` + Delete *DeleteTalkEvent `json:"delete,omitempty"` +} + +// CreateTalkEvent is created when a talk is created +type CreateTalkEvent struct { + ID uint32 `json:"id"` + Name string `json:"name"` + Type TalkType `json:"type"` + Description string `json:"description"` + Week string `json:"week"` +} + +// HideTalkEvent is created when a talk is hidden +type HideTalkEvent struct { + ID uint32 `json:"id"` +} + +// DeleteTalkEvent is created when a talk is deleted +type DeleteTalkEvent struct { + ID uint32 `json:"id"` +} + +// Talk is the resulting type produced by the database type Talk struct { - ID uint32 `gorm:"AUTO_INCREMENT, primary key" json:"id"` - Name string `gorm:"not null" json:"name"` - Type TalkType `gorm:"not null" json:"type"` - Description string `gorm:"not null" json:"description"` - IsHidden bool `gorm:"not null" json:"-"` - Week string `gorm:"index, not null" json:"-"` - Order uint32 `gorm:"not null" json:"-"` // TODO: Talk ordering - CreatedAt time.Time `json:"-"` + ID uint32 `json:"id"` + Name string `json:"name"` + Type TalkType `json:"type"` + Description string `json:"description"` + Week string `json:"week"` + Hidden bool `json:"hidden"` } diff --git a/static/site.js b/static/site.js index ed56361..1bebe56 100644 --- a/static/site.js +++ b/static/site.js @@ -1,9 +1,12 @@ // Resize text area window.onload = function () { - document.getElementById("description").addEventListener("input", (e) => { - e.target.style.height = "auto"; - e.target.style.height = (e.target.scrollHeight) + "px"; - }); + let area = document.getElementById("description"); + if (area) { + area.addEventListener("input", (e) => { + e.target.style.height = "auto"; + e.target.style.height = (e.target.scrollHeight) + "px"; + }); + } } var socket = connect(); @@ -22,7 +25,9 @@ function auth() { const data = { type: 3, - password: password + auth: { + password: password + } } const json = JSON.stringify(data); @@ -44,6 +49,24 @@ function auth() { return promise; } +// Saves the id we saw since the last sync message +var seen = new Set(); +// Sync requests are used to update the table with the latest data +function sync() { + const data = { + type: 4, + sync: { + week: week + } + } + + const json = JSON.stringify(data); + console.log(`Sending: ${json}`); + + seen.clear(); + socket.send(json); +} + // Connects to the websocket endpoint function connect() { var ws_scheme = window.location.protocol == "https:" ? "wss://" : "ws://"; @@ -51,6 +74,7 @@ function connect() { let socket = new WebSocket(ws_scheme + window.location.host + "/ws"); socket.onopen = function (e) { console.log("Connected!", e); + sync(); }; socket.onmessage = function (e) { const data = JSON.parse(e.data); @@ -58,13 +82,17 @@ function connect() { if (data.type == 0) { // Add the talk to the table - addTalk(data); + addTalk(data.new); + seen.add(data.new.id); } else if (data.type == 1 || data.type == 2) { // Hide the talk from the table - hideTalk(data.id); + hideTalk(data.hide); } else if (data.type == 3) { // Receiving an auth message means we have successfully authenticated - handleAuth(data); + handleAuth(data.auth); + } else if (data.type == 4) { + // Remove talks that we didn't see in this sync + hideTalksNotIn(seen); } }; socket.onclose = function (e) { @@ -80,7 +108,7 @@ function connect() { // User triggers talk deletion function del(id) { - confirmed = confirm("Are you sure you want to delete this talk? THIS CANNOT BE UNDONE"); + confirmed = confirm("Are you sure you want to delete this talk?"); if (!confirmed) { return; } @@ -88,7 +116,9 @@ function del(id) { auth().then(() => { const data = { type: 2, - id: id + delete: { + id: id + } } const json = JSON.stringify(data); @@ -105,7 +135,9 @@ function hide(id) { auth().then(() => { const data = { type: 1, - id: id + hide: { + id: id + } }; const json = JSON.stringify(data); @@ -131,10 +163,12 @@ function create() { auth().then(() => { const data = { type: 0, - name: name, - description: description, - talktype: parseInt(talktype), - week: week + new: { + name: name, + description: description, + talktype: typeToString[talktype], + week: week + } }; const json = JSON.stringify(data); @@ -164,27 +198,39 @@ const stringToType = { "after-meeting slot": 4 } -// Hides the talk from the table -function hideTalk(id) { +// Removes all talks that match a predicate +function hideTalks(predicate) { const table = document.getElementById("tb"); const rows = document.getElementById('tb').children; - // Find the row to remove - let rowToRemove = null - for (i = 0; i < rows.length; i++) { - if (rows[i].children[0].innerText == id) { - rowToRemove = rows[i]; - break; + for(let i = rows.length - 1; i >= 0; i--) { + if (rows[i].getAttribute("class") == "event" && predicate(rows[i])) { + table.removeChild(rows[i]); } } +} - if (rowToRemove) { - // Remove the row - table.removeChild(rowToRemove); - } +function hideTalk(id) { + hideTalks((row) => { + return row.children[0].innerText == id; + }); +} + +function hideTalksNotIn(ids) { + console.log("Hiding talks not in", ids); + + hideTalks((row) => { + return !ids.has(parseInt(row.children[0].innerText)); + }); +} + +// Removes all talks from the table +function clearTalks() { + hideTalks((row) => { + return true; + }); } -// {"type":0,"name":"string","description":"string","talktype":0} function addTalk(talk) { if (talk.week != week) { console.log("Skipping new talk because it is for a different week", talk.week); @@ -194,6 +240,14 @@ function addTalk(talk) { const table = document.getElementById("tb"); const rows = document.getElementById('tb').children; + // Check to see if the talk is already in the table + for (let i = 0; i < rows.length; i++) { + if (rows[i].children[0].innerText == talk.id) { + console.log("Skipping new talk because it is already in the table", talk.id); + return + } + } + // Insert the new data into the correct location in the table let i = 0 for (i = 0; i < rows.length - 1; i++) { @@ -219,7 +273,7 @@ function addTalk(talk) { var c2 = row.insertCell(2); c2.setAttribute("class", "type"); - c2.innerHTML = typeToString[talk.talktype]; + c2.innerHTML = talk.talktype; var c3 = row.insertCell(3); c3.setAttribute("class", "description markdown"); diff --git a/utils.go b/utils.go index 4c12424..9371830 100644 --- a/utils.go +++ b/utils.go @@ -21,6 +21,8 @@ func NewNetworks(subnets []string) Networks { } n.nets = append(n.nets, net) } + + log.Printf("[INFO] Trusted subnets: %s", n.nets) return n } diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 0000000..b726167 --- /dev/null +++ b/utils_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "net" + "testing" +) + +func TestContains(t *testing.T) { + n := NewNetworks([]string{"::1/128"}) + if !n.Contains(net.ParseIP("::1")) { + t.Errorf("Expected ::1 to be in ::1/128") + } +}