Move logging, datarithms, and geoip into this git tree.

cache.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/COSI-Lab/logging"
+	"github.com/COSI-Lab/Mirror/logging"
 )
 
 type ProxyWriter struct {

config.go

@@ -13,7 +13,7 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/COSI-Lab/logging"
+	"github.com/COSI-Lab/Mirror/logging"
 	"github.com/xeipuuv/gojsonschema"
 )
 

daily_health.go

@@ -7,7 +7,7 @@ import (
 	"sort"
 	"time"
 
-	"github.com/COSI-Lab/logging"
+	"github.com/COSI-Lab/Mirror/logging"
 	"github.com/influxdata/influxdb-client-go/v2/api"
 	"github.com/wcharczuk/go-chart/v2"
 	"github.com/wcharczuk/go-chart/v2/drawing"

datarithms/README.md

@@ -0,0 +1,3 @@
+# Datarithms
+
+This module includes practical algorithms and data structures for our [Mirror](https://github.com/COSI-Lab/Mirror) project.

datarithms/binarysearch.go

@@ -0,0 +1,56 @@
+package datarithms
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"time"
+)
+
+// BinarySearchFileByDate returns the offset of the first line which date is after lastUpdated
+// this could panic if the parse function isn't guaranteed to work on every line of the file
+func BinarySearchFileByDate(file string, lastUpdated time.Time, parse func(line string) (time.Time, error)) (offset int64, err error) {
+	// Open the log file
+	f, err := os.Open(file)
+	if err != nil {
+		return 0, err
+	}
+
+	// Load all of the line offsets into memory
+	lines := make([]int64, 0)
+
+	// Precalculate the seek offset of each line
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		lines = append(lines, offset)
+		offset += int64(len(scanner.Bytes())) + 1
+	}
+
+	// Run a binary search for the first line which is past the lastUpdated
+	start := 0
+	end := len(lines) - 1
+	for start < end {
+		mid := (start + end) / 2
+
+		// Get the text of the line
+		f.Seek(lines[mid], io.SeekStart)
+		scanner := bufio.NewScanner(f)
+		scanner.Scan()
+		line := scanner.Text()
+
+		tm, err := parse(line)
+		if err != nil {
+			// if for some reason we can't parse the line we increment start and try again
+			start = mid + 1
+			continue
+		}
+
+		if tm.After(lastUpdated) {
+			end = mid
+		} else {
+			start = mid + 1
+		}
+	}
+
+	return lines[start], nil
+}

datarithms/datarithms_test.go

@@ -0,0 +1,155 @@
+package datarithms_test
+
+import (
+	"testing"
+
+	"github.com/COSI-Lab/Mirror/datarithms"
+)
+
+// Test the queue
+func TestQueue(t *testing.T) {
+	// Create a new queue
+	q := datarithms.CircularQueueInit[int](5)
+
+	if q.Capacity() != 5 {
+		t.Error("Capacity is not 5")
+	}
+
+	// Push some elements
+	q.Push(1)
+	q.Push(2)
+	q.Push(3)
+
+	// Check the length
+	if q.Len() != 3 {
+		t.Error("Expected 3, got", q.Len())
+	}
+
+	var element int
+	var err error
+
+	// Pop the first element
+	if element, err = q.Pop(); err == nil && element != 1 {
+		t.Error("Expected 1, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 2 {
+		t.Error("Expected 2, got", q.Len())
+	}
+
+	// Pop the second element
+	if element, err = q.Pop(); err == nil && element != 2 {
+		t.Error("Expected 2, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 1 {
+		t.Error("Expected 1, got", q.Len())
+	}
+
+	// Pop the third element
+	if element, err = q.Pop(); err == nil && element != 3 {
+		t.Error("Expected 3, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 0 {
+		t.Error("Expected 0, got", q.Len())
+	}
+
+	// Pop the fourth element
+	if element, err = q.Pop(); err == nil && element == 0 {
+		t.Error("Expected nil, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 0 {
+		t.Error("Expected 0, got", q.Len())
+	}
+
+	// Push more elements than capacity
+	for i := 0; i < 10; i++ {
+		q.Push(i)
+	}
+
+	// Check the length
+	if q.Len() != 5 {
+		t.Error("Expected 5, got", q.Len())
+	}
+
+	// Pop the first element
+	if element, err = q.Pop(); err != nil && element != 5 {
+		t.Error("Expected 5, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 4 {
+		t.Error("Expected 4, got", q.Len())
+	}
+
+	// Pop the second element
+	if element, err = q.Pop(); err != nil && element != 6 {
+		t.Error("Expected 6, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 3 {
+		t.Error("Expected 3, got", q.Len())
+	}
+
+	// Pop the third element
+	if element, err = q.Pop(); err != nil && element != 7 {
+		t.Error("Expected 7, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 2 {
+		t.Error("Expected 2, got", q.Len())
+	}
+
+	// Pop the fourth element
+	if element, err = q.Pop(); err != nil && element != 8 {
+		t.Error("Expected 8, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 1 {
+		t.Error("Expected 1, got", q.Len())
+	}
+
+	// Pop the fifth element
+	if element, err = q.Pop(); err != nil && element != 9 {
+		t.Error("Expected 9, got", element)
+	}
+
+	// Check the length
+	if q.Len() != 0 {
+		t.Error("Expected 0, got", q.Len())
+	}
+}
+
+func TestSchedule(t *testing.T) {
+	// Create tasks
+	tasks := []datarithms.Task{
+		{Short: "a", Syncs: 1},
+		{Short: "b", Syncs: 2},
+		{Short: "c", Syncs: 4},
+		{Short: "d", Syncs: 8},
+	}
+
+	sched := datarithms.BuildSchedule(tasks)
+	t.Log(sched)
+
+	verify := datarithms.Verify(sched, tasks)
+	if !verify {
+		t.Error("Schedule is invalid")
+	}
+
+	// Next task is in the future
+	_, dt := sched.NextJob()
+
+	if dt < 0 {
+		t.Error("Next task is in the past")
+	}
+}

datarithms/queue.go

@@ -0,0 +1,98 @@
+package datarithms
+
+import (
+	"fmt"
+	"sync"
+)
+
+// Thread Safe circular queue implmentation using a slice for byte slices
+type CircularQueue[T any] struct {
+	lock     sync.RWMutex
+	queue    []T
+	capacity int
+	start    int
+	end      int
+	length   int
+}
+
+// Creates a new circular queue of given capacity
+func CircularQueueInit[T any](capacity int) *CircularQueue[T] {
+	q := new(CircularQueue[T])
+
+	q.queue = make([]T, capacity)
+	q.capacity = capacity
+	q.start = 0
+	q.end = 0
+	q.length = 0
+	q.lock = sync.RWMutex{}
+
+	return q
+}
+
+// Adds a new element to the queue
+func (q *CircularQueue[T]) Push(element T) {
+	q.lock.Lock()
+	q.queue[q.end] = element
+	q.end = (q.end + 1) % q.capacity
+	// If the queue is full, start overwriting from the beginning
+	if q.length == q.capacity {
+		q.start = (q.start + 1) % q.capacity
+	} else {
+		q.length++
+	}
+	q.lock.Unlock()
+}
+
+// Pops the element at the front of the queue
+// If the queue is empty, returns the zero value followed by an error
+func (q *CircularQueue[T]) Pop() (element T, err error) {
+	q.lock.Lock()
+	// If the queue is empty, return nil
+	if q.length == 0 {
+		q.lock.Unlock()
+		return element, fmt.Errorf("CircularQueue is empty")
+	}
+	element = q.queue[q.start]
+	q.start = (q.start + 1) % q.capacity
+	q.length--
+	q.lock.Unlock()
+	return element, nil
+}
+
+// Returns the element at the front of the queue
+func (q *CircularQueue[T]) Front() T {
+	q.lock.RLock()
+	result := q.queue[q.start]
+	q.lock.RUnlock()
+	return result
+}
+
+// Returns the number of elements in the queue
+func (q *CircularQueue[T]) Len() int {
+	q.lock.RLock()
+	result := q.length
+	q.lock.RUnlock()
+	return result
+}
+
+// Returns the capacity of the queue
+func (q *CircularQueue[T]) Capacity() int {
+	q.lock.RLock()
+	result := q.capacity
+	q.lock.RUnlock()
+	return result
+}
+
+// Returns all the elements of the queue
+func (q *CircularQueue[T]) All() []T {
+	q.lock.RLock()
+	result := make([]T, 0, q.length)
+
+	// From start to end
+	for i := q.start; i != q.end; i = (i + 1) % q.capacity {
+		result = append(result, q.queue[i])
+	}
+
+	q.lock.RUnlock()
+	return result
+}