Feature: Use typesense as feed repository

.env.dist

@@ -6,3 +6,6 @@ export UNCONDITIONAL_API_SOURCE_CLIENT_KEY="secret"
 export UNCONDITIONAL_API_LOG_ENV="dev"
 export UNCONDITIONAL_API_BUILD_COMMIT_VERSION=
 export UNCONDITIONAL_API_BUILD_RELEASE_VERSION=
+export UNCONDITIONAL_API_FEED_REPO_INDEX="feeds"
+export UNCONDITIONAL_API_FEED_REPO_HOST="http://localhost:8108"
+export UNCONDITIONAL_API_FEED_REPO_KEY="xyz"

.envrc.dist

@@ -6,3 +6,6 @@ export UNCONDITIONAL_API_SOURCE_CLIENT_KEY="secret"
 export UNCONDITIONAL_API_LOG_ENV="dev"
 export UNCONDITIONAL_API_BUILD_COMMIT_VERSION=
 export UNCONDITIONAL_API_BUILD_RELEASE_VERSION=
+export UNCONDITIONAL_API_FEED_REPO_INDEX="feeds"
+export UNCONDITIONAL_API_FEED_REPO_HOST="http://localhost:8108"
+export UNCONDITIONAL_API_FEED_REPO_KEY="xyz"

Dockerfile

@@ -26,15 +26,24 @@ WORKDIR /data
 COPY --from=builder /app/main /app/main
 
 ARG UNCONDITIONAL_API_SOURCE_REPO
+ARG UNCONDITIONAL_API_SOURCE_CLIENT_KEY
+ARG UNCONDITIONAL_API_FEED_REPO_INDEX
+ARG UNCONDITIONAL_API_FEED_REPO_HOST
+ARG UNCONDITIONAL_API_FEED_REPO_KEY
 ARG UNCONDITIONAL_API_LOG_ENV
+
 ENV UNCONDITIONAL_API_SOURCE_REPO=${UNCONDITIONAL_API_SOURCE_REPO}
+ENV UNCONDITIONAL_API_SOURCE_CLIENT_KEY=${UNCONDITIONAL_API_SOURCE_CLIENT_KEY}
+ENV UNCONDITIONAL_API_FEED_REPO_INDEX=${UNCONDITIONAL_API_FEED_REPO_INDEX}
+ENV UNCONDITIONAL_API_FEED_REPO_HOST=${UNCONDITIONAL_API_FEED_REPO_HOST}
+ENV UNCONDITIONAL_API_FEED_REPO_KEY=${UNCONDITIONAL_API_FEED_REPO_KEY}
 ENV UNCONDITIONAL_API_LOG_ENV=${UNCONDITIONAL_API_LOG_ENV}
 
 RUN --mount=type=secret,id=UNCONDITIONAL_API_SOURCE_CLIENT_KEY \
+    --mount=type=secret,id=UNCONDITIONAL_API_FEED_REPO_KEY \
     UNCONDITIONAL_API_SOURCE_CLIENT_KEY="$(cat /run/secrets/UNCONDITIONAL_API_SOURCE_CLIENT_KEY)" \
-    /app/main source download --path /data/source.json 
-
-RUN /app/main index create --source /data/source.json --name /data/index
+    UNCONDITIONAL_API_FEED_REPO_KEY="$(cat /run/secrets/UNCONDITIONAL_API_FEED_REPO_KEY)" \
+    /app/main index create --name feeds
 
 FROM scratch as release
 COPY --from=certificator /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
@@ -47,12 +56,18 @@ ARG UNCONDITIONAL_API_PORT
 ARG UNCONDITIONAL_API_SOURCE_REPO
 ARG UNCONDITIONAL_API_SOURCE_CLIENT_KEY
 ARG UNCONDITIONAL_API_LOG_ENV
+ARG UNCONDITIONAL_API_FEED_REPO_INDEX
+ARG UNCONDITIONAL_API_FEED_REPO_HOST
+ARG UNCONDITIONAL_API_FEED_REPO_KEY
 
 ENV UNCONDITIONAL_API_ADDRESS=${UNCONDITIONAL_API_ADDRESS}
 ENV UNCONDITIONAL_API_ALLOWED_ORIGINS=${UNCONDITIONAL_API_ALLOWED_ORIGINS}
 ENV UNCONDITIONAL_API_PORT=${UNCONDITIONAL_API_PORT}
 ENV UNCONDITIONAL_API_SOURCE_REPO=${UNCONDITIONAL_API_SOURCE_REPO}
 ENV UNCONDITIONAL_API_SOURCE_CLIENT_KEY=${UNCONDITIONAL_API_SOURCE_CLIENT_KEY}
 ENV UNCONDITIONAL_API_LOG_ENV=${UNCONDITIONAL_API_LOG_ENV}
+ENV UNCONDITIONAL_API_FEED_REPO_INDEX=${UNCONDITIONAL_API_FEED_REPO_INDEX}
+ENV UNCONDITIONAL_API_FEED_REPO_HOST=${UNCONDITIONAL_API_FEED_REPO_HOST}
+ENV UNCONDITIONAL_API_FEED_REPO_KEY=${UNCONDITIONAL_API_FEED_REPO_KEY}
 
-ENTRYPOINT ["./app/main","serve", "--address", "0.0.0.0", "--port","8080", "--index","/data/index"]
+ENTRYPOINT ["./app/main","serve", "--address", "0.0.0.0", "--port","8080"]

cmd/index/create.go

@@ -6,18 +6,21 @@ import (
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 
-	"github.com/unconditionalday/server/internal/app"
 	"github.com/unconditionalday/server/internal/container"
+	"github.com/unconditionalday/server/internal/repository/typesense"
 	"github.com/unconditionalday/server/internal/service"
 	cobrax "github.com/unconditionalday/server/internal/x/cobra"
-	iox "github.com/unconditionalday/server/internal/x/io"
+	typesensex "github.com/unconditionalday/server/internal/x/typesense"
 )
 
 var (
-	ErrIndexNotProvided           = errors.New("index not provided, please provide it using --index flag")
-	ErrSourceNotProvided          = errors.New("source not provided, please provide it using --source flag")
-	ErrSourceClientKeyNotProvided = errors.New("source client-key not provided, please provide it using --source-client-key flag")
-	ErrLogEnvNotProvided          = errors.New("log-env not provided, please provide it using --log-env flag")
+	ErrIndexNotProvided            = errors.New("index not provided, please provide it using --index flag")
+	ErrFeedRepoHostNotProvided     = errors.New("feed repository host not provided, please provide it using --feed-repo-host flag")
+	ErrFeedRepoKeyNotProvided      = errors.New("feed repository key not provided, please provide it using --feed-repo-key flag")
+	ErrSourceNotProvided           = errors.New("source not provided, please provide it using --source flag")
+	ErrSourceClientKeyNotProvided  = errors.New("source client-key not provided, please provide it using --source-client-key flag")
+	ErrSourceRepositoryNotProvided = errors.New("source repo not provided, please provide it using --source-repo flag")
+	ErrLogEnvNotProvided           = errors.New("log-env not provided, please provide it using --log-env flag")
 )
 
 func NewCreateCommand() *cobra.Command {
@@ -31,9 +34,24 @@ func NewCreateCommand() *cobra.Command {
 				return ErrIndexNotProvided
 			}
 
-			s := cobrax.Flag[string](cmd, "source").(string)
-			if s == "" {
-				return ErrSourceNotProvided
+			frh := cobrax.Flag[string](cmd, "feed-repo-host").(string)
+			if i == "" {
+				return ErrFeedRepoHostNotProvided
+			}
+
+			frk := cobrax.Flag[string](cmd, "feed-repo-key").(string)
+			if frk == "" {
+				return ErrFeedRepoKeyNotProvided
+			}
+
+			sr := cobrax.Flag[string](cmd, "source-repo").(string)
+			if sr == "" {
+				return ErrSourceRepositoryNotProvided
+			}
+
+			sk := cobrax.Flag[string](cmd, "source-client-key").(string)
+			if sk == "" {
+				return ErrSourceClientKeyNotProvided
 			}
 
 			l := cobrax.Flag[string](cmd, "log-env").(string)
@@ -42,20 +60,30 @@ func NewCreateCommand() *cobra.Command {
 			}
 
 			params := container.NewDefaultParameters()
-			params.FeedIndex = i
+			params.FeedRepositoryIndex = i
+			params.FeedRepositoryHost = frh
+			params.FeedRepositoryKey = frk
+			params.SourceClientKey = sk
+			params.SourceRepository = sr
 			params.LogEnv = l
 
 			c, _ := container.NewContainer(params)
 
+			t := c.GetTypesenseClient()
+
+			feedSchema := typesense.GetFeedSchema(t)
+			if err := typesensex.CreateOrUpdateCollection(t, feedSchema); err != nil {
+				return err
+			}
+
 			sourceService := service.NewSource(c.GetSourceClient(), c.GetParser(), c.GetVersioning(), c.GetLogger())
 
-			var source app.Source
-			source, err := iox.ReadJSON(s, source)
+			s, err := sourceService.Fetch()
 			if err != nil {
 				return err
 			}
 
-			feeds, err := sourceService.FetchFeeds(source)
+			feeds, err := sourceService.FetchFeeds(s.Data)
 			if err != nil {
 				c.GetLogger().Error("Can't fetch feeds", zap.Error(err))
 			}
@@ -75,9 +103,12 @@ func NewCreateCommand() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringP("source", "s", "", "Source Path")
 	cmd.Flags().StringP("name", "n", "", "Index Name")
 	cmd.Flags().StringP("log-env", "l", "", "Log Env")
+	cmd.Flags().StringP("feed-repo-host", "", "", "Feed's repository host")
+	cmd.Flags().StringP("feed-repo-key", "", "", "Feed's repository API's key")
+	cmd.Flags().String("source-repo", "", "Source Repository")
+	cmd.Flags().String("source-client-key", "", "Source Client Key")
 
 	envPrefix := "UNCONDITIONAL_API"
 	cobrax.BindFlags(cmd, cobrax.InitEnvs(envPrefix), envPrefix)

cmd/serve.go

@@ -13,7 +13,9 @@ import (
 )
 
 var (
-	ErrIndexNotProvided            = errors.New("index not provided, please provide it using --index flag")
+	ErrFeedRepoIndexNotProvided    = errors.New("feed repository index not provided, please provide it using --feed-repo-index flag")
+	ErrFeedRepoHostNotProvided     = errors.New("feed repository host not provided, please provide it using --feed-repo-host flag")
+	ErrFeedRepoKeyNotProvided      = errors.New("feed repository key not provided, please provide it using --feed-repo-key flag")
 	ErrAddressNotProvided          = errors.New("server address not provided, please provide it using --address flag")
 	ErrPortNotProvided             = errors.New("server port not provided, please provide it using --port flag")
 	ErrLogEnvNotProvided           = errors.New("server log-env not provided, please provide it using --log-env flag")
@@ -22,15 +24,25 @@ var (
 	ErrAllowedOriginsNotProvided   = errors.New("server allowed origins not provided, please provide it using --allowed-origins flag")
 )
 
-func NewServeCommand(version version.Build) *cobra.Command {
+func NewServeCommand(buildVersion version.Build) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "serve",
 		Short: "Starts the server",
 		Long:  `Starts the server`,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			i := cobrax.Flag[string](cmd, "index").(string)
+			i := cobrax.Flag[string](cmd, "feed-repo-index").(string)
 			if i == "" {
-				return ErrIndexNotProvided
+				return ErrFeedRepoIndexNotProvided
+			}
+
+			frh := cobrax.Flag[string](cmd, "feed-repo-host").(string)
+			if i == "" {
+				return ErrFeedRepoHostNotProvided
+			}
+
+			frk := cobrax.Flag[string](cmd, "feed-repo-key").(string)
+			if i == "" {
+				return ErrFeedRepoKeyNotProvided
 			}
 
 			l := cobrax.Flag[string](cmd, "log-env").(string)
@@ -44,7 +56,7 @@ func NewServeCommand(version version.Build) *cobra.Command {
 			}
 
 			sk := cobrax.Flag[string](cmd, "source-client-key").(string)
-			if s == "" {
+			if sk == "" {
 				return ErrSourceClientKeyNotProvided
 			}
 
@@ -63,7 +75,7 @@ func NewServeCommand(version version.Build) *cobra.Command {
 				return ErrAllowedOriginsNotProvided
 			}
 
-			params := container.NewParameters(a, i, s, sk, l, p, ao, version)
+			params := container.NewParameters(a, i, frh, frk, s, sk, l, p, ao, buildVersion)
 			c, _ := container.NewContainer(params)
 
 			sourceService := service.NewSource(c.GetSourceClient(), c.GetParser(), c.GetVersioning(), c.GetLogger())
@@ -83,7 +95,9 @@ func NewServeCommand(version version.Build) *cobra.Command {
 
 	cmd.Flags().StringP("address", "a", "localhost", "Server address")
 	cmd.Flags().IntP("port", "p", 8080, "Server port")
-	cmd.Flags().StringP("index", "s", "", "Index path")
+	cmd.Flags().StringP("feed-repo-index", "s", "", "Index path")
+	cmd.Flags().StringP("feed-repo-host", "", "", "Feed's repository host")
+	cmd.Flags().StringP("feed-repo-key", "", "", "Feed's repository API's key")
 	cmd.Flags().String("allowed-origins", "", "Allowed Origins")
 	cmd.Flags().String("source-repo", "", "Source Repository")
 	cmd.Flags().String("source-client-key", "", "Source Client Key")

go.mod

@@ -96,6 +96,7 @@ require (
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/schollz/closestmatch v2.1.0+incompatible // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
@@ -129,5 +130,6 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.8.4
+	github.com/typesense/typesense-go v1.0.0
 	golang.org/x/sys v0.15.0 // indirect
 )

go.sum

@@ -239,6 +239,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
+github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
@@ -275,6 +277,8 @@ github.com/tdewolff/test v1.0.9 h1:SswqJCmeN4B+9gEAi/5uqT0qpi1y2/2O47V/1hhGZT0=
 github.com/tdewolff/test v1.0.9/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/typesense/typesense-go v1.0.0 h1:/8Lr1yf9YjmUKdn/xbTNy+OhwOvBd0noBTRkcB22Uhw=
+github.com/typesense/typesense-go v1.0.0/go.mod h1:4mq4FYHzU7csU/KHaZoyG2bCSKl7GrCeyAr2YhXT1/0=
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=

internal/container/container.go

@@ -1,22 +1,19 @@
 package container
 
 import (
-	"errors"
 	"net/http"
 
-	"github.com/blevesearch/bleve/v2"
-	"github.com/blevesearch/bleve/v2/mapping"
 	"go.uber.org/zap"
 
+	"github.com/typesense/typesense-go/typesense"
 	"github.com/unconditionalday/server/internal/app"
 	"github.com/unconditionalday/server/internal/client/github"
 	"github.com/unconditionalday/server/internal/client/wikipedia"
 	"github.com/unconditionalday/server/internal/parser"
-	bleveRepo "github.com/unconditionalday/server/internal/repository/bleve"
+	typesenseRepo "github.com/unconditionalday/server/internal/repository/typesense"
 	"github.com/unconditionalday/server/internal/search"
 	"github.com/unconditionalday/server/internal/version"
 	"github.com/unconditionalday/server/internal/webserver"
-	blevex "github.com/unconditionalday/server/internal/x/bleve"
 	calverx "github.com/unconditionalday/server/internal/x/calver"
 	netx "github.com/unconditionalday/server/internal/x/net"
 )
@@ -28,12 +25,12 @@ func NewDefaultParameters() Parameters {
 		ServerAllowedOrigins: []string{"*"},
 		SourceRepository:     "source",
 		SourceClientKey:      "secret",
-		FeedIndex:            "feed.index",
+		FeedRepositoryIndex:  "feed.index",
 		LogEnv:               "dev",
 	}
 }
 
-func NewParameters(serverAddress, feedIndex, sourceRepository, sourceClientKey, logEnv string, serverPort int, serverAllowedOrigins []string, buildVersion version.Build) Parameters {
+func NewParameters(serverAddress, feedIndex, feedRepoHost, feedRepoKey, sourceRepository, sourceClientKey, logEnv string, serverPort int, serverAllowedOrigins []string, buildVersion version.Build) Parameters {
 	return Parameters{
 		ServerAddress:        serverAddress,
 		ServerPort:           serverPort,
@@ -44,7 +41,9 @@ func NewParameters(serverAddress, feedIndex, sourceRepository, sourceClientKey,
 
 		BuildVersion: buildVersion,
 
-		FeedIndex: feedIndex,
+		FeedRepositoryIndex: feedIndex,
+		FeedRepositoryHost:  feedRepoHost,
+		FeedRepositoryKey:   feedRepoKey,
 
 		LogEnv: logEnv,
 	}
@@ -61,20 +60,23 @@ type Parameters struct {
 
 	BuildVersion version.Build
 
-	FeedIndex string
+	FeedRepositoryIndex string
+	FeedRepositoryHost  string
+	FeedRepositoryKey   string
 
 	LogEnv string
 }
 
 type Services struct {
-	apiServer      *webserver.Server
-	feedRepository *bleveRepo.FeedRepository
-	sourceClient   *github.Client
-	searchClient   *wikipedia.Client
-	httpClient     *netx.HttpClient
-	logger         *zap.Logger
-	parser         *parser.Parser
-	versioning     *calverx.CalVer
+	apiServer       *webserver.Server
+	feedRepository  *typesenseRepo.FeedRepository
+	sourceClient    *github.Client
+	searchClient    *wikipedia.Client
+	httpClient      *netx.HttpClient
+	typesenseClient *typesense.Client
+	logger          *zap.Logger
+	parser          *parser.Parser
+	versioning      *calverx.CalVer
 }
 
 func NewContainer(p Parameters) (*Container, error) {
@@ -104,23 +106,29 @@ func (c *Container) GetFeedRepository() app.FeedRepository {
 		return c.feedRepository
 	}
 
-	b, err := blevex.NewIndex(c.FeedIndex, mapping.NewIndexMapping())
-	if err != nil {
-		if errors.Is(bleve.ErrorIndexPathExists, err) {
-			b, err = blevex.New(c.FeedIndex)
-			if err != nil {
-				panic(err)
-			}
-		} else {
-			panic(err)
-		}
-	}
+	client := typesense.NewClient(
+		typesense.WithServer(c.FeedRepositoryHost),
+		typesense.WithAPIKey(c.FeedRepositoryKey))
 
-	c.feedRepository = bleveRepo.NewFeedRepository(b)
+	c.feedRepository = typesenseRepo.NewFeedRepository(client)
 
 	return c.feedRepository
 }
 
+func (c *Container) GetTypesenseClient() *typesense.Client {
+	if c.typesenseClient != nil {
+		return c.typesenseClient
+	}
+
+	client := typesense.NewClient(
+		typesense.WithServer(c.FeedRepositoryHost),
+		typesense.WithAPIKey(c.FeedRepositoryKey))
+
+	c.typesenseClient = client
+
+	return c.typesenseClient
+}
+
 func (c *Container) GetSourceClient() app.SourceClient {
 	if c.sourceClient != nil {
 		return c.sourceClient