diff --git a/.env.dist b/.env.dist index dcd9358..08dbd48 100644 --- a/.env.dist +++ b/.env.dist @@ -6,3 +6,6 @@ export UNCONDITIONAL_API_SOURCE_CLIENT_KEY="secret" export UNCONDITIONAL_API_LOG_ENV="dev" export UNCONDITIONAL_API_BUILD_COMMIT_VERSION= export UNCONDITIONAL_API_BUILD_RELEASE_VERSION= +export UNCONDITIONAL_API_FEED_REPO_INDEX="feeds" +export UNCONDITIONAL_API_FEED_REPO_HOST="http://localhost:8108" +export UNCONDITIONAL_API_FEED_REPO_KEY="xyz" diff --git a/.envrc.dist b/.envrc.dist index dcd9358..08dbd48 100644 --- a/.envrc.dist +++ b/.envrc.dist @@ -6,3 +6,6 @@ export UNCONDITIONAL_API_SOURCE_CLIENT_KEY="secret" export UNCONDITIONAL_API_LOG_ENV="dev" export UNCONDITIONAL_API_BUILD_COMMIT_VERSION= export UNCONDITIONAL_API_BUILD_RELEASE_VERSION= +export UNCONDITIONAL_API_FEED_REPO_INDEX="feeds" +export UNCONDITIONAL_API_FEED_REPO_HOST="http://localhost:8108" +export UNCONDITIONAL_API_FEED_REPO_KEY="xyz" diff --git a/Dockerfile b/Dockerfile index c649ae1..5547a83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,15 +26,24 @@ WORKDIR /data COPY --from=builder /app/main /app/main ARG UNCONDITIONAL_API_SOURCE_REPO +ARG UNCONDITIONAL_API_SOURCE_CLIENT_KEY +ARG UNCONDITIONAL_API_FEED_REPO_INDEX +ARG UNCONDITIONAL_API_FEED_REPO_HOST +ARG UNCONDITIONAL_API_FEED_REPO_KEY ARG UNCONDITIONAL_API_LOG_ENV + ENV UNCONDITIONAL_API_SOURCE_REPO=${UNCONDITIONAL_API_SOURCE_REPO} +ENV UNCONDITIONAL_API_SOURCE_CLIENT_KEY=${UNCONDITIONAL_API_SOURCE_CLIENT_KEY} +ENV UNCONDITIONAL_API_FEED_REPO_INDEX=${UNCONDITIONAL_API_FEED_REPO_INDEX} +ENV UNCONDITIONAL_API_FEED_REPO_HOST=${UNCONDITIONAL_API_FEED_REPO_HOST} +ENV UNCONDITIONAL_API_FEED_REPO_KEY=${UNCONDITIONAL_API_FEED_REPO_KEY} ENV UNCONDITIONAL_API_LOG_ENV=${UNCONDITIONAL_API_LOG_ENV} RUN --mount=type=secret,id=UNCONDITIONAL_API_SOURCE_CLIENT_KEY \ + --mount=type=secret,id=UNCONDITIONAL_API_FEED_REPO_KEY \ UNCONDITIONAL_API_SOURCE_CLIENT_KEY="$(cat /run/secrets/UNCONDITIONAL_API_SOURCE_CLIENT_KEY)" \ - /app/main source download --path /data/source.json - -RUN /app/main index create --source /data/source.json --name /data/index + UNCONDITIONAL_API_FEED_REPO_KEY="$(cat /run/secrets/UNCONDITIONAL_API_FEED_REPO_KEY)" \ + /app/main index create --name feeds FROM scratch as release COPY --from=certificator /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ @@ -47,6 +56,9 @@ ARG UNCONDITIONAL_API_PORT ARG UNCONDITIONAL_API_SOURCE_REPO ARG UNCONDITIONAL_API_SOURCE_CLIENT_KEY ARG UNCONDITIONAL_API_LOG_ENV +ARG UNCONDITIONAL_API_FEED_REPO_INDEX +ARG UNCONDITIONAL_API_FEED_REPO_HOST +ARG UNCONDITIONAL_API_FEED_REPO_KEY ENV UNCONDITIONAL_API_ADDRESS=${UNCONDITIONAL_API_ADDRESS} ENV UNCONDITIONAL_API_ALLOWED_ORIGINS=${UNCONDITIONAL_API_ALLOWED_ORIGINS} @@ -54,5 +66,8 @@ ENV UNCONDITIONAL_API_PORT=${UNCONDITIONAL_API_PORT} ENV UNCONDITIONAL_API_SOURCE_REPO=${UNCONDITIONAL_API_SOURCE_REPO} ENV UNCONDITIONAL_API_SOURCE_CLIENT_KEY=${UNCONDITIONAL_API_SOURCE_CLIENT_KEY} ENV UNCONDITIONAL_API_LOG_ENV=${UNCONDITIONAL_API_LOG_ENV} +ENV UNCONDITIONAL_API_FEED_REPO_INDEX=${UNCONDITIONAL_API_FEED_REPO_INDEX} +ENV UNCONDITIONAL_API_FEED_REPO_HOST=${UNCONDITIONAL_API_FEED_REPO_HOST} +ENV UNCONDITIONAL_API_FEED_REPO_KEY=${UNCONDITIONAL_API_FEED_REPO_KEY} -ENTRYPOINT ["./app/main","serve", "--address", "0.0.0.0", "--port","8080", "--index","/data/index"] +ENTRYPOINT ["./app/main","serve", "--address", "0.0.0.0", "--port","8080"] diff --git a/cmd/index/create.go b/cmd/index/create.go index 1164207..c75c286 100644 --- a/cmd/index/create.go +++ b/cmd/index/create.go @@ -6,18 +6,21 @@ import ( "github.com/spf13/cobra" "go.uber.org/zap" - "github.com/unconditionalday/server/internal/app" "github.com/unconditionalday/server/internal/container" + "github.com/unconditionalday/server/internal/repository/typesense" "github.com/unconditionalday/server/internal/service" cobrax "github.com/unconditionalday/server/internal/x/cobra" - iox "github.com/unconditionalday/server/internal/x/io" + typesensex "github.com/unconditionalday/server/internal/x/typesense" ) var ( - ErrIndexNotProvided = errors.New("index not provided, please provide it using --index flag") - ErrSourceNotProvided = errors.New("source not provided, please provide it using --source flag") - ErrSourceClientKeyNotProvided = errors.New("source client-key not provided, please provide it using --source-client-key flag") - ErrLogEnvNotProvided = errors.New("log-env not provided, please provide it using --log-env flag") + ErrIndexNotProvided = errors.New("index not provided, please provide it using --index flag") + ErrFeedRepoHostNotProvided = errors.New("feed repository host not provided, please provide it using --feed-repo-host flag") + ErrFeedRepoKeyNotProvided = errors.New("feed repository key not provided, please provide it using --feed-repo-key flag") + ErrSourceNotProvided = errors.New("source not provided, please provide it using --source flag") + ErrSourceClientKeyNotProvided = errors.New("source client-key not provided, please provide it using --source-client-key flag") + ErrSourceRepositoryNotProvided = errors.New("source repo not provided, please provide it using --source-repo flag") + ErrLogEnvNotProvided = errors.New("log-env not provided, please provide it using --log-env flag") ) func NewCreateCommand() *cobra.Command { @@ -31,9 +34,24 @@ func NewCreateCommand() *cobra.Command { return ErrIndexNotProvided } - s := cobrax.Flag[string](cmd, "source").(string) - if s == "" { - return ErrSourceNotProvided + frh := cobrax.Flag[string](cmd, "feed-repo-host").(string) + if i == "" { + return ErrFeedRepoHostNotProvided + } + + frk := cobrax.Flag[string](cmd, "feed-repo-key").(string) + if frk == "" { + return ErrFeedRepoKeyNotProvided + } + + sr := cobrax.Flag[string](cmd, "source-repo").(string) + if sr == "" { + return ErrSourceRepositoryNotProvided + } + + sk := cobrax.Flag[string](cmd, "source-client-key").(string) + if sk == "" { + return ErrSourceClientKeyNotProvided } l := cobrax.Flag[string](cmd, "log-env").(string) @@ -42,20 +60,30 @@ func NewCreateCommand() *cobra.Command { } params := container.NewDefaultParameters() - params.FeedIndex = i + params.FeedRepositoryIndex = i + params.FeedRepositoryHost = frh + params.FeedRepositoryKey = frk + params.SourceClientKey = sk + params.SourceRepository = sr params.LogEnv = l c, _ := container.NewContainer(params) + t := c.GetTypesenseClient() + + feedSchema := typesense.GetFeedSchema(t) + if err := typesensex.CreateOrUpdateCollection(t, feedSchema); err != nil { + return err + } + sourceService := service.NewSource(c.GetSourceClient(), c.GetParser(), c.GetVersioning(), c.GetLogger()) - var source app.Source - source, err := iox.ReadJSON(s, source) + s, err := sourceService.Fetch() if err != nil { return err } - feeds, err := sourceService.FetchFeeds(source) + feeds, err := sourceService.FetchFeeds(s.Data) if err != nil { c.GetLogger().Error("Can't fetch feeds", zap.Error(err)) } @@ -75,9 +103,12 @@ func NewCreateCommand() *cobra.Command { }, } - cmd.Flags().StringP("source", "s", "", "Source Path") cmd.Flags().StringP("name", "n", "", "Index Name") cmd.Flags().StringP("log-env", "l", "", "Log Env") + cmd.Flags().StringP("feed-repo-host", "", "", "Feed's repository host") + cmd.Flags().StringP("feed-repo-key", "", "", "Feed's repository API's key") + cmd.Flags().String("source-repo", "", "Source Repository") + cmd.Flags().String("source-client-key", "", "Source Client Key") envPrefix := "UNCONDITIONAL_API" cobrax.BindFlags(cmd, cobrax.InitEnvs(envPrefix), envPrefix) diff --git a/cmd/serve.go b/cmd/serve.go index a1ce61e..3e3adec 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -13,7 +13,9 @@ import ( ) var ( - ErrIndexNotProvided = errors.New("index not provided, please provide it using --index flag") + ErrFeedRepoIndexNotProvided = errors.New("feed repository index not provided, please provide it using --feed-repo-index flag") + ErrFeedRepoHostNotProvided = errors.New("feed repository host not provided, please provide it using --feed-repo-host flag") + ErrFeedRepoKeyNotProvided = errors.New("feed repository key not provided, please provide it using --feed-repo-key flag") ErrAddressNotProvided = errors.New("server address not provided, please provide it using --address flag") ErrPortNotProvided = errors.New("server port not provided, please provide it using --port flag") ErrLogEnvNotProvided = errors.New("server log-env not provided, please provide it using --log-env flag") @@ -22,15 +24,25 @@ var ( ErrAllowedOriginsNotProvided = errors.New("server allowed origins not provided, please provide it using --allowed-origins flag") ) -func NewServeCommand(version version.Build) *cobra.Command { +func NewServeCommand(buildVersion version.Build) *cobra.Command { cmd := &cobra.Command{ Use: "serve", Short: "Starts the server", Long: `Starts the server`, RunE: func(cmd *cobra.Command, _ []string) error { - i := cobrax.Flag[string](cmd, "index").(string) + i := cobrax.Flag[string](cmd, "feed-repo-index").(string) if i == "" { - return ErrIndexNotProvided + return ErrFeedRepoIndexNotProvided + } + + frh := cobrax.Flag[string](cmd, "feed-repo-host").(string) + if i == "" { + return ErrFeedRepoHostNotProvided + } + + frk := cobrax.Flag[string](cmd, "feed-repo-key").(string) + if i == "" { + return ErrFeedRepoKeyNotProvided } l := cobrax.Flag[string](cmd, "log-env").(string) @@ -44,7 +56,7 @@ func NewServeCommand(version version.Build) *cobra.Command { } sk := cobrax.Flag[string](cmd, "source-client-key").(string) - if s == "" { + if sk == "" { return ErrSourceClientKeyNotProvided } @@ -63,7 +75,7 @@ func NewServeCommand(version version.Build) *cobra.Command { return ErrAllowedOriginsNotProvided } - params := container.NewParameters(a, i, s, sk, l, p, ao, version) + params := container.NewParameters(a, i, frh, frk, s, sk, l, p, ao, buildVersion) c, _ := container.NewContainer(params) sourceService := service.NewSource(c.GetSourceClient(), c.GetParser(), c.GetVersioning(), c.GetLogger()) @@ -83,7 +95,9 @@ func NewServeCommand(version version.Build) *cobra.Command { cmd.Flags().StringP("address", "a", "localhost", "Server address") cmd.Flags().IntP("port", "p", 8080, "Server port") - cmd.Flags().StringP("index", "s", "", "Index path") + cmd.Flags().StringP("feed-repo-index", "s", "", "Index path") + cmd.Flags().StringP("feed-repo-host", "", "", "Feed's repository host") + cmd.Flags().StringP("feed-repo-key", "", "", "Feed's repository API's key") cmd.Flags().String("allowed-origins", "", "Allowed Origins") cmd.Flags().String("source-repo", "", "Source Repository") cmd.Flags().String("source-client-key", "", "Source Client Key") diff --git a/go.mod b/go.mod index 906248d..371edfa 100644 --- a/go.mod +++ b/go.mod @@ -96,6 +96,7 @@ require ( github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect + github.com/sony/gobreaker v0.5.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect @@ -129,5 +130,6 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 + github.com/typesense/typesense-go v1.0.0 golang.org/x/sys v0.15.0 // indirect ) diff --git a/go.sum b/go.sum index e5f3bdb..6f50c94 100644 --- a/go.sum +++ b/go.sum @@ -239,6 +239,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -275,6 +277,8 @@ github.com/tdewolff/test v1.0.9 h1:SswqJCmeN4B+9gEAi/5uqT0qpi1y2/2O47V/1hhGZT0= github.com/tdewolff/test v1.0.9/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/typesense/typesense-go v1.0.0 h1:/8Lr1yf9YjmUKdn/xbTNy+OhwOvBd0noBTRkcB22Uhw= +github.com/typesense/typesense-go v1.0.0/go.mod h1:4mq4FYHzU7csU/KHaZoyG2bCSKl7GrCeyAr2YhXT1/0= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= diff --git a/internal/container/container.go b/internal/container/container.go index 63fceb2..2bb181c 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -1,22 +1,19 @@ package container import ( - "errors" "net/http" - "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/mapping" "go.uber.org/zap" + "github.com/typesense/typesense-go/typesense" "github.com/unconditionalday/server/internal/app" "github.com/unconditionalday/server/internal/client/github" "github.com/unconditionalday/server/internal/client/wikipedia" "github.com/unconditionalday/server/internal/parser" - bleveRepo "github.com/unconditionalday/server/internal/repository/bleve" + typesenseRepo "github.com/unconditionalday/server/internal/repository/typesense" "github.com/unconditionalday/server/internal/search" "github.com/unconditionalday/server/internal/version" "github.com/unconditionalday/server/internal/webserver" - blevex "github.com/unconditionalday/server/internal/x/bleve" calverx "github.com/unconditionalday/server/internal/x/calver" netx "github.com/unconditionalday/server/internal/x/net" ) @@ -28,12 +25,12 @@ func NewDefaultParameters() Parameters { ServerAllowedOrigins: []string{"*"}, SourceRepository: "source", SourceClientKey: "secret", - FeedIndex: "feed.index", + FeedRepositoryIndex: "feed.index", LogEnv: "dev", } } -func NewParameters(serverAddress, feedIndex, sourceRepository, sourceClientKey, logEnv string, serverPort int, serverAllowedOrigins []string, buildVersion version.Build) Parameters { +func NewParameters(serverAddress, feedIndex, feedRepoHost, feedRepoKey, sourceRepository, sourceClientKey, logEnv string, serverPort int, serverAllowedOrigins []string, buildVersion version.Build) Parameters { return Parameters{ ServerAddress: serverAddress, ServerPort: serverPort, @@ -44,7 +41,9 @@ func NewParameters(serverAddress, feedIndex, sourceRepository, sourceClientKey, BuildVersion: buildVersion, - FeedIndex: feedIndex, + FeedRepositoryIndex: feedIndex, + FeedRepositoryHost: feedRepoHost, + FeedRepositoryKey: feedRepoKey, LogEnv: logEnv, } @@ -61,20 +60,23 @@ type Parameters struct { BuildVersion version.Build - FeedIndex string + FeedRepositoryIndex string + FeedRepositoryHost string + FeedRepositoryKey string LogEnv string } type Services struct { - apiServer *webserver.Server - feedRepository *bleveRepo.FeedRepository - sourceClient *github.Client - searchClient *wikipedia.Client - httpClient *netx.HttpClient - logger *zap.Logger - parser *parser.Parser - versioning *calverx.CalVer + apiServer *webserver.Server + feedRepository *typesenseRepo.FeedRepository + sourceClient *github.Client + searchClient *wikipedia.Client + httpClient *netx.HttpClient + typesenseClient *typesense.Client + logger *zap.Logger + parser *parser.Parser + versioning *calverx.CalVer } func NewContainer(p Parameters) (*Container, error) { @@ -104,23 +106,29 @@ func (c *Container) GetFeedRepository() app.FeedRepository { return c.feedRepository } - b, err := blevex.NewIndex(c.FeedIndex, mapping.NewIndexMapping()) - if err != nil { - if errors.Is(bleve.ErrorIndexPathExists, err) { - b, err = blevex.New(c.FeedIndex) - if err != nil { - panic(err) - } - } else { - panic(err) - } - } + client := typesense.NewClient( + typesense.WithServer(c.FeedRepositoryHost), + typesense.WithAPIKey(c.FeedRepositoryKey)) - c.feedRepository = bleveRepo.NewFeedRepository(b) + c.feedRepository = typesenseRepo.NewFeedRepository(client) return c.feedRepository } +func (c *Container) GetTypesenseClient() *typesense.Client { + if c.typesenseClient != nil { + return c.typesenseClient + } + + client := typesense.NewClient( + typesense.WithServer(c.FeedRepositoryHost), + typesense.WithAPIKey(c.FeedRepositoryKey)) + + c.typesenseClient = client + + return c.typesenseClient +} + func (c *Container) GetSourceClient() app.SourceClient { if c.sourceClient != nil { return c.sourceClient diff --git a/internal/repository/typesense/feed.go b/internal/repository/typesense/feed.go new file mode 100644 index 0000000..cf80f62 --- /dev/null +++ b/internal/repository/typesense/feed.go @@ -0,0 +1,113 @@ +package typesense + +import ( + "context" + "time" + + "github.com/typesense/typesense-go/typesense" + "github.com/typesense/typesense-go/typesense/api" + "github.com/unconditionalday/server/internal/app" +) + +type FeedRepository struct { + client *typesense.Client + ctx context.Context +} + +func NewFeedRepository(client *typesense.Client) *FeedRepository { + return &FeedRepository{ + client: client, + ctx: context.Background(), + } +} + +func (f *FeedRepository) Find(query string) ([]app.Feed, error) { + searchParameters := &api.SearchCollectionParams{ + Q: query, + QueryBy: "title, summary", + } + searchResult, err := f.client.Collection("feeds").Documents().Search(f.ctx, searchParameters) + if err != nil { + return nil, err + } + + feeds := make([]app.Feed, len(*searchResult.Hits)) + for i, x := range *searchResult.Hits { + doc := *x.Document + + date, err := time.Parse(time.RFC3339, doc["date"].(string)) + if err != nil { + return nil, err + } + + f := app.Feed{ + Title: doc["title"].(string), + Link: doc["link"].(string), + Source: doc["source"].(string), + Language: doc["language"].(string), + Summary: doc["summary"].(string), + Date: date, + } + + feeds[i] = f + } + + return feeds, nil +} + +func (f *FeedRepository) Save(doc app.Feed) error { + docMap := map[string]interface{}{ + "title": doc.Title, + "link": doc.Link, + "source": doc.Source, + "language": doc.Language, + "summary": doc.Summary, + "date": doc.Date.Format(time.RFC3339), + } + + // Perform the save/indexing operation + _, err := f.client.Collection("feeds").Documents().Create(f.ctx, docMap) + if err != nil { + return err + } + + return nil +} + +func (f *FeedRepository) Update(doc app.Feed) error { + // Convert app.Feed to map[string]interface{} for updating + docMap := map[string]interface{}{ + "title": doc.Title, + "link": doc.Link, + "source": doc.Source, + "language": doc.Language, + "summary": doc.Summary, + "date": doc.Date.Format(time.RFC3339), + } + + // Perform the update operation + _, err := f.client.Collection("feeds").Documents().Upsert(f.ctx, docMap) + if err != nil { + return err + } + + return nil +} + +func (f *FeedRepository) Count() uint64 { + // Perform the operation to get document count + coll, err := f.client.Collection("feeds").Retrieve(f.ctx) + if err != nil || coll.NumDocuments == nil { + return 0 + } + + return uint64(*coll.NumDocuments) +} + +func (f *FeedRepository) Delete(doc app.Feed) error { + if _, err := f.client.Collection("feeds").Document(doc.Link).Delete(f.ctx); err != nil { + return err + } + + return nil +} diff --git a/internal/repository/typesense/schema.go b/internal/repository/typesense/schema.go new file mode 100644 index 0000000..2172b5e --- /dev/null +++ b/internal/repository/typesense/schema.go @@ -0,0 +1,63 @@ +package typesense + +import ( + "github.com/typesense/typesense-go/typesense" + "github.com/typesense/typesense-go/typesense/api" +) + +func GetFeedSchema(client *typesense.Client) *api.CollectionSchema { + schema := &api.CollectionSchema{ + Fields: []api.Field{ + { + Name: "title", + Type: "string", + }, + { + Name: "link", + Type: "string", + }, + { + Name: "source", + Type: "string", + }, + { + Name: "language", + Type: "string", + }, + { + Name: "summary", + Type: "string", + }, + { + Name: "title_summary_embedding", + Type: "float[]", + Embed: &struct { + From []string "json:\"from\"" + ModelConfig struct { + AccessToken *string "json:\"access_token,omitempty\"" + ApiKey *string "json:\"api_key,omitempty\"" + ClientId *string "json:\"client_id,omitempty\"" + ClientSecret *string "json:\"client_secret,omitempty\"" + ModelName string "json:\"model_name\"" + ProjectId *string "json:\"project_id,omitempty\"" + } "json:\"model_config\"" + }{ + From: []string{"title", "summary"}, + ModelConfig: struct { + AccessToken *string "json:\"access_token,omitempty\"" + ApiKey *string "json:\"api_key,omitempty\"" + ClientId *string "json:\"client_id,omitempty\"" + ClientSecret *string "json:\"client_secret,omitempty\"" + ModelName string "json:\"model_name\"" + ProjectId *string "json:\"project_id,omitempty\"" + }{ + ModelName: "ts/all-MiniLM-L12-v2", + }, + }, + }, + }, + Name: "feeds", + } + + return schema +} diff --git a/internal/service/index/typesense.go b/internal/service/index/typesense.go new file mode 100644 index 0000000..f035453 --- /dev/null +++ b/internal/service/index/typesense.go @@ -0,0 +1,138 @@ +package index + +import ( + "context" + + "github.com/typesense/typesense-go/typesense" + "github.com/typesense/typesense-go/typesense/api" + "github.com/unconditionalday/server/internal/app" +) + +type FeedService struct { + client *typesense.Client + ctx context.Context +} + +func (f *FeedService) Create(doc app.Feed) error { + schema := &api.CollectionSchema{ + Fields: []api.Field{ + { + Name: "title", + Type: "string", + }, + { + Name: "link", + Type: "string", + }, + { + Name: "source", + Type: "string", + }, + { + Name: "language", + Type: "string", + }, + { + Name: "summary", + Type: "string", + }, + { + Name: "title_summary_embedding", + Type: "float[]", + Embed: &struct { + From []string "json:\"from\"" + ModelConfig struct { + AccessToken *string "json:\"access_token,omitempty\"" + ApiKey *string "json:\"api_key,omitempty\"" + ClientId *string "json:\"client_id,omitempty\"" + ClientSecret *string "json:\"client_secret,omitempty\"" + ModelName string "json:\"model_name\"" + ProjectId *string "json:\"project_id,omitempty\"" + } "json:\"model_config\"" + }{ + From: []string{"title", "summary"}, + ModelConfig: struct { + AccessToken *string "json:\"access_token,omitempty\"" + ApiKey *string "json:\"api_key,omitempty\"" + ClientId *string "json:\"client_id,omitempty\"" + ClientSecret *string "json:\"client_secret,omitempty\"" + ModelName string "json:\"model_name\"" + ProjectId *string "json:\"project_id,omitempty\"" + }{ + ModelName: "ts/all-MiniLM-L12-v2", + }, + }, + }, + }, + Name: "feeds", + } + + if _, err := f.client.Collections().Create(context.Background(), schema); err != nil { + return err + } + + return nil +} + +func (f *FeedService) Migrate(doc app.Feed) error { + c := f.client.Collection("feeds") + s := &api.CollectionUpdateSchema{ + Fields: []api.Field{ + { + Name: "title", + Type: "string", + }, + { + Name: "link", + Type: "string", + }, + { + Name: "source", + Type: "string", + }, + { + Name: "language", + Type: "string", + }, + { + Name: "summary", + Type: "string", + }, + { + Name: "title_summary_embedding", + Type: "float[]", + Embed: &struct { + From []string "json:\"from\"" + ModelConfig struct { + AccessToken *string "json:\"access_token,omitempty\"" + ApiKey *string "json:\"api_key,omitempty\"" + ClientId *string "json:\"client_id,omitempty\"" + ClientSecret *string "json:\"client_secret,omitempty\"" + ModelName string "json:\"model_name\"" + ProjectId *string "json:\"project_id,omitempty\"" + } "json:\"model_config\"" + }{ + From: []string{"title", "summary"}, + ModelConfig: struct { + AccessToken *string "json:\"access_token,omitempty\"" + ApiKey *string "json:\"api_key,omitempty\"" + ClientId *string "json:\"client_id,omitempty\"" + ClientSecret *string "json:\"client_secret,omitempty\"" + ModelName string "json:\"model_name\"" + ProjectId *string "json:\"project_id,omitempty\"" + }{ + ModelName: "ts/all-MiniLM-L12-v2", + }, + }, + }, + }, + } + + _, err := c.Update(f.ctx, s) + + if err != nil { + return err + } + + return nil +} diff --git a/internal/x/typesense/schema.go b/internal/x/typesense/schema.go new file mode 100644 index 0000000..a1ca5f5 --- /dev/null +++ b/internal/x/typesense/schema.go @@ -0,0 +1,36 @@ +package typesense + +import ( + "context" + "errors" + "strings" + + "github.com/typesense/typesense-go/typesense" + "github.com/typesense/typesense-go/typesense/api" +) + +var ( + ErrCollectionAlreadyExists = errors.New("the collection already exists") +) + +func CreateOrUpdateCollection(client *typesense.Client, schema *api.CollectionSchema) error { + if _, err := client.Collections().Create(context.Background(), schema); err != nil { + if strings.Contains(err.Error(), "already exists") { + return updateCollection(client, schema) + } + } + + return nil +} + +func updateCollection(client *typesense.Client, schema *api.CollectionSchema) error { + u := &api.CollectionUpdateSchema{ + Fields: schema.Fields, + } + + if _, err := client.Collection(schema.Name).Update(context.Background(), u); err != nil{ + return err + } + + return nil +} diff --git a/scripts/deploy.sh b/scripts/deploy.sh old mode 100644 new mode 100755 index 20cb69a..c64f89b --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -8,6 +8,10 @@ if ! command -v flyctl &> /dev/null; then fi # Deploy -flyctl deploy --build-secret UNCONDITIONAL_API_SOURCE_CLIENT_KEY="$UNCONDITIONAL_API_SOURCE_CLIENT_KEY" \ - --build-arg UNCONDITIONAL_API_BUILD_COMMIT_VERSION="$UNCONDITIONAL_API_BUILD_COMMIT_VERSION" \ - --build-arg UNCONDITIONAL_API_BUILD_RELEASE_VERSION="$UNCONDITIONAL_API_BUILD_RELEASE_VERSION" \ +flyctl deploy \ + --build-secret UNCONDITIONAL_API_SOURCE_CLIENT_KEY="$UNCONDITIONAL_API_SOURCE_CLIENT_KEY" \ + --build-secret UNCONDITIONAL_API_FEED_REPO_KEY="$UNCONDITIONAL_API_FEED_REPO_KEY" \ + --build-arg UNCONDITIONAL_API_FEED_REPO_HOST="$UNCONDITIONAL_API_FEED_REPO_HOST" \ + --build-arg UNCONDITIONAL_API_FEED_REPO_INDEX="$UNCONDITIONAL_API_FEED_REPO_INDEX" \ + --build-arg UNCONDITIONAL_API_BUILD_COMMIT_VERSION="$UNCONDITIONAL_API_BUILD_COMMIT_VERSION" \ + --build-arg UNCONDITIONAL_API_BUILD_RELEASE_VERSION="$UNCONDITIONAL_API_BUILD_RELEASE_VERSION"