Fix has_expiration and other bugs, add more filters

docs/spec/paths/integrations@rarime-points-svc@v1@public@event_types.yaml

@@ -17,6 +17,16 @@ get:
         items:
           type: string
           example: "passport_scan"
+    - in: query
+      name: 'filter[name][not]'
+      description: |
+        Inverted filter by type name: excludes provided values
+      required: false
+      schema:
+        type: array
+        items:
+          type: string
+          example: "referral_specific"
     - in: query
       name: 'filter[flag]'
       description: Filter by configuration flags. Values are disjunctive (OR).

docs/spec/paths/integrations@rarime-points-svc@v1@public@events.yaml

@@ -28,6 +28,16 @@ get:
         items:
           type: string
           example: "passport_scan"
+    - in: query
+      name: 'filter[meta.static.name][not]'
+      description: |
+        Inverted filter by event type name: excludes provided values
+      required: false
+      schema:
+        type: array
+        items:
+          type: string
+          example: "referral_specific"
     - in: query
       name: 'filter[has_expiration]'
       description: Filter events by type which has or hasn't expiration.

internal/data/events.go

@@ -61,6 +61,7 @@ type EventsQ interface {
 	FilterByNullifier(string) EventsQ
 	FilterByStatus(...EventStatus) EventsQ
 	FilterByType(...string) EventsQ
+	FilterByNotType(types ...string) EventsQ
 	FilterByUpdatedAtBefore(int64) EventsQ
 	FilterByExternalID(string) EventsQ
 	FilterInactiveNotClaimed(types ...string) EventsQ

internal/data/pg/events.go

@@ -168,12 +168,12 @@ func (q *events) SelectAbsentTypes(allTypes ...string) ([]data.ReopenableEvent,
 		)
 		SELECT u.nullifier, t.type
 		FROM (
-    		SELECT DISTINCT nullifier FROM %s
+    		SELECT nullifier FROM %s
 		) u
 		CROSS JOIN types t
 		LEFT JOIN %s e ON e.nullifier = u.nullifier AND e.type = t.type
 		WHERE e.type IS NULL;
-	`, strings.Join(values, ", "), eventsTable, eventsTable)
+	`, strings.Join(values, ", "), balancesTable, eventsTable)
 
 	var res []data.ReopenableEvent
 	if err := q.db.SelectRaw(&res, query); err != nil {
@@ -208,6 +208,13 @@ func (q *events) FilterByType(types ...string) data.EventsQ {
 	return q.applyCondition(squirrel.Eq{"type": types})
 }
 
+func (q *events) FilterByNotType(types ...string) data.EventsQ {
+	if len(types) == 0 {
+		return q
+	}
+	return q.applyCondition(squirrel.NotEq{"type": types})
+}
+
 func (q *events) FilterByExternalID(id string) data.EventsQ {
 	return q.applyCondition(squirrel.Eq{"external_id": id})
 }

internal/service/handlers/list_event_types.go

@@ -20,6 +20,9 @@ func ListEventTypes(w http.ResponseWriter, r *http.Request) {
 	types := EventTypes(r).List(
 		evtypes.FilterByNames(req.FilterName...),
 		evtypes.FilterByFlags(req.FilterFlag...),
+		func(ev evtypes.EventConfig) bool {
+			return len(req.FilterNotName) > 0 && !evtypes.FilterByNames(req.FilterNotName...)(ev)
+		},
 	)
 
 	resTypes := make([]resources.EventType, len(types))

internal/service/handlers/list_events.go

@@ -33,7 +33,13 @@ func ListEvents(w http.ResponseWriter, r *http.Request) {
 			filter = func(ev evtypes.EventConfig) bool { return ev.ExpiresAt == nil }
 		}
 
-		req.FilterType = append(req.FilterType, EventTypes(r).Names(filter)...)
+		types := EventTypes(r).Names(filter)
+		if len(types) == 0 {
+			// filter won't be correctly applied if there are no types matching the condition
+			ape.Render(w, newEventsResponse(nil, nil))
+			return
+		}
+		req.FilterType = append(req.FilterType, types...)
 	}
 
 	inactiveTypes := EventTypes(r).Names(func(ev evtypes.EventConfig) bool {
@@ -44,6 +50,7 @@ func ListEvents(w http.ResponseWriter, r *http.Request) {
 		FilterByNullifier(*req.FilterNullifier).
 		FilterByStatus(req.FilterStatus...).
 		FilterByType(req.FilterType...).
+		FilterByNotType(req.FilterNotType...).
 		FilterInactiveNotClaimed(inactiveTypes...).
 		Page(&req.OffsetPageParams).
 		Select()

internal/service/handlers/verify_passport.go

@@ -145,15 +145,11 @@ func doPassportScanUpdates(r *http.Request, balance data.Balance, proof zkptypes
 	}
 
 	// Type not filtered as inactive because expired events can be claimed
-	evTypeRef := EventTypes(r).Get(evtypes.TypeReferralSpecific)
+	evTypeRef := EventTypes(r).Get(evtypes.TypeReferralSpecific, evtypes.FilterInactive)
 	if evTypeRef == nil {
 		Log(r).Debug("Referral specific event type is inactive")
 		return nil
 	}
-	if evTypeRef.Disabled {
-		Log(r).Infof("Event type %s is disabled", evtypes.TypeReferralSpecific)
-		return nil
-	}
 
 	// Claim events for invited friends who scanned the passport.
 	// This is possible when the user registered in the referral
@@ -169,11 +165,6 @@ func doPassportScanUpdates(r *http.Request, balance data.Balance, proof zkptypes
 		return fmt.Errorf("failed to claim referral specific events: %w", err)
 	}
 
-	if evtypes.FilterInactive(*evTypeRef) {
-		Log(r).Debug("Referral specific event type is inactive: event not added")
-		return nil
-	}
-
 	// Adds a friend event for the referrer. If the event
 	// is inactive, then nothing happens. If active, the
 	// fulfilled event is added and, if possible, the event claimed
@@ -281,9 +272,11 @@ func claimReferralSpecificEvents(r *http.Request, evTypeRef *evtypes.EventConfig
 		return nil
 	}
 
-	events, err := EventsQ(r).FilterByNullifier(balance.Nullifier).
+	events, err := EventsQ(r).
+		FilterByNullifier(balance.Nullifier).
 		FilterByType(evtypes.TypeReferralSpecific).
-		FilterByStatus(data.EventFulfilled).Select()
+		FilterByStatus(data.EventFulfilled).
+		Select()
 	if err != nil {
 		return fmt.Errorf("get fulfilled referral specific events: %w", err)
 	}

internal/service/requests/list_event_types.go

@@ -9,8 +9,9 @@ import (
 )
 
 type ListExpiredEvents struct {
-	FilterName []string `filter:"name"`
-	FilterFlag []string `filter:"flag"`
+	FilterName    []string `filter:"name"`
+	FilterFlag    []string `filter:"flag"`
+	FilterNotName []string `url:"filter[name][not]"`
 }
 
 func NewListEventTypes(r *http.Request) (req ListExpiredEvents, err error) {
@@ -25,7 +26,9 @@ func NewListEventTypes(r *http.Request) (req ListExpiredEvents, err error) {
 			evtypes.FlagNotStarted,
 			evtypes.FlagExpired,
 			evtypes.FlagDisabled,
-		)))}.Filter()
+		))),
+		"filter[name][not]": val.Validate(req.FilterNotName, val.When(len(req.FilterName) > 0, val.Nil, val.Empty)),
+	}.Filter()
 
 	return
 }

internal/service/requests/list_events.go

@@ -4,7 +4,7 @@ import (
 	"net/http"
 	"strings"
 
-	validation "github.com/go-ozzo/ozzo-validation/v4"
+	val "github.com/go-ozzo/ozzo-validation/v4"
 	"github.com/rarimo/rarime-points-svc/internal/data"
 	"github.com/rarimo/rarime-points-svc/internal/service/page"
 	"gitlab.com/distributed_lab/urlval/v4"
@@ -16,6 +16,7 @@ type ListEvents struct {
 	FilterStatus        []data.EventStatus `filter:"status"`
 	FilterType          []string           `filter:"meta.static.name"`
 	FilterHasExpiration *bool              `filter:"has_expiration"`
+	FilterNotType       []string           `url:"filter[meta.static.name][not]"`
 	Count               bool               `url:"count"`
 }
 
@@ -32,9 +33,10 @@ func NewListEvents(r *http.Request) (req ListEvents, err error) {
 		*req.FilterNullifier = strings.ToLower(*req.FilterNullifier)
 	}
 
-	err = validation.Errors{
-		"filter[nullifier]": validation.Validate(req.FilterNullifier, validation.Required, validation.Match(nullifierRegexp)),
-		"filter[status]":    validation.Validate(req.FilterStatus, validation.Each(validation.In(data.EventOpen, data.EventFulfilled, data.EventClaimed))),
+	err = val.Errors{
+		"filter[nullifier]":             val.Validate(req.FilterNullifier, val.Required, val.Match(nullifierRegexp)),
+		"filter[status]":                val.Validate(req.FilterStatus, val.Each(val.In(data.EventOpen, data.EventFulfilled, data.EventClaimed))),
+		"filter[meta.static.name][not]": val.Validate(req.FilterNotType, val.When(len(req.FilterType) > 0, val.Nil, val.Empty)),
 	}.Filter()
 	return
 }

internal/service/router.go

@@ -27,23 +27,23 @@ func Run(ctx context.Context, cfg config.Config) {
 	)
 	r.Route("/integrations/rarime-points-svc/v1", func(r chi.Router) {
 		r.Route("/public", func(r chi.Router) {
-			r.Route("/balances/{nullifier}", func(r chi.Router) {
+			r.Route("/balances", func(r chi.Router) {
 				r.Use(handlers.AuthMiddleware(cfg.Auth(), cfg.Log()))
-				r.Get("/", handlers.GetBalance)
-				r.Post("/verifypassport", handlers.VerifyPassport)
-				r.Get("/withdrawals", handlers.ListWithdrawals)
-				r.Post("/withdrawals", handlers.Withdraw)
+				r.Post("/", handlers.CreateBalance)
+				r.Route("/{nullifier}", func(r chi.Router) {
+					r.Get("/", handlers.GetBalance)
+					r.Post("/verifypassport", handlers.VerifyPassport)
+					r.Get("/withdrawals", handlers.ListWithdrawals)
+					r.Post("/withdrawals", handlers.Withdraw)
+				})
 			})
 			r.Route("/events", func(r chi.Router) {
 				r.Use(handlers.AuthMiddleware(cfg.Auth(), cfg.Log()))
 				r.Get("/", handlers.ListEvents)
 				r.Get("/{id}", handlers.GetEvent)
 				r.Patch("/{id}", handlers.ClaimEvent)
 			})
-			r.Route("/balances", func(r chi.Router) {
-				r.Get("/", handlers.Leaderboard)
-				r.Post("/", handlers.CreateBalance)
-			})
+			r.Get("/balances", handlers.Leaderboard)
 			r.Get("/point_price", handlers.GetPointPrice)
 			r.Get("/countries_config", handlers.GetCountriesConfig)
 			r.Get("/event_types", handlers.ListEventTypes)

internal/service/workers/nooneisforgotten/main.go

@@ -1,6 +1,7 @@
 package nooneisforgotten
 
 import (
+	"errors"
 	"fmt"
 	"math"
 	"sort"
@@ -30,7 +31,7 @@ func Run(cfg config.Config, sig chan struct{}) {
 	if err := pg.NewEvents(db).Transaction(func() error {
 		return claimReferralSpecificEvents(db, cfg.EventTypes(), cfg.Levels())
 	}); err != nil {
-		panic(fmt.Errorf("failed to claim referral specific events"))
+		panic(fmt.Errorf("failed to claim referral specific events: %w", err))
 	}
 
 	sig <- struct{}{}
@@ -55,7 +56,7 @@ func updatePassportScanEvents(db *pgdb.DB, types evtypes.Types, levels config.Le
 		return nil
 	}
 
-	if !evType.AutoClaim && evtypes.FilterInactive(*evType) {
+	if evtypes.FilterInactive(*evType) {
 		return nil
 	}
 
@@ -94,6 +95,10 @@ func updatePassportScanEvents(db *pgdb.DB, types evtypes.Types, levels config.Le
 		return nil
 	}
 
+	if len(countriesList) == 0 {
+		return nil
+	}
+
 	countries, err := pg.NewCountries(db).FilterByCodes(countriesList...).Select()
 	if err != nil {
 		return fmt.Errorf("failed to select countries: %w", err)
@@ -231,10 +236,6 @@ func claimReferralSpecificEvents(db *pgdb.DB, types evtypes.Types, levels config
 		return fmt.Errorf("failed to select passport scan events: %w", err)
 	}
 
-	if len(events) == 0 {
-		return nil
-	}
-
 	// we need to have maps which link nullifiers to events slice and countries to balances slice
 	nullifiersEventsMap := make(map[string][]data.Event, len(events))
 	nullifiers := make([]string, 0, len(events))
@@ -246,16 +247,23 @@ func claimReferralSpecificEvents(db *pgdb.DB, types evtypes.Types, levels config
 		nullifiersEventsMap[event.Nullifier] = append(nullifiersEventsMap[event.Nullifier], event)
 	}
 
-	balances, err := pg.NewBalances(db).FilterByNullifier(nullifiers...).FilterDisabled().Select()
+	if len(nullifiers) == 0 {
+		return nil
+	}
+
+	balances, err := pg.NewBalances(db).FilterByNullifier(nullifiers...).Select()
 	if err != nil {
 		return fmt.Errorf("failed to select balances for claim passport scan event: %w", err)
 	}
 	if len(balances) == 0 {
-		return fmt.Errorf("critical: events present, but no balances with nullifier")
+		return errors.New("critical: events present, but no balances with nullifier")
 	}
 
 	countriesBalancesMap := make(map[string][]data.Balance, len(balances))
 	for _, balance := range balances {
+		if !balance.ReferredBy.Valid {
+			continue
+		}
 		// country can't be nil because of db query logic
 		if _, ok := countriesBalancesMap[*balance.Country]; !ok {
 			countriesBalancesMap[*balance.Country] = make([]data.Balance, 0, len(balances))
@@ -312,6 +320,10 @@ func claimReferralSpecificEvents(db *pgdb.DB, types evtypes.Types, levels config
 		}
 	}
 
+	if len(toClaim) == 0 {
+		return nil
+	}
+
 	_, err = pg.NewEvents(db).FilterByID(toClaim...).Update(data.EventClaimed, nil, &evType.Reward)
 	if err != nil {
 		return fmt.Errorf("update event status: %w", err)