From 9f42b0a0e10efae679aba7819b9ec45a7028e172 Mon Sep 17 00:00:00 2001 From: Dinko Korunic Date: Mon, 22 Jan 2024 14:36:55 +0100 Subject: [PATCH] Rate limit refactorings - Swap unmaintained gopkg.in/mail.v2 for Shopify/gomail - Better rate-limiting with time windows/frequency for APIs and mail - Rate limit Gmail for 20 per hour (non-public information) --- go.mod | 5 +++-- go.sum | 8 ++++++-- messenger/calendar.go | 5 +++-- messenger/discord.go | 5 +++-- messenger/mail.go | 9 +++++---- messenger/slack.go | 5 +++-- messenger/telegram.go | 5 +++-- 7 files changed, 26 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index cfc5826..5a7f02b 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,9 @@ go 1.21.6 require ( github.com/BurntSushi/toml v1.3.2 - github.com/KimMachineGun/automemlimit v0.4.0 + github.com/KimMachineGun/automemlimit v0.5.0 github.com/PuerkitoBio/goquery v1.8.1 + github.com/Shopify/gomail v0.0.0-20220729171026-0784ece65e69 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/avast/retry-go/v4 v4.5.1 github.com/blang/semver/v4 v4.0.0 @@ -32,7 +33,6 @@ require ( go.uber.org/ratelimit v0.3.0 golang.org/x/oauth2 v0.16.0 google.golang.org/api v0.157.0 - gopkg.in/mail.v2 v2.3.1 ) require ( @@ -79,6 +79,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/go.sum b/go.sum index 743ac30..9e22a12 100644 --- a/go.sum +++ b/go.sum @@ -6,10 +6,12 @@ cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2Aawl github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/KimMachineGun/automemlimit v0.4.0 h1:qOjSDbAUENEL6fiKmRKuAVhPaLijpoEHFDTE+I+prp0= -github.com/KimMachineGun/automemlimit v0.4.0/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= +github.com/KimMachineGun/automemlimit v0.5.0 h1:BeOe+BbJc8L5chL3OwzVYjVzyvPALdd5wxVVOWuUZmQ= +github.com/KimMachineGun/automemlimit v0.5.0/go.mod h1:di3GCKiu9Y+1fs92erCbUvKzPkNyViN3mA0vti/ykEQ= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/Shopify/gomail v0.0.0-20220729171026-0784ece65e69 h1:gPoXdwo3sKq8qcfMu/Nc/wkJMLKwe7kaG9Uo8tOj3cU= +github.com/Shopify/gomail v0.0.0-20220729171026-0784ece65e69/go.mod h1:RS+Gaowa0M+gCuiFAiRMGBCMqxLrNA7TESTU/Wbblm8= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= @@ -198,6 +200,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/peterbourgon/ff/v4 v4.0.0-alpha.4 h1:aiqS8aBlF9PsAKeMddMSfbwp3smONCn3UO8QfUg0Z7Y= diff --git a/messenger/calendar.go b/messenger/calendar.go index d24d199..0b879b9 100644 --- a/messenger/calendar.go +++ b/messenger/calendar.go @@ -42,7 +42,8 @@ import ( const ( CalendarAPILimit = 5 // 5 req/s per user - CalendarMinDelay = 1 * time.Second / CalendarAPILimit + CalendarWindow = 1 * time.Second + CalendarMinDelay = CalendarWindow / CalendarAPILimit CalendarMaxResults = 100 CalendarCredentials = "assets/calendar_credentials.json" // embedded Google Calendar credentials file @@ -76,7 +77,7 @@ func Calendar(ctx context.Context, ch <-chan interface{}, name, tokFile string, logger.Debug().Msg("Creating exams with Google Calendar API") now := time.Now() - rl := ratelimit.New(CalendarAPILimit) + rl := ratelimit.New(CalendarAPILimit, ratelimit.Per(CalendarWindow)) // process all messages for o := range ch { diff --git a/messenger/discord.go b/messenger/discord.go index 46659ba..00cdd5c 100644 --- a/messenger/discord.go +++ b/messenger/discord.go @@ -38,7 +38,8 @@ import ( const ( DiscordAPILimit = 50 // 50 API req/s per user/IP - DiscordMinDelay = 1 * time.Second / DiscordAPILimit + DiscordWindow = 1 * time.Second + DiscordMinDelay = DiscordWindow / DiscordAPILimit ) var ( @@ -83,7 +84,7 @@ func Discord(ctx context.Context, ch <-chan interface{}, token string, userIDs [ logger.Debug().Msg("Sending a message through Discord") - rl := ratelimit.New(DiscordAPILimit) + rl := ratelimit.New(DiscordAPILimit, ratelimit.Per(DiscordWindow)) // process all messages for o := range ch { diff --git a/messenger/mail.go b/messenger/mail.go index 1e40119..9775825 100644 --- a/messenger/mail.go +++ b/messenger/mail.go @@ -27,17 +27,18 @@ import ( "strconv" "time" + mail "github.com/Shopify/gomail" "github.com/avast/retry-go/v4" "github.com/dkorunic/e-dnevnik-bot/format" "github.com/dkorunic/e-dnevnik-bot/logger" "github.com/dkorunic/e-dnevnik-bot/msgtypes" "go.uber.org/ratelimit" - "gopkg.in/mail.v2" ) const ( - MailSendLimit = 50 // 50 emails per second - MailMinDelay = 1 * time.Second / MailSendLimit + MailSendLimit = 20 // 20 emails per 1 hour + MailWindow = 1 * time.Hour + MailMinDelay = MailWindow / MailSendLimit MailSubject = "Nova ocjena iz e-Dnevnika" ) @@ -71,7 +72,7 @@ func Mail(ctx context.Context, ch <-chan interface{}, server, port, username, pa portInt = 465 } - rl := ratelimit.New(MailSendLimit) + rl := ratelimit.New(MailSendLimit, ratelimit.Per(MailWindow)) // process all messages for o := range ch { diff --git a/messenger/slack.go b/messenger/slack.go index f4eaf32..56f4090 100644 --- a/messenger/slack.go +++ b/messenger/slack.go @@ -37,7 +37,8 @@ import ( const ( SlackAPILImit = 1 // typically 1 req/s per user - SlackMinDelay = 1 * time.Second / SlackAPILImit + SlackWindow = 1 * time.Second + SlackMinDelay = SlackWindow / SlackAPILImit ) var ( @@ -68,7 +69,7 @@ func Slack(ctx context.Context, ch <-chan interface{}, token string, chatIDs []s logger.Debug().Msg("Sending message through Slack") - rl := ratelimit.New(SlackAPILImit) + rl := ratelimit.New(SlackAPILImit, ratelimit.Per(SlackWindow)) var err error diff --git a/messenger/telegram.go b/messenger/telegram.go index 50f2128..a816827 100644 --- a/messenger/telegram.go +++ b/messenger/telegram.go @@ -38,7 +38,8 @@ import ( const ( TelegramAPILimit = 30 // 30 API req/s per user - TelegramMinDelay = 1 * time.Second / TelegramAPILimit + TelegramWindow = 1 * time.Second + TelegramMinDelay = TelegramWindow / TelegramAPILimit ) var ( @@ -77,7 +78,7 @@ func Telegram(ctx context.Context, ch <-chan interface{}, apiKey string, chatIDs logger.Debug().Msg("Sending message through Telegram") - rl := ratelimit.New(TelegramAPILimit) + rl := ratelimit.New(TelegramAPILimit, ratelimit.Per(TelegramWindow)) // process all messages for o := range ch {