From 96a9f4aa915ddb47981088b13545d837f8d61c1a Mon Sep 17 00:00:00 2001 From: rxdn <29165304+rxdn@users.noreply.github.com> Date: Fri, 3 Sep 2021 11:44:12 +0100 Subject: [PATCH] Overhaul ratelimits --- app/http/endpoints/api/reloadguilds.go | 7 +- app/http/endpoints/api/ticket/closeticket.go | 4 +- .../whitelabelcreateinteractions.go | 6 +- .../api/whitelabel/whitelabelpost.go | 4 +- .../api/whitelabel/whitelabelstatuspost.go | 4 +- app/http/middleware/logging.go | 40 +++++ app/http/middleware/multireadbody.go | 12 ++ app/http/middleware/ratelimit.go | 109 ++++++++++++ app/http/server.go | 166 ++++++++---------- app/http/session/redisstore.go | 12 +- botcontext/botcontext.go | 4 +- botcontext/get.go | 4 +- cmd/panel/main.go | 21 ++- config/config.go | 3 + envvars.md | 1 + go.mod | 13 +- {messagequeue => redis}/panelcreate.go | 4 +- {messagequeue => redis}/redis.go | 11 +- {messagequeue => redis}/ticketclose.go | 4 +- rpc/ratelimit.go | 50 ++++++ 20 files changed, 348 insertions(+), 131 deletions(-) create mode 100644 app/http/middleware/logging.go create mode 100644 app/http/middleware/multireadbody.go create mode 100644 app/http/middleware/ratelimit.go rename {messagequeue => redis}/panelcreate.go (75%) rename {messagequeue => redis}/redis.go (68%) rename {messagequeue => redis}/ticketclose.go (86%) create mode 100644 rpc/ratelimit.go diff --git a/app/http/endpoints/api/reloadguilds.go b/app/http/endpoints/api/reloadguilds.go index fcfd1a6..ac5c332 100644 --- a/app/http/endpoints/api/reloadguilds.go +++ b/app/http/endpoints/api/reloadguilds.go @@ -3,7 +3,8 @@ package api import ( "fmt" "github.com/TicketsBot/GoPanel/app/http/session" - "github.com/TicketsBot/GoPanel/messagequeue" + "github.com/TicketsBot/GoPanel/redis" + wrapper "github.com/TicketsBot/GoPanel/redis" "github.com/TicketsBot/GoPanel/utils" "github.com/TicketsBot/GoPanel/utils/discord" "github.com/gin-gonic/gin" @@ -14,14 +15,14 @@ func ReloadGuildsHandler(ctx *gin.Context) { userId := ctx.Keys["userid"].(uint64) key := fmt.Sprintf("tickets:dashboard:guildreload:%d", userId) - res, err := messagequeue.Client.SetNX(key, 1, time.Second*10).Result() + res, err := redis.Client.SetNX(wrapper.DefaultContext(), key, 1, time.Second*10).Result() if err != nil { ctx.JSON(500, utils.ErrorJson(err)) return } if !res { - ttl, err := messagequeue.Client.TTL(key).Result() + ttl, err := redis.Client.TTL(wrapper.DefaultContext(), key).Result() if err != nil { ctx.JSON(500, utils.ErrorJson(err)) return diff --git a/app/http/endpoints/api/ticket/closeticket.go b/app/http/endpoints/api/ticket/closeticket.go index 3a7d848..cd67d85 100644 --- a/app/http/endpoints/api/ticket/closeticket.go +++ b/app/http/endpoints/api/ticket/closeticket.go @@ -2,7 +2,7 @@ package api import ( "github.com/TicketsBot/GoPanel/database" - "github.com/TicketsBot/GoPanel/messagequeue" + "github.com/TicketsBot/GoPanel/redis" "github.com/TicketsBot/GoPanel/utils" "github.com/TicketsBot/common/closerelay" "github.com/gin-gonic/gin" @@ -61,7 +61,7 @@ func CloseTicket(ctx *gin.Context) { Reason: body.Reason, } - if err := closerelay.Publish(messagequeue.Client.Client, data); err != nil { + if err := closerelay.Publish(redis.Client.Client, data); err != nil { ctx.JSON(500, utils.ErrorJson(err)) return } diff --git a/app/http/endpoints/api/whitelabel/whitelabelcreateinteractions.go b/app/http/endpoints/api/whitelabel/whitelabelcreateinteractions.go index fb3a963..fe1831d 100644 --- a/app/http/endpoints/api/whitelabel/whitelabelcreateinteractions.go +++ b/app/http/endpoints/api/whitelabel/whitelabelcreateinteractions.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/TicketsBot/GoPanel/botcontext" "github.com/TicketsBot/GoPanel/database" - "github.com/TicketsBot/GoPanel/messagequeue" + "github.com/TicketsBot/GoPanel/redis" "github.com/TicketsBot/worker/bot/command/impl/admin" "github.com/TicketsBot/worker/bot/command/manager" "github.com/gin-gonic/gin" @@ -45,7 +45,7 @@ func GetWhitelabelCreateInteractions() func(*gin.Context) { key := fmt.Sprintf("tickets:interaction-create-cooldown:%d", bot.BotId) // try to set first, prevent race condition - wasSet, err := messagequeue.Client.SetNX(key, 1, time.Minute).Result() + wasSet, err := redis.Client.SetNX(redis.DefaultContext(), key, 1, time.Minute).Result() if err != nil { ctx.JSON(500, gin.H{ "success": false, @@ -56,7 +56,7 @@ func GetWhitelabelCreateInteractions() func(*gin.Context) { // on cooldown, tell user how long left if !wasSet { - expiration, err := messagequeue.Client.TTL(key).Result() + expiration, err := redis.Client.TTL(redis.DefaultContext(), key).Result() if err != nil { ctx.JSON(500, gin.H{ "success": false, diff --git a/app/http/endpoints/api/whitelabel/whitelabelpost.go b/app/http/endpoints/api/whitelabel/whitelabelpost.go index 2311bb6..f049001 100644 --- a/app/http/endpoints/api/whitelabel/whitelabelpost.go +++ b/app/http/endpoints/api/whitelabel/whitelabelpost.go @@ -2,7 +2,7 @@ package api import ( dbclient "github.com/TicketsBot/GoPanel/database" - "github.com/TicketsBot/GoPanel/messagequeue" + "github.com/TicketsBot/GoPanel/redis" "github.com/TicketsBot/common/tokenchange" "github.com/TicketsBot/database" "github.com/gin-gonic/gin" @@ -86,7 +86,7 @@ func WhitelabelPost(ctx *gin.Context) { OldId: existing.BotId, } - if err := tokenchange.PublishTokenChange(messagequeue.Client.Client, tokenChangeData); err != nil { + if err := tokenchange.PublishTokenChange(redis.Client.Client, tokenChangeData); err != nil { ctx.JSON(500, gin.H{ "success": false, "error": err.Error(), diff --git a/app/http/endpoints/api/whitelabel/whitelabelstatuspost.go b/app/http/endpoints/api/whitelabel/whitelabelstatuspost.go index 06a8f8a..58d9596 100644 --- a/app/http/endpoints/api/whitelabel/whitelabelstatuspost.go +++ b/app/http/endpoints/api/whitelabel/whitelabelstatuspost.go @@ -2,7 +2,7 @@ package api import ( "github.com/TicketsBot/GoPanel/database" - "github.com/TicketsBot/GoPanel/messagequeue" + "github.com/TicketsBot/GoPanel/redis" "github.com/TicketsBot/common/statusupdates" "github.com/gin-gonic/gin" ) @@ -68,7 +68,7 @@ func WhitelabelStatusPost(ctx *gin.Context) { return } - go statusupdates.Publish(messagequeue.Client.Client, bot.BotId) + go statusupdates.Publish(redis.Client.Client, bot.BotId) ctx.JSON(200, gin.H{ "success": true, diff --git a/app/http/middleware/logging.go b/app/http/middleware/logging.go new file mode 100644 index 0000000..4164889 --- /dev/null +++ b/app/http/middleware/logging.go @@ -0,0 +1,40 @@ +package middleware + +import ( + "fmt" + "github.com/getsentry/sentry-go" + "github.com/gin-gonic/gin" + "io/ioutil" + "strconv" +) + +func Logging(ctx *gin.Context) { + defer ctx.Next() + + statusCode := ctx.Writer.Status() + + level := sentry.LevelInfo + if statusCode >= 500 { + level = sentry.LevelError + } + + body, _ := ioutil.ReadAll(ctx.Request.Body) + + sentry.CaptureEvent(&sentry.Event{ + Extra: map[string]interface{}{ + "status_code": strconv.Itoa(statusCode), + "method": ctx.Request.Method, + "path": ctx.Request.URL.Path, + "guild_id": ctx.Keys["guildid"], + "user_id": ctx.Keys["user_id"], + "body": string(body), + }, + Level: level, + Message: fmt.Sprintf("HTTP %d on %s %s", statusCode, ctx.Request.Method, ctx.FullPath()), + Tags: map[string]string{ + "status_code": strconv.Itoa(statusCode), + "method": ctx.Request.Method, + "path": ctx.Request.URL.Path, + }, + }) +} diff --git a/app/http/middleware/multireadbody.go b/app/http/middleware/multireadbody.go new file mode 100644 index 0000000..6d9d087 --- /dev/null +++ b/app/http/middleware/multireadbody.go @@ -0,0 +1,12 @@ +package middleware + +import ( + "bytes" + "github.com/gin-gonic/gin" + "io/ioutil" +) + +func MultiReadBody(ctx *gin.Context) { + body, _ := ioutil.ReadAll(ctx.Request.Body) + ctx.Request.Body = ioutil.NopCloser(bytes.NewBuffer(body)) +} diff --git a/app/http/middleware/ratelimit.go b/app/http/middleware/ratelimit.go new file mode 100644 index 0000000..1703a22 --- /dev/null +++ b/app/http/middleware/ratelimit.go @@ -0,0 +1,109 @@ +package middleware + +import ( + "fmt" + "github.com/TicketsBot/GoPanel/redis" + "github.com/TicketsBot/GoPanel/utils" + "github.com/gin-gonic/gin" + "github.com/go-redis/redis_rate/v9" + "hash/fnv" + "strconv" + "time" +) + +type RateLimitType uint8 + +const ( + RateLimitTypeIp RateLimitType = iota + RateLimitTypeUser + RateLimitTypeGuild +) + +func CreateRateLimiter(rlType RateLimitType, max int, period time.Duration) gin.HandlerFunc { + limiter := redis_rate.NewLimiter(redis.Client) + + return func(ctx *gin.Context) { + limit := redis_rate.Limit{ + Rate: max, + Burst: max, + Period: period, + } + + name, skip := getKey(ctx, rlType, limit) + if skip { + ctx.Next() + return + } + + res, err := limiter.Allow(redis.DefaultContext(), name, limit) + if err != nil { + ctx.AbortWithStatusJSON(500, utils.ErrorJson(err)) + Logging(ctx) + return + } + + // Use smallest remaining for ratelimit headers + smallestRemaining := ctx.Keys["rl_sr"] + if smallestRemaining == nil { + writeHeaders(ctx, res) + } else { + rem := smallestRemaining.(int) + if res.Remaining < rem { + writeHeaders(ctx, res) + } + } + + ctx.Header("X-RateLimit-Limit", strconv.Itoa(res.Limit.Rate)) + ctx.Header("X-RateLimit-Remaining", strconv.Itoa(res.Remaining)) + ctx.Header("X-RateLimit-Reset-After", strconv.FormatInt(res.ResetAfter.Milliseconds(), 10)) + + if res.Allowed <= 0 { + ctx.AbortWithStatusJSON(429, utils.ErrorStr("You are being ratelimited")) + Logging(ctx) + return + } + + ctx.Next() + } +} + +func writeHeaders(ctx *gin.Context, res *redis_rate.Result) { + ctx.Keys["rl_sr"] = res.Remaining + fmt.Println(res.Remaining) + ctx.Header("X-RateLimit-Limit", strconv.Itoa(res.Limit.Rate)) + ctx.Header("X-RateLimit-Remaining", strconv.Itoa(res.Remaining)) + ctx.Header("X-RateLimit-Reset-After", strconv.FormatInt(res.ResetAfter.Milliseconds(), 10)) +} + +// Returns (key, skip) +func getKey(ctx *gin.Context, rlType RateLimitType, limit redis_rate.Limit) (string, bool) { + userId := ctx.Keys["userid"] + guildId := ctx.Keys["guildid"] + + if (rlType == RateLimitTypeUser && userId == nil) || (rlType == RateLimitTypeGuild && guildId == nil) { + ctx.Next() + return "", true + } + + var key string + switch rlType { + case RateLimitTypeIp: + key = ctx.ClientIP() + case RateLimitTypeUser: + key = strconv.FormatUint(userId.(uint64), 10) + case RateLimitTypeGuild: + key = strconv.FormatUint(guildId.(uint64), 10) + } + + target := fmt.Sprintf("%d:%s", rlType, key) + bucket := fmt.Sprintf("%d/%d", limit.Rate, limit.Period.Milliseconds()) + full := fmt.Sprintf("%s:%s:%s", target, bucket, ctx.FullPath()) + + return strconv.FormatUint(uint64(hash(full)), 16), false +} + +func hash(str string) uint32 { + h := fnv.New32a() + h.Write([]byte(str)) + return h.Sum32() +} diff --git a/app/http/server.go b/app/http/server.go index d7b9168..86f9cc3 100644 --- a/app/http/server.go +++ b/app/http/server.go @@ -15,13 +15,10 @@ import ( "github.com/TicketsBot/GoPanel/app/http/middleware" "github.com/TicketsBot/GoPanel/app/http/session" "github.com/TicketsBot/GoPanel/config" - "github.com/TicketsBot/GoPanel/utils" "github.com/TicketsBot/common/permission" + sentrygin "github.com/getsentry/sentry-go/gin" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" - "github.com/ulule/limiter/v3" - mgin "github.com/ulule/limiter/v3/drivers/middleware/gin" - "github.com/ulule/limiter/v3/drivers/store/memory" "log" "time" ) @@ -43,93 +40,110 @@ func StartServer() { router.Use(static.Serve("/assets/", static.LocalFile("./public/static", false))) router.Use(gin.Recovery()) - router.Use(createLimiter(600, time.Minute*10)) + router.Use(middleware.MultiReadBody) + router.Use(sentrygin.New(sentrygin.Options{})) // Defaults are ok + + router.Use(rl(middleware.RateLimitTypeIp, 60, time.Minute)) + router.Use(rl(middleware.RateLimitTypeUser, 60, time.Minute)) + router.Use(rl(middleware.RateLimitTypeGuild, 600, time.Minute*5)) router.Use(middleware.Cors(config.Conf)) - router.GET("/webchat", root.WebChatWs) + router.GET("/webchat", root.WebChatWs, middleware.Logging) - router.POST("/callback", middleware.VerifyXTicketsHeader, root.CallbackHandler) - router.POST("/logout", middleware.VerifyXTicketsHeader, middleware.AuthenticateToken, root.LogoutHandler) + router.POST("/callback", middleware.VerifyXTicketsHeader, root.CallbackHandler, middleware.Logging) + router.POST("/logout", middleware.VerifyXTicketsHeader, middleware.AuthenticateToken, root.LogoutHandler, middleware.Logging) apiGroup := router.Group("/api", middleware.VerifyXTicketsHeader, middleware.AuthenticateToken) { - apiGroup.GET("/session", api.SessionHandler) + apiGroup.GET("/session", api.SessionHandler, middleware.Logging) } guildAuthApiAdmin := apiGroup.Group("/:id", middleware.AuthenticateGuild(true, permission.Admin)) guildAuthApiSupport := apiGroup.Group("/:id", middleware.AuthenticateGuild(true, permission.Support)) guildApiNoAuth := apiGroup.Group("/:id", middleware.ParseGuildId) { - guildAuthApiSupport.GET("/channels", api.ChannelsHandler) - guildAuthApiSupport.GET("/premium", api.PremiumHandler) - guildAuthApiSupport.GET("/user/:user", api.UserHandler) - guildAuthApiSupport.GET("/roles", api.RolesHandler) - guildAuthApiSupport.GET("/members/search", createLimiter(5, time.Second), createLimiter(10, time.Second * 30), createLimiter(75, time.Minute * 30), api.SearchMembers) + guildAuthApiSupport.GET("/channels", api.ChannelsHandler, middleware.Logging) + guildAuthApiSupport.GET("/premium", api.PremiumHandler, middleware.Logging) + guildAuthApiSupport.GET("/user/:user", api.UserHandler, middleware.Logging) + guildAuthApiSupport.GET("/roles", api.RolesHandler, middleware.Logging) + guildAuthApiSupport.GET("/members/search", + rl(middleware.RateLimitTypeGuild, 5, time.Second), + rl(middleware.RateLimitTypeGuild, 10, time.Second*30), + rl(middleware.RateLimitTypeGuild, 75, time.Minute*30), + api.SearchMembers, + middleware.Logging, + ) - guildAuthApiAdmin.GET("/settings", api_settings.GetSettingsHandler) - guildAuthApiAdmin.POST("/settings", api_settings.UpdateSettingsHandler) + guildAuthApiAdmin.GET("/settings", api_settings.GetSettingsHandler, middleware.Logging) + guildAuthApiAdmin.POST("/settings", api_settings.UpdateSettingsHandler, middleware.Logging) - guildAuthApiSupport.GET("/blacklist", api_blacklist.GetBlacklistHandler) - guildAuthApiSupport.POST("/blacklist/:user", api_blacklist.AddBlacklistHandler) - guildAuthApiSupport.DELETE("/blacklist/:user", api_blacklist.RemoveBlacklistHandler) + guildAuthApiSupport.GET("/blacklist", api_blacklist.GetBlacklistHandler, middleware.Logging) + guildAuthApiSupport.POST("/blacklist/:user", api_blacklist.AddBlacklistHandler, middleware.Logging) + guildAuthApiSupport.DELETE("/blacklist/:user", api_blacklist.RemoveBlacklistHandler, middleware.Logging) - guildAuthApiAdmin.GET("/panels", api_panels.ListPanels) - guildAuthApiAdmin.POST("/panels", api_panels.CreatePanel) - guildAuthApiAdmin.PATCH("/panels/:panelid", api_panels.UpdatePanel) - guildAuthApiAdmin.DELETE("/panels/:panelid", api_panels.DeletePanel) + guildAuthApiAdmin.GET("/panels", api_panels.ListPanels, middleware.Logging) + guildAuthApiAdmin.POST("/panels", api_panels.CreatePanel, middleware.Logging) + guildAuthApiAdmin.PATCH("/panels/:panelid", api_panels.UpdatePanel, middleware.Logging) + guildAuthApiAdmin.DELETE("/panels/:panelid", api_panels.DeletePanel, middleware.Logging) - guildAuthApiAdmin.GET("/multipanels", api_panels.MultiPanelList) - guildAuthApiAdmin.POST("/multipanels", api_panels.MultiPanelCreate) - guildAuthApiAdmin.PATCH("/multipanels/:panelid", api_panels.MultiPanelUpdate) - guildAuthApiAdmin.DELETE("/multipanels/:panelid", api_panels.MultiPanelDelete) + guildAuthApiAdmin.GET("/multipanels", api_panels.MultiPanelList, middleware.Logging) + guildAuthApiAdmin.POST("/multipanels", api_panels.MultiPanelCreate, middleware.Logging) + guildAuthApiAdmin.PATCH("/multipanels/:panelid", api_panels.MultiPanelUpdate, middleware.Logging) + guildAuthApiAdmin.DELETE("/multipanels/:panelid", api_panels.MultiPanelDelete, middleware.Logging) // Should be a GET, but easier to take a body for development purposes - guildAuthApiSupport.POST("/transcripts", createLimiter(5, 5 * time.Second), createLimiter(20, time.Minute), api_transcripts.ListTranscripts) + guildAuthApiSupport.POST("/transcripts", + rl(middleware.RateLimitTypeUser, 5, 5*time.Second), + rl(middleware.RateLimitTypeUser, 20, time.Minute), + api_transcripts.ListTranscripts, + middleware.Logging, + ) + // Allow regular users to get their own transcripts, make sure you check perms inside - guildApiNoAuth.GET("/transcripts/:ticketId", createLimiter(10, 10 * time.Second), api_transcripts.GetTranscriptHandler) + guildApiNoAuth.GET("/transcripts/:ticketId", rl(middleware.RateLimitTypeGuild, 10, 10*time.Second), api_transcripts.GetTranscriptHandler, middleware.Logging) - guildAuthApiSupport.GET("/tickets", api_ticket.GetTickets) - guildAuthApiSupport.GET("/tickets/:ticketId", api_ticket.GetTicket) - guildAuthApiSupport.POST("/tickets/:ticketId", createLimiter(5, time.Second * 5), api_ticket.SendMessage) - guildAuthApiSupport.DELETE("/tickets/:ticketId", api_ticket.CloseTicket) + guildAuthApiSupport.GET("/tickets", api_ticket.GetTickets, middleware.Logging) + guildAuthApiSupport.GET("/tickets/:ticketId", api_ticket.GetTicket, middleware.Logging) + guildAuthApiSupport.POST("/tickets/:ticketId", rl(middleware.RateLimitTypeGuild, 5, time.Second*5), api_ticket.SendMessage, middleware.Logging) + guildAuthApiSupport.DELETE("/tickets/:ticketId", api_ticket.CloseTicket, middleware.Logging) - guildAuthApiSupport.GET("/tags", api_tags.TagsListHandler) - guildAuthApiSupport.PUT("/tags", api_tags.CreateTag) - guildAuthApiSupport.DELETE("/tags", api_tags.DeleteTag) + guildAuthApiSupport.GET("/tags", api_tags.TagsListHandler, middleware.Logging) + guildAuthApiSupport.PUT("/tags", api_tags.CreateTag, middleware.Logging) + guildAuthApiSupport.DELETE("/tags", api_tags.DeleteTag, middleware.Logging) - guildAuthApiAdmin.GET("/claimsettings", api_settings.GetClaimSettings) - guildAuthApiAdmin.POST("/claimsettings", api_settings.PostClaimSettings) + guildAuthApiAdmin.GET("/claimsettings", api_settings.GetClaimSettings, middleware.Logging) + guildAuthApiAdmin.POST("/claimsettings", api_settings.PostClaimSettings, middleware.Logging) - guildAuthApiAdmin.GET("/autoclose", api_autoclose.GetAutoClose) - guildAuthApiAdmin.POST("/autoclose", api_autoclose.PostAutoClose) + guildAuthApiAdmin.GET("/autoclose", api_autoclose.GetAutoClose, middleware.Logging) + guildAuthApiAdmin.POST("/autoclose", api_autoclose.PostAutoClose, middleware.Logging) - guildAuthApiAdmin.GET("/team", api_team.GetTeams) - guildAuthApiAdmin.GET("/team/:teamid", createLimiter(10, time.Second * 30), api_team.GetMembers) - guildAuthApiAdmin.POST("/team", createLimiter(10, time.Minute), api_team.CreateTeam) - guildAuthApiAdmin.PUT("/team/:teamid/:snowflake", createLimiter(5, time.Second * 10), api_team.AddMember) - guildAuthApiAdmin.DELETE("/team/:teamid", api_team.DeleteTeam) - guildAuthApiAdmin.DELETE("/team/:teamid/:snowflake", createLimiter(30, time.Minute), api_team.RemoveMember) + guildAuthApiAdmin.GET("/team", api_team.GetTeams, middleware.Logging) + guildAuthApiAdmin.GET("/team/:teamid", rl(middleware.RateLimitTypeUser ,10, time.Second*30), api_team.GetMembers, middleware.Logging) + guildAuthApiAdmin.POST("/team", rl(middleware.RateLimitTypeUser, 10, time.Minute), api_team.CreateTeam, middleware.Logging) + guildAuthApiAdmin.PUT("/team/:teamid/:snowflake", rl(middleware.RateLimitTypeGuild, 5, time.Second*10), api_team.AddMember, middleware.Logging) + guildAuthApiAdmin.DELETE("/team/:teamid", api_team.DeleteTeam, middleware.Logging) + guildAuthApiAdmin.DELETE("/team/:teamid/:snowflake", rl(middleware.RateLimitTypeGuild, 30, time.Minute), api_team.RemoveMember, middleware.Logging) } userGroup := router.Group("/user", middleware.AuthenticateToken) { - userGroup.GET("/guilds", api.GetGuilds) - userGroup.POST("/guilds/reload", api.ReloadGuildsHandler) - userGroup.GET("/permissionlevel", api.GetPermissionLevel) + userGroup.GET("/guilds", api.GetGuilds, middleware.Logging) + userGroup.POST("/guilds/reload", api.ReloadGuildsHandler, middleware.Logging) + userGroup.GET("/permissionlevel", api.GetPermissionLevel, middleware.Logging) { whitelabelGroup := userGroup.Group("/whitelabel", middleware.VerifyWhitelabel(true)) - whitelabelGroup.GET("/", api_whitelabel.WhitelabelGet) - whitelabelGroup.GET("/errors", api_whitelabel.WhitelabelGetErrors) - whitelabelGroup.GET("/guilds", api_whitelabel.WhitelabelGetGuilds) - whitelabelGroup.GET("/public-key", api_whitelabel.WhitelabelGetPublicKey) - whitelabelGroup.POST("/public-key", api_whitelabel.WhitelabelPostPublicKey) - whitelabelGroup.POST("/create-interactions", api_whitelabel.GetWhitelabelCreateInteractions()) + whitelabelGroup.GET("/", api_whitelabel.WhitelabelGet, middleware.Logging) + whitelabelGroup.GET("/errors", api_whitelabel.WhitelabelGetErrors, middleware.Logging) + whitelabelGroup.GET("/guilds", api_whitelabel.WhitelabelGetGuilds, middleware.Logging) + whitelabelGroup.GET("/public-key", api_whitelabel.WhitelabelGetPublicKey, middleware.Logging) + whitelabelGroup.POST("/public-key", api_whitelabel.WhitelabelPostPublicKey, middleware.Logging) + whitelabelGroup.POST("/create-interactions", api_whitelabel.GetWhitelabelCreateInteractions(), middleware.Logging) - whitelabelGroup.POST("/", createLimiter(10, time.Minute), api_whitelabel.WhitelabelPost) - whitelabelGroup.POST("/status", createLimiter(1, time.Second*5), api_whitelabel.WhitelabelStatusPost) + whitelabelGroup.POST("/", rl(middleware.RateLimitTypeUser, 10, time.Minute), api_whitelabel.WhitelabelPost, middleware.Logging) + whitelabelGroup.POST("/status", rl(middleware.RateLimitTypeUser, 1, time.Second*5), api_whitelabel.WhitelabelStatusPost, middleware.Logging) } } @@ -138,40 +152,6 @@ func StartServer() { } } -func serveTemplate(templateName string) func(*gin.Context) { - return func(ctx *gin.Context) { - guildId := ctx.Keys["guildid"].(uint64) - userId := ctx.Keys["userid"].(uint64) - - store, err := session.Store.Get(userId) - if err != nil { - if err == session.ErrNoSession { - ctx.JSON(401, gin.H{ - "success": false, - "auth": true, - }) - } else { - ctx.JSON(500, utils.ErrorJson(err)) - } - - return - } - - ctx.HTML(200, templateName, gin.H{ - "name": store.Name, - "guildId": guildId, - "avatar": store.Avatar, - "baseUrl": config.Conf.Server.BaseUrl, - }) - } -} - -func createLimiter(limit int64, period time.Duration) func(*gin.Context) { - store := memory.NewStore() - rate := limiter.Rate{ - Period: period, - Limit: limit, - } - - return mgin.NewMiddleware(limiter.New(store, rate)) +func rl(rlType middleware.RateLimitType, limit int, period time.Duration) func(*gin.Context) { + return middleware.CreateRateLimiter(rlType, limit, period) } diff --git a/app/http/session/redisstore.go b/app/http/session/redisstore.go index ecca6d1..9f4e670 100644 --- a/app/http/session/redisstore.go +++ b/app/http/session/redisstore.go @@ -4,8 +4,8 @@ import ( "encoding/json" "errors" "fmt" - "github.com/TicketsBot/GoPanel/messagequeue" - "github.com/go-redis/redis" + wrapper "github.com/TicketsBot/GoPanel/redis" + "github.com/go-redis/redis/v8" ) var ErrNoSession = errors.New("no session data found") @@ -16,14 +16,14 @@ type RedisStore struct { func NewRedisStore() *RedisStore { return &RedisStore{ - client: messagequeue.Client.Client, + client: wrapper.Client.Client, } } var keyPrefix = "panel:session:" func (s *RedisStore) Get(userId uint64) (SessionData, error) { - raw, err := s.client.Get(fmt.Sprintf("%s:%d", keyPrefix, userId)).Result() + raw, err := s.client.Get(wrapper.DefaultContext(), fmt.Sprintf("%s:%d", keyPrefix, userId)).Result() if err != nil { if err == redis.Nil { err = ErrNoSession @@ -46,9 +46,9 @@ func (s *RedisStore) Set(userId uint64, data SessionData) error { return err } - return s.client.Set(fmt.Sprintf("%s:%d", keyPrefix, userId), encoded, 0).Err() + return s.client.Set(wrapper.DefaultContext(), fmt.Sprintf("%s:%d", keyPrefix, userId), encoded, 0).Err() } func (s *RedisStore) Clear(userId uint64) error { - return s.client.Del(fmt.Sprintf("%s:%d", keyPrefix, userId)).Err() + return s.client.Del(wrapper.DefaultContext(), fmt.Sprintf("%s:%d", keyPrefix, userId)).Err() } \ No newline at end of file diff --git a/botcontext/botcontext.go b/botcontext/botcontext.go index ad1c2bc..8dd5500 100644 --- a/botcontext/botcontext.go +++ b/botcontext/botcontext.go @@ -3,7 +3,7 @@ package botcontext import ( "github.com/TicketsBot/GoPanel/config" dbclient "github.com/TicketsBot/GoPanel/database" - "github.com/TicketsBot/GoPanel/messagequeue" + "github.com/TicketsBot/GoPanel/redis" "github.com/TicketsBot/GoPanel/rpc/cache" "github.com/TicketsBot/common/permission" "github.com/TicketsBot/database" @@ -26,7 +26,7 @@ func (ctx BotContext) Db() *database.Database { } func (ctx BotContext) Cache() permission.PermissionCache { - return permission.NewRedisCache(messagequeue.Client.Client) + return permission.NewRedisCache(redis.Client.Client) } func (ctx BotContext) IsBotAdmin(userId uint64) bool { diff --git a/botcontext/get.go b/botcontext/get.go index 8b95f64..45f2382 100644 --- a/botcontext/get.go +++ b/botcontext/get.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/TicketsBot/GoPanel/config" dbclient "github.com/TicketsBot/GoPanel/database" - "github.com/TicketsBot/GoPanel/messagequeue" + "github.com/TicketsBot/GoPanel/redis" "github.com/rxdn/gdl/rest/ratelimit" ) @@ -32,7 +32,7 @@ func ContextForGuild(guildId uint64) (ctx BotContext, err error) { } // TODO: Large sharding buckets - ctx.RateLimiter = ratelimit.NewRateLimiter(ratelimit.NewRedisStore(messagequeue.Client.Client, keyPrefix), 1) + ctx.RateLimiter = ratelimit.NewRateLimiter(ratelimit.NewRedisStore(redis.Client.Client, keyPrefix), 1) return } diff --git a/cmd/panel/main.go b/cmd/panel/main.go index 5becc37..736224c 100644 --- a/cmd/panel/main.go +++ b/cmd/panel/main.go @@ -8,7 +8,7 @@ import ( "github.com/TicketsBot/GoPanel/app/http/endpoints/root" "github.com/TicketsBot/GoPanel/config" "github.com/TicketsBot/GoPanel/database" - "github.com/TicketsBot/GoPanel/messagequeue" + "github.com/TicketsBot/GoPanel/redis" "github.com/TicketsBot/GoPanel/rpc" "github.com/TicketsBot/GoPanel/rpc/cache" "github.com/TicketsBot/GoPanel/utils" @@ -17,6 +17,7 @@ import ( "github.com/TicketsBot/common/premium" "github.com/TicketsBot/worker/i18n" "github.com/apex/log" + "github.com/getsentry/sentry-go" "github.com/rxdn/gdl/rest/request" "math/rand" "time" @@ -34,6 +35,16 @@ func main() { config.LoadConfig() + sentryOpts := sentry.ClientOptions{ + Dsn: config.Conf.SentryDsn, + Debug: config.Conf.Debug, + AttachStacktrace: true, + + } + if err := sentry.Init(sentryOpts); err != nil { + fmt.Printf("Error initialising sentry: %s", err.Error()) + } + database.ConnectToDatabase() cache.Instance = cache.NewCache() @@ -47,13 +58,13 @@ func main() { request.RegisterHook(utils.ProxyHook) } - messagequeue.Client = messagequeue.NewRedisClient() - go ListenChat(messagequeue.Client) + redis.Client = redis.NewRedisClient() + go ListenChat(redis.Client) if !config.Conf.Debug { rpc.PremiumClient = premium.NewPremiumLookupClient( premium.NewPatreonClient(config.Conf.Bot.PremiumLookupProxyUrl, config.Conf.Bot.PremiumLookupProxyKey), - messagequeue.Client.Client, + redis.Client.Client, cache.Instance.PgCache, database.Client, ) @@ -65,7 +76,7 @@ func main() { http.StartServer() } -func ListenChat(client messagequeue.RedisClient) { +func ListenChat(client redis.RedisClient) { ch := make(chan chatrelay.MessageData) go chatrelay.Listen(client.Client, ch) diff --git a/config/config.go b/config/config.go index 640235b..ba3d69c 100644 --- a/config/config.go +++ b/config/config.go @@ -13,6 +13,7 @@ type ( Admins []uint64 ForceWhitelabel []uint64 Debug bool + SentryDsn string Server Server Oauth Oauth Database Database @@ -97,6 +98,7 @@ func fromToml() { } } +// TODO: Proper env package func fromEnvvar() { var admins []uint64 for _, id := range strings.Split(os.Getenv("ADMINS"), ",") { @@ -128,6 +130,7 @@ func fromEnvvar() { Admins: admins, ForceWhitelabel: forcedWhitelabel, Debug: os.Getenv("DEBUG") != "", + SentryDsn: os.Getenv("SENTRY_DSN"), Server: Server{ Host: os.Getenv("SERVER_ADDR"), BaseUrl: os.Getenv("BASE_URL"), diff --git a/envvars.md b/envvars.md index 902e421..2925b73 100644 --- a/envvars.md +++ b/envvars.md @@ -10,6 +10,7 @@ - ADMINS - FORCED_WHITELABEL +- SENTRY_DSN - SERVER_ADDR - BASE_URL - MAIN_SITE diff --git a/go.mod b/go.mod index 23a2185..0f92d29 100644 --- a/go.mod +++ b/go.mod @@ -5,23 +5,26 @@ go 1.14 require ( github.com/BurntSushi/toml v0.3.1 github.com/TicketsBot/archiverclient v0.0.0-20210220155137-a562b2f1bbbb - github.com/TicketsBot/common v0.0.0-20210727134627-35eb7ed03a44 - github.com/TicketsBot/database v0.0.0-20210901195306-fcfc088bc2f9 - github.com/TicketsBot/worker v0.0.0-20210901194011-edcd400f1ba0 + github.com/TicketsBot/common v0.0.0-20210903095620-eb02b87cb4ca + github.com/TicketsBot/database v0.0.0-20210902205640-76b8973364e8 + github.com/TicketsBot/worker v0.0.0-20210903100019-6e6eab3a3196 github.com/apex/log v1.1.2 github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect + github.com/getsentry/sentry-go v0.11.0 github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2 github.com/gin-gonic/contrib v0.0.0-20191209060500-d6e26eeaa607 github.com/gin-gonic/gin v1.7.2-0.20210726235953-11aa11a65618 github.com/go-redis/redis v6.15.9+incompatible + github.com/go-redis/redis/v8 v8.11.3 + github.com/go-redis/redis_rate/v9 v9.1.1 github.com/golang-jwt/jwt v3.2.1+incompatible github.com/gorilla/sessions v1.2.0 // indirect github.com/gorilla/websocket v1.4.2 github.com/jackc/pgx/v4 v4.7.1 github.com/pasztorpisti/qs v0.0.0-20171216220353-8d6c33ee906c github.com/pkg/errors v0.9.1 - github.com/rxdn/gdl v0.0.0-20210901190929-59c42354a637 + github.com/rxdn/gdl v0.0.0-20210903095530-5a1c35525d2a github.com/sirupsen/logrus v1.5.0 github.com/ulule/limiter/v3 v3.5.0 - golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 ) diff --git a/messagequeue/panelcreate.go b/redis/panelcreate.go similarity index 75% rename from messagequeue/panelcreate.go rename to redis/panelcreate.go index ed2b920..3193756 100644 --- a/messagequeue/panelcreate.go +++ b/redis/panelcreate.go @@ -1,4 +1,4 @@ -package messagequeue +package redis import ( "encoding/json" @@ -12,6 +12,6 @@ func (c *RedisClient) PublishPanelCreate(settings database.Panel) { return } - c.Publish("tickets:panel:create", string(encoded)) + c.Publish(DefaultContext(), "tickets:panel:create", string(encoded)) } diff --git a/messagequeue/redis.go b/redis/redis.go similarity index 68% rename from messagequeue/redis.go rename to redis/redis.go index f33b329..5781e3f 100644 --- a/messagequeue/redis.go +++ b/redis/redis.go @@ -1,9 +1,11 @@ -package messagequeue +package redis import ( + "context" "fmt" "github.com/TicketsBot/GoPanel/config" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v8" + "time" ) type RedisClient struct { @@ -23,3 +25,8 @@ func NewRedisClient() RedisClient { client, } } + +func DefaultContext() context.Context { + ctx, _ := context.WithTimeout(context.Background(), time.Second*3) + return ctx +} diff --git a/messagequeue/ticketclose.go b/redis/ticketclose.go similarity index 86% rename from messagequeue/ticketclose.go rename to redis/ticketclose.go index 85c931b..9a24c7d 100644 --- a/messagequeue/ticketclose.go +++ b/redis/ticketclose.go @@ -1,4 +1,4 @@ -package messagequeue +package redis import ( "encoding/json" @@ -26,5 +26,5 @@ func (c *RedisClient) PublishTicketClose(guildId uint64, ticketId int, userId ui return } - c.Publish("tickets:close", string(encoded)) + c.Publish(DefaultContext(), "tickets:close", string(encoded)) } diff --git a/rpc/ratelimit.go b/rpc/ratelimit.go new file mode 100644 index 0000000..d81a69f --- /dev/null +++ b/rpc/ratelimit.go @@ -0,0 +1,50 @@ +package rpc + +import ( + "fmt" + "github.com/go-redis/redis" + "time" +) + +var script = redis.NewScript(` +local current = redis.call("GET", KEYS[1]) +local notExists = (not current) + +if not current then + current = 0 +else + current = tonumber(current) +end + +local success = 0 +if current < tonumber(ARGV[1]) then + redis.call("INCR", KEYS[1]) + + if notExists then + redis.call("EXPIRE", KEYS[1], ARGV[2]) + end + + success = 1 +end + +return success +`) + +var TicketOpenLimit = 10 +var TicketOpenLimitInterval = time.Second * 30 + +func TakeTicketRateLimitToken(client *redis.Client, guildId uint64) (bool, error) { + key := fmt.Sprintf("tickets:openratelimit:%d", guildId) + + res, err := script.Run(client, []string{key}, TicketOpenLimit, TicketOpenLimitInterval.Seconds()).Result() + if err != nil { + return false, err + } + + i, ok := res.(int64) + if !ok { + return false, fmt.Errorf("ratelimit token returned %v, not an int64", res) + } + + return i == 1, nil +}