From d5404c9a7e45006149ffae5e96d998ec232481f3 Mon Sep 17 00:00:00 2001 From: rxdn <29165304+rxdn@users.noreply.github.com> Date: Sun, 27 Mar 2022 04:23:05 +0100 Subject: [PATCH] Archives v2 --- app/http/endpoints/api/transcripts/render.go | 10 +-- app/http/middleware/cors.go | 2 +- app/http/middleware/logging.go | 82 +++++++++++--------- app/http/middleware/ratelimit.go | 5 +- app/http/server.go | 7 +- chatreplica/{convert.go => convert_v1.go} | 0 chatreplica/convert_v2.go | 14 ++++ chatreplica/proxy.go | 29 ++++--- chatreplica/structs.go | 66 +++++++++++++++- cmd/panel/main.go | 2 +- go.mod | 3 +- go.sum | 9 ++- 12 files changed, 164 insertions(+), 65 deletions(-) rename chatreplica/{convert.go => convert_v1.go} (100%) create mode 100644 chatreplica/convert_v2.go diff --git a/app/http/endpoints/api/transcripts/render.go b/app/http/endpoints/api/transcripts/render.go index 0ad8e8b..4e5661f 100644 --- a/app/http/endpoints/api/transcripts/render.go +++ b/app/http/endpoints/api/transcripts/render.go @@ -53,7 +53,7 @@ func GetTranscriptRenderHandler(ctx *gin.Context) { } // retrieve ticket messages from bucket - messages, err := utils.ArchiverClient.Get(guildId, ticketId) + transcript, err := utils.ArchiverClient.Get(guildId, ticketId) if err != nil { if errors.Is(err, archiverclient.ErrExpired) { ctx.JSON(404, utils.ErrorStr("Transcript not found")) @@ -65,12 +65,12 @@ func GetTranscriptRenderHandler(ctx *gin.Context) { } // Render - payload := chatreplica.FromArchiveMessages(messages, ticketId) + payload := chatreplica.FromTranscript(transcript, ticketId) html, err := chatreplica.Render(payload) if err != nil { - ctx.JSON(500, utils.ErrorJson(err)) - return - } + ctx.JSON(500, utils.ErrorJson(err)) + return + } ctx.Data(200, "text/html", html) } diff --git a/app/http/middleware/cors.go b/app/http/middleware/cors.go index 98e518c..21fbb86 100644 --- a/app/http/middleware/cors.go +++ b/app/http/middleware/cors.go @@ -22,4 +22,4 @@ func Cors(config config.Config) func(*gin.Context) { ctx.AbortWithStatus(http.StatusNoContent) } } -} \ No newline at end of file +} diff --git a/app/http/middleware/logging.go b/app/http/middleware/logging.go index d46b8b3..99e2637 100644 --- a/app/http/middleware/logging.go +++ b/app/http/middleware/logging.go @@ -8,46 +8,52 @@ import ( "strconv" ) -func Logging(ctx *gin.Context) { - ctx.Next() +func Logging(minLevel sentry.Level) gin.HandlerFunc { + return func(ctx *gin.Context) { + ctx.Next() - statusCode := ctx.Writer.Status() + statusCode := ctx.Writer.Status() - var level sentry.Level - if statusCode >= 500 { - level = sentry.LevelError - } else if statusCode >= 400 { - level = sentry.LevelWarning - } else { - level = sentry.LevelInfo - } - - requestBody, _ := ioutil.ReadAll(ctx.Request.Body) - - var responseBody []byte - if statusCode >= 400 && statusCode <= 599 { - cw, ok := ctx.Writer.(*CustomWriter) - if ok { - responseBody = cw.Read() + var level sentry.Level + if statusCode >= 500 { + level = sentry.LevelError + } else if statusCode >= 400 { + level = sentry.LevelWarning + } else { + level = sentry.LevelInfo } - } - sentry.CaptureEvent(&sentry.Event{ - Extra: map[string]interface{}{ - "status_code": strconv.Itoa(statusCode), - "method": ctx.Request.Method, - "path": ctx.Request.URL.Path, - "guild_id": ctx.Keys["guildid"], - "user_id": ctx.Keys["userid"], - "request_body": string(requestBody), - "response": string(responseBody), - }, - Level: level, - Message: fmt.Sprintf("HTTP %d on %s %s", statusCode, ctx.Request.Method, ctx.FullPath()), - Tags: map[string]string{ - "status_code": strconv.Itoa(statusCode), - "method": ctx.Request.Method, - "path": ctx.Request.URL.Path, - }, - }) + if level < minLevel { + return + } + + requestBody, _ := ioutil.ReadAll(ctx.Request.Body) + + var responseBody []byte + if statusCode >= 400 && statusCode <= 599 { + cw, ok := ctx.Writer.(*CustomWriter) + if ok { + responseBody = cw.Read() + } + } + + sentry.CaptureEvent(&sentry.Event{ + Extra: map[string]interface{}{ + "status_code": strconv.Itoa(statusCode), + "method": ctx.Request.Method, + "path": ctx.Request.URL.Path, + "guild_id": ctx.Keys["guildid"], + "user_id": ctx.Keys["userid"], + "request_body": string(requestBody), + "response": string(responseBody), + }, + Level: level, + Message: fmt.Sprintf("HTTP %d on %s %s", statusCode, ctx.Request.Method, ctx.FullPath()), + Tags: map[string]string{ + "status_code": strconv.Itoa(statusCode), + "method": ctx.Request.Method, + "path": ctx.Request.URL.Path, + }, + }) + } } diff --git a/app/http/middleware/ratelimit.go b/app/http/middleware/ratelimit.go index 52d094f..0e40245 100644 --- a/app/http/middleware/ratelimit.go +++ b/app/http/middleware/ratelimit.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/TicketsBot/GoPanel/redis" "github.com/TicketsBot/GoPanel/utils" + "github.com/getsentry/sentry-go" "github.com/gin-gonic/gin" "github.com/go-redis/redis_rate/v9" "hash/fnv" @@ -38,7 +39,7 @@ func CreateRateLimiter(rlType RateLimitType, max int, period time.Duration) gin. res, err := limiter.Allow(redis.DefaultContext(), name, limit) if err != nil { ctx.AbortWithStatusJSON(500, utils.ErrorJson(err)) - Logging(ctx) + Logging(sentry.LevelError)(ctx) return } @@ -59,7 +60,7 @@ func CreateRateLimiter(rlType RateLimitType, max int, period time.Duration) gin. if res.Allowed <= 0 { ctx.AbortWithStatusJSON(429, utils.ErrorStr("You are being ratelimited")) - Logging(ctx) + Logging(sentry.LevelWarning)(ctx) return } diff --git a/app/http/server.go b/app/http/server.go index 4ae2e3b..6504b67 100644 --- a/app/http/server.go +++ b/app/http/server.go @@ -18,6 +18,7 @@ import ( "github.com/TicketsBot/GoPanel/app/http/session" "github.com/TicketsBot/GoPanel/config" "github.com/TicketsBot/common/permission" + "github.com/getsentry/sentry-go" sentrygin "github.com/getsentry/sentry-go/gin" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" @@ -43,7 +44,7 @@ func StartServer() { router.Use(gin.Recovery()) router.Use(middleware.MultiReadBody, middleware.ReadResponse) - router.Use(middleware.Logging) + router.Use(middleware.Logging(sentry.LevelError)) router.Use(sentrygin.New(sentrygin.Options{})) // Defaults are ok router.Use(rl(middleware.RateLimitTypeIp, 60, time.Minute)) @@ -79,7 +80,7 @@ func StartServer() { rl(middleware.RateLimitTypeGuild, 10, time.Second*30), rl(middleware.RateLimitTypeGuild, 75, time.Minute*30), api.SearchMembers, - middleware.Logging, + middleware.Logging(sentry.LevelError), // TODO: Remove? ) // Must be readable to load transcripts page @@ -116,7 +117,7 @@ func StartServer() { rl(middleware.RateLimitTypeUser, 5, 5*time.Second), rl(middleware.RateLimitTypeUser, 20, time.Minute), api_transcripts.ListTranscripts, - middleware.Logging, + middleware.Logging(sentry.LevelError), // TODO: Remove? ) // Allow regular users to get their own transcripts, make sure you check perms inside diff --git a/chatreplica/convert.go b/chatreplica/convert_v1.go similarity index 100% rename from chatreplica/convert.go rename to chatreplica/convert_v1.go diff --git a/chatreplica/convert_v2.go b/chatreplica/convert_v2.go new file mode 100644 index 0000000..b608072 --- /dev/null +++ b/chatreplica/convert_v2.go @@ -0,0 +1,14 @@ +package chatreplica + +import ( + "fmt" + v2 "github.com/TicketsBot/logarchiver/model/v2" +) + +func FromTranscript(transcript v2.Transcript, ticketId int) Payload { + return Payload{ + Entities: EntitiesFromTranscript(transcript.Entities), + Messages: MessagesFromTranscript(transcript.Messages), + ChannelName: fmt.Sprintf("ticket-%d", ticketId), + } +} diff --git a/chatreplica/proxy.go b/chatreplica/proxy.go index 6db99f4..40cb83f 100644 --- a/chatreplica/proxy.go +++ b/chatreplica/proxy.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/TicketsBot/GoPanel/config" + "github.com/getsentry/sentry-go" "io/ioutil" "net/http" "time" @@ -20,25 +21,33 @@ var client = &http.Client{ func Render(payload Payload) ([]byte, error) { encoded, err := json.Marshal(payload) if err != nil { - return nil, err - } + return nil, err + } res, err := client.Post(config.Conf.Bot.RenderServiceUrl, "application/json", bytes.NewBuffer(encoded)) if err != nil { - return nil, err - } - - fmt.Println(string(encoded)) + return nil, err + } if res.StatusCode != 200 { - return nil, fmt.Errorf("render service returned status code %d", res.StatusCode) - } + fmt.Println(string(encoded)) + + sentry.CaptureEvent(&sentry.Event{ + Extra: map[string]interface{}{ + "request_body": string(encoded), + }, + Level: sentry.LevelError, + Message: fmt.Sprintf("Render service returned status code %d", res.StatusCode), + }) + + return nil, fmt.Errorf("render service returned status code %d", res.StatusCode) + } bytes, err := ioutil.ReadAll(res.Body) defer res.Body.Close() if err != nil { - return nil, err - } + return nil, err + } return bytes, nil } diff --git a/chatreplica/structs.go b/chatreplica/structs.go index 8e18a17..c82983b 100644 --- a/chatreplica/structs.go +++ b/chatreplica/structs.go @@ -1,10 +1,12 @@ package chatreplica import ( + v2 "github.com/TicketsBot/logarchiver/model/v2" "github.com/rxdn/gdl/objects/channel" "github.com/rxdn/gdl/objects/channel/embed" "github.com/rxdn/gdl/objects/channel/message" "github.com/rxdn/gdl/objects/user" + "strconv" ) type ( @@ -54,6 +56,68 @@ const ( BadgeBot Badge = "bot" ) +// TODO: Use a generic ptr func func badgePtr(b Badge) *Badge { - return &b + return &b +} + +func MessagesFromTranscript(messages []v2.Message) []Message { + // Can't assign length as we might filter + var wrappedMessages []Message + + for _, msg := range messages { + if msg.Content == "" && len(msg.Embeds) == 0 && len(msg.Attachments) == 0 { + continue + } + + wrappedMessages = append(wrappedMessages, Message{ + Id: msg.Id, + Type: message.MessageTypeDefault, + Author: msg.AuthorId, + Time: msg.Timestamp.UnixMilli(), + Content: msg.Content, + Embeds: msg.Embeds, + Attachments: msg.Attachments, + }) + } + + return wrappedMessages +} + +func EntitiesFromTranscript(entities v2.Entities) Entities { + users := make(map[string]User) + for _, user := range entities.Users { + var badge *Badge + if user.Bot { + badge = badgePtr(BadgeBot) + } + + users[strconv.FormatUint(user.Id, 10)] = User{ + Avatar: user.AvatarUrl(256), + Username: user.Username, + Discriminator: user.Discriminator, + Badge: badge, + } + } + + channels := make(map[string]Channel) + for _, channel := range entities.Channels { + channels[strconv.FormatUint(channel.Id, 10)] = Channel{ + Name: channel.Name, + } + } + + roles := make(map[string]Role) + for _, role := range entities.Roles { + roles[strconv.FormatUint(role.Id, 10)] = Role{ + Name: role.Name, + Color: int(role.Colour), + } + } + + return Entities{ + Users: users, + Channels: channels, + Roles: roles, + } } diff --git a/cmd/panel/main.go b/cmd/panel/main.go index 090ba78..968087d 100644 --- a/cmd/panel/main.go +++ b/cmd/panel/main.go @@ -39,8 +39,8 @@ func main() { Dsn: config.Conf.SentryDsn, Debug: config.Conf.Debug, AttachStacktrace: true, - } + if err := sentry.Init(sentryOpts); err != nil { fmt.Printf("Error initialising sentry: %s", err.Error()) } diff --git a/go.mod b/go.mod index 44ded32..7a47e2f 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.14 require ( github.com/BurntSushi/toml v0.3.1 - github.com/TicketsBot/archiverclient v0.0.0-20211126155247-fb9aac46bd0c + github.com/TicketsBot/archiverclient v0.0.0-20220326163414-558fd52746dc github.com/TicketsBot/common v0.0.0-20220311020409-8068ba1c2ea3 github.com/TicketsBot/database v0.0.0-20220217133004-d190910ad66f + github.com/TicketsBot/logarchiver v0.0.0-20220326162808-cdf0310f5e1c // indirect github.com/TicketsBot/worker v0.0.0-20220318232743-c143004d6762 github.com/apex/log v1.1.2 github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect diff --git a/go.sum b/go.sum index fc7eff6..5675f9c 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,9 @@ github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMd github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/ReneKroon/ttlcache v1.6.0/go.mod h1:DG6nbhXKUQhrExfwwLuZUdH7UnRDDRA1IW+nBuCssvs= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/TicketsBot/archiverclient v0.0.0-20211126155247-fb9aac46bd0c h1:5JgU8LkEbeZ/kZg/T0SUGq5mUA+ne9b4lyOkGZ3En+g= github.com/TicketsBot/archiverclient v0.0.0-20211126155247-fb9aac46bd0c/go.mod h1:MHUPvDnKd9SIsfg4/kQcC9Lyzq9Q6WDeRV4k+F7nris= +github.com/TicketsBot/archiverclient v0.0.0-20220326163414-558fd52746dc h1:n15W8Eg+ik3/0yqPzZVRP2oZJcIZCIgQ071cZleedKo= +github.com/TicketsBot/archiverclient v0.0.0-20220326163414-558fd52746dc/go.mod h1:2KcfHS0JnSsgcxZBs3NyWMXNQzEo67mBSGOyzHPWOCc= github.com/TicketsBot/common v0.0.0-20201015164440-602656bff706/go.mod h1:KVuvTmhUhMBVTMmjEpSzitKnODu6wMkFa3pK/W2NuiY= github.com/TicketsBot/common v0.0.0-20220311020409-8068ba1c2ea3 h1:wcmBzDWg68gumBKDdMPr+CFiFM03YKB2pBNy6Vcq+To= github.com/TicketsBot/common v0.0.0-20220311020409-8068ba1c2ea3/go.mod h1:SVwX6gKkxRCMbp+qwJIgvQiy/Ut0fUddexEqRB/NTzc= @@ -15,8 +16,9 @@ github.com/TicketsBot/database v0.0.0-20200516170158-fd8a949aec2c/go.mod h1:eky4 github.com/TicketsBot/database v0.0.0-20210902172951-4e1f8ced84b7/go.mod h1:A4T2uQFIWC/ttCYpfgv7AkPjR09mMRgzG13lgoV/+aI= github.com/TicketsBot/database v0.0.0-20220217133004-d190910ad66f h1:kBIHaAxyIGqxgwz/ZRggNGjyIdZ3ctWZqjJ9Svrn4L4= github.com/TicketsBot/database v0.0.0-20220217133004-d190910ad66f/go.mod h1:72oWvH/Gq1iKeXCZhVRZn1JFbNVC5iAgERZWTrEarEo= -github.com/TicketsBot/logarchiver v0.0.0-20211126155104-4d78c720975c h1:TTeqSyA2b3OyPtZrjlnxcapXJMOBnO4Aft0aAXPudu4= github.com/TicketsBot/logarchiver v0.0.0-20211126155104-4d78c720975c/go.mod h1:a9ldKejmFXgqsisXXNewDs2kgiZwDNRA7oDL4Yoh3PU= +github.com/TicketsBot/logarchiver v0.0.0-20220326162808-cdf0310f5e1c h1:OqGjFH6mbE6gd+NqI2ARJdtH3UUvhiAkD0r0fhGJK2s= +github.com/TicketsBot/logarchiver v0.0.0-20220326162808-cdf0310f5e1c/go.mod h1:jgi2OXQKsd5nUnTIRkwvPmeuD/i7OhN68LKMssuQY1c= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 h1:NHD5GB6cjlkpZFjC76Yli2S63/J2nhr8MuE6KlYJpQM= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261/go.mod h1:2zPxDAN2TAPpxUPjxszjs3QFKreKrQh5al/R3cMXmYk= github.com/TicketsBot/worker v0.0.0-20220318232743-c143004d6762 h1:T3xT2PK3S8IigEZq0wOiJINjjr5IvsGLjqdz7uR8prs= @@ -573,8 +575,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= nhooyr.io/websocket v1.8.4 h1:P43INlkmY2eCxLvHeiMFK/ROUiOm0NdzRGGDtURbe58= nhooyr.io/websocket v1.8.4/go.mod h1:LiqdCg1Cu7TPWxEvPjPa0TGYxCsy4pHNTN9gGluwBpQ=