Refactor

2024-11-11 22:59:24 +00:00 · 2024-11-11 22:59:24 +00:00 · 4c405bfd73
commit 4c405bfd73
parent d940be634d
9 changed files with 144 additions and 288 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,4 @@
 config.toml
 *.iml
 .idea
+.env
--- a/app/http/middleware/logging.go
+++ b/app/http/middleware/logging.go
@ -1,87 +1,48 @@
 package middleware

 import (
-	"fmt"
-	"github.com/getsentry/sentry-go"
 	"github.com/gin-gonic/gin"
-	"io/ioutil"
-	"runtime/debug"
-	"strconv"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"time"
 )

-type Level uint8
+func Logging(logger *zap.Logger) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		start := time.Now()
+		path := c.Request.URL.Path
+		raw := c.Request.URL.RawQuery

-const (
-	LevelDebug Level = iota
-	LevelInfo
-	LevelWarning
-	LevelError
-	LevelFatal
-)
+		// Process request
+		c.Next()

-func (l Level) sentryLevel() sentry.Level {
-	switch l {
-	case LevelDebug:
-		return sentry.LevelDebug
-	case LevelInfo:
-		return sentry.LevelInfo
-	case LevelWarning:
-		return sentry.LevelWarning
-	case LevelError:
-		return sentry.LevelError
-	case LevelFatal:
-		return sentry.LevelFatal
-	default:
-		return sentry.LevelDebug
-	}
-}
+		statusCode := c.Writer.Status()

-func Logging(minLevel Level) gin.HandlerFunc {
-	return func(ctx *gin.Context) {
-		ctx.Next()
-
-		statusCode := ctx.Writer.Status()
-
-		level := LevelInfo
+		level := zapcore.InfoLevel
 		if statusCode >= 500 {
-			level = LevelError
+			level = zapcore.ErrorLevel
 		} else if statusCode >= 400 {
-			level = LevelWarning
+			level = zapcore.WarnLevel
 		}

-		if level < minLevel {
-			return
+		fields := []zap.Field{
+			zap.String("method", c.Request.Method),
+			zap.String("path", path),
+			zap.String("query", raw),
+			zap.Int("status", c.Writer.Status()),
+			zap.String("timestamp", start.String()),
+			zap.Duration("latency", time.Now().Sub(start)),
+			zap.String("client_ip", c.ClientIP()),
 		}

-		requestBody, _ := ioutil.ReadAll(ctx.Request.Body)
-
-		var responseBody []byte
-		if statusCode >= 400 && statusCode <= 599 {
-			cw, ok := ctx.Writer.(*CustomWriter)
-			if ok {
-				responseBody = cw.Read()
-			}
+		if guildId, ok := c.Keys["guildid"]; ok {
+			fields = append(fields, zap.Uint64("guild_id", guildId.(uint64)))
 		}

-		sentry.CaptureEvent(&sentry.Event{
-			Extra: map[string]interface{}{
-				"status_code":  strconv.Itoa(statusCode),
-				"method":       ctx.Request.Method,
-				"path":         ctx.Request.URL.Path,
-				"query":        ctx.Request.URL.RawQuery,
-				"guild_id":     ctx.Keys["guildid"],
-				"user_id":      ctx.Keys["userid"],
-				"request_body": string(requestBody),
-				"response":     string(responseBody),
-				"stacktrace":   string(debug.Stack()),
-			},
-			Level:   level.sentryLevel(),
-			Message: fmt.Sprintf("HTTP %d on %s %s", statusCode, ctx.Request.Method, ctx.FullPath()),
-			Tags: map[string]string{
-				"status_code": strconv.Itoa(statusCode),
-				"method":      ctx.Request.Method,
-				"path":        ctx.Request.URL.Path,
-			},
-		})
+		if userId, ok := c.Keys["userid"]; ok {
+			fields = append(fields, zap.Uint64("user_id", userId.(uint64)))
+		}
+
+		logger.Log(level, "Incoming HTTP request", fields...)
 	}
 }
--- a/app/http/middleware/ratelimit.go
+++ b/app/http/middleware/ratelimit.go
@ -38,7 +38,6 @@ func CreateRateLimiter(rlType RateLimitType, max int, period time.Duration) gin.
 		res, err := limiter.Allow(redis.DefaultContext(), name, limit)
 		if err != nil {
 			ctx.AbortWithStatusJSON(500, utils.ErrorJson(err))
-			Logging(LevelError)(ctx)
 			return
 		}

--- a/app/http/server.go
+++ b/app/http/server.go
@ -23,14 +23,16 @@ import (
 	"github.com/TicketsBot/common/permission"
 	"github.com/gin-gonic/gin"
 	"github.com/penglongli/gin-metrics/ginmetrics"
-	"log"
+	"go.uber.org/zap"
 	"time"
 )

-func StartServer(sm *livechat.SocketManager) {
-	log.Println("Starting HTTP server")
+func StartServer(logger *zap.Logger, sm *livechat.SocketManager) {
+	logger.Info("Starting HTTP server")

-	router := gin.Default()
+	router := gin.New()
+	router.Use(gin.Recovery())
+	router.Use(middleware.Logging(logger))

 	router.RemoteIPHeaders = config.Conf.Server.RealIpHeaders
 	if err := router.SetTrustedProxies(config.Conf.Server.TrustedProxies); err != nil {
@ -40,10 +42,6 @@ func StartServer(sm *livechat.SocketManager) {
 	// Sessions
 	session.Store = session.NewRedisStore()

-	router.Use(gin.Recovery())
-	router.Use(middleware.MultiReadBody, middleware.ReadResponse)
-	router.Use(middleware.Logging(middleware.LevelError))
-
 	router.Use(rl(middleware.RateLimitTypeIp, 60, time.Minute))
 	router.Use(rl(middleware.RateLimitTypeIp, 20, time.Second*10))
 	router.Use(rl(middleware.RateLimitTypeUser, 60, time.Minute))
@ -57,7 +55,10 @@ func StartServer(sm *livechat.SocketManager) {
 		monitor.UseWithoutExposingEndpoint(router)
 		monitor.SetMetricPath("/metrics")

-		metricRouter := gin.Default()
+		metricRouter := gin.New()
+		metricRouter.Use(gin.Recovery())
+		metricRouter.Use(middleware.Logging(logger))
+		
 		monitor.Expose(metricRouter)

 		go func() {
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@ -13,11 +13,14 @@ import (
 	"github.com/TicketsBot/archiverclient"
 	"github.com/TicketsBot/common/chatrelay"
 	"github.com/TicketsBot/common/model"
+	"github.com/TicketsBot/common/observability"
 	"github.com/TicketsBot/common/premium"
 	"github.com/TicketsBot/common/secureproxy"
 	"github.com/TicketsBot/worker/i18n"
 	"github.com/getsentry/sentry-go"
 	"github.com/rxdn/gdl/rest/request"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 	"net/http"
 	"net/http/pprof"
 )
@ -25,27 +28,53 @@ import (
 func main() {
 	startPprof()

-	config.LoadConfig()
+	cfg, err := config.LoadConfig()
+	utils.Must(err)
+	config.Conf = cfg

-	sentryOpts := sentry.ClientOptions{
-		Dsn:              config.Conf.SentryDsn,
-		Debug:            config.Conf.Debug,
-		AttachStacktrace: true,
-		EnableTracing:    true,
-		TracesSampleRate: 0.1,
+	if config.Conf.SentryDsn != nil {
+		sentryOpts := sentry.ClientOptions{
+			Dsn:              *config.Conf.SentryDsn,
+			Debug:            config.Conf.Debug,
+			AttachStacktrace: true,
+			EnableTracing:    true,
+			TracesSampleRate: 0.1,
+		}
+
+		if err := sentry.Init(sentryOpts); err != nil {
+			fmt.Printf("Failed to initialise sentry: %s", err.Error())
+		}
 	}

-	if err := sentry.Init(sentryOpts); err != nil {
-		fmt.Printf("Error initialising sentry: %s", err.Error())
+	var logger *zap.Logger
+	if config.Conf.JsonLogs {
+		loggerConfig := zap.NewProductionConfig()
+		loggerConfig.Level.SetLevel(config.Conf.LogLevel)
+
+		logger, err = loggerConfig.Build(
+			zap.AddCaller(),
+			zap.AddStacktrace(zap.ErrorLevel),
+			zap.WrapCore(observability.ZapSentryAdapter(observability.EnvironmentProduction)),
+		)
+	} else {
+		loggerConfig := zap.NewDevelopmentConfig()
+		loggerConfig.Level.SetLevel(config.Conf.LogLevel)
+		loggerConfig.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
+
+		logger, err = loggerConfig.Build(zap.AddCaller(), zap.AddStacktrace(zap.ErrorLevel))
 	}

-	fmt.Println("Connecting to database...")
+	if err != nil {
+		panic(fmt.Errorf("failed to initialise zap logger: %w", err))
+	}
+
+	logger.Info("Connecting to database")
 	database.ConnectToDatabase()

-	fmt.Println("Connecting to cache...")
+	logger.Info("Connecting to cache")
 	cache.Instance = cache.NewCache()

-	fmt.Println("Initialising microservice clients...")
+	logger.Info("Initialising microservice clients")
 	utils.ArchiverClient = archiverclient.NewArchiverClient(archiverclient.NewProxyRetriever(config.Conf.Bot.ObjectStore), []byte(config.Conf.Bot.AesKey))
 	utils.SecureProxyClient = secureproxy.NewSecureProxy(config.Conf.SecureProxyUrl)

@ -57,7 +86,7 @@ func main() {
 		request.RegisterHook(utils.ProxyHook)
 	}

-	fmt.Println("Connecting to Redis...")
+	logger.Info("Connecting to Redis")
 	redis.Client = redis.NewRedisClient()

 	socketManager := livechat.NewSocketManager()
@ -76,8 +105,8 @@ func main() {
 		rpc.PremiumClient = &c
 	}

-	fmt.Println("Starting server...")
-	app.StartServer(socketManager)
+	logger.Info("Starting server")
+	app.StartServer(logger, socketManager)
 }

 func ListenChat(client *redis.RedisClient, sm *livechat.SocketManager) {
--- a/config.toml.example
+++ b/config.toml.example
@ -1,36 +0,0 @@
-admins=[585576154958921739]
-
-[server]
-host="0.0.0.0:3000"
-baseUrl="http://localhost:3000"
-mainSite="https://ticketsbot.net"
-  [server.ratelimit]
-  window=10
-  max=600
-  [server.session]
-  threads=10
-  secret="secret"
-
-[oauth]
-id=
-secret=""
-redirectUri=""
-
-[database]
-uri="postgres://user:pwd@localhost:5432/database?pool_max_conns=10"
-
-[bot]
-token=""
-premium-lookup-proxy-url="http://localhost:3000"
-premium-lookup-proxy-key=""
-objectstore=""
-aes-key=""
-
-[redis]
-host="127.0.0.1"
-port=6379
-password=""
-threads=5
-
-[cache]
-uri="postgres://pwd:user@localhost:5432/db"
--- a/config/config.go
+++ b/config/config.go
@ -2,184 +2,82 @@ package config

 import (
 	"github.com/BurntSushi/toml"
-	"github.com/TicketsBot/common/sentry"
+	"github.com/caarlos0/env/v11"
+	"go.uber.org/zap/zapcore"
 	"os"
-	"strconv"
-	"strings"
 )

-type (
-	Config struct {
-		Admins          []uint64
-		ForceWhitelabel []uint64
-		Debug           bool
-		SentryDsn       string
-		Server          Server
-		Oauth           Oauth
-		Database        Database
-		Bot             Bot
-		Redis           Redis
-		Cache           Cache
-		SecureProxyUrl  string
+type Config struct {
+	Admins          []uint64      `env:"ADMINS"`
+	ForceWhitelabel []uint64      `env:"FORCED_WHITELABEL"`
+	Debug           bool          `env:"DEBUG"`
+	SentryDsn       *string       `env:"SENTRY_DSN"`
+	JsonLogs        bool          `env:"JSON_LOGS" envDefault:"false"`
+	LogLevel        zapcore.Level `env:"LOG_LEVEL" envDefault:"info"`
+	Server          struct {
+		Host       string `env:"SERVER_ADDR,required"`
+		MetricHost string `env:"METRIC_SERVER_ADDR"`
+		BaseUrl    string `env:"BASE_URL,required"`
+		MainSite   string `env:"MAIN_SITE,required"`
+		Ratelimit  struct {
+			Window int `env:"WINDOW,required"`
+			Max    int `env:"MAX,required"`
+		} `envPrefix:"RATELIMIT_"`
+		Secret         string   `env:"JWT_SECRET,required"`
+		RealIpHeaders  []string `env:"REAL_IP_HEADERS"`
+		TrustedProxies []string `env:"TRUSTED_PROXIES"`
 	}
-
-	Server struct {
-		Host           string
-		MetricHost     string
-		BaseUrl        string
-		MainSite       string
-		Ratelimit      Ratelimit
-		Session        Session
-		Secret         string
-		RealIpHeaders  []string
-		TrustedProxies []string
-	}
-
-	Ratelimit struct {
-		Window int
-		Max    int
-	}
-
-	Session struct {
-		Threads int
-		Secret  string
-	}
-
 	Oauth struct {
-		Id          uint64
-		Secret      string
-		RedirectUri string
-	}
-
+		Id          uint64 `env:"ID,required"`
+		Secret      string `env:"SECRET,required"`
+		RedirectUri string `env:"REDIRECT_URI,required"`
+	} `envPrefix:"OAUTH_"`
 	Database struct {
-		Uri string
-	}
-
+		Uri string `env:"URI,required"`
+	} `envPrefix:"DATABASE_"`
 	Bot struct {
-		Id                                   uint64
-		Token                                string
-		PremiumLookupProxyUrl                string `toml:"premium-lookup-proxy-url"`
-		PremiumLookupProxyKey                string `toml:"premium-lookup-proxy-key"`
-		ObjectStore                          string
-		AesKey                               string `toml:"aes-key"`
-		ProxyUrl                             string `toml:"discord-proxy-url"`
-		RenderServiceUrl                     string `toml:"render-service-url"`
-		ImageProxySecret                     string `toml:"image-proxy-secret"`
-		PublicIntegrationRequestWebhookId    uint64 `toml:"public-integration-request-webhook-id"`
-		PublicIntegrationRequestWebhookToken string `toml:"public-integration-request-webhook-token"`
+		Id                                   uint64 `env:"BOT_ID,required"`
+		Token                                string `env:"BOT_TOKEN,required"`
+		ObjectStore                          string `env:"LOG_ARCHIVER_URL"`
+		AesKey                               string `env:"LOG_AES_KEY" toml:"aes-key"`
+		ProxyUrl                             string `env:"DISCORD_PROXY_URL" toml:"discord-proxy-url"`
+		RenderServiceUrl                     string `env:"RENDER_SERVICE_URL" toml:"render-service-url"`
+		ImageProxySecret                     string `env:"IMAGE_PROXY_SECRET" toml:"image-proxy-secret"`
+		PublicIntegrationRequestWebhookId    uint64 `env:"PUBLIC_INTEGRATION_REQUEST_WEBHOOK_ID" toml:"public-integration-request-webhook-id"`
+		PublicIntegrationRequestWebhookToken string `env:"PUBLIC_INTEGRATION_REQUEST_WEBHOOK_TOKEN" toml:"public-integration-request-webhook-token"`
 	}
-
 	Redis struct {
-		Host     string
-		Port     int
-		Password string
-		Threads  int
-	}
-
+		Host     string `env:"HOST,required"`
+		Port     int    `env:"PORT,required"`
+		Password string `env:"PASSWORD"`
+		Threads  int    `env:"THREADS,required"`
+	} `envPrefix:"REDIS_"`
 	Cache struct {
-		Uri string
-	}
-)
+		Uri string `env:"URI,required"`
+	} `envPrefix:"CACHE_"`
+	SecureProxyUrl string `env:"SECURE_PROXY_URL"`
+}

-var (
-	Conf Config
-)
+// TODO: Don't use a global variable
+var Conf Config

-func LoadConfig() {
+func LoadConfig() (Config, error) {
 	if _, err := os.Stat("config.toml"); err == nil {
-		fromToml()
+		return fromToml()
 	} else {
-		fromEnvvar()
+		return fromEnvvar()
 	}
 }

-func fromToml() {
+func fromToml() (Config, error) {
+	var config Config
 	if _, err := toml.DecodeFile("config.toml", &Conf); err != nil {
-		panic(err)
+		return Config{}, err
 	}
+
+	return config, nil
 }

-// TODO: Proper env package
-func fromEnvvar() {
-	var admins []uint64
-	for _, id := range strings.Split(os.Getenv("ADMINS"), ",") {
-		if parsed, err := strconv.ParseUint(id, 10, 64); err == nil {
-			admins = append(admins, parsed)
-		} else {
-			sentry.Error(err)
-		}
-	}
-
-	var forcedWhitelabel []uint64
-	for _, id := range strings.Split(os.Getenv("FORCED_WHITELABEL"), ",") {
-		if parsed, err := strconv.ParseUint(id, 10, 64); err == nil {
-			forcedWhitelabel = append(forcedWhitelabel, parsed)
-		} else {
-			sentry.Error(err)
-		}
-	}
-
-	rateLimitWindow, _ := strconv.Atoi(os.Getenv("RATELIMIT_WINDOW"))
-	rateLimitMax, _ := strconv.Atoi(os.Getenv("RATELIMIT_MAX"))
-	sessionThreads, _ := strconv.Atoi(os.Getenv("SESSION_DB_THREADS"))
-	oauthId, _ := strconv.ParseUint(os.Getenv("OAUTH_ID"), 10, 64)
-	botId, _ := strconv.ParseUint(os.Getenv("BOT_ID"), 10, 64)
-	redisPort, _ := strconv.Atoi(os.Getenv("REDIS_PORT"))
-	redisThreads, _ := strconv.Atoi(os.Getenv("REDIS_THREADS"))
-	publicIntegrationRequestWebhookId, _ := strconv.ParseUint(os.Getenv("PUBLIC_INTEGRATION_REQUEST_WEBHOOK_ID"), 10, 64)
-
-	Conf = Config{
-		Admins:          admins,
-		ForceWhitelabel: forcedWhitelabel,
-		Debug:           os.Getenv("DEBUG") != "",
-		SentryDsn:       os.Getenv("SENTRY_DSN"),
-		Server: Server{
-			Host:       os.Getenv("SERVER_ADDR"),
-			MetricHost: os.Getenv("METRIC_SERVER_ADDR"),
-			BaseUrl:    os.Getenv("BASE_URL"),
-			MainSite:   os.Getenv("MAIN_SITE"),
-			Ratelimit: Ratelimit{
-				Window: rateLimitWindow,
-				Max:    rateLimitMax,
-			},
-			Session: Session{
-				Threads: sessionThreads,
-				Secret:  os.Getenv("SESSION_SECRET"),
-			},
-			Secret:         os.Getenv("JWT_SECRET"),
-			TrustedProxies: strings.Split(os.Getenv("TRUSTED_PROXIES"), ","),
-			RealIpHeaders:  strings.Split(os.Getenv("REAL_IP_HEADERS"), ","),
-		},
-		Oauth: Oauth{
-			Id:          oauthId,
-			Secret:      os.Getenv("OAUTH_SECRET"),
-			RedirectUri: os.Getenv("OAUTH_REDIRECT_URI"),
-		},
-		Database: Database{
-			Uri: os.Getenv("DATABASE_URI"),
-		},
-		Bot: Bot{
-			Id:                                   botId,
-			Token:                                os.Getenv("BOT_TOKEN"),
-			PremiumLookupProxyUrl:                os.Getenv("PREMIUM_PROXY_URL"),
-			PremiumLookupProxyKey:                os.Getenv("PREMIUM_PROXY_KEY"),
-			ObjectStore:                          os.Getenv("LOG_ARCHIVER_URL"),
-			AesKey:                               os.Getenv("LOG_AES_KEY"),
-			ProxyUrl:                             os.Getenv("DISCORD_PROXY_URL"),
-			RenderServiceUrl:                     os.Getenv("RENDER_SERVICE_URL"),
-			ImageProxySecret:                     os.Getenv("IMAGE_PROXY_SECRET"),
-			PublicIntegrationRequestWebhookId:    publicIntegrationRequestWebhookId,
-			PublicIntegrationRequestWebhookToken: os.Getenv("PUBLIC_INTEGRATION_REQUEST_WEBHOOK_TOKEN"),
-		},
-		Redis: Redis{
-			Host:     os.Getenv("REDIS_HOST"),
-			Port:     redisPort,
-			Password: os.Getenv("REDIS_PASSWORD"),
-			Threads:  redisThreads,
-		},
-		Cache: Cache{
-			Uri: os.Getenv("CACHE_URI"),
-		},
-		SecureProxyUrl: os.Getenv("SECURE_PROXY_URL"),
-	}
+func fromEnvvar() (Config, error) {
+	return env.ParseAs[Config]()
 }
--- a/go.mod
+++ b/go.mod
@ -49,6 +49,7 @@ require (
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/caarlos0/env v3.5.0+incompatible // indirect
 	github.com/caarlos0/env/v10 v10.0.0 // indirect
+	github.com/caarlos0/env/v11 v11.2.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
--- a/go.sum
+++ b/go.sum
@ -88,6 +88,8 @@ github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yi
 github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y=
 github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA=
 github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18=
+github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg=
+github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=