Refactor

.gitignore

@@ -1,3 +1,4 @@
 config.toml
 *.iml
 .idea
+.env

app/http/middleware/logging.go

@@ -1,87 +1,48 @@
 package middleware
 
 import (
-	"fmt"
-	"github.com/getsentry/sentry-go"
 	"github.com/gin-gonic/gin"
-	"io/ioutil"
+	"go.uber.org/zap"
-	"runtime/debug"
+	"go.uber.org/zap/zapcore"
-	"strconv"
+	"time"
 )
 
-type Level uint8
+func Logging(logger *zap.Logger) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		start := time.Now()
+		path := c.Request.URL.Path
+		raw := c.Request.URL.RawQuery
 
-const (
+		// Process request
-	LevelDebug Level = iota
+		c.Next()
-	LevelInfo
-	LevelWarning
-	LevelError
-	LevelFatal
-)
 
-func (l Level) sentryLevel() sentry.Level {
+		statusCode := c.Writer.Status()
-	switch l {
-	case LevelDebug:
-		return sentry.LevelDebug
-	case LevelInfo:
-		return sentry.LevelInfo
-	case LevelWarning:
-		return sentry.LevelWarning
-	case LevelError:
-		return sentry.LevelError
-	case LevelFatal:
-		return sentry.LevelFatal
-	default:
-		return sentry.LevelDebug
-	}
-}
 
-func Logging(minLevel Level) gin.HandlerFunc {
+		level := zapcore.InfoLevel
-	return func(ctx *gin.Context) {
-		ctx.Next()
-
-		statusCode := ctx.Writer.Status()
-
-		level := LevelInfo
 		if statusCode >= 500 {
-			level = LevelError
+			level = zapcore.ErrorLevel
 		} else if statusCode >= 400 {
-			level = LevelWarning
+			level = zapcore.WarnLevel
 		}
 
-		if level < minLevel {
+		fields := []zap.Field{
-			return
+			zap.String("method", c.Request.Method),
+			zap.String("path", path),
+			zap.String("query", raw),
+			zap.Int("status", c.Writer.Status()),
+			zap.String("timestamp", start.String()),
+			zap.Duration("latency", time.Now().Sub(start)),
+			zap.String("client_ip", c.ClientIP()),
 		}
 
-		requestBody, _ := ioutil.ReadAll(ctx.Request.Body)
+		if guildId, ok := c.Keys["guildid"]; ok {
-
+			fields = append(fields, zap.Uint64("guild_id", guildId.(uint64)))
-		var responseBody []byte
-		if statusCode >= 400 && statusCode <= 599 {
-			cw, ok := ctx.Writer.(*CustomWriter)
-			if ok {
-				responseBody = cw.Read()
-			}
 		}
 
-		sentry.CaptureEvent(&sentry.Event{
+		if userId, ok := c.Keys["userid"]; ok {
-			Extra: map[string]interface{}{
+			fields = append(fields, zap.Uint64("user_id", userId.(uint64)))
-				"status_code":  strconv.Itoa(statusCode),
+		}
-				"method":       ctx.Request.Method,
+
-				"path":         ctx.Request.URL.Path,
+		logger.Log(level, "Incoming HTTP request", fields...)
-				"query":        ctx.Request.URL.RawQuery,
-				"guild_id":     ctx.Keys["guildid"],
-				"user_id":      ctx.Keys["userid"],
-				"request_body": string(requestBody),
-				"response":     string(responseBody),
-				"stacktrace":   string(debug.Stack()),
-			},
-			Level:   level.sentryLevel(),
-			Message: fmt.Sprintf("HTTP %d on %s %s", statusCode, ctx.Request.Method, ctx.FullPath()),
-			Tags: map[string]string{
-				"status_code": strconv.Itoa(statusCode),
-				"method":      ctx.Request.Method,
-				"path":        ctx.Request.URL.Path,
-			},
-		})
 	}
 }

app/http/middleware/ratelimit.go

@@ -38,7 +38,6 @@ func CreateRateLimiter(rlType RateLimitType, max int, period time.Duration) gin.
 		res, err := limiter.Allow(redis.DefaultContext(), name, limit)
 		if err != nil {
 			ctx.AbortWithStatusJSON(500, utils.ErrorJson(err))
-			Logging(LevelError)(ctx)
 			return
 		}
 

app/http/server.go

@@ -23,14 +23,16 @@ import (
 	"github.com/TicketsBot/common/permission"
 	"github.com/gin-gonic/gin"
 	"github.com/penglongli/gin-metrics/ginmetrics"
-	"log"
+	"go.uber.org/zap"
 	"time"
 )
 
-func StartServer(sm *livechat.SocketManager) {
+func StartServer(logger *zap.Logger, sm *livechat.SocketManager) {
-	log.Println("Starting HTTP server")
+	logger.Info("Starting HTTP server")
 
-	router := gin.Default()
+	router := gin.New()
+	router.Use(gin.Recovery())
+	router.Use(middleware.Logging(logger))
 
 	router.RemoteIPHeaders = config.Conf.Server.RealIpHeaders
 	if err := router.SetTrustedProxies(config.Conf.Server.TrustedProxies); err != nil {
@@ -40,10 +42,6 @@ func StartServer(sm *livechat.SocketManager) {
 	// Sessions
 	session.Store = session.NewRedisStore()
 
-	router.Use(gin.Recovery())
-	router.Use(middleware.MultiReadBody, middleware.ReadResponse)
-	router.Use(middleware.Logging(middleware.LevelError))
-
 	router.Use(rl(middleware.RateLimitTypeIp, 60, time.Minute))
 	router.Use(rl(middleware.RateLimitTypeIp, 20, time.Second*10))
 	router.Use(rl(middleware.RateLimitTypeUser, 60, time.Minute))
@@ -57,7 +55,10 @@ func StartServer(sm *livechat.SocketManager) {
 		monitor.UseWithoutExposingEndpoint(router)
 		monitor.SetMetricPath("/metrics")
 
-		metricRouter := gin.Default()
+		metricRouter := gin.New()
+		metricRouter.Use(gin.Recovery())
+		metricRouter.Use(middleware.Logging(logger))
+		
 		monitor.Expose(metricRouter)
 
 		go func() {

cmd/api/main.go

@@ -13,11 +13,14 @@ import (
 	"github.com/TicketsBot/archiverclient"
 	"github.com/TicketsBot/common/chatrelay"
 	"github.com/TicketsBot/common/model"
+	"github.com/TicketsBot/common/observability"
 	"github.com/TicketsBot/common/premium"
 	"github.com/TicketsBot/common/secureproxy"
 	"github.com/TicketsBot/worker/i18n"
 	"github.com/getsentry/sentry-go"
 	"github.com/rxdn/gdl/rest/request"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 	"net/http"
 	"net/http/pprof"
 )
@@ -25,10 +28,13 @@ import (
 func main() {
 	startPprof()
 
-	config.LoadConfig()
+	cfg, err := config.LoadConfig()
+	utils.Must(err)
+	config.Conf = cfg
 
+	if config.Conf.SentryDsn != nil {
 		sentryOpts := sentry.ClientOptions{
-		Dsn:              config.Conf.SentryDsn,
+			Dsn:              *config.Conf.SentryDsn,
 			Debug:            config.Conf.Debug,
 			AttachStacktrace: true,
 			EnableTracing:    true,
@@ -36,16 +42,39 @@ func main() {
 		}
 
 		if err := sentry.Init(sentryOpts); err != nil {
-		fmt.Printf("Error initialising sentry: %s", err.Error())
+			fmt.Printf("Failed to initialise sentry: %s", err.Error())
+		}
 	}
 
-	fmt.Println("Connecting to database...")
+	var logger *zap.Logger
+	if config.Conf.JsonLogs {
+		loggerConfig := zap.NewProductionConfig()
+		loggerConfig.Level.SetLevel(config.Conf.LogLevel)
+
+		logger, err = loggerConfig.Build(
+			zap.AddCaller(),
+			zap.AddStacktrace(zap.ErrorLevel),
+			zap.WrapCore(observability.ZapSentryAdapter(observability.EnvironmentProduction)),
+		)
+	} else {
+		loggerConfig := zap.NewDevelopmentConfig()
+		loggerConfig.Level.SetLevel(config.Conf.LogLevel)
+		loggerConfig.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
+
+		logger, err = loggerConfig.Build(zap.AddCaller(), zap.AddStacktrace(zap.ErrorLevel))
+	}
+
+	if err != nil {
+		panic(fmt.Errorf("failed to initialise zap logger: %w", err))
+	}
+
+	logger.Info("Connecting to database")
 	database.ConnectToDatabase()
 
-	fmt.Println("Connecting to cache...")
+	logger.Info("Connecting to cache")
 	cache.Instance = cache.NewCache()
 
-	fmt.Println("Initialising microservice clients...")
+	logger.Info("Initialising microservice clients")
 	utils.ArchiverClient = archiverclient.NewArchiverClient(archiverclient.NewProxyRetriever(config.Conf.Bot.ObjectStore), []byte(config.Conf.Bot.AesKey))
 	utils.SecureProxyClient = secureproxy.NewSecureProxy(config.Conf.SecureProxyUrl)
 
@@ -57,7 +86,7 @@ func main() {
 		request.RegisterHook(utils.ProxyHook)
 	}
 
-	fmt.Println("Connecting to Redis...")
+	logger.Info("Connecting to Redis")
 	redis.Client = redis.NewRedisClient()
 
 	socketManager := livechat.NewSocketManager()
@@ -76,8 +105,8 @@ func main() {
 		rpc.PremiumClient = &c
 	}
 
-	fmt.Println("Starting server...")
+	logger.Info("Starting server")
-	app.StartServer(socketManager)
+	app.StartServer(logger, socketManager)
 }
 
 func ListenChat(client *redis.RedisClient, sm *livechat.SocketManager) {

config.toml.example

@@ -1,36 +0,0 @@
-admins=[585576154958921739]
-
-[server]
-host="0.0.0.0:3000"
-baseUrl="http://localhost:3000"
-mainSite="https://ticketsbot.net"
-  [server.ratelimit]
-  window=10
-  max=600
-  [server.session]
-  threads=10
-  secret="secret"
-
-[oauth]
-id=
-secret=""
-redirectUri=""
-
-[database]
-uri="postgres://user:pwd@localhost:5432/database?pool_max_conns=10"
-
-[bot]
-token=""
-premium-lookup-proxy-url="http://localhost:3000"
-premium-lookup-proxy-key=""
-objectstore=""
-aes-key=""
-
-[redis]
-host="127.0.0.1"
-port=6379
-password=""
-threads=5
-
-[cache]
-uri="postgres://pwd:user@localhost:5432/db"

config/config.go

@@ -2,184 +2,82 @@ package config
 
 import (
 	"github.com/BurntSushi/toml"
-	"github.com/TicketsBot/common/sentry"
+	"github.com/caarlos0/env/v11"
+	"go.uber.org/zap/zapcore"
 	"os"
-	"strconv"
-	"strings"
 )
 
-type (
+type Config struct {
-	Config struct {
+	Admins          []uint64      `env:"ADMINS"`
-		Admins          []uint64
+	ForceWhitelabel []uint64      `env:"FORCED_WHITELABEL"`
-		ForceWhitelabel []uint64
+	Debug           bool          `env:"DEBUG"`
-		Debug           bool
+	SentryDsn       *string       `env:"SENTRY_DSN"`
-		SentryDsn       string
+	JsonLogs        bool          `env:"JSON_LOGS" envDefault:"false"`
-		Server          Server
+	LogLevel        zapcore.Level `env:"LOG_LEVEL" envDefault:"info"`
-		Oauth           Oauth
-		Database        Database
-		Bot             Bot
-		Redis           Redis
-		Cache           Cache
-		SecureProxyUrl  string
-	}
-
 	Server          struct {
-		Host           string
+		Host       string `env:"SERVER_ADDR,required"`
-		MetricHost     string
+		MetricHost string `env:"METRIC_SERVER_ADDR"`
-		BaseUrl        string
+		BaseUrl    string `env:"BASE_URL,required"`
-		MainSite       string
+		MainSite   string `env:"MAIN_SITE,required"`
-		Ratelimit      Ratelimit
-		Session        Session
-		Secret         string
-		RealIpHeaders  []string
-		TrustedProxies []string
-	}
-
 		Ratelimit  struct {
-		Window int
+			Window int `env:"WINDOW,required"`
-		Max    int
+			Max    int `env:"MAX,required"`
+		} `envPrefix:"RATELIMIT_"`
+		Secret         string   `env:"JWT_SECRET,required"`
+		RealIpHeaders  []string `env:"REAL_IP_HEADERS"`
+		TrustedProxies []string `env:"TRUSTED_PROXIES"`
 	}
-
-	Session struct {
-		Threads int
-		Secret  string
-	}
-
 	Oauth struct {
-		Id          uint64
+		Id          uint64 `env:"ID,required"`
-		Secret      string
+		Secret      string `env:"SECRET,required"`
-		RedirectUri string
+		RedirectUri string `env:"REDIRECT_URI,required"`
-	}
+	} `envPrefix:"OAUTH_"`
-
 	Database struct {
-		Uri string
+		Uri string `env:"URI,required"`
-	}
+	} `envPrefix:"DATABASE_"`
-
 	Bot struct {
-		Id                                   uint64
+		Id                                   uint64 `env:"BOT_ID,required"`
-		Token                                string
+		Token                                string `env:"BOT_TOKEN,required"`
-		PremiumLookupProxyUrl                string `toml:"premium-lookup-proxy-url"`
+		ObjectStore                          string `env:"LOG_ARCHIVER_URL"`
-		PremiumLookupProxyKey                string `toml:"premium-lookup-proxy-key"`
+		AesKey                               string `env:"LOG_AES_KEY" toml:"aes-key"`
-		ObjectStore                          string
+		ProxyUrl                             string `env:"DISCORD_PROXY_URL" toml:"discord-proxy-url"`
-		AesKey                               string `toml:"aes-key"`
+		RenderServiceUrl                     string `env:"RENDER_SERVICE_URL" toml:"render-service-url"`
-		ProxyUrl                             string `toml:"discord-proxy-url"`
+		ImageProxySecret                     string `env:"IMAGE_PROXY_SECRET" toml:"image-proxy-secret"`
-		RenderServiceUrl                     string `toml:"render-service-url"`
+		PublicIntegrationRequestWebhookId    uint64 `env:"PUBLIC_INTEGRATION_REQUEST_WEBHOOK_ID" toml:"public-integration-request-webhook-id"`
-		ImageProxySecret                     string `toml:"image-proxy-secret"`
+		PublicIntegrationRequestWebhookToken string `env:"PUBLIC_INTEGRATION_REQUEST_WEBHOOK_TOKEN" toml:"public-integration-request-webhook-token"`
-		PublicIntegrationRequestWebhookId    uint64 `toml:"public-integration-request-webhook-id"`
-		PublicIntegrationRequestWebhookToken string `toml:"public-integration-request-webhook-token"`
 	}
-
 	Redis struct {
-		Host     string
+		Host     string `env:"HOST,required"`
-		Port     int
+		Port     int    `env:"PORT,required"`
-		Password string
+		Password string `env:"PASSWORD"`
-		Threads  int
+		Threads  int    `env:"THREADS,required"`
-	}
+	} `envPrefix:"REDIS_"`
-
 	Cache struct {
-		Uri string
+		Uri string `env:"URI,required"`
-	}
+	} `envPrefix:"CACHE_"`
-)
+	SecureProxyUrl string `env:"SECURE_PROXY_URL"`
+}
 
-var (
+// TODO: Don't use a global variable
-	Conf Config
+var Conf Config
-)
 
-func LoadConfig() {
+func LoadConfig() (Config, error) {
 	if _, err := os.Stat("config.toml"); err == nil {
-		fromToml()
+		return fromToml()
 	} else {
-		fromEnvvar()
+		return fromEnvvar()
 	}
 }
 
-func fromToml() {
+func fromToml() (Config, error) {
+	var config Config
 	if _, err := toml.DecodeFile("config.toml", &Conf); err != nil {
-		panic(err)
+		return Config{}, err
 	}
+
+	return config, nil
 }
 
-// TODO: Proper env package
+func fromEnvvar() (Config, error) {
-func fromEnvvar() {
+	return env.ParseAs[Config]()
-	var admins []uint64
-	for _, id := range strings.Split(os.Getenv("ADMINS"), ",") {
-		if parsed, err := strconv.ParseUint(id, 10, 64); err == nil {
-			admins = append(admins, parsed)
-		} else {
-			sentry.Error(err)
-		}
-	}
-
-	var forcedWhitelabel []uint64
-	for _, id := range strings.Split(os.Getenv("FORCED_WHITELABEL"), ",") {
-		if parsed, err := strconv.ParseUint(id, 10, 64); err == nil {
-			forcedWhitelabel = append(forcedWhitelabel, parsed)
-		} else {
-			sentry.Error(err)
-		}
-	}
-
-	rateLimitWindow, _ := strconv.Atoi(os.Getenv("RATELIMIT_WINDOW"))
-	rateLimitMax, _ := strconv.Atoi(os.Getenv("RATELIMIT_MAX"))
-	sessionThreads, _ := strconv.Atoi(os.Getenv("SESSION_DB_THREADS"))
-	oauthId, _ := strconv.ParseUint(os.Getenv("OAUTH_ID"), 10, 64)
-	botId, _ := strconv.ParseUint(os.Getenv("BOT_ID"), 10, 64)
-	redisPort, _ := strconv.Atoi(os.Getenv("REDIS_PORT"))
-	redisThreads, _ := strconv.Atoi(os.Getenv("REDIS_THREADS"))
-	publicIntegrationRequestWebhookId, _ := strconv.ParseUint(os.Getenv("PUBLIC_INTEGRATION_REQUEST_WEBHOOK_ID"), 10, 64)
-
-	Conf = Config{
-		Admins:          admins,
-		ForceWhitelabel: forcedWhitelabel,
-		Debug:           os.Getenv("DEBUG") != "",
-		SentryDsn:       os.Getenv("SENTRY_DSN"),
-		Server: Server{
-			Host:       os.Getenv("SERVER_ADDR"),
-			MetricHost: os.Getenv("METRIC_SERVER_ADDR"),
-			BaseUrl:    os.Getenv("BASE_URL"),
-			MainSite:   os.Getenv("MAIN_SITE"),
-			Ratelimit: Ratelimit{
-				Window: rateLimitWindow,
-				Max:    rateLimitMax,
-			},
-			Session: Session{
-				Threads: sessionThreads,
-				Secret:  os.Getenv("SESSION_SECRET"),
-			},
-			Secret:         os.Getenv("JWT_SECRET"),
-			TrustedProxies: strings.Split(os.Getenv("TRUSTED_PROXIES"), ","),
-			RealIpHeaders:  strings.Split(os.Getenv("REAL_IP_HEADERS"), ","),
-		},
-		Oauth: Oauth{
-			Id:          oauthId,
-			Secret:      os.Getenv("OAUTH_SECRET"),
-			RedirectUri: os.Getenv("OAUTH_REDIRECT_URI"),
-		},
-		Database: Database{
-			Uri: os.Getenv("DATABASE_URI"),
-		},
-		Bot: Bot{
-			Id:                                   botId,
-			Token:                                os.Getenv("BOT_TOKEN"),
-			PremiumLookupProxyUrl:                os.Getenv("PREMIUM_PROXY_URL"),
-			PremiumLookupProxyKey:                os.Getenv("PREMIUM_PROXY_KEY"),
-			ObjectStore:                          os.Getenv("LOG_ARCHIVER_URL"),
-			AesKey:                               os.Getenv("LOG_AES_KEY"),
-			ProxyUrl:                             os.Getenv("DISCORD_PROXY_URL"),
-			RenderServiceUrl:                     os.Getenv("RENDER_SERVICE_URL"),
-			ImageProxySecret:                     os.Getenv("IMAGE_PROXY_SECRET"),
-			PublicIntegrationRequestWebhookId:    publicIntegrationRequestWebhookId,
-			PublicIntegrationRequestWebhookToken: os.Getenv("PUBLIC_INTEGRATION_REQUEST_WEBHOOK_TOKEN"),
-		},
-		Redis: Redis{
-			Host:     os.Getenv("REDIS_HOST"),
-			Port:     redisPort,
-			Password: os.Getenv("REDIS_PASSWORD"),
-			Threads:  redisThreads,
-		},
-		Cache: Cache{
-			Uri: os.Getenv("CACHE_URI"),
-		},
-		SecureProxyUrl: os.Getenv("SECURE_PROXY_URL"),
-	}
 }

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/caarlos0/env v3.5.0+incompatible // indirect
 	github.com/caarlos0/env/v10 v10.0.0 // indirect
+	github.com/caarlos0/env/v11 v11.2.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

go.sum

@@ -88,6 +88,8 @@ github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yi
 github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y=
 github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA=
 github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18=
+github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg=
+github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=