fix: Importing (offset transcripts)

Signed-off-by: Ben Hall <ben@benh.codes>

app/http/endpoints/api/export/import.go

@@ -11,21 +11,25 @@ import (
 	"os"
 	"time"
 
+	database2 "github.com/TicketsBot-cloud/database"
 	"github.com/TicketsBot/GoPanel/app/http/endpoints/api/export/validator"
 	"github.com/TicketsBot/GoPanel/botcontext"
+	"github.com/TicketsBot/GoPanel/config"
 	dbclient "github.com/TicketsBot/GoPanel/database"
 	"github.com/TicketsBot/GoPanel/rpc"
+	"github.com/TicketsBot/GoPanel/s3"
 	"github.com/TicketsBot/GoPanel/utils"
 	"github.com/TicketsBot/common/premium"
 	"github.com/TicketsBot/database"
 	"github.com/gin-gonic/gin"
+	"github.com/minio/minio-go/v7"
 	"golang.org/x/sync/errgroup"
 )
 
+//	func ImportHandler(ctx *gin.Context) {
+//		ctx.JSON(401, "This endpoint is disabled")
+//	}
 func ImportHandler(ctx *gin.Context) {
-	ctx.JSON(401, "This endpoint is disabled")
-}
-func Importv2Handler(ctx *gin.Context) {
 	// Parse request body from multipart form
 	queryCtx, cancel := context.WithTimeout(context.Background(), time.Minute*1500)
 	defer cancel()
@@ -33,11 +37,11 @@ func Importv2Handler(ctx *gin.Context) {
 	var transcriptOutput *validator.GuildTranscriptsOutput
 	var data *validator.GuildData
 
-	dataFile, _, err := ctx.Request.FormFile("data_file")
+	dataFile, _, dataErr := ctx.Request.FormFile("data_file")
-	dataFileExists := err == nil
+	dataFileExists := dataErr == nil
 
-	transcriptsFile, _, err := ctx.Request.FormFile("transcripts_file")
+	transcriptsFile, _, transcriptsErr := ctx.Request.FormFile("transcripts_file")
-	transcriptFileExists := err == nil
+	transcriptFileExists := transcriptsErr == nil
 
 	// Decrypt file
 	publicKeyBlock, _ := pem.Decode([]byte(os.Getenv("V1_PUBLIC_KEY")))
@@ -81,6 +85,8 @@ func Importv2Handler(ctx *gin.Context) {
 		}
 	}
 
+	guildId, selfId := ctx.Keys["guildid"].(uint64), ctx.Keys["userid"].(uint64)
+
 	if transcriptFileExists {
 		defer transcriptsFile.Close()
 
@@ -96,9 +102,15 @@ func Importv2Handler(ctx *gin.Context) {
 			ctx.JSON(400, utils.ErrorJson(err))
 			return
 		}
-	}
 
-	guildId, selfId := ctx.Keys["guildid"].(uint64), ctx.Keys["userid"].(uint64)
+		// Upload transcripts
+		if transcriptFileExists {
+			if _, err := s3.S3Client.PutObject(ctx, config.Conf.S3Import.Bucket, fmt.Sprintf("transcripts/%d.zip", guildId), transcriptReader, transcriptReader.Size(), minio.PutObjectOptions{}); err != nil {
+				ctx.JSON(500, utils.ErrorStr("Failed to upload transcripts"))
+				return
+			}
+		}
+	}
 
 	botCtx, err := botcontext.ContextForGuild(guildId)
 	if err != nil {
@@ -139,6 +151,7 @@ func Importv2Handler(ctx *gin.Context) {
 		ticketIdMap    = mapping["ticket"]
 		formIdMap      = mapping["form"]
 		formInputIdMap = mapping["form_input"]
+		panelIdMap     = mapping["panel"]
 	)
 
 	if ticketIdMap == nil {
@@ -153,6 +166,10 @@ func Importv2Handler(ctx *gin.Context) {
 		formInputIdMap = make(map[int]int)
 	}
 
+	if panelIdMap == nil {
+		panelIdMap = make(map[int]int)
+	}
+
 	if dataFileExists {
 
 		group, _ := errgroup.WithContext(queryCtx)
@@ -489,8 +506,6 @@ func Importv2Handler(ctx *gin.Context) {
 		}
 		panelCount := len(existingPanels)
 
-		panelIdMap := make(map[int]int)
-
 		for _, panel := range data.Panels {
 			if premiumTier < premium.Premium && panelCount > 2 {
 				panel.ForceDisabled = true
@@ -512,8 +527,12 @@ func Importv2Handler(ctx *gin.Context) {
 				panel.WelcomeMessageEmbed = &newEmbedId
 			}
 
+			// TODO: Fix this permanently
+			panel.MessageId = panel.MessageId - 1
+
 			panelId, err := dbclient.Client.Panel.CreateWithTx(queryCtx, panelTx, panel)
 			if err != nil {
+				fmt.Println(err)
 				ctx.JSON(500, utils.ErrorJson(err))
 				return
 			}
@@ -598,70 +617,45 @@ func Importv2Handler(ctx *gin.Context) {
 			return
 		}
 
-		// Import tickets
+		ticketCount, err := dbclient.Client.Tickets.GetTotalTicketCount(queryCtx, guildId)
-		for _, ticket := range data.Tickets {
+		if err != nil {
+			ctx.JSON(500, utils.ErrorJson(err))
+			return
+		}
+		ticketsToCreate := make([]database2.Ticket, len(data.Tickets))
+		ticketIdMap = make(map[int]int)
+
+		for i, ticket := range data.Tickets {
 			if _, ok := ticketIdMap[ticket.Id]; !ok {
 				var panelId *int
 				if ticket.PanelId != nil {
 					a := panelIdMap[*ticket.PanelId]
 					panelId = &a
 				}
-				newTicketId, err := dbclient.Client.Tickets.Create(queryCtx, guildId, ticket.UserId, ticket.IsThread, panelId)
+				ticketsToCreate[i] = database2.Ticket{
-				if err != nil {
+					Id:               ticket.Id + ticketCount,
-					ctx.JSON(500, utils.ErrorJson(err))
+					GuildId:          guildId,
-					return
+					ChannelId:        ticket.ChannelId,
+					UserId:           ticket.UserId,
+					Open:             ticket.Open,
+					OpenTime:         ticket.OpenTime,
+					WelcomeMessageId: ticket.WelcomeMessageId,
+					PanelId:          panelId,
+					HasTranscript:    ticket.HasTranscript,
+					CloseTime:        ticket.CloseTime,
+					IsThread:         ticket.IsThread,
+					JoinMessageId:    ticket.JoinMessageId,
+					NotesThreadId:    ticket.NotesThreadId,
 				}
 
-				ticketIdMap[ticket.Id] = newTicketId
+				ticketIdMap[ticket.Id] = ticket.Id + ticketCount
-
-				if ticket.Open {
-					if err := dbclient.Client.Tickets.SetOpen(queryCtx, guildId, newTicketId); err != nil {
-						ctx.JSON(500, utils.ErrorJson(err))
-						return
-					}
-				} else {
-					if err := dbclient.Client.Tickets.Close(queryCtx, newTicketId, guildId); err != nil {
-						ctx.JSON(500, utils.ErrorJson(err))
-						return
-					}
-				}
-
-				if ticket.ChannelId != nil {
-					if err := dbclient.Client.Tickets.SetChannelId(queryCtx, guildId, newTicketId, *ticket.ChannelId); err != nil {
-						ctx.JSON(500, utils.ErrorJson(err))
-						return
-					}
-				}
-
-				if err := dbclient.Client.Tickets.SetHasTranscript(queryCtx, guildId, newTicketId, ticket.HasTranscript); err != nil {
-					ctx.JSON(500, utils.ErrorJson(err))
-					return
-				}
-				if ticket.NotesThreadId != nil {
-					if err := dbclient.Client.Tickets.SetNotesThreadId(queryCtx, guildId, newTicketId, *ticket.NotesThreadId); err != nil {
-						ctx.JSON(500, utils.ErrorJson(err))
-						return
-					}
-				}
-				if err := dbclient.Client.Tickets.SetStatus(queryCtx, guildId, newTicketId, ticket.Status); err != nil {
-					ctx.JSON(500, utils.ErrorJson(err))
-					return
-				}
 			}
 		}
-	}
 
-	// Upload transcripts
+		if err := dbclient.Client2.Tickets.BulkImport(queryCtx, guildId, ticketsToCreate); err != nil {
-	if transcriptFileExists {
+			ctx.JSON(500, utils.ErrorJson(err))
-		for ticketId, transcript := range transcriptOutput.Transcripts {
+			return
-			if err := utils.ArchiverClient.ImportTranscript(queryCtx, guildId, ticketIdMap[ticketId], transcript); err != nil {
-				ctx.JSON(500, utils.ErrorJson(err))
-				return
-			}
 		}
-	}
-
-	if dataFileExists {
 
 		ticketsExtrasGroup, _ := errgroup.WithContext(queryCtx)
 
@@ -679,7 +673,22 @@ func Importv2Handler(ctx *gin.Context) {
 		// Import ticket last messages
 		ticketsExtrasGroup.Go(func() (err error) {
 			for _, msg := range data.TicketLastMessages {
-				err = dbclient.Client.TicketLastMessage.Set(queryCtx, guildId, ticketIdMap[msg.TicketId], *msg.Data.LastMessageId, *msg.Data.UserId, *msg.Data.UserIsStaff)
+				lastMessageId := uint64(0)
+				if msg.Data.LastMessageId != nil {
+					lastMessageId = *msg.Data.LastMessageId
+				}
+
+				userId := uint64(0)
+				if msg.Data.UserId != nil {
+					userId = *msg.Data.UserId
+				}
+
+				userIsStaff := false
+				if msg.Data.UserIsStaff != nil {
+					userIsStaff = *msg.Data.UserIsStaff
+				}
+
+				err = dbclient.Client.TicketLastMessage.Set(queryCtx, guildId, ticketIdMap[msg.TicketId], lastMessageId, userId, userIsStaff)
 			}
 			return
 		})
@@ -764,6 +773,7 @@ func Importv2Handler(ctx *gin.Context) {
 	newMapping["ticket"] = ticketIdMap
 	newMapping["form"] = formIdMap
 	newMapping["form_input"] = formInputIdMap
+	newMapping["panel"] = panelIdMap
 
 	for area, m := range newMapping {
 		for sourceId, targetId := range m {

cmd/api/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/TicketsBot/GoPanel/redis"
 	"github.com/TicketsBot/GoPanel/rpc"
 	"github.com/TicketsBot/GoPanel/rpc/cache"
+	"github.com/TicketsBot/GoPanel/s3"
 	"github.com/TicketsBot/GoPanel/utils"
 	"github.com/TicketsBot/common/chatrelay"
 	"github.com/TicketsBot/common/model"
@@ -77,6 +78,9 @@ func main() {
 	logger.Info("Connecting to cache")
 	cache.Instance = cache.NewCache()
 
+	logger.Info("Connecting to import S3")
+	s3.ConnectS3(config.Conf.S3Import.Endpoint, config.Conf.S3Import.AccessKey, config.Conf.S3Import.SecretKey)
+
 	logger.Info("Initialising microservice clients")
 	utils.ArchiverClient = archiverclient.NewArchiverClient(archiverclient.NewProxyRetriever(config.Conf.Bot.ObjectStore), []byte(config.Conf.Bot.AesKey))
 	utils.SecureProxyClient = secureproxy.NewSecureProxy(config.Conf.SecureProxyUrl)

config/config.go

@@ -1,10 +1,11 @@
 package config
 
 import (
+	"os"
+
 	"github.com/BurntSushi/toml"
 	"github.com/caarlos0/env/v11"
 	"go.uber.org/zap/zapcore"
-	"os"
 )
 
 type Config struct {
@@ -56,6 +57,12 @@ type Config struct {
 		Uri string `env:"URI,required"`
 	} `envPrefix:"CACHE_"`
 	SecureProxyUrl string `env:"SECURE_PROXY_URL"`
+	S3Import       struct {
+		Endpoint  string `env:"ENDPOINT,required"`
+		AccessKey string `env:"ACCESS_KEY,required"`
+		SecretKey string `env:"SECRET_KEY,required"`
+		Bucket    string `env:"BUCKET,required"`
+	} `envPrefix:"S3_IMPORT_"`
 }
 
 // TODO: Don't use a global variable

frontend/src/components/manage/ImportModal.svelte

@@ -101,7 +101,7 @@
         queryLoading = false;
 
         dispatchClose();
-        notifySuccess('Imported settings successfully');
+        notifySuccess('Imported settings successfully - Your transcripts will be processed separately and may take some time to appear.');
     }
 
     function handleKeydown(e) {

frontend/src/includes/ManageSidebar.svelte

@@ -28,12 +28,12 @@
             <ManageSidebarLink {currentRoute} title="Blacklist" icon="fa-ban" href="/manage/{guildId}/blacklist" />
             <ManageSidebarLink {currentRoute} title="Tags" icon="fa-tags" href="/manage/{guildId}/tags" />
 
-            <!-- {#if isAdmin}
+            {#if isAdmin}
                 <hr />
                 <div class="row" style="align-items: center; justify-content: center;">
                 <ManageSidebarButton on:click={() => importModal = true} title="Import Data" icon="fa-file-import" />
             </div>
-            {/if} -->
+            {/if}
         </ul>
     </nav>
     <nav class="bottom">

go.mod

@@ -71,7 +71,7 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-redsync/redsync/v4 v4.12.1 // indirect
-	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/goccy/go-json v0.10.4 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
@@ -90,13 +90,13 @@ require (
 	github.com/jedib0t/go-pretty/v6 v6.5.6 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/ratelimit v1.0.2 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.73 // indirect
+	github.com/minio/minio-go/v7 v7.0.85 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -111,7 +111,7 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
-	github.com/rs/xid v1.5.0 // indirect
+	github.com/rs/xid v1.6.0 // indirect
 	github.com/schollz/progressbar/v3 v3.8.2 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect

go.sum

@@ -192,6 +192,8 @@ github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM=
+github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -365,10 +367,14 @@ github.com/klauspost/compress v1.10.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
 github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
+github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -409,6 +415,8 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo=
 github.com/minio/minio-go/v7 v7.0.73/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
+github.com/minio/minio-go/v7 v7.0.85 h1:9psTLS/NTvC3MWoyjhjXpwcKoNbkongaCSF3PNpSuXo=
+github.com/minio/minio-go/v7 v7.0.85/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -495,6 +503,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/rxdn/gdl v0.0.0-20241201120412-8fd61c53dd96 h1:CIEWvDLeku/Wf+FepJeL3V8pRjtTXf2EkOZ5KDVbih0=

s3/s3.go

@@ -0,0 +1,23 @@
+package s3
+
+import (
+	"log"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+var S3Client *minio.Client
+
+func ConnectS3(endpoint, accessKey, secretKey string) {
+	// Initialize minio client object.
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
+		Secure: true,
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	S3Client = minioClient
+}