push transcripts straight to s3 rather than our api

Signed-off-by: Ben Hall <ben@benh.codes>

app/http/endpoints/api/export/import.go

@@ -8,6 +8,8 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io"
+	"net/http"
+	"net/url"
 	"os"
 	"time"
 
@@ -32,6 +34,41 @@ import (
 //	func ImportHandler(ctx *gin.Context) {
 //		ctx.JSON(401, "This endpoint is disabled")
 //	}
+
+func PresignTranscriptURL(ctx *gin.Context) {
+	guildId, userId := ctx.Keys["guildid"].(uint64), ctx.Keys["userid"].(uint64)
+
+	botCtx, err := botcontext.ContextForGuild(guildId)
+	if err != nil {
+		ctx.JSON(500, utils.ErrorJson(err))
+		return
+	}
+
+	guild, err := botCtx.GetGuild(context.Background(), guildId)
+	if err != nil {
+		ctx.JSON(500, utils.ErrorJson(err))
+		return
+	}
+
+	if guild.OwnerId != userId {
+		ctx.JSON(403, utils.ErrorStr("Only the server owner can import transcripts"))
+		return
+	}
+
+	// Presign URL
+	url, err := s3.S3Client.PresignHeader(ctx, "PUT", config.Conf.S3Import.Bucket, fmt.Sprintf("transcripts/%d.zip", guildId), time.Minute*1, url.Values{}, http.Header{
+		"Content-Type": []string{"application/x-zip-compressed"},
+	})
+	if err != nil {
+		ctx.JSON(500, utils.ErrorJson(err))
+		return
+	}
+
+	ctx.JSON(200, gin.H{
+		"url": url.String(),
+	})
+}
+
 func ImportHandler(ctx *gin.Context) {
 	// Return slices
 	successfulItems := make([]string, 0)

app/http/server.go

@@ -123,6 +123,7 @@ func StartServer(logger *zap.Logger, sm *livechat.SocketManager) {
 		guildAuthApiAdmin.POST("/settings", api_settings.UpdateSettingsHandler)
 
 		guildAuthApiAdmin.POST("/import", api_import.ImportHandler)
+		guildAuthApiAdmin.GET("/import/presign", api_import.PresignTranscriptURL)
 
 		guildAuthApiSupport.GET("/blacklist", api_blacklist.GetBlacklistHandler)
 		guildAuthApiSupport.POST("/blacklist", api_blacklist.AddBlacklistHandler)

frontend/src/views/Import.svelte

@@ -139,9 +139,9 @@
         dispatch("close", {});
     }
 
+
     async function dispatchConfirm() {
         let dataFileInput = document.getElementById("import_data");
-
         let transcriptFileInput = document.getElementById("import_transcripts");
 
         if (
@@ -158,14 +158,6 @@
         if (dataFileInput.files.length > 0) {
             frmData.append("data_file", dataFileInput.files[0]);
         }
-        if (transcriptFileInput.files.length > 0) {
-            if(transcriptFileInput.files[0].size > 100 * 1024 * 1024) {
-                transcriptFileInput.value = "";
-                notifyError("Transcript file is too large, please upload a file smaller than 100MB or wait until we support larger files.");
-                return;
-            }
-            frmData.append("transcripts_file", transcriptFileInput.files[0]);
-        }
 
         queryLoading = true;
         setTimeout(() => {
@@ -176,24 +168,50 @@
                 );
             }
         }, 60 * 1000);
-        const res = await axios.post(
+
-            `${API_URL}/api/${guildId}/import`,
+        if (transcriptFileInput.files.length > 0) {
-            frmData,
+            const presignRes = await axios.get(`${API_URL}/api/${guildId}/import/presign`);
-            {
+            if (presignRes.status !== 200) {
+                notifyError(`Failed to upload transcripts: ${presignRes.data.error}`);
+                queryLoading = false;
+                return;
+            }
+            
+            await fetch(presignRes.data.url, {
+                method: "PUT",
+                body: transcriptFileInput.files[0],
                 headers: {
-                    "Content-Type": "multipart/form-data",
+                    "Content-Type": transcriptFileInput.files[0].type,
                 },
-            },
+            }).then((res) => {
-        );
+                if (res.status !== 200) {
-        if (res.status !== 200) {
+                    notifyError(`Failed to upload transcripts: ${res.data.error}`);
-            notifyError(`Failed to import settings: ${res.data.error}`);
+                    queryLoading = false;
-            queryLoading = false;
+                    return;
-            return;
+                }
+            });
+        }
+
+        if (dataFileInput.files.length > 0) {
+            const res = await axios.post(
+                `${API_URL}/api/${guildId}/import`,
+                frmData,
+                {
+                    headers: {
+                        "Content-Type": "multipart/form-data",
+                    },
+                },
+            );
+            if (res.status !== 200) {
+                notifyError(`Failed to import settings: ${res.data.error}`);
+                queryLoading = false;
+                return;
+            }
+            dataReturned = true;
+            resData = res.data;
         }
 
         queryLoading = false;
-        dataReturned = true;
-        resData = res.data;
 
         dispatchClose();
         notifySuccess(