fix: rewrite for memory efficiency

front_api/routes/openapi_impl.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -23,6 +23,9 @@ import (
 	"github.com/zhenghaoz/gorse/client"
 )
 
+// You can customize this to whichever value you see fit
+const maxThumbnailSize = 500 * 1024 // 500 KB
+
 type Server struct {
 	r RouteHandler
 	c *redis.RedisStore
@@ -119,76 +122,104 @@ func (s Server) Upload(ctx echo.Context, params UploadParams) error {
 		return ctx.String(http.StatusForbidden, "Insufficient user status")
 	}
 
-	title := params.Title
-	description := params.Description
-	tags := params.Tags
-
+	// Open files first to validate and get file handles
+  // Close them immediately after; memory efficiency
 	thumbFileHeader, err := ctx.FormFile(thumbnailKey)
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if thumbFile, err := thumbFileHeader.Open(); err == nil {
+			thumbFile.Close()
+		}
+	}()
 
-	thumbFile, err := thumbFileHeader.Open()
+	videoFileHeader, err := ctx.FormFile(videoKey)
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if videoFile, err := videoFileHeader.Open(); err == nil {
+			videoFile.Close()
+		}
+	}()
 
-	videoFileHeader, err := ctx.FormFile(videoKey)
+	// Prepare upload client
+	uploadClient, err := s.r.v.UploadVideo(context.Background())
 	if err != nil {
 		return err
 	}
 
-	videoFile, err := videoFileHeader.Open()
-	if err != nil {
-		return err
+	// Send metadata chunk
+	metaChunk := &videoproto.InputVideoChunk{
+		Payload: &videoproto.InputVideoChunk_Meta{
+			Meta: &videoproto.InputFileMetadata{
+				Title:             params.Title,
+				Description:       params.Description,
+				AuthorUID:         "0", // TODO: Verify this placeholder
+				OriginalVideoLink: "0",
+				AuthorUsername:    profile.Username,
+				OriginalSite:      "blank", // TODO: Replace with actual source
+				OriginalID:        "0",
+				DomesticAuthorID:  profile.UserID,
+				Tags:              params.Tags,
+				Category:          params.Category,
+			},
+		},
 	}
 
-	// TODO: rewrite me, this isn't memory efficient
-	videoBytes, err := ioutil.ReadAll(videoFile)
+	err = uploadClient.Send(metaChunk)
 	if err != nil {
 		return err
 	}
 
-	thumbBytes, err := ioutil.ReadAll(thumbFile)
+	// Stream thumbnail separately
+	thumbFile, err := thumbFileHeader.Open()
 	if err != nil {
 		return err
 	}
+	defer thumbFile.Close()
 
-	uploadClient, err := s.r.v.UploadVideo(context.Background())
+	thumbBytes, err := io.ReadAll(io.LimitReader(thumbFile, maxThumbnailSize))
 	if err != nil {
 		return err
 	}
 
-	metaChunk := &videoproto.InputVideoChunk{
+	// Update metadata with thumbnail
+	updateMetaChunk := &videoproto.InputVideoChunk{
 		Payload: &videoproto.InputVideoChunk_Meta{
 			Meta: &videoproto.InputFileMetadata{
-				Title:             title,
-				Description:       description,
-				AuthorUID:         "0", // TODO can't accept a blank foreign author id??
-				OriginalVideoLink: "0",
-				AuthorUsername:    profile.Username,
-				OriginalSite:      "blank", // todo AAAAAAAAAAAAAAAAAa
-				OriginalID:        "0",
-				DomesticAuthorID:  profile.UserID,
-				Tags:              tags,
-				Thumbnail:         thumbBytes,
-				Category:          params.Category,
+				Thumbnail: thumbBytes,
 			},
 		},
 	}
+	err = uploadClient.Send(updateMetaChunk)
+	if err != nil {
+		return err
+	}
 
-	err = uploadClient.Send(metaChunk)
+	// Stream video in chunks
+	videoFile, err := videoFileHeader.Open()
 	if err != nil {
 		return err
 	}
+  // reuse one buffer chunk for streaming therefore more memory efficient
+	defer videoFile.Close()
+
+	chunkBuffer := make([]byte, fileUploadChunkSize)
+	for {
+		bytesRead, err := videoFile.Read(chunkBuffer)
+		if err != nil && err != io.EOF {
+			return err
+		}
+		if bytesRead == 0 {
+			break
+		}
 
-	for byteInd := 0; byteInd < len(videoBytes); byteInd += fileUploadChunkSize {
-		videoByteSlice := videoBytes[byteInd:min(len(videoBytes), byteInd+fileUploadChunkSize)]
-		log.Infof("uploading byte %d", byteInd)
 		videoChunk := &videoproto.InputVideoChunk{
 			Payload: &videoproto.InputVideoChunk_Content{
 				Content: &videoproto.FileContent{
-					Data: videoByteSlice,
+					Data: chunkBuffer[:bytesRead],
 				},
 			},
 		}
@@ -199,12 +230,12 @@ func (s Server) Upload(ctx echo.Context, params UploadParams) error {
 		}
 	}
 
+	// Close and receive response
 	resp, err := uploadClient.CloseAndRecv()
 	if err != nil {
 		return err
 	}
 
-	// Redirect to the new video
 	return ctx.JSON(http.StatusOK, resp.VideoID)
 }