How to Add S3 File Uploads to a Gin App

// files/files.go
package files

import (
	"errors"
	"io"
	"net/http"
	"path"
	"strings"
	"time"

	"encore.dev/beta/auth"
	"encore.dev/storage/cache"
	"encore.dev/storage/objects"
	"github.com/gin-gonic/gin"
	"github.com/google/uuid"
)

// S3 on AWS and Cloud Storage on GCP. `encore run` keeps a local bucket.
var uploads = objects.NewBucket("uploads", objects.BucketConfig{})

// SignedUploadURL only binds a key and a TTL, so this limits how often a
// caller can mint one. The counter is shared by every instance.
var uploadLimits = cache.NewCluster("upload-limits", cache.ClusterConfig{
	EvictionPolicy: cache.AllKeysLRU,
})

var signedUploads = cache.NewIntKeyspace[auth.UID](uploadLimits, cache.KeyspaceConfig{
	KeyPattern:    "signed-upload/:key",
	DefaultExpiry: cache.ExpireIn(10 * time.Minute),
})

const maxUploadBytes = 10 << 20 // 10 MiB

//encore:service
type Service struct {
	router *gin.Engine
}

func initService() (*Service, error) {
	r := gin.New()
	r.Use(gin.Recovery())
	r.MaxMultipartMemory = 8 << 20
	r.POST("/files", uploadFile)
	r.GET("/files/:key", downloadFile)
	r.POST("/files/upload-url", uploadURL)
	return &Service{router: r}, nil
}

//encore:api auth raw path=/!fallback
func (s *Service) Fallback(w http.ResponseWriter, req *http.Request) {
	s.router.ServeHTTP(w, req)
}

func objectKey(name string) string {
	uid, _ := auth.UserID()
	return string(uid) + "-" + uuid.NewString() + path.Ext(name)
}

func uploadFile(c *gin.Context) {
	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxUploadBytes)
	fh, err := c.FormFile("file")
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": "multipart field 'file' is required (max 10 MiB)"})
		return
	}
	src, err := fh.Open()
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": "could not read file"})
		return
	}
	defer src.Close()

	// Generate the key on the server so clients cannot overwrite other objects.
	key := objectKey(fh.Filename)
	w := uploads.Upload(c.Request.Context(), key, objects.WithUploadAttrs(objects.UploadAttrs{
		ContentType: "application/octet-stream",
	}))
	if _, err := io.Copy(w, src); err != nil {
		w.Abort(err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "upload failed"})
		return
	}
	if err := w.Close(); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": "upload failed"})
		return
	}
	c.JSON(http.StatusCreated, gin.H{"key": key, "size": fh.Size})
}

func downloadFile(c *gin.Context) {
	ctx := c.Request.Context()
	uid, _ := auth.UserID()
	key := c.Param("key")
	if !strings.HasPrefix(key, string(uid)+"-") {
		c.JSON(http.StatusNotFound, gin.H{"error": "file not found"})
		return
	}
	attrs, err := uploads.Attrs(ctx, key)
	if errors.Is(err, objects.ErrObjectNotFound) {
		c.JSON(http.StatusNotFound, gin.H{"error": "file not found"})
		return
	} else if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not load file"})
		return
	}
	r := uploads.Download(ctx, key)
	if err := r.Err(); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not load file"})
		return
	}
	defer r.Close()
	c.Header("Content-Disposition", "attachment")
	c.Header("X-Content-Type-Options", "nosniff")
	c.DataFromReader(http.StatusOK, attrs.Size, "application/octet-stream", r, nil)
}

type uploadURLRequest struct {
	Filename string `json:"filename" binding:"required"`
}

// uploadURL returns a short-lived signed URL so large files go straight
// from the client to the bucket instead of through the Gin handler.
func uploadURL(c *gin.Context) {
	var req uploadURLRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": "filename is required"})
		return
	}
	uid, _ := auth.UserID()
	n, err := signedUploads.Increment(c.Request.Context(), uid, 1)
	if err != nil || n > 5 {
		c.JSON(http.StatusTooManyRequests, gin.H{"error": "upload url limit exceeded"})
		return
	}
	key := objectKey(req.Filename)
	url, err := uploads.SignedUploadURL(c.Request.Context(), key, objects.WithTTL(15*time.Minute))
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not sign url"})
		return
	}
	c.JSON(http.StatusOK, gin.H{"key": key, "upload_url": url.URL})
}

var (
	bucket   = os.Getenv("UPLOADS_BUCKET")
	s3Client *s3.Client
	uploader *manager.Uploader
	presign  *s3.PresignClient
)

func init() {
	cfg, err := config.LoadDefaultConfig(context.Background())
	if err != nil {
		log.Fatal(err)
	}
	s3Client = s3.NewFromConfig(cfg)
	uploader = manager.NewUploader(s3Client)
	presign = s3.NewPresignClient(s3Client)
}

func uploadFile(c *gin.Context) {
	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 10<<20)
	fh, err := c.FormFile("file")
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": "file is required"})
		return
	}
	src, err := fh.Open()
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": "could not read file"})
		return
	}
	defer src.Close()
	key := uuid.NewString() + path.Ext(fh.Filename)
	_, err = uploader.Upload(c.Request.Context(), &s3.PutObjectInput{
		Bucket:      aws.String(bucket),
		Key:         aws.String(key),
		Body:        src,
		ContentType: aws.String(fh.Header.Get("Content-Type")),
	})
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": "upload failed"})
		return
	}
	c.JSON(http.StatusCreated, gin.H{"key": key})
}