How to empty a S3 bucket using the AWS SDK for GOlang?

Question

Goal: To empty an existing S3 bucket using the AWS SDK for GOlang.

Answer 1

AWS SDK now has BatchDeleteIterator that can do the job. Example provided via Amazon docs.

package main

import (
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
    "github.com/aws/aws-sdk-go/service/s3/s3manager"

    "fmt"
    "os"
)

//    go run s3_delete_objects BUCKET

func main() {
    if len(os.Args) != 2 {
        exitErrorf("Bucket name required\nUsage: %s BUCKET", os.Args[0])
    }

    bucket := os.Args[1]

    // Initialize a session in us-west-2 that the SDK will use to load
    // credentials from the shared credentials file ~/.aws/credentials.
    sess, _ := session.NewSession(&aws.Config{
        Region: aws.String("us-west-2")},
    )

    // Create S3 service client
    svc := s3.New(sess)

    // Setup BatchDeleteIterator to iterate through a list of objects.
    iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{
        Bucket: aws.String(bucket),
    })

    // Traverse iterator deleting each object
    if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil {
        exitErrorf("Unable to delete objects from bucket %q, %v", bucket, err)
    }

    fmt.Printf("Deleted object(s) from bucket: %s", bucket)
}

func exitErrorf(msg string, args ...interface{}) {
    fmt.Fprintf(os.Stderr, msg+"\n", args...)
    os.Exit(1)
}

Answer 2

NOTE: These are code snippets that might require YOU to make changes on YOUR SIDE to make it run.

You will need to implement the below method:

  //EmptyBucket empties the Amazon S3 bucket
    func (s awsS3) EmptyBucket(bucket string) error {
        log.Info("removing objects from S3 bucket : ", bucket)
        params := &s3.ListObjectsInput{
            Bucket: aws.String(bucket),
        }
        for {
            //Requesting for batch of objects from s3 bucket
            objects, err := s.Client.ListObjects(params)
            if err != nil {
                return err
            }
            //Checks if the bucket is already empty
            if len((*objects).Contents) == 0 {
                log.Info("Bucket is already empty")
                return nil
             }
            log.Info("First object in batch | ", *(objects.Contents[0].Key))

            //creating an array of pointers of ObjectIdentifier
            objectsToDelete := make([]*s3.ObjectIdentifier, 0, 1000)
            for _, object := range (*objects).Contents {
                obj := s3.ObjectIdentifier{
                    Key: object.Key,
                }
                objectsToDelete = append(objectsToDelete, &obj)
            }
            //Creating JSON payload for bulk delete
            deleteArray := s3.Delete{Objects: objectsToDelete}
            deleteParams := &s3.DeleteObjectsInput{
                Bucket: aws.String(bucket),
                Delete: &deleteArray,
            }
            //Running the Bulk delete job (limit 1000)
            _, err = s.Client.DeleteObjects(deleteParams)
            if err != nil {
                return err
            }
            if *(*objects).IsTruncated { //if there are more objects in the bucket, IsTruncated = true
                params.Marker = (*deleteParams).Delete.Objects[len((*deleteParams).Delete.Objects)-1].Key
                log.Info("Requesting next batch | ", *(params.Marker))
            } else { //if all objects in the bucket have been cleaned up.
                break
            }
        }
        log.Info("Emptied S3 bucket : ", bucket)
        return nil
    } 

UPDATE : The latest version of AWS SDK for GO has resolved the prior issue I had.