feat(prune): add prune files feature

This feature allows you to specify the n latest backups to keep in the
bucket.

helpers/helpers.go

@@ -0,0 +1,49 @@
+package helpers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"os"
+)
+
+func GetCredentials() (string, string, error) {
+	// get credentials from env vars
+	keyName := os.Getenv("KEY_ID")
+	if keyName == "" {
+		return "", "", errors.New("missing or empty KEY_ID")
+	}
+	applicationKey := os.Getenv("APPLICATION_KEY")
+	if applicationKey == "" {
+		return "", "", errors.New("missing or empty APPLICATION_KEY")
+	}
+	return keyName, applicationKey, nil
+}
+
+func CreateClient(accessKeyID, secretAccessKey, endpoint string) (*minio.Client, error) {
+	// Initialize minio client object.
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return minioClient, nil
+}
+
+func VerifyBucket(client *minio.Client, ctx context.Context, bucketName string) error {
+	exists, err := client.BucketExists(ctx, bucketName)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return errors.New(fmt.Sprintf("bucket %s does not exist\n", bucketName))
+	}
+
+	return nil
+}

main.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"backup-tool/pruner"
 	"backup-tool/uploader"
+	"flag"
 	"github.com/joho/godotenv"
 	"log"
 	"os"
@@ -11,13 +13,22 @@ const (
 	ENDPOINT = "s3.us-west-004.backblazeb2.com"
 )
 
+var skipUpload bool
+var keepNum int
+
+func init() {
+	flag.BoolVar(&skipUpload, "skip-upload", false, "do not upload file to S3/Backblaze")
+	flag.IntVar(&keepNum, "keep-backups", 7, "number of backups to keep; 0 to keep all")
+}
+
 func main() {
+	flag.Parse()
 	err := godotenv.Load()
 	if err != nil {
 		log.Fatalf("error loading .env file: (%s)", err.Error())
 	}
 
-	if len(os.Args) < 2 {
+	if len(os.Args) < 2 && !skipUpload {
 		log.Fatalln("missing filename parameter")
 	}
 
@@ -26,8 +37,14 @@ func main() {
 		log.Fatalln("missing or empty BUCKET_NAME")
 	}
 
-	err = uploader.UploadFile(bucketName, ENDPOINT, os.Args[1])
-	if err != nil {
-		log.Fatalln(err.Error())
+	if !skipUpload {
+		err = uploader.UploadFile(bucketName, ENDPOINT, os.Args[1])
+		if err != nil {
+			log.Fatalln(err.Error())
+		}
+	}
+
+	if keepNum > 0 {
+		err = pruner.PruneFiles(bucketName, ENDPOINT, keepNum)
 	}
 }

pruner/pruner.go

@@ -0,0 +1,52 @@
+package pruner
+
+import (
+	"backup-tool/helpers"
+	"context"
+	"github.com/minio/minio-go/v7"
+	"log"
+	"sort"
+)
+
+func PruneFiles(bucket, endpoint string, keep int) error {
+	ctx := context.Background()
+	accessKeyID, secretAccessKey, err := helpers.GetCredentials()
+	if err != nil {
+		return err
+	}
+	client, err := helpers.CreateClient(accessKeyID, secretAccessKey, endpoint)
+	if err != nil {
+		return err
+	}
+	err = helpers.VerifyBucket(client, ctx, bucket)
+	if err != nil {
+		return err
+	}
+
+	var objects []minio.ObjectInfo
+	for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{}) {
+		objects = append(objects, obj)
+	}
+
+	if len(objects) < keep {
+		return nil
+	}
+
+	sort.Sort(BucketObjects(objects)) // last modified object will be at position 0
+	for _, object := range objects[keep:] {
+		err := client.RemoveObject(ctx, bucket, object.Key, minio.RemoveObjectOptions{})
+		if err != nil {
+			log.Printf("error deleting object %q: %s", object.Key, err.Error())
+		} else {
+			log.Printf("deleted object %q", object.Key)
+		}
+	}
+
+	return nil
+}
+
+type BucketObjects []minio.ObjectInfo
+
+func (b BucketObjects) Len() int           { return len(b) }
+func (b BucketObjects) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b BucketObjects) Less(i, j int) bool { return b[i].LastModified.After(b[j].LastModified) } // return newest object first in list

uploader/uploader.go

@@ -1,11 +1,10 @@
 package uploader
 
 import (
+	"backup-tool/helpers"
 	"context"
 	"errors"
-	"fmt"
 	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"log"
 	"os"
 )
@@ -13,15 +12,15 @@ import (
 func UploadFile(bucketName, endpoint, fileName string) error {
 
 	ctx := context.Background()
-	accessKeyID, secretAccessKey, err := getCredentials()
+	accessKeyID, secretAccessKey, err := helpers.GetCredentials()
 	if err != nil {
 		return (err)
 	}
-	client, err := createClient(accessKeyID, secretAccessKey, endpoint)
+	client, err := helpers.CreateClient(accessKeyID, secretAccessKey, endpoint)
 	if err != nil {
 		return (err)
 	}
-	err = verifyBucket(client, ctx, bucketName)
+	err = helpers.VerifyBucket(client, ctx, bucketName)
 	if err != nil {
 		return err
 	}
@@ -51,42 +50,3 @@ func validateUploadFile(fileName string) (string, string, error) {
 
 	return fileName, file.Name(), nil
 }
-
-func verifyBucket(client *minio.Client, ctx context.Context, bucketName string) error {
-	exists, err := client.BucketExists(ctx, bucketName)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		return errors.New(fmt.Sprintf("bucket %s does not exist\n", bucketName))
-	}
-
-	return nil
-}
-
-func createClient(accessKeyID, secretAccessKey, endpoint string) (*minio.Client, error) {
-	// Initialize minio client object.
-	minioClient, err := minio.New(endpoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
-		Secure: true,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return minioClient, nil
-}
-
-func getCredentials() (string, string, error) {
-	// get credentials from env vars
-	keyName := os.Getenv("KEY_ID")
-	if keyName == "" {
-		return "", "", errors.New("missing or empty KEY_ID")
-	}
-	applicationKey := os.Getenv("APPLICATION_KEY")
-	if applicationKey == "" {
-		return "", "", errors.New("missing or empty APPLICATION_KEY")
-	}
-	return keyName, applicationKey, nil
-}