feat(prune): add prune files feature

This feature allows you to specify the n latest backups to keep in the
bucket.

go.mod

@@ -1,6 +1,6 @@
 module backup-tool
 
-go 1.21
+go 1.18
 
 require (
 	github.com/dustin/go-humanize v1.0.1 // indirect

helpers/helpers.go

@@ -0,0 +1,49 @@
+package helpers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"os"
+)
+
+func GetCredentials() (string, string, error) {
+	// get credentials from env vars
+	keyName := os.Getenv("KEY_ID")
+	if keyName == "" {
+		return "", "", errors.New("missing or empty KEY_ID")
+	}
+	applicationKey := os.Getenv("APPLICATION_KEY")
+	if applicationKey == "" {
+		return "", "", errors.New("missing or empty APPLICATION_KEY")
+	}
+	return keyName, applicationKey, nil
+}
+
+func CreateClient(accessKeyID, secretAccessKey, endpoint string) (*minio.Client, error) {
+	// Initialize minio client object.
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return minioClient, nil
+}
+
+func VerifyBucket(client *minio.Client, ctx context.Context, bucketName string) error {
+	exists, err := client.BucketExists(ctx, bucketName)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return errors.New(fmt.Sprintf("bucket %s does not exist\n", bucketName))
+	}
+
+	return nil
+}

main.go

@@ -1,43 +1,50 @@
 package main
 
 import (
-	"backup-tool/s3helper"
+	"backup-tool/pruner"
-	"context"
+	"backup-tool/uploader"
-	"fmt"
+	"flag"
+	"github.com/joho/godotenv"
 	"log"
 	"os"
-
-	"github.com/joho/godotenv"
 )
 
-func main() {
+const (
+	ENDPOINT = "s3.us-west-004.backblazeb2.com"
+)
 
+var skipUpload bool
+var keepNum int
+
+func init() {
+	flag.BoolVar(&skipUpload, "skip-upload", false, "do not upload file to S3/Backblaze")
+	flag.IntVar(&keepNum, "keep-backups", 7, "number of backups to keep; 0 to keep all")
+}
+
+func main() {
+	flag.Parse()
 	err := godotenv.Load()
 	if err != nil {
 		log.Fatalf("error loading .env file: (%s)", err.Error())
 	}
 
-	if len(os.Args) > 1 {
+	if len(os.Args) < 2 && !skipUpload {
-		// only do this if we have been given a file to upload (for now)
+		log.Fatalln("missing filename parameter")
-		// TODO: have this tool take the backup of the DB itself
+	}
-		b2Creds, err := getCredentials([]string{"B2_KEY_ID", "B2_APPLICATION_KEY"})
+
+	bucketName := os.Getenv("BUCKET_NAME")
+	if bucketName == "" {
+		log.Fatalln("missing or empty BUCKET_NAME")
+	}
+
+	if !skipUpload {
+		err = uploader.UploadFile(bucketName, ENDPOINT, os.Args[1])
 		if err != nil {
-			log.Fatalf("could not get B2 credentials: %s", err)
+			log.Fatalln(err.Error())
 		}
-		s3helper.UploadFile(context.Background(), b2Creds)
 	}
 
-}
+	if keepNum > 0 {
-
+		err = pruner.PruneFiles(bucketName, ENDPOINT, keepNum)
-func getCredentials(credNames []string) (map[string]string, error) {
-	creds := map[string]string{}
-	for _, name := range credNames {
-		value := os.Getenv(name)
-		if value == "" {
-			return nil, fmt.Errorf("missing or empty ENV var: %s", name)
-		}
-		creds[name] = value
 	}
-
-	return creds, nil
 }

pruner/pruner.go

@@ -0,0 +1,52 @@
+package pruner
+
+import (
+	"backup-tool/helpers"
+	"context"
+	"github.com/minio/minio-go/v7"
+	"log"
+	"sort"
+)
+
+func PruneFiles(bucket, endpoint string, keep int) error {
+	ctx := context.Background()
+	accessKeyID, secretAccessKey, err := helpers.GetCredentials()
+	if err != nil {
+		return err
+	}
+	client, err := helpers.CreateClient(accessKeyID, secretAccessKey, endpoint)
+	if err != nil {
+		return err
+	}
+	err = helpers.VerifyBucket(client, ctx, bucket)
+	if err != nil {
+		return err
+	}
+
+	var objects []minio.ObjectInfo
+	for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{}) {
+		objects = append(objects, obj)
+	}
+
+	if len(objects) < keep {
+		return nil
+	}
+
+	sort.Sort(BucketObjects(objects)) // last modified object will be at position 0
+	for _, object := range objects[keep:] {
+		err := client.RemoveObject(ctx, bucket, object.Key, minio.RemoveObjectOptions{})
+		if err != nil {
+			log.Printf("error deleting object %q: %s", object.Key, err.Error())
+		} else {
+			log.Printf("deleted object %q", object.Key)
+		}
+	}
+
+	return nil
+}
+
+type BucketObjects []minio.ObjectInfo
+
+func (b BucketObjects) Len() int           { return len(b) }
+func (b BucketObjects) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b BucketObjects) Less(i, j int) bool { return b[i].LastModified.After(b[j].LastModified) } // return newest object first in list

s3helper/s3helper.go

@@ -1,72 +0,0 @@
-package s3helper
-
-import (
-	"context"
-	"log"
-	"os"
-
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-)
-
-const (
-	ENDPOINT   = "s3.us-west-004.backblazeb2.com"
-	BUCKETNAME = "ducimon-db-backups"
-)
-
-func UploadFile(ctx context.Context, b2Creds map[string]string) error {
-
-	client := createClient(b2Creds["B2_KEY_ID"], b2Creds["B2_APPLICATION_KEY"])
-	verifyBucket(client, ctx)
-	absolutePath, basename := validateUploadFile()
-	_, err := client.FPutObject(ctx, BUCKETNAME, "test", absolutePath, minio.PutObjectOptions{})
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	log.Printf("upload of %s complete\n", basename)
-
-	return nil
-}
-
-func createClient(accessKeyID string, secretAccessKey string) *minio.Client {
-	// Initialize minio client object.
-	minioClient, err := minio.New(ENDPOINT, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
-		Secure: true,
-	})
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	return minioClient
-}
-
-func verifyBucket(client *minio.Client, ctx context.Context) {
-	exists, err := client.BucketExists(ctx, BUCKETNAME)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	if !exists {
-		log.Fatalf("bucket %s does not exist\n", BUCKETNAME)
-	}
-}
-
-func validateUploadFile() (string, string) {
-	if len(os.Args) < 2 {
-		log.Fatalln("upload file not specified")
-	}
-
-	file, err := os.Stat(os.Args[1])
-
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	if file.IsDir() {
-		log.Fatalln("upload of directories is not supported")
-	}
-
-	return os.Args[1], file.Name()
-}

uploader/uploader.go

@@ -0,0 +1,52 @@
+package uploader
+
+import (
+	"backup-tool/helpers"
+	"context"
+	"errors"
+	"github.com/minio/minio-go/v7"
+	"log"
+	"os"
+)
+
+func UploadFile(bucketName, endpoint, fileName string) error {
+
+	ctx := context.Background()
+	accessKeyID, secretAccessKey, err := helpers.GetCredentials()
+	if err != nil {
+		return (err)
+	}
+	client, err := helpers.CreateClient(accessKeyID, secretAccessKey, endpoint)
+	if err != nil {
+		return (err)
+	}
+	err = helpers.VerifyBucket(client, ctx, bucketName)
+	if err != nil {
+		return err
+	}
+	absolutePath, basename, err := validateUploadFile(fileName)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.FPutObject(ctx, bucketName, basename, absolutePath, minio.PutObjectOptions{})
+	if err != nil {
+		return err
+	}
+
+	log.Printf("upload of %s complete\n", basename)
+	return nil
+}
+
+func validateUploadFile(fileName string) (string, string, error) {
+	file, err := os.Stat(fileName)
+	if err != nil {
+		return "", "", err
+	}
+
+	if file.IsDir() {
+		return "", "", errors.New("upload of directories is not supported")
+	}
+
+	return fileName, file.Name(), nil
+}