From 21db79a33399064dd49a6ee88368337cdc8362bc Mon Sep 17 00:00:00 2001 From: Alex Csengery Date: Thu, 9 Nov 2023 21:56:15 -0500 Subject: [PATCH] feat(prune): add prune files feature This feature allows you to specify the n latest backups to keep in the bucket. --- go.mod | 2 +- helpers/helpers.go | 49 +++++++++++++++++++++++++++++++++++++++++ main.go | 25 +++++++++++++++++---- pruner/pruner.go | 52 ++++++++++++++++++++++++++++++++++++++++++++ uploader/uploader.go | 48 ++++------------------------------------ 5 files changed, 127 insertions(+), 49 deletions(-) create mode 100644 helpers/helpers.go create mode 100644 pruner/pruner.go diff --git a/go.mod b/go.mod index 4de1df6..d8c2a04 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module backup-tool -go 1.21 +go 1.18 require ( github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/helpers/helpers.go b/helpers/helpers.go new file mode 100644 index 0000000..4d49ecd --- /dev/null +++ b/helpers/helpers.go @@ -0,0 +1,49 @@ +package helpers + +import ( + "context" + "errors" + "fmt" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "os" +) + +func GetCredentials() (string, string, error) { + // get credentials from env vars + keyName := os.Getenv("KEY_ID") + if keyName == "" { + return "", "", errors.New("missing or empty KEY_ID") + } + applicationKey := os.Getenv("APPLICATION_KEY") + if applicationKey == "" { + return "", "", errors.New("missing or empty APPLICATION_KEY") + } + return keyName, applicationKey, nil +} + +func CreateClient(accessKeyID, secretAccessKey, endpoint string) (*minio.Client, error) { + // Initialize minio client object. + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: true, + }) + if err != nil { + return nil, err + } + + return minioClient, nil +} + +func VerifyBucket(client *minio.Client, ctx context.Context, bucketName string) error { + exists, err := client.BucketExists(ctx, bucketName) + if err != nil { + return err + } + + if !exists { + return errors.New(fmt.Sprintf("bucket %s does not exist\n", bucketName)) + } + + return nil +} diff --git a/main.go b/main.go index 2d86934..1704bc7 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,9 @@ package main import ( + "backup-tool/pruner" "backup-tool/uploader" + "flag" "github.com/joho/godotenv" "log" "os" @@ -11,13 +13,22 @@ const ( ENDPOINT = "s3.us-west-004.backblazeb2.com" ) +var skipUpload bool +var keepNum int + +func init() { + flag.BoolVar(&skipUpload, "skip-upload", false, "do not upload file to S3/Backblaze") + flag.IntVar(&keepNum, "keep-backups", 7, "number of backups to keep; 0 to keep all") +} + func main() { + flag.Parse() err := godotenv.Load() if err != nil { log.Fatalf("error loading .env file: (%s)", err.Error()) } - if len(os.Args) < 2 { + if len(os.Args) < 2 && !skipUpload { log.Fatalln("missing filename parameter") } @@ -26,8 +37,14 @@ func main() { log.Fatalln("missing or empty BUCKET_NAME") } - err = uploader.UploadFile(bucketName, ENDPOINT, os.Args[1]) - if err != nil { - log.Fatalln(err.Error()) + if !skipUpload { + err = uploader.UploadFile(bucketName, ENDPOINT, os.Args[1]) + if err != nil { + log.Fatalln(err.Error()) + } + } + + if keepNum > 0 { + err = pruner.PruneFiles(bucketName, ENDPOINT, keepNum) } } diff --git a/pruner/pruner.go b/pruner/pruner.go new file mode 100644 index 0000000..ce5e648 --- /dev/null +++ b/pruner/pruner.go @@ -0,0 +1,52 @@ +package pruner + +import ( + "backup-tool/helpers" + "context" + "github.com/minio/minio-go/v7" + "log" + "sort" +) + +func PruneFiles(bucket, endpoint string, keep int) error { + ctx := context.Background() + accessKeyID, secretAccessKey, err := helpers.GetCredentials() + if err != nil { + return err + } + client, err := helpers.CreateClient(accessKeyID, secretAccessKey, endpoint) + if err != nil { + return err + } + err = helpers.VerifyBucket(client, ctx, bucket) + if err != nil { + return err + } + + var objects []minio.ObjectInfo + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{}) { + objects = append(objects, obj) + } + + if len(objects) < keep { + return nil + } + + sort.Sort(BucketObjects(objects)) // last modified object will be at position 0 + for _, object := range objects[keep:] { + err := client.RemoveObject(ctx, bucket, object.Key, minio.RemoveObjectOptions{}) + if err != nil { + log.Printf("error deleting object %q: %s", object.Key, err.Error()) + } else { + log.Printf("deleted object %q", object.Key) + } + } + + return nil +} + +type BucketObjects []minio.ObjectInfo + +func (b BucketObjects) Len() int { return len(b) } +func (b BucketObjects) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b BucketObjects) Less(i, j int) bool { return b[i].LastModified.After(b[j].LastModified) } // return newest object first in list diff --git a/uploader/uploader.go b/uploader/uploader.go index 1329b1e..112ea28 100644 --- a/uploader/uploader.go +++ b/uploader/uploader.go @@ -1,11 +1,10 @@ package uploader import ( + "backup-tool/helpers" "context" "errors" - "fmt" "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "log" "os" ) @@ -13,15 +12,15 @@ import ( func UploadFile(bucketName, endpoint, fileName string) error { ctx := context.Background() - accessKeyID, secretAccessKey, err := getCredentials() + accessKeyID, secretAccessKey, err := helpers.GetCredentials() if err != nil { return (err) } - client, err := createClient(accessKeyID, secretAccessKey, endpoint) + client, err := helpers.CreateClient(accessKeyID, secretAccessKey, endpoint) if err != nil { return (err) } - err = verifyBucket(client, ctx, bucketName) + err = helpers.VerifyBucket(client, ctx, bucketName) if err != nil { return err } @@ -51,42 +50,3 @@ func validateUploadFile(fileName string) (string, string, error) { return fileName, file.Name(), nil } - -func verifyBucket(client *minio.Client, ctx context.Context, bucketName string) error { - exists, err := client.BucketExists(ctx, bucketName) - if err != nil { - return err - } - - if !exists { - return errors.New(fmt.Sprintf("bucket %s does not exist\n", bucketName)) - } - - return nil -} - -func createClient(accessKeyID, secretAccessKey, endpoint string) (*minio.Client, error) { - // Initialize minio client object. - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: true, - }) - if err != nil { - return nil, err - } - - return minioClient, nil -} - -func getCredentials() (string, string, error) { - // get credentials from env vars - keyName := os.Getenv("KEY_ID") - if keyName == "" { - return "", "", errors.New("missing or empty KEY_ID") - } - applicationKey := os.Getenv("APPLICATION_KEY") - if applicationKey == "" { - return "", "", errors.New("missing or empty APPLICATION_KEY") - } - return keyName, applicationKey, nil -}