From 5f033b28cf48de67977db277c126993ee0584515 Mon Sep 17 00:00:00 2001 From: LPOCHOLLE <> Date: Fri, 22 Oct 2021 11:42:33 +0200 Subject: [PATCH] Add choice of save mode --- SELECT.sql | 3 ++ Backup.py => backup.py | 85 +++++++++++++++++++++++++++++------------- main.py | 19 ++++------ 3 files changed, 69 insertions(+), 38 deletions(-) create mode 100644 SELECT.sql rename Backup.py => backup.py (78%) diff --git a/SELECT.sql b/SELECT.sql new file mode 100644 index 0000000..2ea38ed --- /dev/null +++ b/SELECT.sql @@ -0,0 +1,3 @@ +SELECT name, path, size, m_date, c_date, crypt.id, compress_mode +FROM files +inner join CRYPT on files.crypt_id = crypt.id \ No newline at end of file diff --git a/Backup.py b/backup.py similarity index 78% rename from Backup.py rename to backup.py index a1f3441..ce08708 100644 --- a/Backup.py +++ b/backup.py @@ -11,21 +11,42 @@ from b2sdk.v2 import B2Api from crypt import encrypt_file, decrypt_file +ZFILL = 5 +BACKBLAZE = 100 +LOCAL = 101 + class Backup: key = None + buk = None tarball_size = "50M" - save_location = None + save_location = "" - def __init__(self, key, bdd, app_key_id=None, app_key=None, bucket_id=None): + def __init__(self, key, bdd=None, save_mode=None, save_location=None, app_key_id=None, app_key=None, bucket_id=None): self.key = key - self.bdd = DataBase(bdd) - self.b2 = B2Api() - self.b2.authorize_account("production", app_key_id, app_key) - self.buk = self.b2.get_bucket_by_id(bucket_id) + if save_mode is None: + raise Exception("No save_mode") + elif save_mode == BACKBLAZE: + if None in (app_key_id, app_key, bucket_id): + raise Exception("Some arguments are not filled") + self.b2 = B2Api() + self.b2.authorize_account("production", app_key_id, app_key) + self.buk = self.b2.get_bucket_by_id(bucket_id) + self.bdd = DataBase(bdd) + elif save_mode == LOCAL: + if None in (save_location, bdd): + raise Exception("Some arguments are not filled") + self.save_location = save_location + self.bdd = DataBase(bdd) - def recurse(self, path): + self.save_mode = save_mode + + def save(self, path, recurse=True): + self.__save(path, recurse=recurse) + return self + + def __save(self, path, recurse=True): tarball_size = parse_size(self.tarball_size) files = [] for f in os.listdir(path): @@ -45,16 +66,15 @@ class Backup: print("Proceed", uri) enc = crypt(compress(uri), self.key) print(" Size : ", get_size(enc)) - self.buk.upload_bytes(enc.read(), crypt_name) - # save(enc, os.path.join(self.save_location, crypt_name)) + save_crypted_file(enc, self.save_mode, file_name=os.path.join(self.save_location, crypt_name), bucket=self.buk) else: files.append({'name': f, 'path': pathlib.Path(uri).as_posix(), 'size': size, 'm_date': m_date, 'c_date': c_date}) - elif os.path.isdir(uri): - self.recurse(uri) + elif os.path.isdir(uri) and recurse: + self.__save(uri, recurse=recurse) if len(files) > 0: crypt_name = self.bdd.add(files, compress_mode="tar.gz") if crypt_name is not None: @@ -62,18 +82,34 @@ class Backup: tarball = tar([file['path'] for file in files]) enc = crypt(compress(tarball), self.key) print(" Size : ", get_size(enc)) - self.buk.upload_bytes(enc.read(), crypt_name) - # save(enc, os.path.join(self.save_location, crypt_name)) + save_crypted_file(enc, self.save_mode, file_name=os.path.join(self.save_location, crypt_name), bucket=self.buk) def recover_file(self, paths, parents=False, save_path=os.getcwd()): files = self.bdd.get_crypt_file(paths) for file in files: - dl = tempfile.SpooledTemporaryFile() - self.buk.download_file_by_name(file['crypt']).save(dl) + if file['crypt'] is not None: + crypted_file = get_crypted_file(self.save_mode, os.path.join(self.save_location, file['crypt']), bucket=self.buk) - if parents: - save_path = os.path.join(save_path, file['path']) - untar(uncompress(uncrypt(dl, self.key)), file['path'], save_path) + if parents: + save_path = os.path.join(save_path, file['path']) + untar(uncompress(uncrypt(crypted_file, self.key)), file['name'], save_path) + + +def save_crypted_file(file, mode, file_name, bucket=None): + if mode == BACKBLAZE: + bucket.upload_bytes(file.read(), file_name) + elif mode == LOCAL: + save(file, file_name) + + +def get_crypted_file(mode, file, bucket=None): + dl = tempfile.SpooledTemporaryFile() + if mode == BACKBLAZE: + bucket.download_file_by_name(file).save(dl) + elif mode == LOCAL: + with open(file, 'rb') as infile: + dl.write(infile.read()) + return dl def get_size(in_file): @@ -86,12 +122,12 @@ def get_size(in_file): in_file.seek(0) return human_size(filesize, decimal_places=1, unit='si') + def tar(files): tarball = tempfile.SpooledTemporaryFile() with tarfile.open(fileobj=tarball, mode='w') as zipfile: for file in files: - # zipfile.add(os.path.join(file['path'], file['name'])) - zipfile.add(file) + zipfile.add(file, arcname=os.path.basename(file)) return tarball @@ -103,10 +139,7 @@ def untar(tar_file, file, save_path): zipfile = tarfile.open(tar_file, 'r') if not os.path.isdir(save_path): os.mkdir(save_path) - for member in zipfile.getmembers(): - if member.name == file: - member.name = os.path.basename(member.name) # remove the path - zipfile.extract(member, path=save_path) + zipfile.extract(file, path=save_path) zipfile.close() @@ -210,7 +243,7 @@ class DataBase: path = pathlib.Path(path).as_posix() cursor.execute("""SELECT crypt_id FROM files WHERE path=?""", (path,)) try: - crypt_list.append({'path': path, 'crypt': str(cursor.fetchone()['crypt_id']).zfill(5)}) + crypt_list.append({'name': os.path.basename(path),'path': path, 'crypt': str(cursor.fetchone()['crypt_id']).zfill(ZFILL)}) except TypeError: crypt_list.append({'path': path, 'crypt': None}) return crypt_list @@ -267,7 +300,7 @@ class DataBase: if proceed: cursor.execute("""INSERT INTO crypt VALUES(?, ?)""", (crypt_id, compress_mode)) self.conn.commit() - return str(crypt_id).zfill(5) + return str(crypt_id).zfill(ZFILL) else: return None diff --git a/main.py b/main.py index f2aaf3b..171cd12 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,8 @@ from Crypto.Random import get_random_bytes import os -from Backup import Backup + +import backup +from backup import Backup if not os.path.exists("key"): key = get_random_bytes(32) # 32 bytes * 8 = 256 bits (1 byte = 8 bits) @@ -17,18 +19,11 @@ application_key = 'K003RNvGfy+pazc6pD97xuUzPcDEqS0' bucket_id = '6a1a9000075465fe7cc4021a' bdd = "bdd.db" -bck = Backup(key, bdd, app_key_id=application_key_id, app_key=application_key, bucket_id=bucket_id) -bck.save_location = "crypted" +# bck = Backup(key, save_mode=backup.BACKBLAZE, bdd=bdd, app_key_id=application_key_id, app_key=application_key, bucket_id=bucket_id) +bck = Backup(key, save_mode=backup.LOCAL, bdd=bdd, save_location="crypted") rootdir = "test" -bck.recurse(rootdir) +bck.save(rootdir).save("test2") -bck.recover_file(paths=["test\depth1\depth2\\1316614572_leopard.jpg","test/depth1/depth1_text - Copie.txt"], save_path="recovered") - -# base = DataBase(bdd) -# file1 = {'name': "testname", 'path': "pathtest"} -# file2 = {'name': "secondname", 'path': "secondpath"} -# truc = [] -# truc.append(file1) -# base.add(truc) \ No newline at end of file +bck.recover_file(paths=["test\\depth1\\depth2\\1316614572_leopard.jpg", "test/depth1/depth1_text - Copie.txt"], save_path="recovered")