From 155dcf48550c438aa20ac40877ec13b03aa739ff Mon Sep 17 00:00:00 2001 From: LPOCHOLLE <> Date: Mon, 18 Oct 2021 17:00:28 +0200 Subject: [PATCH] backblaze upload support --- Backup.py | 82 ++++++++++++++++++++++++++++++++++++++++++++----------- main.py | 7 ++++- 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/Backup.py b/Backup.py index bb72a26..556ec83 100644 --- a/Backup.py +++ b/Backup.py @@ -7,6 +7,9 @@ import mgzip as gzip import tarfile import sqlite3 import pathlib +import hashlib +import b2sdk.v2 +from b2sdk.v2 import B2Api from crypt import * @@ -16,10 +19,14 @@ class Backup: tarball_size = "50M" save_location = None - def __init__(self, key, bdd): + def __init__(self, key, bdd, app_key_id=None, app_key=None, bucket_id=None): self.key = key self.bdd = DataBase(bdd) + self.b2 = B2Api() + self.b2.authorize_account("production", app_key_id, app_key) + self.buk = self.b2.get_bucket_by_id(bucket_id) + def recurse(self, path): tarball_size = parse_size(self.tarball_size) files = [] @@ -31,7 +38,7 @@ class Backup: c_date = datetime.fromtimestamp(os.path.getctime(uri)).strftime("%Y-%m-%d %H:%M:%S.%f") if size > tarball_size: crypt_name = self.bdd.add([{'name': f, - 'path': path, + 'path': uri, 'size': size, 'm_date': m_date, 'c_date': c_date}], @@ -39,10 +46,12 @@ class Backup: if crypt_name is not None: print("Proceed", uri) enc = crypt(compress(uri), self.key) + print(" Size : ", get_size(enc)) + upload_b2(self.buk, enc, crypt_name) save(enc, os.path.join(self.save_location, crypt_name)) else: files.append({'name': f, - 'path': path, + 'path': uri, 'size': size, 'm_date': m_date, 'c_date': c_date}) @@ -54,13 +63,44 @@ class Backup: print("Proceed", path, ":", [file['name'] for file in files]) tarball = tar(files) enc = crypt(compress(tarball), self.key) + print(" Size : ", get_size(enc)) + upload_b2(self.buk, enc, crypt_name) save(enc, os.path.join(self.save_location, crypt_name)) +def get_size(in_file): + if type(in_file) is str: + filesize = os.path.getsize(in_file) + + elif type(in_file) is io.BufferedRandom or tempfile.SpooledTemporaryFile: + in_file.seek(0, 2) + filesize = in_file.tell() + in_file.seek(0) + return human_size(filesize, decimal_places=1, unit='si') + + +def upload_b2(buk, file, save_name, chunksize=64 * 1024): + if type(file) is io.BufferedRandom or tempfile.SpooledTemporaryFile: + file.seek(0) + sha1 = hashlib.sha1(file.read()).hexdigest() + file.seek(0) + + input_sources = b2sdk.v2.UploadSourceBytes(file.read(), content_sha1=sha1) + buk.upload(input_sources, save_name) + + # while chunk := file.read(chunksize): + # input_sources = b2sdk.v2.UploadSourceBytes(chunk, content_sha1=sha1) + # buk.upload(input_sources, save_name) + else: + print("Unable to save " + str(file) + " of type " + str(type(file))) + return + + def tar(files): tarball = tempfile.SpooledTemporaryFile() with tarfile.open(fileobj=tarball, mode='w') as zipfile: for file in files: + # zipfile.add(os.path.join(file['path'], file['name'])) zipfile.add(file['path']) return tarball @@ -140,6 +180,12 @@ class DataBase: def __create_table(self): cursor = self.conn.cursor() + cursor.execute("""DROP TABLE IF EXISTS files""") + + cursor.execute("""DROP TABLE IF EXISTS crypt""") + + self.conn.commit() + cursor.execute(""" CREATE TABLE IF NOT EXISTS files( id INTEGER PRIMARY KEY UNIQUE NOT NULL, @@ -173,11 +219,9 @@ class DataBase: try: crypt_id_list.append(cursor.fetchone()['crypt_id']) except TypeError: - pass - if len(crypt_id_list) > 0: - return most_frequent(crypt_id_list) - else: - return None + cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as crypt_id FROM crypt""") + return cursor.fetchone()['crypt_id'] + return most_frequent(crypt_id_list) def exist(self, file): cursor = self.conn.cursor() @@ -201,9 +245,6 @@ class DataBase: def add(self, list_file, compress_mode=None): cursor = self.conn.cursor() crypt_id = self.get_crypt_id(list_file) - if crypt_id is None: - cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as crypt_id FROM crypt""") - crypt_id = cursor.fetchone()['crypt_id'] cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as files_id FROM files""") file_id = cursor.fetchone()['files_id'] proceed = False @@ -219,18 +260,27 @@ class DataBase: file_id += 1 proceed = True if proceed: - cursor.execute("""INSERT OR IGNORE INTO crypt VALUES(?, ?)""", (crypt_id, compress_mode)) + cursor.execute("""INSERT INTO crypt VALUES(?, ?)""", (crypt_id, compress_mode)) self.conn.commit() return str(crypt_id).zfill(5) else: return None -def human_size(size, decimal_places=0): - for unit in ['B', 'K', 'M', 'G', 'T']: - if size < 1024.0: +def human_size(size, decimal_places=0, unit=None): + unit_tab = ['B', 'K', 'M', 'G', 'T'] + format = 1024.0 + if unit == 'iec': + format = 1024.0 + unit_tab = ['B', 'KiB', 'MiB', 'GiB', 'TiB'] + elif unit == 'si': + format = 1000.0 + unit_tab = ['B', 'KB', 'MB', 'GB', 'TB'] + for unit in unit_tab: + if size < format: break - size /= 1024.0 + size /= format + return f"{size:.{decimal_places}f}{unit}" diff --git a/main.py b/main.py index 2a01316..46fd8e6 100644 --- a/main.py +++ b/main.py @@ -11,8 +11,13 @@ else: print("Recovered") print(key) + +application_key_id = '003aa00745ec42a0000000003' +application_key = 'K003zMa5e07FheUrB38/fqKFfHSlXok' +bucket_id = '6a1a9000075465fe7cc4021a' + bdd = "bdd.db" -bck = Backup(key, bdd) +bck = Backup(key, bdd, app_key_id=application_key_id, app_key=application_key, bucket_id=bucket_id) bck.save_location = "crypted"