From 4c79d283b0fdb5566d78a375a85860173f6e8cae Mon Sep 17 00:00:00 2001 From: toto <> Date: Thu, 28 Oct 2021 15:57:34 +0200 Subject: [PATCH] hash + set crypt attr --- backup.py | 55 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/backup.py b/backup.py index 059cb4f..fb5de36 100644 --- a/backup.py +++ b/backup.py @@ -7,8 +7,9 @@ import mgzip as gzip import tarfile import sqlite3 import pathlib -from b2sdk.v2 import B2Api +import hashlib +from b2sdk.v2 import B2Api from crypt import encrypt_file, decrypt_file ZFILL = 5 @@ -56,17 +57,17 @@ class Backup: m_date = datetime.fromtimestamp(os.path.getmtime(uri)).strftime("%Y-%m-%d %H:%M:%S.%f") c_date = datetime.fromtimestamp(os.path.getctime(uri)).strftime("%Y-%m-%d %H:%M:%S.%f") if size > tarball_size: - crypt_name = self.bdd.add([{'name': f, + crypt_id = self.bdd.add([{'name': f, 'path': pathlib.Path(uri).as_posix(), 'size': size, 'm_date': m_date, - 'c_date': c_date}], - compress_mode="gz") - if crypt_name is not None: - print("Proceed", uri) + 'c_date': c_date}]) + if crypt_id is not None: + print("Proceed", uri, ' ==> ', crypt_id) enc = crypt(compress(uri), self.key) + self.bdd.set_crypt_attr(crypt_id, compress_mode="gz", sha1sum=hashlib.sha1(enc.read()).hexdigest()) print(" Size : ", get_size(enc)) - upload_file(enc, self.save_mode, file_name=os.path.join(self.save_location, crypt_name), bucket=self.buk) + upload_file(enc, self.save_mode, file_name=os.path.join(self.save_location, str(crypt_id).zfill(ZFILL)), bucket=self.buk) else: files.append({'name': f, 'path': pathlib.Path(uri).as_posix(), @@ -76,19 +77,20 @@ class Backup: elif os.path.isdir(uri) and recurse: self.__save(uri, recurse=recurse) if len(files) > 0: - crypt_name = self.bdd.add(files, compress_mode="tar.gz") - if crypt_name is not None: - print("Proceed", path, ":", [file['name'] for file in files]) + crypt_id = self.bdd.add(files) + if crypt_id is not None: + print("Proceed", path, ":", [file['name'] for file in files], ' ==> ', crypt_id) tarball = tar([file['path'] for file in files]) enc = crypt(compress(tarball), self.key) + self.bdd.set_crypt_attr(crypt_id, compress_mode="tar.gz", sha1sum=hashlib.sha1(enc.read()).hexdigest()) print(" Size : ", get_size(enc)) - upload_file(enc, self.save_mode, file_name=os.path.join(self.save_location, crypt_name), bucket=self.buk) + upload_file(enc, self.save_mode, file_name=os.path.join(self.save_location, str(crypt_id).zfill(ZFILL)), bucket=self.buk) def recover_file(self, paths, parents=False, save_path=os.getcwd()): files = self.bdd.get_crypt_name(paths) for file in files: - if file['crypt'] is not None: - encrypted_file = download_file(self.save_mode, os.path.join(self.save_location, file['crypt']), bucket=self.buk) + if file['crypt_id'] is not None: + encrypted_file = download_file(self.save_mode, os.path.join(self.save_location, str(file['crypt_id']).zfill(ZFILL)), bucket=self.buk) if parents: save_path = os.path.join(save_path, file['path']) @@ -236,7 +238,8 @@ class DataBase: cursor.execute(""" CREATE TABLE IF NOT EXISTS crypt( id INTEGER PRIMARY KEY UNIQUE NOT NULL, - compress_mode TEXT + compress_mode TEXT, + sha1sum TEXT ) """) @@ -258,7 +261,7 @@ class DataBase: try: crypt_list.append({'name': os.path.basename(path), 'path': path, - 'crypt': str(retval['crypt_id']).zfill(ZFILL), + 'crypt_id': retval['crypt_id'], 'compress_mode': retval['compress_mode']}) except TypeError: crypt_list.append({'path': path, 'crypt': None}) @@ -275,6 +278,8 @@ class DataBase: except TypeError: pass try: + if len(list(set(crypt_id_list))) == 1: + return crypt_id_list[0] id = most_frequent(crypt_id_list) except ValueError: cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as crypt_id FROM crypt""") @@ -283,14 +288,17 @@ class DataBase: 'name': ', '.join([f"'{file['name']}'" for file in list_file]), 'path': ', '.join([f"'{file['path']}'" for file in list_file])} cursor.execute("""SELECT 1 FROM files - WHERE crypt_id={id} - AND name NOT IN ({name}) - AND path NOT IN ({path})""".format(**params)) + WHERE crypt_id={id} + AND name NOT IN ({name}) + AND path NOT IN ({path})""".format(**params)) neighbour = cursor.fetchall() if len(neighbour) > 0: cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as crypt_id FROM crypt""") return cursor.fetchone()['crypt_id'] else: + cursor.execute("""UPDATE files SET crypt_id={id} + WHERE name IN ({name}) + AND path IN ({path})""".format(**params)) return id def exist(self, file): @@ -312,7 +320,12 @@ class DataBase: else: return True - def add(self, list_file, compress_mode=None): + def set_crypt_attr(self, crypt_id, compress_mode=None, sha1sum=None): + cursor = self.conn.cursor() + cursor.execute("""UPDATE crypt SET compress_mode='{mode}', sha1sum='{sum}' WHERE id='{id}'""".format(id=crypt_id, mode=compress_mode, sum=sha1sum)) + self.conn.commit() + + def add(self, list_file): cursor = self.conn.cursor() crypt_id = self.__get_crypt_id(list_file) cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as files_id FROM files""") @@ -330,9 +343,9 @@ class DataBase: file_id += 1 proceed = True if proceed: - cursor.execute("""INSERT OR IGNORE INTO crypt VALUES(?, ?)""", (crypt_id, compress_mode)) + cursor.execute("""INSERT OR IGNORE INTO crypt (id) VALUES(?)""", (crypt_id,)) self.conn.commit() - return str(crypt_id).zfill(ZFILL) + return crypt_id else: return None