Add choice of save mode

This commit is contained in:
LPOCHOLLE 2021-10-22 11:42:33 +02:00
parent 2f272f5486
commit 5f033b28cf
3 changed files with 69 additions and 38 deletions

3
SELECT.sql Normal file
View File

@ -0,0 +1,3 @@
SELECT name, path, size, m_date, c_date, crypt.id, compress_mode
FROM files
inner join CRYPT on files.crypt_id = crypt.id

View File

@ -11,21 +11,42 @@ from b2sdk.v2 import B2Api
from crypt import encrypt_file, decrypt_file from crypt import encrypt_file, decrypt_file
ZFILL = 5
BACKBLAZE = 100
LOCAL = 101
class Backup: class Backup:
key = None key = None
buk = None
tarball_size = "50M" tarball_size = "50M"
save_location = None save_location = ""
def __init__(self, key, bdd, app_key_id=None, app_key=None, bucket_id=None): def __init__(self, key, bdd=None, save_mode=None, save_location=None, app_key_id=None, app_key=None, bucket_id=None):
self.key = key self.key = key
self.bdd = DataBase(bdd)
self.b2 = B2Api() if save_mode is None:
self.b2.authorize_account("production", app_key_id, app_key) raise Exception("No save_mode")
self.buk = self.b2.get_bucket_by_id(bucket_id) elif save_mode == BACKBLAZE:
if None in (app_key_id, app_key, bucket_id):
raise Exception("Some arguments are not filled")
self.b2 = B2Api()
self.b2.authorize_account("production", app_key_id, app_key)
self.buk = self.b2.get_bucket_by_id(bucket_id)
self.bdd = DataBase(bdd)
elif save_mode == LOCAL:
if None in (save_location, bdd):
raise Exception("Some arguments are not filled")
self.save_location = save_location
self.bdd = DataBase(bdd)
def recurse(self, path): self.save_mode = save_mode
def save(self, path, recurse=True):
self.__save(path, recurse=recurse)
return self
def __save(self, path, recurse=True):
tarball_size = parse_size(self.tarball_size) tarball_size = parse_size(self.tarball_size)
files = [] files = []
for f in os.listdir(path): for f in os.listdir(path):
@ -45,16 +66,15 @@ class Backup:
print("Proceed", uri) print("Proceed", uri)
enc = crypt(compress(uri), self.key) enc = crypt(compress(uri), self.key)
print(" Size : ", get_size(enc)) print(" Size : ", get_size(enc))
self.buk.upload_bytes(enc.read(), crypt_name) save_crypted_file(enc, self.save_mode, file_name=os.path.join(self.save_location, crypt_name), bucket=self.buk)
# save(enc, os.path.join(self.save_location, crypt_name))
else: else:
files.append({'name': f, files.append({'name': f,
'path': pathlib.Path(uri).as_posix(), 'path': pathlib.Path(uri).as_posix(),
'size': size, 'size': size,
'm_date': m_date, 'm_date': m_date,
'c_date': c_date}) 'c_date': c_date})
elif os.path.isdir(uri): elif os.path.isdir(uri) and recurse:
self.recurse(uri) self.__save(uri, recurse=recurse)
if len(files) > 0: if len(files) > 0:
crypt_name = self.bdd.add(files, compress_mode="tar.gz") crypt_name = self.bdd.add(files, compress_mode="tar.gz")
if crypt_name is not None: if crypt_name is not None:
@ -62,18 +82,34 @@ class Backup:
tarball = tar([file['path'] for file in files]) tarball = tar([file['path'] for file in files])
enc = crypt(compress(tarball), self.key) enc = crypt(compress(tarball), self.key)
print(" Size : ", get_size(enc)) print(" Size : ", get_size(enc))
self.buk.upload_bytes(enc.read(), crypt_name) save_crypted_file(enc, self.save_mode, file_name=os.path.join(self.save_location, crypt_name), bucket=self.buk)
# save(enc, os.path.join(self.save_location, crypt_name))
def recover_file(self, paths, parents=False, save_path=os.getcwd()): def recover_file(self, paths, parents=False, save_path=os.getcwd()):
files = self.bdd.get_crypt_file(paths) files = self.bdd.get_crypt_file(paths)
for file in files: for file in files:
dl = tempfile.SpooledTemporaryFile() if file['crypt'] is not None:
self.buk.download_file_by_name(file['crypt']).save(dl) crypted_file = get_crypted_file(self.save_mode, os.path.join(self.save_location, file['crypt']), bucket=self.buk)
if parents: if parents:
save_path = os.path.join(save_path, file['path']) save_path = os.path.join(save_path, file['path'])
untar(uncompress(uncrypt(dl, self.key)), file['path'], save_path) untar(uncompress(uncrypt(crypted_file, self.key)), file['name'], save_path)
def save_crypted_file(file, mode, file_name, bucket=None):
if mode == BACKBLAZE:
bucket.upload_bytes(file.read(), file_name)
elif mode == LOCAL:
save(file, file_name)
def get_crypted_file(mode, file, bucket=None):
dl = tempfile.SpooledTemporaryFile()
if mode == BACKBLAZE:
bucket.download_file_by_name(file).save(dl)
elif mode == LOCAL:
with open(file, 'rb') as infile:
dl.write(infile.read())
return dl
def get_size(in_file): def get_size(in_file):
@ -86,12 +122,12 @@ def get_size(in_file):
in_file.seek(0) in_file.seek(0)
return human_size(filesize, decimal_places=1, unit='si') return human_size(filesize, decimal_places=1, unit='si')
def tar(files): def tar(files):
tarball = tempfile.SpooledTemporaryFile() tarball = tempfile.SpooledTemporaryFile()
with tarfile.open(fileobj=tarball, mode='w') as zipfile: with tarfile.open(fileobj=tarball, mode='w') as zipfile:
for file in files: for file in files:
# zipfile.add(os.path.join(file['path'], file['name'])) zipfile.add(file, arcname=os.path.basename(file))
zipfile.add(file)
return tarball return tarball
@ -103,10 +139,7 @@ def untar(tar_file, file, save_path):
zipfile = tarfile.open(tar_file, 'r') zipfile = tarfile.open(tar_file, 'r')
if not os.path.isdir(save_path): if not os.path.isdir(save_path):
os.mkdir(save_path) os.mkdir(save_path)
for member in zipfile.getmembers(): zipfile.extract(file, path=save_path)
if member.name == file:
member.name = os.path.basename(member.name) # remove the path
zipfile.extract(member, path=save_path)
zipfile.close() zipfile.close()
@ -210,7 +243,7 @@ class DataBase:
path = pathlib.Path(path).as_posix() path = pathlib.Path(path).as_posix()
cursor.execute("""SELECT crypt_id FROM files WHERE path=?""", (path,)) cursor.execute("""SELECT crypt_id FROM files WHERE path=?""", (path,))
try: try:
crypt_list.append({'path': path, 'crypt': str(cursor.fetchone()['crypt_id']).zfill(5)}) crypt_list.append({'name': os.path.basename(path),'path': path, 'crypt': str(cursor.fetchone()['crypt_id']).zfill(ZFILL)})
except TypeError: except TypeError:
crypt_list.append({'path': path, 'crypt': None}) crypt_list.append({'path': path, 'crypt': None})
return crypt_list return crypt_list
@ -267,7 +300,7 @@ class DataBase:
if proceed: if proceed:
cursor.execute("""INSERT INTO crypt VALUES(?, ?)""", (crypt_id, compress_mode)) cursor.execute("""INSERT INTO crypt VALUES(?, ?)""", (crypt_id, compress_mode))
self.conn.commit() self.conn.commit()
return str(crypt_id).zfill(5) return str(crypt_id).zfill(ZFILL)
else: else:
return None return None

19
main.py
View File

@ -1,6 +1,8 @@
from Crypto.Random import get_random_bytes from Crypto.Random import get_random_bytes
import os import os
from Backup import Backup
import backup
from backup import Backup
if not os.path.exists("key"): if not os.path.exists("key"):
key = get_random_bytes(32) # 32 bytes * 8 = 256 bits (1 byte = 8 bits) key = get_random_bytes(32) # 32 bytes * 8 = 256 bits (1 byte = 8 bits)
@ -17,18 +19,11 @@ application_key = 'K003RNvGfy+pazc6pD97xuUzPcDEqS0'
bucket_id = '6a1a9000075465fe7cc4021a' bucket_id = '6a1a9000075465fe7cc4021a'
bdd = "bdd.db" bdd = "bdd.db"
bck = Backup(key, bdd, app_key_id=application_key_id, app_key=application_key, bucket_id=bucket_id)
bck.save_location = "crypted"
# bck = Backup(key, save_mode=backup.BACKBLAZE, bdd=bdd, app_key_id=application_key_id, app_key=application_key, bucket_id=bucket_id)
bck = Backup(key, save_mode=backup.LOCAL, bdd=bdd, save_location="crypted")
rootdir = "test" rootdir = "test"
bck.recurse(rootdir) bck.save(rootdir).save("test2")
bck.recover_file(paths=["test\depth1\depth2\\1316614572_leopard.jpg","test/depth1/depth1_text - Copie.txt"], save_path="recovered") bck.recover_file(paths=["test\\depth1\\depth2\\1316614572_leopard.jpg", "test/depth1/depth1_text - Copie.txt"], save_path="recovered")
# base = DataBase(bdd)
# file1 = {'name': "testname", 'path': "pathtest"}
# file2 = {'name': "secondname", 'path': "secondpath"}
# truc = []
# truc.append(file1)
# base.add(truc)