backblaze upload support
This commit is contained in:
LPOCHOLLE
parent
1827794ec5
commit
155dcf4855
82
Backup.py
82
Backup.py
@ -7,6 +7,9 @@ import mgzip as gzip
|
||||
import tarfile
|
||||
import sqlite3
|
||||
import pathlib
|
||||
import hashlib
|
||||
import b2sdk.v2
|
||||
from b2sdk.v2 import B2Api
|
||||
|
||||
from crypt import *
|
||||
|
||||
@ -16,10 +19,14 @@ class Backup:
|
||||
tarball_size = "50M"
|
||||
save_location = None
|
||||
|
||||
def __init__(self, key, bdd):
|
||||
def __init__(self, key, bdd, app_key_id=None, app_key=None, bucket_id=None):
|
||||
self.key = key
|
||||
self.bdd = DataBase(bdd)
|
||||
|
||||
self.b2 = B2Api()
|
||||
self.b2.authorize_account("production", app_key_id, app_key)
|
||||
self.buk = self.b2.get_bucket_by_id(bucket_id)
|
||||
|
||||
def recurse(self, path):
|
||||
tarball_size = parse_size(self.tarball_size)
|
||||
files = []
|
||||
@ -31,7 +38,7 @@ class Backup:
|
||||
c_date = datetime.fromtimestamp(os.path.getctime(uri)).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||
if size > tarball_size:
|
||||
crypt_name = self.bdd.add([{'name': f,
|
||||
'path': path,
|
||||
'path': uri,
|
||||
'size': size,
|
||||
'm_date': m_date,
|
||||
'c_date': c_date}],
|
||||
@ -39,10 +46,12 @@ class Backup:
|
||||
if crypt_name is not None:
|
||||
print("Proceed", uri)
|
||||
enc = crypt(compress(uri), self.key)
|
||||
print(" Size : ", get_size(enc))
|
||||
upload_b2(self.buk, enc, crypt_name)
|
||||
save(enc, os.path.join(self.save_location, crypt_name))
|
||||
else:
|
||||
files.append({'name': f,
|
||||
'path': path,
|
||||
'path': uri,
|
||||
'size': size,
|
||||
'm_date': m_date,
|
||||
'c_date': c_date})
|
||||
@ -54,13 +63,44 @@ class Backup:
|
||||
print("Proceed", path, ":", [file['name'] for file in files])
|
||||
tarball = tar(files)
|
||||
enc = crypt(compress(tarball), self.key)
|
||||
print(" Size : ", get_size(enc))
|
||||
upload_b2(self.buk, enc, crypt_name)
|
||||
save(enc, os.path.join(self.save_location, crypt_name))
|
||||
|
||||
|
||||
def get_size(in_file):
|
||||
if type(in_file) is str:
|
||||
filesize = os.path.getsize(in_file)
|
||||
|
||||
elif type(in_file) is io.BufferedRandom or tempfile.SpooledTemporaryFile:
|
||||
in_file.seek(0, 2)
|
||||
filesize = in_file.tell()
|
||||
in_file.seek(0)
|
||||
return human_size(filesize, decimal_places=1, unit='si')
|
||||
|
||||
|
||||
def upload_b2(buk, file, save_name, chunksize=64 * 1024):
|
||||
if type(file) is io.BufferedRandom or tempfile.SpooledTemporaryFile:
|
||||
file.seek(0)
|
||||
sha1 = hashlib.sha1(file.read()).hexdigest()
|
||||
file.seek(0)
|
||||
|
||||
input_sources = b2sdk.v2.UploadSourceBytes(file.read(), content_sha1=sha1)
|
||||
buk.upload(input_sources, save_name)
|
||||
|
||||
# while chunk := file.read(chunksize):
|
||||
# input_sources = b2sdk.v2.UploadSourceBytes(chunk, content_sha1=sha1)
|
||||
# buk.upload(input_sources, save_name)
|
||||
else:
|
||||
print("Unable to save " + str(file) + " of type " + str(type(file)))
|
||||
return
|
||||
|
||||
|
||||
def tar(files):
|
||||
tarball = tempfile.SpooledTemporaryFile()
|
||||
with tarfile.open(fileobj=tarball, mode='w') as zipfile:
|
||||
for file in files:
|
||||
# zipfile.add(os.path.join(file['path'], file['name']))
|
||||
zipfile.add(file['path'])
|
||||
return tarball
|
||||
|
||||
@ -140,6 +180,12 @@ class DataBase:
|
||||
def __create_table(self):
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
cursor.execute("""DROP TABLE IF EXISTS files""")
|
||||
|
||||
cursor.execute("""DROP TABLE IF EXISTS crypt""")
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS files(
|
||||
id INTEGER PRIMARY KEY UNIQUE NOT NULL,
|
||||
@ -173,11 +219,9 @@ class DataBase:
|
||||
try:
|
||||
crypt_id_list.append(cursor.fetchone()['crypt_id'])
|
||||
except TypeError:
|
||||
pass
|
||||
if len(crypt_id_list) > 0:
|
||||
return most_frequent(crypt_id_list)
|
||||
else:
|
||||
return None
|
||||
cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as crypt_id FROM crypt""")
|
||||
return cursor.fetchone()['crypt_id']
|
||||
return most_frequent(crypt_id_list)
|
||||
|
||||
def exist(self, file):
|
||||
cursor = self.conn.cursor()
|
||||
@ -201,9 +245,6 @@ class DataBase:
|
||||
def add(self, list_file, compress_mode=None):
|
||||
cursor = self.conn.cursor()
|
||||
crypt_id = self.get_crypt_id(list_file)
|
||||
if crypt_id is None:
|
||||
cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as crypt_id FROM crypt""")
|
||||
crypt_id = cursor.fetchone()['crypt_id']
|
||||
cursor.execute("""SELECT IFNULL(max(id) + 1, 0) as files_id FROM files""")
|
||||
file_id = cursor.fetchone()['files_id']
|
||||
proceed = False
|
||||
@ -219,18 +260,27 @@ class DataBase:
|
||||
file_id += 1
|
||||
proceed = True
|
||||
if proceed:
|
||||
cursor.execute("""INSERT OR IGNORE INTO crypt VALUES(?, ?)""", (crypt_id, compress_mode))
|
||||
cursor.execute("""INSERT INTO crypt VALUES(?, ?)""", (crypt_id, compress_mode))
|
||||
self.conn.commit()
|
||||
return str(crypt_id).zfill(5)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def human_size(size, decimal_places=0):
|
||||
for unit in ['B', 'K', 'M', 'G', 'T']:
|
||||
if size < 1024.0:
|
||||
def human_size(size, decimal_places=0, unit=None):
|
||||
unit_tab = ['B', 'K', 'M', 'G', 'T']
|
||||
format = 1024.0
|
||||
if unit == 'iec':
|
||||
format = 1024.0
|
||||
unit_tab = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
|
||||
elif unit == 'si':
|
||||
format = 1000.0
|
||||
unit_tab = ['B', 'KB', 'MB', 'GB', 'TB']
|
||||
for unit in unit_tab:
|
||||
if size < format:
|
||||
break
|
||||
size /= 1024.0
|
||||
size /= format
|
||||
|
||||
return f"{size:.{decimal_places}f}{unit}"
|
||||
|
||||
|
||||
|
||||
7
main.py
7
main.py
@ -11,8 +11,13 @@ else:
|
||||
print("Recovered")
|
||||
|
||||
print(key)
|
||||
|
||||
application_key_id = '003aa00745ec42a0000000003'
|
||||
application_key = 'K003zMa5e07FheUrB38/fqKFfHSlXok'
|
||||
bucket_id = '6a1a9000075465fe7cc4021a'
|
||||
|
||||
bdd = "bdd.db"
|
||||
bck = Backup(key, bdd)
|
||||
bck = Backup(key, bdd, app_key_id=application_key_id, app_key=application_key, bucket_id=bucket_id)
|
||||
bck.save_location = "crypted"
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user