From f18cb1a3ef9bbf54364373adbbf2994de0c3a5c5 Mon Sep 17 00:00:00 2001 From: RaSan <34002411+RaSan147@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:38:57 +0600 Subject: [PATCH 1/2] save files using thread for faster read from client --- dev_src/_fs_utils.py | 102 ++++++++++++++++++++++++++++++++ dev_src/local_server_pyrobox.py | 71 +++++++++++++--------- dev_src/pyroboxCore.py | 3 + 3 files changed, 148 insertions(+), 28 deletions(-) diff --git a/dev_src/_fs_utils.py b/dev_src/_fs_utils.py index 37f5082..046ac88 100644 --- a/dev_src/_fs_utils.py +++ b/dev_src/_fs_utils.py @@ -15,9 +15,12 @@ > Thanks for your help! """ +from io import BufferedWriter import os from queue import Queue import re +import time +import traceback import urllib.parse @@ -476,3 +479,102 @@ def write(location): write(location) + +class UploadHandler: + def __init__(self, uid): + self.serial_io = Queue() + # format [[temp_file_obj, mode, data?], ...] + # if mode is 's' then data is [os_f_path, overwrite] + # if mode is 'w' then data is the data to write + self.active = True + self.waited = 0 + self.done = False + self.uid = uid + self.error = False + self.nap_time = 1 + + self.stop_on_error = True + + def upload(self, temp_file, mode, data=None): + """ + * format `[[temp_file_obj, mode, data?], ...]` + * if `mode` is `s` then data is [os_f_path, overwrite] + * if `mode` is `w` then binary data is the data to write + """ + if not self.done: + self.serial_io.put([temp_file, mode, data]) + + def start(self, server): + try: + self._start(server) + except Exception as e: + traceback.print_exc() + server.log_error("Upload Failed") + self.kill() + + def err(self, error_msg): + self.error = error_msg + if self.stop_on_error: + self.kill() + + def sleep(self): + time.sleep(self.nap_time) + + + + def _start(self, server): + while self.active or not self.serial_io.empty(): + if not self.serial_io.empty(): + self.waited = 0 + req_data = self.serial_io.get() + file:BufferedWriter = req_data[0] + mode = req_data[1] + data = req_data[2] + if mode == "w": # write + file.write(data) + if mode == "s": #save + temp_fn = file.name + if not file.closed: + file.close() + os_f_path, overwrite = data + os_fn = os.path.basename(os_f_path) + + name, ext = os.path.splitext(os_f_path) + while (not overwrite) and os.path.isfile(os_f_path): + n = 1 + os_f_path = f"{name}({n}){ext}" + n += 1 + + + try: + os.rename(temp_fn, os_f_path) + except Exception as e: + server.log_error(f'Failed to replace {temp_fn} with {os_f_path} by {self.uid}') + server.log_error(traceback.format_exc()) + self.err(f"Failed to upload {os_fn}") + + break + + else: + self.waited += 1 + self.sleep() + if self.waited > 100: + self.active = False + break + + def kill(self): + self.active = False + self.done = True + + for f in self.serial_io.queue: + name = f[0].name + if not f[0].closed: + f[0].close() + + if os.path.exists(name): + os.remove(name) + + self.serial_io.queue.clear() + + + diff --git a/dev_src/local_server_pyrobox.py b/dev_src/local_server_pyrobox.py index 2c025dc..ab68d76 100644 --- a/dev_src/local_server_pyrobox.py +++ b/dev_src/local_server_pyrobox.py @@ -14,6 +14,7 @@ import datetime +import threading import urllib.parse import urllib.request @@ -27,7 +28,7 @@ from pyroboxCore import config as CoreConfig, logger, DealPostData as DPD, run as run_server, tools, reload_server, __version__ -from _fs_utils import get_titles, dir_navigator, get_dir_size, get_stat, get_tree_count_n_size, fmbytes, humanbytes +from _fs_utils import get_titles, dir_navigator, get_dir_size, get_stat, get_tree_count_n_size, fmbytes, humanbytes, UploadHandler from _arg_parser import main as arg_parser import _page_templates as pt from _exceptions import LimitExceed @@ -1144,7 +1145,22 @@ def upload(self: SH, *args, **kwargs): return None - uploaded_files = [] # Uploaded folder list + # uploaded_files = [] # Uploaded folder list + + upload_handler = UploadHandler(uid) + + upload_thread = threading.Thread(target=upload_handler.start, args=(self,)) + upload_thread.start() + + def remove_from_temp(temp_fn): + try: + upload_handler.kill() + except OSError: + pass + + finally: + if temp_fn in CoreConfig.temp_file: + CoreConfig.temp_file.remove(temp_fn) @@ -1184,43 +1200,42 @@ def upload(self: SH, *args, **kwargs): # ORIGINAL FILE STARTS FROM HERE try: - with open(temp_fn, 'wb') as out: - preline = post.get() - while post.remainbytes > 0: - line = post.get() - if post.boundary in line: + out = open(temp_fn, 'wb') + preline = post.get() + while post.remainbytes > 0 and not upload_handler.error: + line = post.get() + if post.boundary in line: + preline = preline[0:-1] + if preline.endswith(b'\r'): preline = preline[0:-1] - if preline.endswith(b'\r'): - preline = preline[0:-1] - out.write(preline) - uploaded_files.append(rltv_path,) - break - else: - out.write(preline) - preline = line + upload_handler.upload(out, 'w', preline) + # out.write(preline) + # uploaded_files.append(rltv_path,) + break + else: + upload_handler.upload(out, 'w', preline) + # out.write(preline) + preline = line + + upload_handler.upload(out, 's', (os_f_path, user.MODIFY)) - while (not user.MODIFY) and os.path.isfile(os_f_path): - n = 1 - name, ext = os.path.splitext(os_f_path) - fn = f"{name}({n}){ext}" - n += 1 - os.replace(temp_fn, os_f_path) + if upload_handler.error and not upload_handler.active: + remove_from_temp(temp_fn) + return self.send_error(upload_handler.error, HTTPStatus.INTERNAL_SERVER_ERROR, cookie=cookie) except (IOError, OSError): traceback.print_exc() - return self.send_txt("Can't create file to write, do you have permission to write?", HTTPStatus.SERVICE_UNAVAILABLE, cookie=cookie) - finally: - try: - os.remove(temp_fn) - CoreConfig.temp_file.remove(temp_fn) - except OSError: - pass + upload_handler.active = False # will take no further inputs + upload_thread.join() + + if upload_handler.error: + return self.send_error(upload_handler.error, HTTPStatus.INTERNAL_SERVER_ERROR, cookie=cookie) diff --git a/dev_src/pyroboxCore.py b/dev_src/pyroboxCore.py index 0b83254..eac558d 100644 --- a/dev_src/pyroboxCore.py +++ b/dev_src/pyroboxCore.py @@ -1684,6 +1684,7 @@ def get(self, show=F, strip=F, Timeout=10, chunk_size=0): if chunk_size <= 0: chunk_size = self.remainbytes + waited = 0 for _ in range(Timeout*2): if self.is_multipart(): line = req.rfile.readline() @@ -1692,6 +1693,8 @@ def get(self, show=F, strip=F, Timeout=10, chunk_size=0): if line: break time.sleep(.5) + waited +=.5 + print(f"Waited for {waited}s") else: raise ConnectionAbortedError From ff3502cfbd83c208042cc039b0efc2d99d258d19 Mon Sep 17 00:00:00 2001 From: RaSan <34002411+RaSan147@users.noreply.github.com> Date: Thu, 4 Apr 2024 19:13:14 +0600 Subject: [PATCH 2/2] remove timeout to fix unwanted connection lost --- dev_src/_fs_utils.py | 4 ++- dev_src/script_file_list.js | 60 ++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/dev_src/_fs_utils.py b/dev_src/_fs_utils.py index 046ac88..93fc739 100644 --- a/dev_src/_fs_utils.py +++ b/dev_src/_fs_utils.py @@ -547,6 +547,8 @@ def _start(self, server): try: + if os.path.exists(os_f_path): # if overwrite is disabled then the previous part will handle the new filename. + os.remove(os_f_path) os.rename(temp_fn, os_f_path) except Exception as e: server.log_error(f'Failed to replace {temp_fn} with {os_f_path} by {self.uid}') @@ -566,7 +568,7 @@ def kill(self): self.active = False self.done = True - for f in self.serial_io.queue: + for f in tuple(self.serial_io): name = f[0].name if not f[0].closed: f[0].close() diff --git a/dev_src/script_file_list.js b/dev_src/script_file_list.js index 673e81b..ca6a8c1 100644 --- a/dev_src/script_file_list.js +++ b/dev_src/script_file_list.js @@ -98,14 +98,14 @@ class UploadManager { let file_list = byId("content_container") this.drag_pop_open = false; - file_list.ondragover = (event)=>{ + file_list.ondragover = async (event)=>{ event.preventDefault(); //preventing from default behaviour if(that.drag_pop_open){ return; } that.drag_pop_open = true; - form = upload_man.new() + form = await upload_man.new() popup_msg.createPopup("Upload Files", form, true, onclose=()=>{ that.drag_pop_open = false; }) @@ -125,7 +125,7 @@ class UploadManager { }; } - new() { + async new() { //selecting all required elements let that = this; let index = this.last_index; @@ -335,7 +335,7 @@ class UploadManager { // const filenames = formData.getAll('files').map(v => v.name).join(', ') request.open(e.target.method, e.target.action); - request.timeout = 60 * 1000; + // request.timeout = 60 * 1000; // in case wifi have no internet, it will stop after 60 seconds request.onreadystatechange = () => { if (request.readyState === XMLHttpRequest.DONE) { msg = `${request.status}: ${request.statusText}`; @@ -420,44 +420,44 @@ class UploadManager { upload_pop_status.innerText = msg; } - /** - * Checks if a file is already selected or not. - * @param {File} file - The file to check. - * @returns {number} - Returns the index+1 of the file if it exists, otherwise returns 0. - */ - function uploader_exist(file) { - for (let i = 0; i < selected_files.files.length; i++) { - const f = selected_files.files[i]; - if (f.name == file.name) { - return i+1; // 0 is false, so we add 1 to make it true - } - }; - return 0; // false; + function truncate_file_name(name){ + // if bigger than 20, truncate, veryvery...last6chars + if(name.length > 20){ + return name.slice(0, 7) + "..." + name.slice(-6); + } + return name; } - - /** - * Adds files to the selected files list and replaces any existing file with the same name. - * @param {FileList} files - The list of files to add. - */ - function addFiles(files) { - var exist = false; - + function remove_duplicates(files) { for (let i = 0; i < files.length; i++) { + var selected_fnames = []; const file = files[i]; - exist = uploader_exist(file); + + let exist = [...selected_files.files].findIndex(f => f.name === file.name); - if (exist) { + if (exist > -1) { // if file already selected, // remove that and replace with // new one, because, when uploading // last file will remain in host server, // so we need to replace it with new one - toaster.toast("File already selected", 1500); + toaster.toast(truncate_file_name(file.name) + " already selected", 1500); selected_files.items.remove(exist-1); } selected_files.items.add(file); }; + } + + + /** + * Adds files to the selected files list and replaces any existing file with the same name. + * @param {FileList} files - The list of files to add. + */ + function addFiles(files) { + + remove_duplicates(files); + + log("selected "+ selected_files.items.length+ " files"); uploader_showFiles(); } @@ -641,8 +641,8 @@ class FileManager { popup_msg.open_popup(); } - Show_upload_files() { - let form = upload_man.new() + async Show_upload_files() { + let form = await upload_man.new() popup_msg.createPopup("Upload Files", form); popup_msg.open_popup(); }