From 18a3e3d3f28612b40076b252ab39e707984934c0 Mon Sep 17 00:00:00 2001 From: Hell13Cat <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 09:48:56 +0300 Subject: [PATCH 1/9] 0.1.3b --- system_module/db_old.py | 104 ---------------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 system_module/db_old.py diff --git a/system_module/db_old.py b/system_module/db_old.py deleted file mode 100644 index ddcc998..0000000 --- a/system_module/db_old.py +++ /dev/null @@ -1,104 +0,0 @@ -from tinydb import TinyDB, Query -import os -import json -from collections import Counter -from system_module import gelbooru -from datetime import datetime - -class DB: - def __init__(self, debug: bool): - if debug: - db_path = "data_test" - else: - db_path = "data" - if not os.path.exists(os.path.join(db_path, "db.json")): - json.dump({}, open(os.path.join(db_path, "db.json"), "w", encoding="utf-8")) - self.db = TinyDB(db_path+"/db.json") - self.File = Query() - - def add(self, data_add: dict): - self.db.insert(data_add) - - def add_raw(self, data_add: dict): - is_video = False - is_gif = False - if data_add["image"].split(".")[-1] in ["webm", "mp4", "avi"]: - is_video = True - if data_add["image"].split(".")[-1] == "gif": - is_gif = True - add_data = { - "md5": data_add["md5"], - "local_filename": data_add["image"], - "is_video": is_video, - "is_gif": is_gif, - "tags": data_add["tags"].split(" "), - "rating": data_add["rating"], - "score": data_add["score"], - "timestamp": data_add["created_at"], - "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]), - "source": data_add["source"], - "width": data_add["width"], - "height": data_add["height"] - } - self.db.insert(add_data) - - def get_all(self): - sorted_records = sorted( - self.db.all(), - key=lambda x: datetime.strptime( - x["timestamp"], - "%a %b %d %H:%M:%S %z %Y" - ), - reverse=True - ) - return sorted_records - - def get_search(self, rating_tmp: list[str], tags: list[str]): - mapping_rating = { - "e": "explicit", - "g": "general", - "s": "sensitive", - "q": "questionable" - } - rating = [mapping_rating.get(x, x) for x in rating_tmp] - if rating: - query = self.File.rating.one_of(rating) - if tags != [""]: - tag_query = self.File.tags.test(lambda t: set(tags).issubset(set(t or []))) - query = tag_query if query is None else query & tag_query - results = self.db.search(query) if query else self.db.all() - sorted_records = sorted( - results, - key=lambda x: datetime.strptime( - x["timestamp"], - "%a %b %d %H:%M:%S %z %Y" - ), - reverse=True - ) - return sorted_records - - def get_unknow(self): - results = self.db.search(self.File.tags == []) - return results - - def get_autocomplete(self, tag_slise: str): - counter = Counter() - for record in self.db.all(): - counter.update(record.get("tags", [])) - results = [ - { - "tag": tag, - "count": count - } - for tag, count in counter.items() - if tag.lower().startswith(tag_slise) - ] - results.sort(key=lambda x: x["count"], reverse=True) - top_10 = results[:10] - return top_10 - - def edit_data(self, id_file: str, edit_data: dict): - self.db.update(edit_data, self.File.md5 == id_file) - - def get_from_id(self, id_file: str): - return self.db.get(self.File.md5 == id_file) \ No newline at end of file From 7fe9b73d4936ae0f146ac27f4d2631e1c60f5058 Mon Sep 17 00:00:00 2001 From: Hell13Cat <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 10:32:38 +0300 Subject: [PATCH 2/9] 0.1.4s --- .gitignore | 2 + data/analytic.json | 64 -------------------------------- run.py | 15 +++++--- system_module/db.py | 16 ++++---- system_module/gelbooru.py | 19 +++++----- system_module/image_processor.py | 11 +++--- system_module/logger.py | 2 +- 7 files changed, 36 insertions(+), 93 deletions(-) delete mode 100644 data/analytic.json diff --git a/.gitignore b/.gitignore index 10e2c22..bc633ee 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,10 @@ .vscode old/ data_test/ +tmp/ config_gb.json migration.py +change_log.txt # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/data/analytic.json b/data/analytic.json deleted file mode 100644 index 527c676..0000000 --- a/data/analytic.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "total_images_analyzed": 14003, - "average_weight": { - "bytes": 2980950.54, - "formatted": "2.84 MB" - }, - "storage_stats": { - "folders": { - "img": "38.88 GB", - "json_db": "24.39 MB", - "thumb": "331.45 MB", - "logs": "14.44 MB" - }, - "types_in_img": { - "images": "38.56 GB", - "videos": "140.80 MB", - "gifs": "181.30 MB", - "logs": "14.44 MB" - }, - "grand_total": "39.24 GB" - }, - "ratings_stats": { - "explicit": { - "count": 1605, - "size_formatted": "4.52 GB" - }, - "sensitive": { - "count": 6623, - "size_formatted": "19.35 GB" - }, - "questionable": { - "count": 1591, - "size_formatted": "4.72 GB" - }, - "general": { - "count": 4184, - "size_formatted": "10.29 GB" - } - }, - "resolution_stats": { - "largest": { - "filename": "5adc7e0e773417f9c379accf.png", - "md5": "5a63c28906dc7e0e7734c379accf", - "resolution": "9921x14031" - }, - "smallest": { - "filename": "8fc614e712ad21d2151ccde8a3c87.gif", - "md5": "8fc614e712ae22151ccde8a3c87", - "resolution": "240x280" - } - }, - "weight_stats": { - "largest": { - "filename": "4cf0a32c61c7e7914095d1fe.png", - "md5": "4ab6acf0a32c61c7e79fe", - "size_formatted": "65.21 MB" - }, - "smallest": { - "filename": "2e1c8a5870b90c3f538d7.png", - "md5": "28c679ea3470b90c3f538d7", - "size_formatted": "6.49 KB" - } - } -} \ No newline at end of file diff --git a/run.py b/run.py index 1f3aa04..db61bb3 100644 --- a/run.py +++ b/run.py @@ -1,11 +1,14 @@ -from system_module import db, gelbooru, logger, image_processor -from flask import Flask, render_template, jsonify, send_from_directory, render_template_string, request import json -import requests -from pathlib import Path +import os import subprocess import traceback -import os +from pathlib import Path + +import requests +from flask import (Flask, jsonify, render_template, render_template_string, + request, send_from_directory) + +from system_module import db, gelbooru, image_processor, logger if Path("config_gb.json").exists(): CONFIG_FILE = "config_gb.json" @@ -119,7 +122,7 @@ def api(): return jsonify(results_send) elif data["type"] == "autocomplete": tag_slise = data["q"] - result_autocomplete = database.get_autocomplete(tag_slise) + result_autocomplete = database.get_top_tag(tag_slise, count_tag_get=10) return jsonify(result_autocomplete) elif data["type"] == "post": post_id = data["post_id"] diff --git a/system_module/db.py b/system_module/db.py index b675716..3b491ae 100644 --- a/system_module/db.py +++ b/system_module/db.py @@ -1,8 +1,10 @@ -import sqlite3 -import os import json +import os +import sqlite3 from datetime import datetime -from system_module import gelbooru + +from system_module.gelbooru import gelbooru_date_parse + class DB: def __init__(self, debug: bool): @@ -117,7 +119,7 @@ class DB: "score": data_add["score"], "timestamp": data_add["created_at"], "parsed_time": parsed_time, - "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]), + "display_date": gelbooru_date_parse(data_add["created_at"]), "source": data_add["source"], "width": data_add["width"], "height": data_add["height"] @@ -165,15 +167,15 @@ class DB: cursor = self.conn.execute(query, (md5,)) return cursor.fetchone() is not None - def get_autocomplete(self, tag_slise: str): + def get_top_tag(self, tag_slise: str, count_tag_get: int = 9999999): query = """ SELECT value AS tag, COUNT(*) AS count FROM files, json_each(files.tags) WHERE value LIKE ? GROUP BY value ORDER BY count DESC - LIMIT 10 - """ + LIMIT {count_tag_get} + """.format(count_tag_get=count_tag_get) cursor = self.conn.execute(query, (f"{tag_slise.lower()}%",)) return [{"tag": row["tag"], "count": row["count"]} for row in cursor.fetchall()] diff --git a/system_module/gelbooru.py b/system_module/gelbooru.py index e41897d..0fca573 100644 --- a/system_module/gelbooru.py +++ b/system_module/gelbooru.py @@ -1,8 +1,10 @@ -import requests -from datetime import datetime import os +from datetime import datetime -def gelbooru_date_parse(date_str): +import requests + + +def gelbooru_date_parse(date_str: str): if not date_str: return 0, "Unknown" try: @@ -13,14 +15,14 @@ def gelbooru_date_parse(date_str): return 0, "Unknown" class GB: - def __init__(self, api_id, api_hash, headers, dw_folder): + def __init__(self, api_id: str, api_hash: str, headers: dict[str, str], dw_folder: str): self.api_id = api_id self.api_hash = api_hash self.dw_folder = dw_folder self.root_url = "https://gelbooru.com/index.php" self.HEADERS = headers - def get_from_id(self, id_post): + def get_from_id(self, id_post: int): params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "id": id_post, "api_key":self.api_hash, "user_id":self.api_id} response = requests.get(self.root_url, params=params, headers=self.HEADERS, timeout=15) print(response.url) @@ -28,13 +30,10 @@ class GB: file_name = response.json()["post"][0]["image"] return response.json()["post"][0], url_file, file_name - def get_from_md5(self, md5): + def get_from_md5(self, md5: str): params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "tags":"md5:"+md5, "api_key":self.api_hash, "user_id":self.api_id} response = requests.get(self.root_url, params=params, headers=self.HEADERS, timeout=15) print(response.url) url_file = response.json()["post"][0]["file_url"] file_name = response.json()["post"][0]["image"] - return response.json()["post"][0], url_file, file_name - - def get_binary_file(self, id_post): - pass \ No newline at end of file + return response.json()["post"][0], url_file, file_name \ No newline at end of file diff --git a/system_module/image_processor.py b/system_module/image_processor.py index b390ae9..3d490ba 100644 --- a/system_module/image_processor.py +++ b/system_module/image_processor.py @@ -1,6 +1,7 @@ -import subprocess -from PIL import Image import os +import subprocess + +from PIL import Image EXTENSIONS_IMG = ('jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp') EXTENSIONS_VID = ('mp4', 'webm') @@ -15,7 +16,7 @@ def check_ffmpeg(): except (FileNotFoundError, subprocess.CalledProcessError): return False -def video_gen(input_path_file, output_path_file, TARGET_MIN_SIZE): +def video_gen(input_path_file: str, output_path_file: str, TARGET_MIN_SIZE: int): vf_scale = f"scale='if(gt(iw,ih),-1,{TARGET_MIN_SIZE})':'if(gt(iw,ih),{TARGET_MIN_SIZE},-1)'" cmd = ["ffmpeg", "-y", "-i", input_path_file, "-ss", "00:00:00", "-vframes", "1", "-vf", vf_scale, "-q:v", "2", output_path_file] if os.name == 'nt': creationflags = subprocess.CREATE_NO_WINDOW @@ -24,7 +25,7 @@ def video_gen(input_path_file, output_path_file, TARGET_MIN_SIZE): if result.returncode != 0: raise Exception(f"FFmpeg error: {result.stderr.decode('utf-8', errors='ignore')}") -def image_and_gif_gen(input_path_file, output_path_file, TARGET_MIN_SIZE): +def image_and_gif_gen(input_path_file: str, output_path_file: str, TARGET_MIN_SIZE: int): with Image.open(input_path_file) as img: img_rgba = img.convert("RGBA") background = Image.new("RGB", img_rgba.size, (255, 255, 255)) @@ -43,7 +44,7 @@ def image_and_gif_gen(input_path_file, output_path_file, TARGET_MIN_SIZE): resized_img = final_img.resize((new_width, new_height), Image.Resampling.LANCZOS) resized_img.save(output_path_file, "JPEG", quality=85) -def generate_thumb(input_path_file, THUMB_FOLDER, TARGET_MIN_SIZE): +def generate_thumb(input_path_file: str, THUMB_FOLDER: str, TARGET_MIN_SIZE: int): ext_file = input_path_file.split(".")[-1] if ext_file not in VALID_EXTENSIONS: return {"status":False, "message":"Расширение не поддерживаеться!"} diff --git a/system_module/logger.py b/system_module/logger.py index 32ce271..f1f2abf 100644 --- a/system_module/logger.py +++ b/system_module/logger.py @@ -18,7 +18,7 @@ class Logger: self.logger_cli.setLevel(logging.INFO) self.logger_cli.addHandler(console_handler) - def send(self, level_log, type_log, message): + def send(self, level_log: str, type_log: str, message: str): if type_log == "file": getattr(self.logger_file, level_log)(message) elif type_log == "cli": From bd3036f058663df27ada9846761655b1f844d43c Mon Sep 17 00:00:00 2001 From: Hell13Cat <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 11:21:47 +0300 Subject: [PATCH 3/9] 0.1.5s --- run.py | 8 ++++++-- system_module/logger.py | 38 ++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/run.py b/run.py index db61bb3..3daa8c0 100644 --- a/run.py +++ b/run.py @@ -6,7 +6,7 @@ from pathlib import Path import requests from flask import (Flask, jsonify, render_template, render_template_string, - request, send_from_directory) + request, send_from_directory, make_response) from system_module import db, gelbooru, image_processor, logger @@ -30,6 +30,7 @@ THUMB_FOLDER = CONFIG_ROOT["file"]["thumb"] gb = gelbooru.GB(GB_ID, GB_HASH, GB_HEADERS, FILE_DW_FOLDERS) log = logger.Logger() +log.init_global_exception_hook() database = db.DB(True) app = Flask(__name__) @@ -59,7 +60,10 @@ def post(post_id): @app.route('/file/') def serve_image(filename): folder_path = find_file(filename, FILE_FOLDERS) - return send_from_directory(folder_path, filename) + if folder_path == None: + return make_response("Not found", 404) + else: + return send_from_directory(folder_path, filename) @app.route('/thumb/') def serve_thumb(filename): diff --git a/system_module/logger.py b/system_module/logger.py index f1f2abf..c44ce7b 100644 --- a/system_module/logger.py +++ b/system_module/logger.py @@ -1,28 +1,34 @@ +import sys import logging class Logger: def __init__(self): - self.logger_file = logging.getLogger(__name__) - self.logger_cli = logging.getLogger(__name__) + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') file_handler = logging.FileHandler("logs.log", encoding="utf-8") + file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(formatter) console_handler = logging.StreamHandler() + console_handler.setLevel(logging.DEBUG) console_handler.setFormatter(formatter) - self.logger_file.setLevel(logging.DEBUG) - self.logger_file.addHandler(file_handler) - - self.logger_cli.setLevel(logging.INFO) - self.logger_cli.addHandler(console_handler) - - def send(self, level_log: str, type_log: str, message: str): - if type_log == "file": - getattr(self.logger_file, level_log)(message) - elif type_log == "cli": - getattr(self.logger_cli, level_log)(message) - else: - getattr(self.logger_file, level_log)(message) - getattr(self.logger_cli, level_log)(message) \ No newline at end of file + self.logger.addHandler(file_handler) + self.logger.addHandler(console_handler) + + def init_global_exception_hook(self): + def handle_exception(exc_type, exc_value, exc_traceback): + if issubclass(exc_type, KeyboardInterrupt): + sys.__excepthook__( + exc_type, + exc_value, + exc_traceback + ) + return + self.logger.critical("ERROR:", exc_info=(exc_type, exc_value, exc_traceback)) + sys.excepthook = handle_exception + + def send(self, level_log: str, message: str): + getattr(self.logger, level_log)(message) \ No newline at end of file From 34f144d240b1bea303f7d515a2947754d75ca281 Mon Sep 17 00:00:00 2001 From: Hell13Cat <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 12:05:30 +0300 Subject: [PATCH 4/9] 0.1.6s --- button_script.user.js | 4 +- run.py | 81 +++++++++--------- system_module/tasks.py | 2 + templates/index.html | 2 + templates/task.html | 182 +++++++++++++++++++++++++++++++++++++++++ updater/config.json | 5 ++ updater/run.py | 59 +++++++++++++ 7 files changed, 296 insertions(+), 39 deletions(-) create mode 100644 system_module/tasks.py create mode 100644 templates/task.html create mode 100644 updater/config.json create mode 100644 updater/run.py diff --git a/button_script.user.js b/button_script.user.js index 90f5880..15e18cb 100644 --- a/button_script.user.js +++ b/button_script.user.js @@ -6,7 +6,7 @@ // @grant GM_registerMenuCommand // @grant GM_getValue // @grant GM_setValue -// @version 1.0r +// @version 1.1r // @author https://t.me/Nyako_TW // @downloadURL https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js // @updateURL https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js @@ -107,7 +107,7 @@ if (originalImageLinkElement) { headers: { "Content-Type": "application/json" }, - data: JSON.stringify({ id: imageID }), + data: JSON.stringify({ id: imageID , type: "download"}), onload: function(response) { console.log('Ответ от API:', response.responseText); newDownloadButton.style.outline = "2px solid green"; // Успех diff --git a/run.py b/run.py index 3daa8c0..0681224 100644 --- a/run.py +++ b/run.py @@ -8,7 +8,7 @@ import requests from flask import (Flask, jsonify, render_template, render_template_string, request, send_from_directory, make_response) -from system_module import db, gelbooru, image_processor, logger +from system_module import db, gelbooru, image_processor, logger, tasks if Path("config_gb.json").exists(): CONFIG_FILE = "config_gb.json" @@ -53,6 +53,10 @@ def index(): def analytics(): return render_template("analytics.html") +@app.route("/task") +def task(): + return render_template("task.html") + @app.route("/post/") def post(post_id): return render_template("post.html") @@ -69,42 +73,6 @@ def serve_image(filename): def serve_thumb(filename): return send_from_directory(THUMB_FOLDER, filename) -@app.route("/dw_api", methods=['POST']) -def dw_api(): - data = request.get_json() - if not data or 'id' not in data: - return jsonify({'status': 'error', 'message': 'URL not get'}), 400 - image_id = data["id"] - try: - data_get, url_file, file_name = gb.get_from_id(image_id) - if database.check_uni(data_get["md5"]): - message = f"File '{file_name}' already exists. Skipping." - log.send("warning", "all", message) - return jsonify({'status': 'skipped', 'message': message}) - with requests.get(url_file, stream=True, headers=GB_HEADERS, timeout=30) as r: - r.raise_for_status() - content_type = r.headers.get('Content-Type', '') - if 'text/html' in content_type: - log.send("error", "all", "The server returned HTML instead of an image. Bot protection.") - return False - with open(FILE_DW_FOLDERS+"/"+file_name, 'wb') as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) - database.add_raw(data_get) - res_data = image_processor.generate_thumb(FILE_DW_FOLDERS+"/"+file_name, THUMB_FOLDER, TARGET_MIN_SIZE) - if not res_data["status"]: - print(res_data["message"]) - return jsonify({'status': 'success'}) - except requests.exceptions.RequestException as e: - error_message = f"Network error {image_id}: {e}" - log.send("error", "all", error_message) - return jsonify({'status': 'error', 'message': error_message}), 500 - except Exception as e: - traceback.print_exc() - error_message = f"Unknown error: {e}" - log.send("error", "all", error_message) - return jsonify({'status': 'error', 'message': error_message}), 500 - @app.route("/api", methods=["GET", "POST"]) def api(): data = request.get_json() @@ -132,6 +100,45 @@ def api(): post_id = data["post_id"] result_post = database.get_from_id(post_id) return jsonify(result_post) + elif data["type"] == "task": + if data["task"] == "analytic": + if tasks.calculate_analytic(): + return "", 200 + else: + return "", 500 + elif data["type"] == "download": + if not data or 'id' not in data: + return jsonify({'status': 'error', 'message': 'URL not get'}), 400 + image_id = data["id"] + try: + data_get, url_file, file_name = gb.get_from_id(image_id) + if database.check_uni(data_get["md5"]): + message = f"File '{file_name}' already exists. Skipping." + log.send("warning", "all", message) + return jsonify({'status': 'skipped', 'message': message}) + with requests.get(url_file, stream=True, headers=GB_HEADERS, timeout=30) as r: + r.raise_for_status() + content_type = r.headers.get('Content-Type', '') + if 'text/html' in content_type: + log.send("error", "all", "The server returned HTML instead of an image. Bot protection.") + return False + with open(FILE_DW_FOLDERS+"/"+file_name, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + database.add_raw(data_get) + res_data = image_processor.generate_thumb(FILE_DW_FOLDERS+"/"+file_name, THUMB_FOLDER, TARGET_MIN_SIZE) + if not res_data["status"]: + print(res_data["message"]) + return jsonify({'status': 'success'}) + except requests.exceptions.RequestException as e: + error_message = f"Network error {image_id}: {e}" + log.send("error", "all", error_message) + return jsonify({'status': 'error', 'message': error_message}), 500 + except Exception as e: + traceback.print_exc() + error_message = f"Unknown error: {e}" + log.send("error", "all", error_message) + return jsonify({'status': 'error', 'message': error_message}), 500 else: pass return "", 204 diff --git a/system_module/tasks.py b/system_module/tasks.py new file mode 100644 index 0000000..5a520ce --- /dev/null +++ b/system_module/tasks.py @@ -0,0 +1,2 @@ +def calculate_analytic(): + pass \ No newline at end of file diff --git a/templates/index.html b/templates/index.html index f912482..1d6df03 100644 --- a/templates/index.html +++ b/templates/index.html @@ -18,6 +18,7 @@ header { display: flex; align-items: center; background-color: var(--bg-header); padding: 10px 20px; border-bottom: 1px solid var(--border-color); gap: 15px; flex-wrap: wrap; } .logo { font-weight: bold; font-size: 18px; color: white; text-decoration: none; } .analytics { color: #88bece; text-decoration: none; font-size: 12px; } + .tasks { color: #88bece; text-decoration: none; font-size: 12px; } .search-wrapper { flex-grow: 1; max-width: 400px; position: relative; } .search-wrapper input { width: 100%; background-color: var(--bg-input); border: 1px solid var(--border-color); color: white; padding: 6px 10px; border-radius: 3px; outline: none; } @@ -66,6 +67,7 @@ + ⚙️ Tasks
    diff --git a/templates/task.html b/templates/task.html new file mode 100644 index 0000000..c58608b --- /dev/null +++ b/templates/task.html @@ -0,0 +1,182 @@ + + + + + + Tasks - LocalBooru + + + +
    + +
    + +
      +
      +
      + +
      +

      System Tasks

      + +
      + + +
      +
      +

      Пересчёт аналитики

      +

      Обновляет статистику (количество файлов, занимаемое место). Рекомендуется запускать после массового добавления или удаления постов.

      +
      +
      + + +
      +
      + + + + +
      +
      + + + + \ No newline at end of file diff --git a/updater/config.json b/updater/config.json new file mode 100644 index 0000000..65ae636 --- /dev/null +++ b/updater/config.json @@ -0,0 +1,5 @@ +{ + "host":"https://git.nekono.su", + "user":"Nyako", + "repo":"Local-Booru" +} \ No newline at end of file diff --git a/updater/run.py b/updater/run.py new file mode 100644 index 0000000..9a66c5a --- /dev/null +++ b/updater/run.py @@ -0,0 +1,59 @@ +import os +import json +import shutil +import tempfile +import zipfile +import urllib.request + +CONFIG_PATH = os.path.join(os.path.dirname(__file__), "config.json") + +EXCLUDE_PATHS = { + "data", + "config.json", + "logs.log", + "updater" +} + +def load_config(): + with open(CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f) + +def get_latest_release(cfg): + url = f"{cfg['host']}/api/v1/repos/{cfg['user']}/{cfg['repo']}/releases/latest" + with urllib.request.urlopen(url) as r: + return json.loads(r.read().decode("utf-8")) + +def download(url, path): + urllib.request.urlretrieve(url, path) + +def should_exclude(relative_path: str) -> bool: + parts = set(relative_path.split(os.sep)) + return any(ex in parts for ex in EXCLUDE_PATHS) + +def apply_update(src_root, target_root): + for root, dirs, files in os.walk(src_root): + rel_dir = os.path.relpath(root, src_root) + if rel_dir == ".": + rel_dir = "" + for file in files: + rel_path = os.path.normpath(os.path.join(rel_dir, file)) + if should_exclude(rel_path): + continue + src_file = os.path.join(root, file) + dst_file = os.path.join(target_root, rel_path) + os.makedirs(os.path.dirname(dst_file), exist_ok=True) + shutil.copy2(src_file, dst_file) + +cfg = load_config() +release = get_latest_release(cfg) +zip_url = release["zipball_url"] +tmp = tempfile.mkdtemp() +zip_path = os.path.join(tmp, "release.zip") +print("Downloading:", release["tag_name"]) +download(zip_url, zip_path) +with zipfile.ZipFile(zip_path, "r") as z: + z.extractall(tmp) +extracted_root = os.path.join(tmp, os.listdir(tmp)[0]) +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +apply_update(extracted_root, project_root) +print("Update success!") From d1bc3fe8368acb46b98e2a19d1d7865132ee6748 Mon Sep 17 00:00:00 2001 From: Nyako <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 12:50:35 +0300 Subject: [PATCH 5/9] 0.1.7s --- run.py | 13 +---- system_module/tasks.py | 114 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 13 deletions(-) diff --git a/run.py b/run.py index 0681224..cee9d0c 100644 --- a/run.py +++ b/run.py @@ -34,15 +34,6 @@ log.init_global_exception_hook() database = db.DB(True) app = Flask(__name__) -def find_file(filename, file_folders): - for folder in file_folders: - folder_path = Path(folder) - if not folder_path.exists(): - continue - for file_path in folder_path.rglob(filename): - return folder_path - return None - if not image_processor.check_ffmpeg(): print("WARNING!\n\nУ вас не установлен ffmpeg. Генерация превью для видео не доступна!") @app.route("/") @@ -63,7 +54,7 @@ def post(post_id): @app.route('/file/') def serve_image(filename): - folder_path = find_file(filename, FILE_FOLDERS) + folder_path = task.find_file(filename, FILE_FOLDERS) if folder_path == None: return make_response("Not found", 404) else: @@ -102,7 +93,7 @@ def api(): return jsonify(result_post) elif data["type"] == "task": if data["task"] == "analytic": - if tasks.calculate_analytic(): + if tasks.calculate_analytic(database, CONFIG_ROOT): return "", 200 else: return "", 500 diff --git a/system_module/tasks.py b/system_module/tasks.py index 5a520ce..8bdafa0 100644 --- a/system_module/tasks.py +++ b/system_module/tasks.py @@ -1,2 +1,112 @@ -def calculate_analytic(): - pass \ No newline at end of file +import os +from pathlib import Path + +def find_file(filename, file_folders): + for folder in file_folders: + folder_path = Path(folder) + if not folder_path.exists(): + continue + for file_path in folder_path.rglob(filename): + return folder_path + return None + +def get_size_format(b, factor=1024, suffix="B"): + for unit in ["", "K", "M", "G", "T"]: + if b < factor: + return f"{b:.2f} {unit}{suffix}" + b /= factor + return f"{b:.2f} P{suffix}" + +def get_dir_size(path): + total = 0 + if os.path.exists(path): + for f in os.listdir(path): + fp = os.path.join(path, f) + if os.path.isfile(fp): + total += os.path.getsize(fp) + return total + +def get_file_list_size(file_list: list[str]): + total = 0 + for fp in file_list: + if os.path.isfile(fp): + total += os.path.getsize(fp) + return total + +def calculate_analytic(database, config): + all_db = database.get_all() + explicit_db = database.get_search(["e"], []) + sensitive_db = database.get_search(["s"], []) + questionable_db = database.get_search(["q"], []) + general_db = database.get_search(["g"], []) + rating_list_file_path = {"explicit":[], "sensitive":[], "questionable":[], "general":[]} + for one_post in all_db: + file_path = find_file(one_post["local_filename"], config["file"]["repos"]) + if file_path != None: + rating_list_file_path[one_post["rating"]].append(file_path) + ratings_stats = { + "explicit": { + "count": len(explicit_db), + "size_formatted": get_size_format(get_file_list_size(rating_list_file_path["explicit"])) + }, + "sensitive": { + "count": len(sensitive_db), + "size_formatted": get_size_format(get_file_list_size(rating_list_file_path["sensitive"])) + }, + "questionable": { + "count": len(questionable_db), + "size_formatted": get_size_format(get_file_list_size(rating_list_file_path["questionable"])) + }, + "general": { + "count": len(general_db), + "size_formatted": get_size_format(get_file_list_size(rating_list_file_path["general"])) + } + }, + result = { + "total_images_analyzed": len(all_db), + "average_weight": { + "bytes": round(avg_size_bytes, 2), + "formatted": get_size_format(avg_size_bytes) + }, + "storage_stats": { + "folders": { + "img": get_size_format(size_img), + "json_db": get_size_format(size_json), + "thumb": get_size_format(size_thumb), + "logs": get_size_format(size_logs) + }, + "types_in_img": { + "images": get_size_format(type_sizes["IMG"]), + "videos": get_size_format(type_sizes["VID"]), + "gifs": get_size_format(type_sizes["GIF"]), + "logs": get_size_format(type_sizes["LOGS"]) + }, + "grand_total": get_size_format(grand_total) + }, + "ratings_stats": ratings_stats, + "resolution_stats": { + "largest": { + "filename": max_res["filename"], + "md5": max_res["md5"], + "resolution": f'{max_res["width"]}x{max_res["height"]}' + }, + "smallest": { + "filename": min_res["filename"], + "md5": min_res["md5"] if min_res["area"] != float('inf') else "", + "resolution": f'{min_res["width"]}x{min_res["height"]}' if min_res["area"] != float('inf') else "N/A" + } + }, + "weight_stats": { + "largest": { + "filename": max_weight["filename"], + "md5": max_weight["md5"], + "size_formatted": get_size_format(max_weight["size"]) + }, + "smallest": { + "filename": min_weight["filename"], + "md5": min_weight["md5"] if min_weight["size"] != float('inf') else "", + "size_formatted": get_size_format(min_weight["size"]) if min_weight["size"] != float('inf') else "N/A" + } + }, + "tag_top":database.get_top_tag("") + } \ No newline at end of file From 36a044c8daef65b5bdfaeff615abbf6b4d657e64 Mon Sep 17 00:00:00 2001 From: Nyako <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 13:30:10 +0300 Subject: [PATCH 6/9] 0.1.8s --- config.json | 3 +- system_module/tasks.py | 85 +++++++++++++++++++++++++++++------------- 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/config.json b/config.json index d8c5d31..4528e85 100644 --- a/config.json +++ b/config.json @@ -16,6 +16,7 @@ "file":{ "repos":["data/files"], "dw":"data/files", - "thumb":"data/thumb" + "thumb":"data/thumb", + "data":"data" } } \ No newline at end of file diff --git a/system_module/tasks.py b/system_module/tasks.py index 8bdafa0..a7e73a7 100644 --- a/system_module/tasks.py +++ b/system_module/tasks.py @@ -1,4 +1,5 @@ import os +import json from pathlib import Path def find_file(filename, file_folders): @@ -17,7 +18,7 @@ def get_size_format(b, factor=1024, suffix="B"): b /= factor return f"{b:.2f} P{suffix}" -def get_dir_size(path): +def get_dir_size(path: str): total = 0 if os.path.exists(path): for f in os.listdir(path): @@ -33,6 +34,12 @@ def get_file_list_size(file_list: list[str]): total += os.path.getsize(fp) return total +def get_size_edge(type_searсh: bool, file_list: list[str]): + if type_searсh: + return min(file_list, key=os.path.getsize) + else: + return max(file_list, key=os.path.getsize) + def calculate_analytic(database, config): all_db = database.get_all() explicit_db = database.get_search(["e"], []) @@ -40,48 +47,72 @@ def calculate_analytic(database, config): questionable_db = database.get_search(["q"], []) general_db = database.get_search(["g"], []) rating_list_file_path = {"explicit":[], "sensitive":[], "questionable":[], "general":[]} + type_list_file_path = {"images":[], "videos":[], "gifs":[]} + files_not_found = 0 for one_post in all_db: file_path = find_file(one_post["local_filename"], config["file"]["repos"]) if file_path != None: rating_list_file_path[one_post["rating"]].append(file_path) + if one_post["is_gif"] == 1: + type_list_file_path["gifs"].append(file_path) + elif one_post["is_video"] == 1: + type_list_file_path["videos"].append(file_path) + else: + type_list_file_path["images"].append(file_path) + else: + files_not_found += 1 + file_size_dict = {"explicit":get_file_list_size(rating_list_file_path["explicit"]), + "sensitive":get_file_list_size(rating_list_file_path["sensitive"]), + "questionable":get_file_list_size(rating_list_file_path["questionable"]), + "general":get_file_list_size(rating_list_file_path["general"]) + } + type_size_dict = {"images":get_file_list_size(type_list_file_path["images"]), + "videos":get_file_list_size(type_list_file_path["videos"]), + "gifs":get_file_list_size(type_list_file_path["gifs"])} ratings_stats = { "explicit": { "count": len(explicit_db), - "size_formatted": get_size_format(get_file_list_size(rating_list_file_path["explicit"])) + "size_formatted": get_size_format(file_size_dict["explicit"]) }, "sensitive": { "count": len(sensitive_db), - "size_formatted": get_size_format(get_file_list_size(rating_list_file_path["sensitive"])) + "size_formatted": get_size_format(file_size_dict["sensitive"]) }, "questionable": { "count": len(questionable_db), - "size_formatted": get_size_format(get_file_list_size(rating_list_file_path["questionable"])) + "size_formatted": get_size_format(file_size_dict["questionable"]) }, "general": { "count": len(general_db), - "size_formatted": get_size_format(get_file_list_size(rating_list_file_path["general"])) + "size_formatted": get_size_format(file_size_dict["general"]) } - }, + } + total_file_data_size = file_size_dict["general"]+file_size_dict["questionable"]+file_size_dict["sensitive"]+file_size_dict["explicit"] + total_thumb_size = get_dir_size(config[["file"]["thumb"]]) + total_log_size = get_file_list_size(["logs.log"]) + total_db_size = get_file_list_size([os.path.join("data", "db.sqlite")]) + weight_stats_smallest = get_size_edge(True, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) + weight_stats_largest = get_size_edge(False, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) result = { "total_images_analyzed": len(all_db), + "files_not_found":files_not_found, "average_weight": { - "bytes": round(avg_size_bytes, 2), - "formatted": get_size_format(avg_size_bytes) + "bytes": round(total_file_data_size/(len(rating_list_file_path["explicit"])+len(rating_list_file_path["sensitive"])+len(rating_list_file_path["questionable"])+len(rating_list_file_path["general"])), 2), + "formatted": get_size_format(round(total_file_data_size/(len(rating_list_file_path["explicit"])+len(rating_list_file_path["sensitive"])+len(rating_list_file_path["questionable"])+len(rating_list_file_path["general"])), 2)) }, "storage_stats": { "folders": { - "img": get_size_format(size_img), - "json_db": get_size_format(size_json), - "thumb": get_size_format(size_thumb), - "logs": get_size_format(size_logs) + "data": get_size_format(total_file_data_size), + "db": get_size_format(total_db_size), + "thumb": get_size_format(total_thumb_size), + "logs": get_size_format(total_log_size) }, - "types_in_img": { - "images": get_size_format(type_sizes["IMG"]), - "videos": get_size_format(type_sizes["VID"]), - "gifs": get_size_format(type_sizes["GIF"]), - "logs": get_size_format(type_sizes["LOGS"]) + "types": { + "images": get_size_format(type_size_dict["images"]), + "videos": get_size_format(type_size_dict["videos"]), + "gifs": get_size_format(type_size_dict["gifs"]) }, - "grand_total": get_size_format(grand_total) + "grand_total": get_size_format(total_file_data_size+total_thumb_size+total_log_size+total_db_size) }, "ratings_stats": ratings_stats, "resolution_stats": { @@ -92,21 +123,23 @@ def calculate_analytic(database, config): }, "smallest": { "filename": min_res["filename"], - "md5": min_res["md5"] if min_res["area"] != float('inf') else "", - "resolution": f'{min_res["width"]}x{min_res["height"]}' if min_res["area"] != float('inf') else "N/A" + "md5": min_res["md5"], + "resolution": f'{min_res["width"]}x{min_res["height"]}' } }, "weight_stats": { "largest": { - "filename": max_weight["filename"], + "filename": weight_stats_largest, "md5": max_weight["md5"], "size_formatted": get_size_format(max_weight["size"]) }, "smallest": { - "filename": min_weight["filename"], - "md5": min_weight["md5"] if min_weight["size"] != float('inf') else "", - "size_formatted": get_size_format(min_weight["size"]) if min_weight["size"] != float('inf') else "N/A" + "filename": weight_stats_smallest, + "md5": min_weight["md5"], + "size_formatted": get_size_format(min_weight["size"]) } }, - "tag_top":database.get_top_tag("") - } \ No newline at end of file + "top_tag":database.get_top_tag("") + } + with open(os.path.join(config["file"]["data"], "analytic.json"), 'w', encoding='utf-8') as out_f: + json.dump(result, out_f, indent=4, ensure_ascii=False) \ No newline at end of file From 2c2aa65947938d5bb3e8b3a2e1b4307be60435ee Mon Sep 17 00:00:00 2001 From: Nyako <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 14:52:48 +0300 Subject: [PATCH 7/9] 0.1.9s --- doc/install_en.md | 1 + doc/install_ru.md | 1 + run.py | 9 +- system_module/db.py | 6 +- system_module/tasks.py | 114 ++++++++++++++++++----- templates/analytics.html | 191 +++++++++++++++++++-------------------- templates/index.html | 2 - templates/task.html | 42 --------- 8 files changed, 193 insertions(+), 173 deletions(-) diff --git a/doc/install_en.md b/doc/install_en.md index dee7309..df0bd8a 100644 --- a/doc/install_en.md +++ b/doc/install_en.md @@ -53,6 +53,7 @@ Config explanation: | `file`-`repos` | `["data/files"]` | List of directories where source files are stored | | `file`-`dw` | `"data/files"` | Directory for new downloads | | `file`-`thumb` | `"data/thumb"` | Directory for thumbnails | +| `file`-`data` | `"data"` | Directory for database | 1. You must obtain your Gelbooru User ID and Hash and add them to the config file for the system to function correctly. 2. Install the script in [VioletMonkey](https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js). diff --git a/doc/install_ru.md b/doc/install_ru.md index ef9ff8d..ca74f81 100644 --- a/doc/install_ru.md +++ b/doc/install_ru.md @@ -54,6 +54,7 @@ | `file`-`repos` | `["data/files"]` | Список каталогов где храняться полные файлы | | `file`-`dw` | `"data/files"` | Каталог хранения загрузок | | `file`-`thumb` | `"data/thumb"` | Каталог обложек | +| `file`-`data` | `"data"` | Каталог базы данных | 1. Обязательно надо получить ID и Hash пользователя и внести в конфиг для работы! diff --git a/run.py b/run.py index cee9d0c..17b2a4d 100644 --- a/run.py +++ b/run.py @@ -6,7 +6,7 @@ from pathlib import Path import requests from flask import (Flask, jsonify, render_template, render_template_string, - request, send_from_directory, make_response) + request, send_from_directory, make_response, send_file) from system_module import db, gelbooru, image_processor, logger, tasks @@ -26,12 +26,13 @@ GB_HEADERS = CONFIG_ROOT["gb"]["headers"] FILE_FOLDERS = CONFIG_ROOT["file"]["repos"] FILE_DW_FOLDERS = CONFIG_ROOT["file"]["dw"] THUMB_FOLDER = CONFIG_ROOT["file"]["thumb"] +DATA_FOLDER = CONFIG_ROOT["file"]["data"] gb = gelbooru.GB(GB_ID, GB_HASH, GB_HEADERS, FILE_DW_FOLDERS) log = logger.Logger() log.init_global_exception_hook() -database = db.DB(True) +database = db.DB(DATA_FOLDER) app = Flask(__name__) if not image_processor.check_ffmpeg(): print("WARNING!\n\nУ вас не установлен ffmpeg. Генерация превью для видео не доступна!") @@ -54,7 +55,7 @@ def post(post_id): @app.route('/file/') def serve_image(filename): - folder_path = task.find_file(filename, FILE_FOLDERS) + folder_path = tasks.find_file(filename, FILE_FOLDERS) if folder_path == None: return make_response("Not found", 404) else: @@ -91,6 +92,8 @@ def api(): post_id = data["post_id"] result_post = database.get_from_id(post_id) return jsonify(result_post) + elif data["type"] == "analytic": + return send_file(os.path.join(DATA_FOLDER, "analytic.json"), mimetype="application/json") elif data["type"] == "task": if data["task"] == "analytic": if tasks.calculate_analytic(database, CONFIG_ROOT): diff --git a/system_module/db.py b/system_module/db.py index 3b491ae..2780c1a 100644 --- a/system_module/db.py +++ b/system_module/db.py @@ -7,11 +7,7 @@ from system_module.gelbooru import gelbooru_date_parse class DB: - def __init__(self, debug: bool): - if debug: - db_path = "data_test" - else: - db_path = "data" + def __init__(self, db_path: str): if not os.path.exists(db_path): os.makedirs(db_path) diff --git a/system_module/tasks.py b/system_module/tasks.py index a7e73a7..8755f9b 100644 --- a/system_module/tasks.py +++ b/system_module/tasks.py @@ -2,13 +2,13 @@ import os import json from pathlib import Path -def find_file(filename, file_folders): +def find_file(filename: str, file_folders: list[str], is_folder: bool = True): for folder in file_folders: folder_path = Path(folder) if not folder_path.exists(): continue for file_path in folder_path.rglob(filename): - return folder_path + return str(folder_path) if is_folder else str(file_path) return None def get_size_format(b, factor=1024, suffix="B"): @@ -36,9 +36,10 @@ def get_file_list_size(file_list: list[str]): def get_size_edge(type_searсh: bool, file_list: list[str]): if type_searсh: - return min(file_list, key=os.path.getsize) + result = min(file_list, key=lambda f: Path(f).stat().st_size) else: - return max(file_list, key=os.path.getsize) + result = max(file_list, key=lambda f: Path(f).stat().st_size) + return str(result) def calculate_analytic(database, config): all_db = database.get_all() @@ -50,8 +51,9 @@ def calculate_analytic(database, config): type_list_file_path = {"images":[], "videos":[], "gifs":[]} files_not_found = 0 for one_post in all_db: - file_path = find_file(one_post["local_filename"], config["file"]["repos"]) + file_path = find_file(one_post["local_filename"], config["file"]["repos"], is_folder=False) if file_path != None: + file_path = str(file_path) rating_list_file_path[one_post["rating"]].append(file_path) if one_post["is_gif"] == 1: type_list_file_path["gifs"].append(file_path) @@ -88,11 +90,78 @@ def calculate_analytic(database, config): } } total_file_data_size = file_size_dict["general"]+file_size_dict["questionable"]+file_size_dict["sensitive"]+file_size_dict["explicit"] - total_thumb_size = get_dir_size(config[["file"]["thumb"]]) + total_thumb_size = get_dir_size(config["file"]["thumb"]) total_log_size = get_file_list_size(["logs.log"]) - total_db_size = get_file_list_size([os.path.join("data", "db.sqlite")]) - weight_stats_smallest = get_size_edge(True, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) - weight_stats_largest = get_size_edge(False, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) + total_db_size = get_file_list_size([os.path.join(config["file"]["data"], "db.sqlite")]) + weight_stats_smallest_file = get_size_edge(True, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) + weight_stats_largest_file = get_size_edge(False, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) + weight_stats_data = { + "smallest":{ + "file":"", + "md5":"", + "size":0 + }, + "largest":{ + "file":"", + "md5":"", + "size":0 + } + } + resolution_stats_data = { + "smallest":{ + "file":"", + "md5":"", + "res":{ + "width":0, + "height":0, + "total":99999999999999999999999999 + } + }, + "largest":{ + "file":"", + "md5":"", + "res":{ + "width":0, + "height":0, + "total":0 + } + } + } + for one_post_scan in all_db: + if one_post_scan["local_filename"] == os.path.basename(weight_stats_smallest_file): + weight_stats_data["smallest"] = { + "file":os.path.basename(weight_stats_smallest_file), + "md5":one_post_scan["md5"], + "size":get_file_list_size([weight_stats_smallest_file]) + } + if one_post_scan["local_filename"] == os.path.basename(weight_stats_largest_file): + weight_stats_data["largest"] = { + "file":os.path.basename(weight_stats_largest_file), + "md5":one_post_scan["md5"], + "size":get_file_list_size([weight_stats_largest_file]) + } + print(get_file_list_size([os.path.basename(weight_stats_largest_file)])) + one_post_res_total = one_post_scan["width"]*one_post_scan["height"] + if resolution_stats_data["largest"]["res"]["total"] < one_post_res_total: + resolution_stats_data["largest"] = { + "file":one_post_scan["local_filename"], + "md5":one_post_scan["md5"], + "res":{ + "width":one_post_scan["width"], + "height":one_post_scan["height"], + "total":one_post_res_total + } + } + if resolution_stats_data["smallest"]["res"]["total"] > one_post_res_total: + resolution_stats_data["smallest"] = { + "file":one_post_scan["local_filename"], + "md5":one_post_scan["md5"], + "res":{ + "width":one_post_scan["width"], + "height":one_post_scan["height"], + "total":one_post_res_total + } + } result = { "total_images_analyzed": len(all_db), "files_not_found":files_not_found, @@ -117,29 +186,30 @@ def calculate_analytic(database, config): "ratings_stats": ratings_stats, "resolution_stats": { "largest": { - "filename": max_res["filename"], - "md5": max_res["md5"], - "resolution": f'{max_res["width"]}x{max_res["height"]}' + "filename": resolution_stats_data["largest"]["file"], + "md5": resolution_stats_data["largest"]["md5"], + "resolution": str(resolution_stats_data["largest"]["res"]["width"]) + " x " + str(resolution_stats_data["largest"]["res"]["height"]) }, "smallest": { - "filename": min_res["filename"], - "md5": min_res["md5"], - "resolution": f'{min_res["width"]}x{min_res["height"]}' + "filename": resolution_stats_data["smallest"]["file"], + "md5": resolution_stats_data["smallest"]["md5"], + "resolution": str(resolution_stats_data["smallest"]["res"]["width"]) + " x " + str(resolution_stats_data["smallest"]["res"]["height"]) } }, "weight_stats": { "largest": { - "filename": weight_stats_largest, - "md5": max_weight["md5"], - "size_formatted": get_size_format(max_weight["size"]) + "filename": os.path.basename(weight_stats_data["largest"]["file"]), + "md5": weight_stats_data["largest"]["md5"], + "size_formatted": get_size_format(weight_stats_data["largest"]["size"]) }, "smallest": { - "filename": weight_stats_smallest, - "md5": min_weight["md5"], - "size_formatted": get_size_format(min_weight["size"]) + "filename": os.path.basename(weight_stats_data["smallest"]["file"]), + "md5": weight_stats_data["smallest"]["md5"], + "size_formatted": get_size_format(weight_stats_data["smallest"]["size"]) } }, "top_tag":database.get_top_tag("") } with open(os.path.join(config["file"]["data"], "analytic.json"), 'w', encoding='utf-8') as out_f: - json.dump(result, out_f, indent=4, ensure_ascii=False) \ No newline at end of file + json.dump(result, out_f, indent=4, ensure_ascii=False) + return True \ No newline at end of file diff --git a/templates/analytics.html b/templates/analytics.html index 3fe3ee8..d80e558 100644 --- a/templates/analytics.html +++ b/templates/analytics.html @@ -41,22 +41,21 @@ .ext-block a { color: var(--link-color); text-decoration: none; word-break: break-all; } .ext-block a:hover { text-decoration: underline; } + .loader { color: var(--text-muted); font-size: 16px; } + @media (max-width: 1000px) { .dashboard-grid { grid-template-columns: 1fr 1fr; } .left-col { grid-column: 1/-1; display: grid; grid-template-columns: 1fr 1fr; } } @media (max-width: 700px) { .dashboard-grid, .left-col { grid-template-columns: 1fr; } header { flex-wrap: wrap; } .search-wrapper { order: 3; min-width: 100%; margin-top: 10px;} }
      - - 📊 Analytics -
      - -
        -
        +

        Database Analytics

        +
        Fetching data from API...
        +
        diff --git a/templates/index.html b/templates/index.html index 1d6df03..a557621 100644 --- a/templates/index.html +++ b/templates/index.html @@ -64,9 +64,7 @@
        - ⚙️ Tasks
        diff --git a/templates/task.html b/templates/task.html index c58608b..ece992c 100644 --- a/templates/task.html +++ b/templates/task.html @@ -56,10 +56,6 @@
        -
        - -
          -
          @@ -139,44 +135,6 @@ btn.innerText = "Запустить"; } } - - - // === ЛОГИКА ПОИСКА (оставлена из предыдущей версии) === - const searchInput = document.getElementById('searchInput'); - const list = document.getElementById('autocompleteList'); - let debounceTimer; - - searchInput.addEventListener('input', () => { - clearTimeout(debounceTimer); - const words = searchInput.value.split(' '); - const currentWord = words[words.length - 1]; - if (currentWord.length < 2) { list.style.display = 'none'; return; } - - debounceTimer = setTimeout(async () => { - const req = { type: "autocomplete", q: currentWord }; - const data = await mockAutocompleteAPI(req); - - if (data.length === 0) { list.style.display = 'none'; return; } - - list.innerHTML = data.map(item => `
        • ${item.tag}${item.count}
          • `).join(''); - list.style.display = 'block'; - - list.querySelectorAll('.autocomplete-item').forEach(li => { - li.addEventListener('click', () => { - words[words.length - 1] = li.getAttribute('data-tag'); - searchInput.value = words.join(' ') + ' '; - list.style.display = 'none'; - searchInput.focus(); - }); - }); - }, 200); - }); - - document.addEventListener('click', (e) => { if (e.target !== searchInput && e.target !== list) list.style.display = 'none'; }); - searchInput.addEventListener('keypress', (e) => { if(e.key === 'Enter') window.location.href = `index.html?query=${searchInput.value.trim()}`; }); - - // Заглушка для поиска (оставьте или замените на ваш fetch) - function mockAutocompleteAPI(req) { return new Promise(res => { const db = [{tag: "long_hair", count: 8184}, {tag: "1girl", count: 11061}]; res(db.filter(t => t.tag.includes(req.q.toLowerCase()))); }); } \ No newline at end of file From 21e37776f903be3975d411c573fafaf316e29b83 Mon Sep 17 00:00:00 2001 From: Nyako <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 15:07:26 +0300 Subject: [PATCH 8/9] 0.1.10s --- .gitignore | 1 + button_script.user.js | 2 +- doc/change_log/2b_en.md | 29 ++++++++++++++++++++ doc/change_log/2b_ru.md | 29 ++++++++++++++++++++ system_module/tasks.py | 2 +- updater/config.json | 5 ---- updater/run.py | 59 ----------------------------------------- 7 files changed, 61 insertions(+), 66 deletions(-) create mode 100644 doc/change_log/2b_en.md create mode 100644 doc/change_log/2b_ru.md delete mode 100644 updater/config.json delete mode 100644 updater/run.py diff --git a/.gitignore b/.gitignore index bc633ee..fe4a496 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ tmp/ config_gb.json migration.py change_log.txt +up/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/button_script.user.js b/button_script.user.js index 15e18cb..c80d1d4 100644 --- a/button_script.user.js +++ b/button_script.user.js @@ -95,7 +95,7 @@ if (originalImageLinkElement) { // Получаем настроенный пользователем корневой путь const apiRoot = GM_getValue('nyakonya_gb_api_root', 'http://127.0.2.1'); // Формируем корректный URL для запроса к dw_api - const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/dw_api`; + const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/api`; console.log(`Отправка URL на локальный API: ${imageUrl} (Endpoint: ${apiUrl})`); const originalOutline = newDownloadButton.style.outline; diff --git a/doc/change_log/2b_en.md b/doc/change_log/2b_en.md new file mode 100644 index 0000000..1ed3f4a --- /dev/null +++ b/doc/change_log/2b_en.md @@ -0,0 +1,29 @@ +### 0.2.0b + +> **Fixes** + +1. Added 404 error handling when the source file is not found. + +> **Changes** + +1. Merged `/dw_api` and `/api` into a single `/api` endpoint. + +> **New** + +1. Added a system module for background tasks. +2. Added an analytics page. + +> **Code** + +1. Deleted the legacy `db_old.py` file. +2. Sorted and refactored imports. +3. Added type hinting for input data in `system_module` functions. +4. Removed `data/analytic.json`. +5. Updated the logic for retrieving popular tags to support analytics. + - 5.1. Renamed `get_autocomplete` function to `get_top_tags`. + - 5.2. Added an optional `limit` argument to the `get_top_tags` function. +6. Removed the `get_binary_file()` function from the `gelbooru` module as it was unused. +7. Refactored the logging module. +8. Added comprehensive error logging. +9. Added the `/task` endpoint. +10. Implemented new logic for passing the database path via the configuration file. \ No newline at end of file diff --git a/doc/change_log/2b_ru.md b/doc/change_log/2b_ru.md new file mode 100644 index 0000000..3a791d6 --- /dev/null +++ b/doc/change_log/2b_ru.md @@ -0,0 +1,29 @@ +### 0.2.0b + +> **Иправления** + +1. Добавлен вывод 404 если исходный файл не найден + +> **Изменения** + +1. /dw_api и /api объединены в /api + +> **Новое** + +1. Системный модуль с задачами +2. Страница с аналитикой + +> **Код** + +1. Удалён старый файл db_old.py +2. Отсортированы и изменены импорты +3. Добавлена типизация входных данных в функции расположенные в system_module +4. Удалён data/analytic.json +5. Изменена логика получения списка популярных тегов для адаптации к аналитике + - 5.1. Функция get_autocomplete переименована в get_top_tag + - 5.1. Для функции get_top_tag добален необязательный аргумент для лимита выдачи +6. Функция get_binary_file() из модуля gelbooru удалена из-за отсусвия факта использования +7. Переписан модуль логов +8. Добавлено полное логирование ошибок +9. Добавлен эндпойнт /task +10. Новая логика передачи пути к базе данных через конфиг \ No newline at end of file diff --git a/system_module/tasks.py b/system_module/tasks.py index 8755f9b..c015c76 100644 --- a/system_module/tasks.py +++ b/system_module/tasks.py @@ -11,7 +11,7 @@ def find_file(filename: str, file_folders: list[str], is_folder: bool = True): return str(folder_path) if is_folder else str(file_path) return None -def get_size_format(b, factor=1024, suffix="B"): +def get_size_format(b: int, factor=1024, suffix="B"): for unit in ["", "K", "M", "G", "T"]: if b < factor: return f"{b:.2f} {unit}{suffix}" diff --git a/updater/config.json b/updater/config.json deleted file mode 100644 index 65ae636..0000000 --- a/updater/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "host":"https://git.nekono.su", - "user":"Nyako", - "repo":"Local-Booru" -} \ No newline at end of file diff --git a/updater/run.py b/updater/run.py deleted file mode 100644 index 9a66c5a..0000000 --- a/updater/run.py +++ /dev/null @@ -1,59 +0,0 @@ -import os -import json -import shutil -import tempfile -import zipfile -import urllib.request - -CONFIG_PATH = os.path.join(os.path.dirname(__file__), "config.json") - -EXCLUDE_PATHS = { - "data", - "config.json", - "logs.log", - "updater" -} - -def load_config(): - with open(CONFIG_PATH, "r", encoding="utf-8") as f: - return json.load(f) - -def get_latest_release(cfg): - url = f"{cfg['host']}/api/v1/repos/{cfg['user']}/{cfg['repo']}/releases/latest" - with urllib.request.urlopen(url) as r: - return json.loads(r.read().decode("utf-8")) - -def download(url, path): - urllib.request.urlretrieve(url, path) - -def should_exclude(relative_path: str) -> bool: - parts = set(relative_path.split(os.sep)) - return any(ex in parts for ex in EXCLUDE_PATHS) - -def apply_update(src_root, target_root): - for root, dirs, files in os.walk(src_root): - rel_dir = os.path.relpath(root, src_root) - if rel_dir == ".": - rel_dir = "" - for file in files: - rel_path = os.path.normpath(os.path.join(rel_dir, file)) - if should_exclude(rel_path): - continue - src_file = os.path.join(root, file) - dst_file = os.path.join(target_root, rel_path) - os.makedirs(os.path.dirname(dst_file), exist_ok=True) - shutil.copy2(src_file, dst_file) - -cfg = load_config() -release = get_latest_release(cfg) -zip_url = release["zipball_url"] -tmp = tempfile.mkdtemp() -zip_path = os.path.join(tmp, "release.zip") -print("Downloading:", release["tag_name"]) -download(zip_url, zip_path) -with zipfile.ZipFile(zip_path, "r") as z: - z.extractall(tmp) -extracted_root = os.path.join(tmp, os.listdir(tmp)[0]) -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -apply_update(extracted_root, project_root) -print("Update success!") From b2b2145c37f6f51469fba4cb5bb85b30172bed21 Mon Sep 17 00:00:00 2001 From: Nyako <46496367+Hell13Cat@users.noreply.github.com> Date: Sat, 6 Jun 2026 15:30:28 +0300 Subject: [PATCH 9/9] 0.2.0b --- run.py | 4 ++ system_module/tasks.py | 9 +++ templates/task.html | 136 +++++++++++++++++++++++++++++++++-------- 3 files changed, 123 insertions(+), 26 deletions(-) diff --git a/run.py b/run.py index 17b2a4d..66cdda5 100644 --- a/run.py +++ b/run.py @@ -56,6 +56,7 @@ def post(post_id): @app.route('/file/') def serve_image(filename): folder_path = tasks.find_file(filename, FILE_FOLDERS) + print(filename, FILE_FOLDERS) if folder_path == None: return make_response("Not found", 404) else: @@ -100,6 +101,9 @@ def api(): return "", 200 else: return "", 500 + if data["task"] == "not_found_file": + list_md5_not_found = tasks.get_files_not_found(database, CONFIG_ROOT) + return jsonify({'list': list_md5_not_found}), 200 elif data["type"] == "download": if not data or 'id' not in data: return jsonify({'status': 'error', 'message': 'URL not get'}), 400 diff --git a/system_module/tasks.py b/system_module/tasks.py index c015c76..07f138a 100644 --- a/system_module/tasks.py +++ b/system_module/tasks.py @@ -41,6 +41,15 @@ def get_size_edge(type_searсh: bool, file_list: list[str]): result = max(file_list, key=lambda f: Path(f).stat().st_size) return str(result) +def get_files_not_found(database, config): + all_db = database.get_all() + list_md5 = [] + for one_post in all_db: + file_path = find_file(one_post["local_filename"], config["file"]["repos"], is_folder=False) + if file_path == None: + list_md5.append(one_post["md5"]) + return list_md5 + def calculate_analytic(database, config): all_db = database.get_all() explicit_db = database.get_search(["e"], []) diff --git a/templates/task.html b/templates/task.html index ece992c..2b08645 100644 --- a/templates/task.html +++ b/templates/task.html @@ -17,20 +17,18 @@ /* Search Autocomplete Styles */ .search-wrapper { flex-grow: 1; max-width: 400px; position: relative; } .search-wrapper input { width: 100%; background-color: var(--bg-input); border: 1px solid var(--border-color); color: white; padding: 6px 10px; border-radius: 3px; outline: none; } - .autocomplete-list { position: absolute; top: 100%; left: 0; right: 0; background-color: var(--bg-box); border: 1px solid var(--border-color); border-radius: 3px; margin-top: 4px; list-style: none; z-index: 1000; max-height: 250px; overflow-y: auto; box-shadow: 0 4px 10px rgba(0,0,0,0.5); display: none; } - .autocomplete-item { padding: 8px 10px; cursor: pointer; display: flex; justify-content: space-between; border-bottom: 1px solid #333; } - .autocomplete-item:hover { background-color: var(--bg-input); } - .autocomplete-count { color: var(--text-muted); font-size: 12px; } - + /* Main Content & Tasks Styles */ main { padding: 20px; max-width: 1000px; margin: 0 auto; } h1 { font-size: 20px; margin-bottom: 20px; color: white; border-bottom: 1px solid var(--border-color); padding-bottom: 10px; } .task-list { display: flex; flex-direction: column; gap: 15px; } - .task-card { background-color: var(--bg-panel); border: 1px solid var(--border-color); border-radius: 4px; padding: 15px 20px; display: flex; justify-content: space-between; align-items: center; gap: 20px; transition: background-color 0.2s; } + /* Добавил flex-wrap: wrap чтобы результаты могли падать на новую строку */ + .task-card { background-color: var(--bg-panel); border: 1px solid var(--border-color); border-radius: 4px; padding: 15px 20px; display: flex; justify-content: space-between; align-items: center; gap: 20px; flex-wrap: wrap; transition: background-color 0.2s; } .task-card:hover { background-color: var(--bg-box); } + .task-info { flex: 1; min-width: 250px; } .task-info h3 { font-size: 16px; color: var(--link-color); margin-bottom: 5px; } .task-info p { color: var(--text-muted); font-size: 13px; line-height: 1.4; } @@ -44,6 +42,10 @@ .task-status { font-weight: bold; font-size: 13px; min-width: 90px; text-align: right; } .status-success { color: var(--color-g); } .status-error { color: var(--color-e); } + + /* Стили для вывода результатов */ + .task-result-box { width: 100%; margin-top: 10px; border-top: 1px solid var(--border-color); padding-top: 15px; display: none; } + .task-result-box textarea { width: 100%; height: 150px; background-color: var(--bg-input); color: var(--text-main); border: 1px solid var(--border-color); border-radius: 3px; padding: 10px; margin-top: 10px; resize: vertical; outline: none; font-family: monospace; } @media (max-width: 700px) { header { flex-wrap: wrap; } @@ -75,48 +77,46 @@
          - -
          -

          Новая задача

          -

          Описание новой задачи.

          +

          Поиск недоступных файлов

          +

          Ищет посты, для которых отсутствуют физические файлы на диске. Выводит список ссылок на экран и предлагает скачать их в формате .txt.

          - + +
          + +
          +
          + Найдено битых ссылок: 0 + +
          +
          - --> \ No newline at end of file