diff --git a/.gitignore b/.gitignore index 10e2c22..fe4a496 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,11 @@ .vscode old/ data_test/ +tmp/ config_gb.json migration.py +change_log.txt +up/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/button_script.user.js b/button_script.user.js index 90f5880..c80d1d4 100644 --- a/button_script.user.js +++ b/button_script.user.js @@ -6,7 +6,7 @@ // @grant GM_registerMenuCommand // @grant GM_getValue // @grant GM_setValue -// @version 1.0r +// @version 1.1r // @author https://t.me/Nyako_TW // @downloadURL https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js // @updateURL https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js @@ -95,7 +95,7 @@ if (originalImageLinkElement) { // Получаем настроенный пользователем корневой путь const apiRoot = GM_getValue('nyakonya_gb_api_root', 'http://127.0.2.1'); // Формируем корректный URL для запроса к dw_api - const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/dw_api`; + const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/api`; console.log(`Отправка URL на локальный API: ${imageUrl} (Endpoint: ${apiUrl})`); const originalOutline = newDownloadButton.style.outline; @@ -107,7 +107,7 @@ if (originalImageLinkElement) { headers: { "Content-Type": "application/json" }, - data: JSON.stringify({ id: imageID }), + data: JSON.stringify({ id: imageID , type: "download"}), onload: function(response) { console.log('Ответ от API:', response.responseText); newDownloadButton.style.outline = "2px solid green"; // Успех diff --git a/config.json b/config.json index d8c5d31..4528e85 100644 --- a/config.json +++ b/config.json @@ -16,6 +16,7 @@ "file":{ "repos":["data/files"], "dw":"data/files", - "thumb":"data/thumb" + "thumb":"data/thumb", + "data":"data" } } \ No newline at end of file diff --git a/data/analytic.json b/data/analytic.json deleted file mode 100644 index 527c676..0000000 --- a/data/analytic.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "total_images_analyzed": 14003, - "average_weight": { - "bytes": 2980950.54, - "formatted": "2.84 MB" - }, - "storage_stats": { - "folders": { - "img": "38.88 GB", - "json_db": "24.39 MB", - "thumb": "331.45 MB", - "logs": "14.44 MB" - }, - "types_in_img": { - "images": "38.56 GB", - "videos": "140.80 MB", - "gifs": "181.30 MB", - "logs": "14.44 MB" - }, - "grand_total": "39.24 GB" - }, - "ratings_stats": { - "explicit": { - "count": 1605, - "size_formatted": "4.52 GB" - }, - "sensitive": { - "count": 6623, - "size_formatted": "19.35 GB" - }, - "questionable": { - "count": 1591, - "size_formatted": "4.72 GB" - }, - "general": { - "count": 4184, - "size_formatted": "10.29 GB" - } - }, - "resolution_stats": { - "largest": { - "filename": "5adc7e0e773417f9c379accf.png", - "md5": "5a63c28906dc7e0e7734c379accf", - "resolution": "9921x14031" - }, - "smallest": { - "filename": "8fc614e712ad21d2151ccde8a3c87.gif", - "md5": "8fc614e712ae22151ccde8a3c87", - "resolution": "240x280" - } - }, - "weight_stats": { - "largest": { - "filename": "4cf0a32c61c7e7914095d1fe.png", - "md5": "4ab6acf0a32c61c7e79fe", - "size_formatted": "65.21 MB" - }, - "smallest": { - "filename": "2e1c8a5870b90c3f538d7.png", - "md5": "28c679ea3470b90c3f538d7", - "size_formatted": "6.49 KB" - } - } -} \ No newline at end of file diff --git a/doc/change_log/2b_en.md b/doc/change_log/2b_en.md new file mode 100644 index 0000000..1ed3f4a --- /dev/null +++ b/doc/change_log/2b_en.md @@ -0,0 +1,29 @@ +### 0.2.0b + +> **Fixes** + +1. Added 404 error handling when the source file is not found. + +> **Changes** + +1. Merged `/dw_api` and `/api` into a single `/api` endpoint. + +> **New** + +1. Added a system module for background tasks. +2. Added an analytics page. + +> **Code** + +1. Deleted the legacy `db_old.py` file. +2. Sorted and refactored imports. +3. Added type hinting for input data in `system_module` functions. +4. Removed `data/analytic.json`. +5. Updated the logic for retrieving popular tags to support analytics. + - 5.1. Renamed `get_autocomplete` function to `get_top_tags`. + - 5.2. Added an optional `limit` argument to the `get_top_tags` function. +6. Removed the `get_binary_file()` function from the `gelbooru` module as it was unused. +7. Refactored the logging module. +8. Added comprehensive error logging. +9. Added the `/task` endpoint. +10. Implemented new logic for passing the database path via the configuration file. \ No newline at end of file diff --git a/doc/change_log/2b_ru.md b/doc/change_log/2b_ru.md new file mode 100644 index 0000000..3a791d6 --- /dev/null +++ b/doc/change_log/2b_ru.md @@ -0,0 +1,29 @@ +### 0.2.0b + +> **Иправления** + +1. Добавлен вывод 404 если исходный файл не найден + +> **Изменения** + +1. /dw_api и /api объединены в /api + +> **Новое** + +1. Системный модуль с задачами +2. Страница с аналитикой + +> **Код** + +1. Удалён старый файл db_old.py +2. Отсортированы и изменены импорты +3. Добавлена типизация входных данных в функции расположенные в system_module +4. Удалён data/analytic.json +5. Изменена логика получения списка популярных тегов для адаптации к аналитике + - 5.1. Функция get_autocomplete переименована в get_top_tag + - 5.1. Для функции get_top_tag добален необязательный аргумент для лимита выдачи +6. Функция get_binary_file() из модуля gelbooru удалена из-за отсусвия факта использования +7. Переписан модуль логов +8. Добавлено полное логирование ошибок +9. Добавлен эндпойнт /task +10. Новая логика передачи пути к базе данных через конфиг \ No newline at end of file diff --git a/doc/install_en.md b/doc/install_en.md index dee7309..df0bd8a 100644 --- a/doc/install_en.md +++ b/doc/install_en.md @@ -53,6 +53,7 @@ Config explanation: | `file`-`repos` | `["data/files"]` | List of directories where source files are stored | | `file`-`dw` | `"data/files"` | Directory for new downloads | | `file`-`thumb` | `"data/thumb"` | Directory for thumbnails | +| `file`-`data` | `"data"` | Directory for database | 1. You must obtain your Gelbooru User ID and Hash and add them to the config file for the system to function correctly. 2. Install the script in [VioletMonkey](https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js). diff --git a/doc/install_ru.md b/doc/install_ru.md index ef9ff8d..ca74f81 100644 --- a/doc/install_ru.md +++ b/doc/install_ru.md @@ -54,6 +54,7 @@ | `file`-`repos` | `["data/files"]` | Список каталогов где храняться полные файлы | | `file`-`dw` | `"data/files"` | Каталог хранения загрузок | | `file`-`thumb` | `"data/thumb"` | Каталог обложек | +| `file`-`data` | `"data"` | Каталог базы данных | 1. Обязательно надо получить ID и Hash пользователя и внести в конфиг для работы! diff --git a/run.py b/run.py index 1f3aa04..66cdda5 100644 --- a/run.py +++ b/run.py @@ -1,11 +1,14 @@ -from system_module import db, gelbooru, logger, image_processor -from flask import Flask, render_template, jsonify, send_from_directory, render_template_string, request import json -import requests -from pathlib import Path +import os import subprocess import traceback -import os +from pathlib import Path + +import requests +from flask import (Flask, jsonify, render_template, render_template_string, + request, send_from_directory, make_response, send_file) + +from system_module import db, gelbooru, image_processor, logger, tasks if Path("config_gb.json").exists(): CONFIG_FILE = "config_gb.json" @@ -23,22 +26,15 @@ GB_HEADERS = CONFIG_ROOT["gb"]["headers"] FILE_FOLDERS = CONFIG_ROOT["file"]["repos"] FILE_DW_FOLDERS = CONFIG_ROOT["file"]["dw"] THUMB_FOLDER = CONFIG_ROOT["file"]["thumb"] +DATA_FOLDER = CONFIG_ROOT["file"]["data"] gb = gelbooru.GB(GB_ID, GB_HASH, GB_HEADERS, FILE_DW_FOLDERS) log = logger.Logger() -database = db.DB(True) +log.init_global_exception_hook() +database = db.DB(DATA_FOLDER) app = Flask(__name__) -def find_file(filename, file_folders): - for folder in file_folders: - folder_path = Path(folder) - if not folder_path.exists(): - continue - for file_path in folder_path.rglob(filename): - return folder_path - return None - if not image_processor.check_ffmpeg(): print("WARNING!\n\nУ вас не установлен ffmpeg. Генерация превью для видео не доступна!") @app.route("/") @@ -49,55 +45,27 @@ def index(): def analytics(): return render_template("analytics.html") +@app.route("/task") +def task(): + return render_template("task.html") + @app.route("/post/") def post(post_id): return render_template("post.html") @app.route('/file/') def serve_image(filename): - folder_path = find_file(filename, FILE_FOLDERS) - return send_from_directory(folder_path, filename) + folder_path = tasks.find_file(filename, FILE_FOLDERS) + print(filename, FILE_FOLDERS) + if folder_path == None: + return make_response("Not found", 404) + else: + return send_from_directory(folder_path, filename) @app.route('/thumb/') def serve_thumb(filename): return send_from_directory(THUMB_FOLDER, filename) -@app.route("/dw_api", methods=['POST']) -def dw_api(): - data = request.get_json() - if not data or 'id' not in data: - return jsonify({'status': 'error', 'message': 'URL not get'}), 400 - image_id = data["id"] - try: - data_get, url_file, file_name = gb.get_from_id(image_id) - if database.check_uni(data_get["md5"]): - message = f"File '{file_name}' already exists. Skipping." - log.send("warning", "all", message) - return jsonify({'status': 'skipped', 'message': message}) - with requests.get(url_file, stream=True, headers=GB_HEADERS, timeout=30) as r: - r.raise_for_status() - content_type = r.headers.get('Content-Type', '') - if 'text/html' in content_type: - log.send("error", "all", "The server returned HTML instead of an image. Bot protection.") - return False - with open(FILE_DW_FOLDERS+"/"+file_name, 'wb') as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) - database.add_raw(data_get) - res_data = image_processor.generate_thumb(FILE_DW_FOLDERS+"/"+file_name, THUMB_FOLDER, TARGET_MIN_SIZE) - if not res_data["status"]: - print(res_data["message"]) - return jsonify({'status': 'success'}) - except requests.exceptions.RequestException as e: - error_message = f"Network error {image_id}: {e}" - log.send("error", "all", error_message) - return jsonify({'status': 'error', 'message': error_message}), 500 - except Exception as e: - traceback.print_exc() - error_message = f"Unknown error: {e}" - log.send("error", "all", error_message) - return jsonify({'status': 'error', 'message': error_message}), 500 - @app.route("/api", methods=["GET", "POST"]) def api(): data = request.get_json() @@ -119,12 +87,56 @@ def api(): return jsonify(results_send) elif data["type"] == "autocomplete": tag_slise = data["q"] - result_autocomplete = database.get_autocomplete(tag_slise) + result_autocomplete = database.get_top_tag(tag_slise, count_tag_get=10) return jsonify(result_autocomplete) elif data["type"] == "post": post_id = data["post_id"] result_post = database.get_from_id(post_id) return jsonify(result_post) + elif data["type"] == "analytic": + return send_file(os.path.join(DATA_FOLDER, "analytic.json"), mimetype="application/json") + elif data["type"] == "task": + if data["task"] == "analytic": + if tasks.calculate_analytic(database, CONFIG_ROOT): + return "", 200 + else: + return "", 500 + if data["task"] == "not_found_file": + list_md5_not_found = tasks.get_files_not_found(database, CONFIG_ROOT) + return jsonify({'list': list_md5_not_found}), 200 + elif data["type"] == "download": + if not data or 'id' not in data: + return jsonify({'status': 'error', 'message': 'URL not get'}), 400 + image_id = data["id"] + try: + data_get, url_file, file_name = gb.get_from_id(image_id) + if database.check_uni(data_get["md5"]): + message = f"File '{file_name}' already exists. Skipping." + log.send("warning", "all", message) + return jsonify({'status': 'skipped', 'message': message}) + with requests.get(url_file, stream=True, headers=GB_HEADERS, timeout=30) as r: + r.raise_for_status() + content_type = r.headers.get('Content-Type', '') + if 'text/html' in content_type: + log.send("error", "all", "The server returned HTML instead of an image. Bot protection.") + return False + with open(FILE_DW_FOLDERS+"/"+file_name, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + database.add_raw(data_get) + res_data = image_processor.generate_thumb(FILE_DW_FOLDERS+"/"+file_name, THUMB_FOLDER, TARGET_MIN_SIZE) + if not res_data["status"]: + print(res_data["message"]) + return jsonify({'status': 'success'}) + except requests.exceptions.RequestException as e: + error_message = f"Network error {image_id}: {e}" + log.send("error", "all", error_message) + return jsonify({'status': 'error', 'message': error_message}), 500 + except Exception as e: + traceback.print_exc() + error_message = f"Unknown error: {e}" + log.send("error", "all", error_message) + return jsonify({'status': 'error', 'message': error_message}), 500 else: pass return "", 204 diff --git a/system_module/db.py b/system_module/db.py index b675716..2780c1a 100644 --- a/system_module/db.py +++ b/system_module/db.py @@ -1,15 +1,13 @@ -import sqlite3 -import os import json +import os +import sqlite3 from datetime import datetime -from system_module import gelbooru + +from system_module.gelbooru import gelbooru_date_parse + class DB: - def __init__(self, debug: bool): - if debug: - db_path = "data_test" - else: - db_path = "data" + def __init__(self, db_path: str): if not os.path.exists(db_path): os.makedirs(db_path) @@ -117,7 +115,7 @@ class DB: "score": data_add["score"], "timestamp": data_add["created_at"], "parsed_time": parsed_time, - "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]), + "display_date": gelbooru_date_parse(data_add["created_at"]), "source": data_add["source"], "width": data_add["width"], "height": data_add["height"] @@ -165,15 +163,15 @@ class DB: cursor = self.conn.execute(query, (md5,)) return cursor.fetchone() is not None - def get_autocomplete(self, tag_slise: str): + def get_top_tag(self, tag_slise: str, count_tag_get: int = 9999999): query = """ SELECT value AS tag, COUNT(*) AS count FROM files, json_each(files.tags) WHERE value LIKE ? GROUP BY value ORDER BY count DESC - LIMIT 10 - """ + LIMIT {count_tag_get} + """.format(count_tag_get=count_tag_get) cursor = self.conn.execute(query, (f"{tag_slise.lower()}%",)) return [{"tag": row["tag"], "count": row["count"]} for row in cursor.fetchall()] diff --git a/system_module/db_old.py b/system_module/db_old.py deleted file mode 100644 index ddcc998..0000000 --- a/system_module/db_old.py +++ /dev/null @@ -1,104 +0,0 @@ -from tinydb import TinyDB, Query -import os -import json -from collections import Counter -from system_module import gelbooru -from datetime import datetime - -class DB: - def __init__(self, debug: bool): - if debug: - db_path = "data_test" - else: - db_path = "data" - if not os.path.exists(os.path.join(db_path, "db.json")): - json.dump({}, open(os.path.join(db_path, "db.json"), "w", encoding="utf-8")) - self.db = TinyDB(db_path+"/db.json") - self.File = Query() - - def add(self, data_add: dict): - self.db.insert(data_add) - - def add_raw(self, data_add: dict): - is_video = False - is_gif = False - if data_add["image"].split(".")[-1] in ["webm", "mp4", "avi"]: - is_video = True - if data_add["image"].split(".")[-1] == "gif": - is_gif = True - add_data = { - "md5": data_add["md5"], - "local_filename": data_add["image"], - "is_video": is_video, - "is_gif": is_gif, - "tags": data_add["tags"].split(" "), - "rating": data_add["rating"], - "score": data_add["score"], - "timestamp": data_add["created_at"], - "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]), - "source": data_add["source"], - "width": data_add["width"], - "height": data_add["height"] - } - self.db.insert(add_data) - - def get_all(self): - sorted_records = sorted( - self.db.all(), - key=lambda x: datetime.strptime( - x["timestamp"], - "%a %b %d %H:%M:%S %z %Y" - ), - reverse=True - ) - return sorted_records - - def get_search(self, rating_tmp: list[str], tags: list[str]): - mapping_rating = { - "e": "explicit", - "g": "general", - "s": "sensitive", - "q": "questionable" - } - rating = [mapping_rating.get(x, x) for x in rating_tmp] - if rating: - query = self.File.rating.one_of(rating) - if tags != [""]: - tag_query = self.File.tags.test(lambda t: set(tags).issubset(set(t or []))) - query = tag_query if query is None else query & tag_query - results = self.db.search(query) if query else self.db.all() - sorted_records = sorted( - results, - key=lambda x: datetime.strptime( - x["timestamp"], - "%a %b %d %H:%M:%S %z %Y" - ), - reverse=True - ) - return sorted_records - - def get_unknow(self): - results = self.db.search(self.File.tags == []) - return results - - def get_autocomplete(self, tag_slise: str): - counter = Counter() - for record in self.db.all(): - counter.update(record.get("tags", [])) - results = [ - { - "tag": tag, - "count": count - } - for tag, count in counter.items() - if tag.lower().startswith(tag_slise) - ] - results.sort(key=lambda x: x["count"], reverse=True) - top_10 = results[:10] - return top_10 - - def edit_data(self, id_file: str, edit_data: dict): - self.db.update(edit_data, self.File.md5 == id_file) - - def get_from_id(self, id_file: str): - return self.db.get(self.File.md5 == id_file) \ No newline at end of file diff --git a/system_module/gelbooru.py b/system_module/gelbooru.py index e41897d..0fca573 100644 --- a/system_module/gelbooru.py +++ b/system_module/gelbooru.py @@ -1,8 +1,10 @@ -import requests -from datetime import datetime import os +from datetime import datetime -def gelbooru_date_parse(date_str): +import requests + + +def gelbooru_date_parse(date_str: str): if not date_str: return 0, "Unknown" try: @@ -13,14 +15,14 @@ def gelbooru_date_parse(date_str): return 0, "Unknown" class GB: - def __init__(self, api_id, api_hash, headers, dw_folder): + def __init__(self, api_id: str, api_hash: str, headers: dict[str, str], dw_folder: str): self.api_id = api_id self.api_hash = api_hash self.dw_folder = dw_folder self.root_url = "https://gelbooru.com/index.php" self.HEADERS = headers - def get_from_id(self, id_post): + def get_from_id(self, id_post: int): params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "id": id_post, "api_key":self.api_hash, "user_id":self.api_id} response = requests.get(self.root_url, params=params, headers=self.HEADERS, timeout=15) print(response.url) @@ -28,13 +30,10 @@ class GB: file_name = response.json()["post"][0]["image"] return response.json()["post"][0], url_file, file_name - def get_from_md5(self, md5): + def get_from_md5(self, md5: str): params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "tags":"md5:"+md5, "api_key":self.api_hash, "user_id":self.api_id} response = requests.get(self.root_url, params=params, headers=self.HEADERS, timeout=15) print(response.url) url_file = response.json()["post"][0]["file_url"] file_name = response.json()["post"][0]["image"] - return response.json()["post"][0], url_file, file_name - - def get_binary_file(self, id_post): - pass \ No newline at end of file + return response.json()["post"][0], url_file, file_name \ No newline at end of file diff --git a/system_module/image_processor.py b/system_module/image_processor.py index b390ae9..3d490ba 100644 --- a/system_module/image_processor.py +++ b/system_module/image_processor.py @@ -1,6 +1,7 @@ -import subprocess -from PIL import Image import os +import subprocess + +from PIL import Image EXTENSIONS_IMG = ('jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp') EXTENSIONS_VID = ('mp4', 'webm') @@ -15,7 +16,7 @@ def check_ffmpeg(): except (FileNotFoundError, subprocess.CalledProcessError): return False -def video_gen(input_path_file, output_path_file, TARGET_MIN_SIZE): +def video_gen(input_path_file: str, output_path_file: str, TARGET_MIN_SIZE: int): vf_scale = f"scale='if(gt(iw,ih),-1,{TARGET_MIN_SIZE})':'if(gt(iw,ih),{TARGET_MIN_SIZE},-1)'" cmd = ["ffmpeg", "-y", "-i", input_path_file, "-ss", "00:00:00", "-vframes", "1", "-vf", vf_scale, "-q:v", "2", output_path_file] if os.name == 'nt': creationflags = subprocess.CREATE_NO_WINDOW @@ -24,7 +25,7 @@ def video_gen(input_path_file, output_path_file, TARGET_MIN_SIZE): if result.returncode != 0: raise Exception(f"FFmpeg error: {result.stderr.decode('utf-8', errors='ignore')}") -def image_and_gif_gen(input_path_file, output_path_file, TARGET_MIN_SIZE): +def image_and_gif_gen(input_path_file: str, output_path_file: str, TARGET_MIN_SIZE: int): with Image.open(input_path_file) as img: img_rgba = img.convert("RGBA") background = Image.new("RGB", img_rgba.size, (255, 255, 255)) @@ -43,7 +44,7 @@ def image_and_gif_gen(input_path_file, output_path_file, TARGET_MIN_SIZE): resized_img = final_img.resize((new_width, new_height), Image.Resampling.LANCZOS) resized_img.save(output_path_file, "JPEG", quality=85) -def generate_thumb(input_path_file, THUMB_FOLDER, TARGET_MIN_SIZE): +def generate_thumb(input_path_file: str, THUMB_FOLDER: str, TARGET_MIN_SIZE: int): ext_file = input_path_file.split(".")[-1] if ext_file not in VALID_EXTENSIONS: return {"status":False, "message":"Расширение не поддерживаеться!"} diff --git a/system_module/logger.py b/system_module/logger.py index 32ce271..c44ce7b 100644 --- a/system_module/logger.py +++ b/system_module/logger.py @@ -1,28 +1,34 @@ +import sys import logging class Logger: def __init__(self): - self.logger_file = logging.getLogger(__name__) - self.logger_cli = logging.getLogger(__name__) + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') file_handler = logging.FileHandler("logs.log", encoding="utf-8") + file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(formatter) console_handler = logging.StreamHandler() + console_handler.setLevel(logging.DEBUG) console_handler.setFormatter(formatter) - self.logger_file.setLevel(logging.DEBUG) - self.logger_file.addHandler(file_handler) - - self.logger_cli.setLevel(logging.INFO) - self.logger_cli.addHandler(console_handler) - - def send(self, level_log, type_log, message): - if type_log == "file": - getattr(self.logger_file, level_log)(message) - elif type_log == "cli": - getattr(self.logger_cli, level_log)(message) - else: - getattr(self.logger_file, level_log)(message) - getattr(self.logger_cli, level_log)(message) \ No newline at end of file + self.logger.addHandler(file_handler) + self.logger.addHandler(console_handler) + + def init_global_exception_hook(self): + def handle_exception(exc_type, exc_value, exc_traceback): + if issubclass(exc_type, KeyboardInterrupt): + sys.__excepthook__( + exc_type, + exc_value, + exc_traceback + ) + return + self.logger.critical("ERROR:", exc_info=(exc_type, exc_value, exc_traceback)) + sys.excepthook = handle_exception + + def send(self, level_log: str, message: str): + getattr(self.logger, level_log)(message) \ No newline at end of file diff --git a/system_module/tasks.py b/system_module/tasks.py new file mode 100644 index 0000000..07f138a --- /dev/null +++ b/system_module/tasks.py @@ -0,0 +1,224 @@ +import os +import json +from pathlib import Path + +def find_file(filename: str, file_folders: list[str], is_folder: bool = True): + for folder in file_folders: + folder_path = Path(folder) + if not folder_path.exists(): + continue + for file_path in folder_path.rglob(filename): + return str(folder_path) if is_folder else str(file_path) + return None + +def get_size_format(b: int, factor=1024, suffix="B"): + for unit in ["", "K", "M", "G", "T"]: + if b < factor: + return f"{b:.2f} {unit}{suffix}" + b /= factor + return f"{b:.2f} P{suffix}" + +def get_dir_size(path: str): + total = 0 + if os.path.exists(path): + for f in os.listdir(path): + fp = os.path.join(path, f) + if os.path.isfile(fp): + total += os.path.getsize(fp) + return total + +def get_file_list_size(file_list: list[str]): + total = 0 + for fp in file_list: + if os.path.isfile(fp): + total += os.path.getsize(fp) + return total + +def get_size_edge(type_searсh: bool, file_list: list[str]): + if type_searсh: + result = min(file_list, key=lambda f: Path(f).stat().st_size) + else: + result = max(file_list, key=lambda f: Path(f).stat().st_size) + return str(result) + +def get_files_not_found(database, config): + all_db = database.get_all() + list_md5 = [] + for one_post in all_db: + file_path = find_file(one_post["local_filename"], config["file"]["repos"], is_folder=False) + if file_path == None: + list_md5.append(one_post["md5"]) + return list_md5 + +def calculate_analytic(database, config): + all_db = database.get_all() + explicit_db = database.get_search(["e"], []) + sensitive_db = database.get_search(["s"], []) + questionable_db = database.get_search(["q"], []) + general_db = database.get_search(["g"], []) + rating_list_file_path = {"explicit":[], "sensitive":[], "questionable":[], "general":[]} + type_list_file_path = {"images":[], "videos":[], "gifs":[]} + files_not_found = 0 + for one_post in all_db: + file_path = find_file(one_post["local_filename"], config["file"]["repos"], is_folder=False) + if file_path != None: + file_path = str(file_path) + rating_list_file_path[one_post["rating"]].append(file_path) + if one_post["is_gif"] == 1: + type_list_file_path["gifs"].append(file_path) + elif one_post["is_video"] == 1: + type_list_file_path["videos"].append(file_path) + else: + type_list_file_path["images"].append(file_path) + else: + files_not_found += 1 + file_size_dict = {"explicit":get_file_list_size(rating_list_file_path["explicit"]), + "sensitive":get_file_list_size(rating_list_file_path["sensitive"]), + "questionable":get_file_list_size(rating_list_file_path["questionable"]), + "general":get_file_list_size(rating_list_file_path["general"]) + } + type_size_dict = {"images":get_file_list_size(type_list_file_path["images"]), + "videos":get_file_list_size(type_list_file_path["videos"]), + "gifs":get_file_list_size(type_list_file_path["gifs"])} + ratings_stats = { + "explicit": { + "count": len(explicit_db), + "size_formatted": get_size_format(file_size_dict["explicit"]) + }, + "sensitive": { + "count": len(sensitive_db), + "size_formatted": get_size_format(file_size_dict["sensitive"]) + }, + "questionable": { + "count": len(questionable_db), + "size_formatted": get_size_format(file_size_dict["questionable"]) + }, + "general": { + "count": len(general_db), + "size_formatted": get_size_format(file_size_dict["general"]) + } + } + total_file_data_size = file_size_dict["general"]+file_size_dict["questionable"]+file_size_dict["sensitive"]+file_size_dict["explicit"] + total_thumb_size = get_dir_size(config["file"]["thumb"]) + total_log_size = get_file_list_size(["logs.log"]) + total_db_size = get_file_list_size([os.path.join(config["file"]["data"], "db.sqlite")]) + weight_stats_smallest_file = get_size_edge(True, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) + weight_stats_largest_file = get_size_edge(False, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) + weight_stats_data = { + "smallest":{ + "file":"", + "md5":"", + "size":0 + }, + "largest":{ + "file":"", + "md5":"", + "size":0 + } + } + resolution_stats_data = { + "smallest":{ + "file":"", + "md5":"", + "res":{ + "width":0, + "height":0, + "total":99999999999999999999999999 + } + }, + "largest":{ + "file":"", + "md5":"", + "res":{ + "width":0, + "height":0, + "total":0 + } + } + } + for one_post_scan in all_db: + if one_post_scan["local_filename"] == os.path.basename(weight_stats_smallest_file): + weight_stats_data["smallest"] = { + "file":os.path.basename(weight_stats_smallest_file), + "md5":one_post_scan["md5"], + "size":get_file_list_size([weight_stats_smallest_file]) + } + if one_post_scan["local_filename"] == os.path.basename(weight_stats_largest_file): + weight_stats_data["largest"] = { + "file":os.path.basename(weight_stats_largest_file), + "md5":one_post_scan["md5"], + "size":get_file_list_size([weight_stats_largest_file]) + } + print(get_file_list_size([os.path.basename(weight_stats_largest_file)])) + one_post_res_total = one_post_scan["width"]*one_post_scan["height"] + if resolution_stats_data["largest"]["res"]["total"] < one_post_res_total: + resolution_stats_data["largest"] = { + "file":one_post_scan["local_filename"], + "md5":one_post_scan["md5"], + "res":{ + "width":one_post_scan["width"], + "height":one_post_scan["height"], + "total":one_post_res_total + } + } + if resolution_stats_data["smallest"]["res"]["total"] > one_post_res_total: + resolution_stats_data["smallest"] = { + "file":one_post_scan["local_filename"], + "md5":one_post_scan["md5"], + "res":{ + "width":one_post_scan["width"], + "height":one_post_scan["height"], + "total":one_post_res_total + } + } + result = { + "total_images_analyzed": len(all_db), + "files_not_found":files_not_found, + "average_weight": { + "bytes": round(total_file_data_size/(len(rating_list_file_path["explicit"])+len(rating_list_file_path["sensitive"])+len(rating_list_file_path["questionable"])+len(rating_list_file_path["general"])), 2), + "formatted": get_size_format(round(total_file_data_size/(len(rating_list_file_path["explicit"])+len(rating_list_file_path["sensitive"])+len(rating_list_file_path["questionable"])+len(rating_list_file_path["general"])), 2)) + }, + "storage_stats": { + "folders": { + "data": get_size_format(total_file_data_size), + "db": get_size_format(total_db_size), + "thumb": get_size_format(total_thumb_size), + "logs": get_size_format(total_log_size) + }, + "types": { + "images": get_size_format(type_size_dict["images"]), + "videos": get_size_format(type_size_dict["videos"]), + "gifs": get_size_format(type_size_dict["gifs"]) + }, + "grand_total": get_size_format(total_file_data_size+total_thumb_size+total_log_size+total_db_size) + }, + "ratings_stats": ratings_stats, + "resolution_stats": { + "largest": { + "filename": resolution_stats_data["largest"]["file"], + "md5": resolution_stats_data["largest"]["md5"], + "resolution": str(resolution_stats_data["largest"]["res"]["width"]) + " x " + str(resolution_stats_data["largest"]["res"]["height"]) + }, + "smallest": { + "filename": resolution_stats_data["smallest"]["file"], + "md5": resolution_stats_data["smallest"]["md5"], + "resolution": str(resolution_stats_data["smallest"]["res"]["width"]) + " x " + str(resolution_stats_data["smallest"]["res"]["height"]) + } + }, + "weight_stats": { + "largest": { + "filename": os.path.basename(weight_stats_data["largest"]["file"]), + "md5": weight_stats_data["largest"]["md5"], + "size_formatted": get_size_format(weight_stats_data["largest"]["size"]) + }, + "smallest": { + "filename": os.path.basename(weight_stats_data["smallest"]["file"]), + "md5": weight_stats_data["smallest"]["md5"], + "size_formatted": get_size_format(weight_stats_data["smallest"]["size"]) + } + }, + "top_tag":database.get_top_tag("") + } + with open(os.path.join(config["file"]["data"], "analytic.json"), 'w', encoding='utf-8') as out_f: + json.dump(result, out_f, indent=4, ensure_ascii=False) + return True \ No newline at end of file diff --git a/templates/analytics.html b/templates/analytics.html index 3fe3ee8..d80e558 100644 --- a/templates/analytics.html +++ b/templates/analytics.html @@ -41,22 +41,21 @@ .ext-block a { color: var(--link-color); text-decoration: none; word-break: break-all; } .ext-block a:hover { text-decoration: underline; } + .loader { color: var(--text-muted); font-size: 16px; } + @media (max-width: 1000px) { .dashboard-grid { grid-template-columns: 1fr 1fr; } .left-col { grid-column: 1/-1; display: grid; grid-template-columns: 1fr 1fr; } } @media (max-width: 700px) { .dashboard-grid, .left-col { grid-template-columns: 1fr; } header { flex-wrap: wrap; } .search-wrapper { order: 3; min-width: 100%; margin-top: 10px;} }
- - 📊 Analytics -
- -
    -
    +

    Database Analytics

    +
    Fetching data from API...
    +
    diff --git a/templates/index.html b/templates/index.html index f912482..a557621 100644 --- a/templates/index.html +++ b/templates/index.html @@ -18,6 +18,7 @@ header { display: flex; align-items: center; background-color: var(--bg-header); padding: 10px 20px; border-bottom: 1px solid var(--border-color); gap: 15px; flex-wrap: wrap; } .logo { font-weight: bold; font-size: 18px; color: white; text-decoration: none; } .analytics { color: #88bece; text-decoration: none; font-size: 12px; } + .tasks { color: #88bece; text-decoration: none; font-size: 12px; } .search-wrapper { flex-grow: 1; max-width: 400px; position: relative; } .search-wrapper input { width: 100%; background-color: var(--bg-input); border: 1px solid var(--border-color); color: white; padding: 6px 10px; border-radius: 3px; outline: none; } @@ -63,9 +64,8 @@
    - + ⚙️ Tasks
      diff --git a/templates/task.html b/templates/task.html new file mode 100644 index 0000000..2b08645 --- /dev/null +++ b/templates/task.html @@ -0,0 +1,224 @@ + + + + + + Tasks - LocalBooru + + + +
      + +
      + +
      +

      System Tasks

      + +
      + + +
      +
      +

      Пересчёт аналитики

      +

      Обновляет статистику (количество файлов, занимаемое место). Рекомендуется запускать после массового добавления или удаления постов.

      +
      +
      + + +
      +
      + + +
      +
      +

      Поиск недоступных файлов

      +

      Ищет посты, для которых отсутствуют физические файлы на диске. Выводит список ссылок на экран и предлагает скачать их в формате .txt.

      +
      +
      + + +
      + +
      +
      + Найдено битых ссылок: 0 + +
      + +
      +
      + +
      +
      + + + + \ No newline at end of file