Merge pull request '0.2.0b' (#1) from dev into main

Reviewed-on: #1

.gitignore

@@ -3,8 +3,11 @@
 .vscode
 old/
 data_test/
+tmp/
 config_gb.json
 migration.py
+change_log.txt
+up/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

button_script.user.js

@@ -6,7 +6,7 @@
 // @grant       GM_registerMenuCommand
 // @grant       GM_getValue
 // @grant       GM_setValue
-// @version     1.0r
+// @version     1.1r
 // @author      https://t.me/Nyako_TW
 // @downloadURL  https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js
 // @updateURL    https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js
@@ -95,7 +95,7 @@ if (originalImageLinkElement) {
             // Получаем настроенный пользователем корневой путь
             const apiRoot = GM_getValue('nyakonya_gb_api_root', 'http://127.0.2.1');
             // Формируем корректный URL для запроса к dw_api
-            const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/dw_api`;
+            const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/api`;
 
             console.log(`Отправка URL на локальный API: ${imageUrl} (Endpoint: ${apiUrl})`);
             const originalOutline = newDownloadButton.style.outline;
@@ -107,7 +107,7 @@ if (originalImageLinkElement) {
                 headers: {
                     "Content-Type": "application/json"
                 },
-                data: JSON.stringify({ id: imageID }),
+                data: JSON.stringify({ id: imageID , type: "download"}),
                 onload: function(response) {
                     console.log('Ответ от API:', response.responseText);
                     newDownloadButton.style.outline = "2px solid green"; // Успех

config.json

@@ -16,6 +16,7 @@
     "file":{
         "repos":["data/files"],
         "dw":"data/files",
-        "thumb":"data/thumb"
+        "thumb":"data/thumb",
+        "data":"data"
     }
 }

data/analytic.json

@@ -1,64 +0,0 @@
-{
-    "total_images_analyzed": 14003,
-    "average_weight": {
-        "bytes": 2980950.54,
-        "formatted": "2.84 MB"
-    },
-    "storage_stats": {
-        "folders": {
-            "img": "38.88 GB",
-            "json_db": "24.39 MB",
-            "thumb": "331.45 MB",
-            "logs": "14.44 MB"
-        },
-        "types_in_img": {
-            "images": "38.56 GB",
-            "videos": "140.80 MB",
-            "gifs": "181.30 MB",
-            "logs": "14.44 MB"
-        },
-        "grand_total": "39.24 GB"
-    },
-    "ratings_stats": {
-        "explicit": {
-            "count": 1605,
-            "size_formatted": "4.52 GB"
-        },
-        "sensitive": {
-            "count": 6623,
-            "size_formatted": "19.35 GB"
-        },
-        "questionable": {
-            "count": 1591,
-            "size_formatted": "4.72 GB"
-        },
-        "general": {
-            "count": 4184,
-            "size_formatted": "10.29 GB"
-        }
-    },
-    "resolution_stats": {
-        "largest": {
-            "filename": "5adc7e0e773417f9c379accf.png",
-            "md5": "5a63c28906dc7e0e7734c379accf",
-            "resolution": "9921x14031"
-        },
-        "smallest": {
-            "filename": "8fc614e712ad21d2151ccde8a3c87.gif",
-            "md5": "8fc614e712ae22151ccde8a3c87",
-            "resolution": "240x280"
-        }
-    },
-    "weight_stats": {
-        "largest": {
-            "filename": "4cf0a32c61c7e7914095d1fe.png",
-            "md5": "4ab6acf0a32c61c7e79fe",
-            "size_formatted": "65.21 MB"
-        },
-        "smallest": {
-            "filename": "2e1c8a5870b90c3f538d7.png",
-            "md5": "28c679ea3470b90c3f538d7",
-            "size_formatted": "6.49 KB"
-        }
-    }
-}

doc/change_log/2b_en.md

@@ -0,0 +1,29 @@
+### 0.2.0b
+
+> **Fixes**
+
+1. Added 404 error handling when the source file is not found.
+
+> **Changes**
+
+1. Merged `/dw_api` and `/api` into a single `/api` endpoint.
+
+> **New**
+
+1. Added a system module for background tasks.
+2. Added an analytics page.
+
+> **Code**
+
+1. Deleted the legacy `db_old.py` file.
+2. Sorted and refactored imports.
+3. Added type hinting for input data in `system_module` functions.
+4. Removed `data/analytic.json`.
+5. Updated the logic for retrieving popular tags to support analytics.
+    - 5.1. Renamed `get_autocomplete` function to `get_top_tags`.
+    - 5.2. Added an optional `limit` argument to the `get_top_tags` function.
+6. Removed the `get_binary_file()` function from the `gelbooru` module as it was unused.
+7. Refactored the logging module.
+8. Added comprehensive error logging.
+9. Added the `/task` endpoint.
+10. Implemented new logic for passing the database path via the configuration file.

doc/change_log/2b_ru.md

@@ -0,0 +1,29 @@
+### 0.2.0b
+
+> **Иправления**
+
+1. Добавлен вывод 404 если исходный файл не найден
+
+> **Изменения**
+
+1. /dw_api и /api объединены в /api
+
+> **Новое**
+
+1. Системный модуль с задачами
+2. Страница с аналитикой
+
+> **Код**
+
+1. Удалён старый файл db_old.py
+2. Отсортированы и изменены импорты
+3. Добавлена типизация входных данных в функции расположенные в system_module
+4. Удалён data/analytic.json
+5. Изменена логика получения списка популярных тегов для адаптации к аналитике
+    - 5.1. Функция get_autocomplete переименована в get_top_tag
+    - 5.1. Для функции get_top_tag добален необязательный аргумент для лимита выдачи
+6. Функция get_binary_file() из модуля gelbooru удалена из-за отсусвия факта использования
+7. Переписан модуль логов
+8. Добавлено полное логирование ошибок
+9. Добавлен эндпойнт /task
+10. Новая логика передачи пути к базе данных через конфиг

doc/install_en.md

@@ -53,6 +53,7 @@ Config explanation:
 | `file`-`repos` | `["data/files"]` | List of directories where source files are stored |
 | `file`-`dw` | `"data/files"` | Directory for new downloads |
 | `file`-`thumb` | `"data/thumb"` | Directory for thumbnails |
+| `file`-`data` | `"data"` | Directory for database |
 
 1. You must obtain your Gelbooru User ID and Hash and add them to the config file for the system to function correctly.
 2. Install the script in [VioletMonkey](https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js).

doc/install_ru.md

@@ -54,6 +54,7 @@
 | `file`-`repos` | `["data/files"]` | Список каталогов где храняться полные файлы |
 | `file`-`dw` | `"data/files"` | Каталог хранения загрузок |
 | `file`-`thumb` | `"data/thumb"` | Каталог обложек |
+| `file`-`data` | `"data"` | Каталог базы данных |
 
 1. Обязательно надо получить ID и Hash пользователя и внести в конфиг для работы!
 

run.py

@@ -1,11 +1,14 @@
-from system_module import db, gelbooru, logger, image_processor
-from flask import Flask, render_template, jsonify, send_from_directory, render_template_string, request
 import json
-import requests
+import os
-from pathlib import Path
 import subprocess
 import traceback
-import os
+from pathlib import Path
+
+import requests
+from flask import (Flask, jsonify, render_template, render_template_string,
+                    request, send_from_directory, make_response, send_file)
+
+from system_module import db, gelbooru, image_processor, logger, tasks
 
 if Path("config_gb.json").exists():
     CONFIG_FILE = "config_gb.json"
@@ -23,22 +26,15 @@ GB_HEADERS = CONFIG_ROOT["gb"]["headers"]
 FILE_FOLDERS = CONFIG_ROOT["file"]["repos"]
 FILE_DW_FOLDERS = CONFIG_ROOT["file"]["dw"]
 THUMB_FOLDER = CONFIG_ROOT["file"]["thumb"]
+DATA_FOLDER = CONFIG_ROOT["file"]["data"]
 
 
 gb = gelbooru.GB(GB_ID, GB_HASH, GB_HEADERS, FILE_DW_FOLDERS)
 log = logger.Logger()
-database = db.DB(True)
+log.init_global_exception_hook()
+database = db.DB(DATA_FOLDER)
 app = Flask(__name__)
 
-def find_file(filename, file_folders):
-    for folder in file_folders:
-        folder_path = Path(folder)
-        if not folder_path.exists():
-            continue
-        for file_path in folder_path.rglob(filename):
-            return folder_path
-    return None
-
 if not image_processor.check_ffmpeg(): print("WARNING!\n\nУ вас не установлен ffmpeg. Генерация превью для видео не доступна!")
 
 @app.route("/")
@@ -49,55 +45,27 @@ def index():
 def analytics():
     return render_template("analytics.html")
 
+@app.route("/task")
+def task():
+    return render_template("task.html")
+
 @app.route("/post/<path:post_id>")
 def post(post_id):
     return render_template("post.html")
 
 @app.route('/file/<path:filename>')
 def serve_image(filename):
-    folder_path = find_file(filename, FILE_FOLDERS)
+    folder_path = tasks.find_file(filename, FILE_FOLDERS)
-    return send_from_directory(folder_path, filename)
+    print(filename, FILE_FOLDERS)
+    if folder_path == None:
+        return make_response("Not found", 404)
+    else:
+        return send_from_directory(folder_path, filename)
 
 @app.route('/thumb/<path:filename>')
 def serve_thumb(filename):
     return send_from_directory(THUMB_FOLDER, filename)
 
-@app.route("/dw_api", methods=['POST'])
-def dw_api():
-    data = request.get_json()
-    if not data or 'id' not in data:
-        return jsonify({'status': 'error', 'message': 'URL not get'}), 400
-    image_id = data["id"]
-    try:
-        data_get, url_file, file_name = gb.get_from_id(image_id)
-        if database.check_uni(data_get["md5"]):
-            message = f"File '{file_name}' already exists. Skipping."
-            log.send("warning", "all", message)
-            return jsonify({'status': 'skipped', 'message': message})
-        with requests.get(url_file, stream=True, headers=GB_HEADERS, timeout=30) as r:
-            r.raise_for_status()
-            content_type = r.headers.get('Content-Type', '')
-            if 'text/html' in content_type:
-                log.send("error", "all", "The server returned HTML instead of an image. Bot protection.")
-                return False
-            with open(FILE_DW_FOLDERS+"/"+file_name, 'wb') as f:
-                for chunk in r.iter_content(chunk_size=8192):
-                    f.write(chunk)
-        database.add_raw(data_get)
-        res_data = image_processor.generate_thumb(FILE_DW_FOLDERS+"/"+file_name, THUMB_FOLDER, TARGET_MIN_SIZE)
-        if not res_data["status"]:
-            print(res_data["message"])
-        return jsonify({'status': 'success'})
-    except requests.exceptions.RequestException as e:
-        error_message = f"Network error {image_id}: {e}"
-        log.send("error", "all", error_message)
-        return jsonify({'status': 'error', 'message': error_message}), 500
-    except Exception as e:
-        traceback.print_exc()
-        error_message = f"Unknown error: {e}"
-        log.send("error", "all", error_message)
-        return jsonify({'status': 'error', 'message': error_message}), 500
-
 @app.route("/api", methods=["GET", "POST"])
 def api():
     data = request.get_json()
@@ -119,12 +87,56 @@ def api():
             return jsonify(results_send)
     elif data["type"] == "autocomplete":
         tag_slise = data["q"]
-        result_autocomplete = database.get_autocomplete(tag_slise)
+        result_autocomplete = database.get_top_tag(tag_slise, count_tag_get=10)
         return jsonify(result_autocomplete)
     elif data["type"] == "post":
         post_id = data["post_id"]
         result_post = database.get_from_id(post_id)
         return jsonify(result_post)
+    elif data["type"] == "analytic":
+        return send_file(os.path.join(DATA_FOLDER, "analytic.json"), mimetype="application/json")
+    elif data["type"] == "task":
+        if data["task"] == "analytic":
+            if tasks.calculate_analytic(database, CONFIG_ROOT):
+                return "", 200
+            else:
+                return "", 500
+        if data["task"] == "not_found_file":
+            list_md5_not_found = tasks.get_files_not_found(database, CONFIG_ROOT)
+            return jsonify({'list': list_md5_not_found}), 200
+    elif data["type"] == "download":
+        if not data or 'id' not in data:
+            return jsonify({'status': 'error', 'message': 'URL not get'}), 400
+        image_id = data["id"]
+        try:
+            data_get, url_file, file_name = gb.get_from_id(image_id)
+            if database.check_uni(data_get["md5"]):
+                message = f"File '{file_name}' already exists. Skipping."
+                log.send("warning", "all", message)
+                return jsonify({'status': 'skipped', 'message': message})
+            with requests.get(url_file, stream=True, headers=GB_HEADERS, timeout=30) as r:
+                r.raise_for_status()
+                content_type = r.headers.get('Content-Type', '')
+                if 'text/html' in content_type:
+                    log.send("error", "all", "The server returned HTML instead of an image. Bot protection.")
+                    return False
+                with open(FILE_DW_FOLDERS+"/"+file_name, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=8192):
+                        f.write(chunk)
+            database.add_raw(data_get)
+            res_data = image_processor.generate_thumb(FILE_DW_FOLDERS+"/"+file_name, THUMB_FOLDER, TARGET_MIN_SIZE)
+            if not res_data["status"]:
+                print(res_data["message"])
+            return jsonify({'status': 'success'})
+        except requests.exceptions.RequestException as e:
+            error_message = f"Network error {image_id}: {e}"
+            log.send("error", "all", error_message)
+            return jsonify({'status': 'error', 'message': error_message}), 500
+        except Exception as e:
+            traceback.print_exc()
+            error_message = f"Unknown error: {e}"
+            log.send("error", "all", error_message)
+            return jsonify({'status': 'error', 'message': error_message}), 500
     else:
         pass
     return "", 204

system_module/db.py

@@ -1,15 +1,13 @@
-import sqlite3
-import os
 import json
+import os
+import sqlite3
 from datetime import datetime
-from system_module import gelbooru
+
+from system_module.gelbooru import gelbooru_date_parse
+
 
 class DB:
-    def __init__(self, debug: bool):
+    def __init__(self, db_path: str):
-        if debug: 
-            db_path = "data_test" 
-        else: 
-            db_path = "data"
             
         if not os.path.exists(db_path):
             os.makedirs(db_path)
@@ -117,7 +115,7 @@ class DB:
             "score": data_add["score"],
             "timestamp": data_add["created_at"],
             "parsed_time": parsed_time,
-            "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]),
+            "display_date": gelbooru_date_parse(data_add["created_at"]),
             "source": data_add["source"],
             "width": data_add["width"],
             "height": data_add["height"]
@@ -165,15 +163,15 @@ class DB:
         cursor = self.conn.execute(query, (md5,))
         return cursor.fetchone() is not None
     
-    def get_autocomplete(self, tag_slise: str):
+    def get_top_tag(self, tag_slise: str, count_tag_get: int = 9999999):
         query = """
             SELECT value AS tag, COUNT(*) AS count 
             FROM files, json_each(files.tags) 
             WHERE value LIKE ? 
             GROUP BY value 
             ORDER BY count DESC 
-            LIMIT 10
+            LIMIT {count_tag_get}
-        """
+        """.format(count_tag_get=count_tag_get)
         cursor = self.conn.execute(query, (f"{tag_slise.lower()}%",))
         return [{"tag": row["tag"], "count": row["count"]} for row in cursor.fetchall()]
     

system_module/db_old.py

@@ -1,104 +0,0 @@
-from tinydb import TinyDB, Query
-import os
-import json
-from collections import Counter
-from system_module import gelbooru
-from datetime import datetime
-
-class DB:
-    def __init__(self, debug: bool):
-        if debug: 
-            db_path = "data_test" 
-        else: 
-            db_path = "data"
-        if not os.path.exists(os.path.join(db_path, "db.json")):
-            json.dump({}, open(os.path.join(db_path, "db.json"), "w", encoding="utf-8"))
-        self.db = TinyDB(db_path+"/db.json")
-        self.File = Query()
-        
-    def add(self, data_add: dict):
-        self.db.insert(data_add)
-    
-    def add_raw(self, data_add: dict):
-        is_video = False
-        is_gif = False
-        if data_add["image"].split(".")[-1] in ["webm", "mp4", "avi"]:
-            is_video = True
-        if data_add["image"].split(".")[-1] == "gif":
-            is_gif = True
-        add_data = {
-            "md5": data_add["md5"],
-            "local_filename": data_add["image"],
-            "is_video": is_video,
-            "is_gif": is_gif,
-            "tags": data_add["tags"].split(" "),
-            "rating": data_add["rating"],
-            "score": data_add["score"],
-            "timestamp": data_add["created_at"],
-            "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]),
-            "source": data_add["source"],
-            "width": data_add["width"],
-            "height": data_add["height"]
-        }
-        self.db.insert(add_data)
-    
-    def get_all(self):
-        sorted_records = sorted(
-            self.db.all(),
-            key=lambda x: datetime.strptime(
-                x["timestamp"],
-                "%a %b %d %H:%M:%S %z %Y"
-            ),
-            reverse=True
-        )
-        return sorted_records
-    
-    def get_search(self, rating_tmp: list[str], tags: list[str]):
-        mapping_rating = {
-            "e": "explicit",
-            "g": "general",
-            "s": "sensitive",
-            "q": "questionable"
-        }
-        rating = [mapping_rating.get(x, x) for x in rating_tmp]
-        if rating:
-            query = self.File.rating.one_of(rating)
-        if tags != [""]:
-            tag_query = self.File.tags.test(lambda t: set(tags).issubset(set(t or [])))
-            query = tag_query if query is None else query & tag_query
-        results = self.db.search(query) if query else self.db.all()
-        sorted_records = sorted(
-            results,
-            key=lambda x: datetime.strptime(
-                x["timestamp"],
-                "%a %b %d %H:%M:%S %z %Y"
-            ),
-            reverse=True
-        )
-        return sorted_records
-    
-    def get_unknow(self):
-        results = self.db.search(self.File.tags == [])
-        return results
-    
-    def get_autocomplete(self, tag_slise: str):
-        counter = Counter()
-        for record in self.db.all():
-            counter.update(record.get("tags", []))
-        results = [
-            {
-                "tag": tag,
-                "count": count
-            }
-            for tag, count in counter.items()
-            if tag.lower().startswith(tag_slise)
-        ]
-        results.sort(key=lambda x: x["count"], reverse=True)
-        top_10 = results[:10]
-        return top_10
-    
-    def edit_data(self, id_file: str, edit_data: dict):
-        self.db.update(edit_data, self.File.md5 == id_file)
-    
-    def get_from_id(self, id_file: str):
-        return self.db.get(self.File.md5 == id_file)

system_module/gelbooru.py

@@ -1,8 +1,10 @@
-import requests
-from datetime import datetime
 import os
+from datetime import datetime
 
-def gelbooru_date_parse(date_str):
+import requests
+
+
+def gelbooru_date_parse(date_str: str):
     if not date_str:
         return 0, "Unknown"
     try:
@@ -13,14 +15,14 @@ def gelbooru_date_parse(date_str):
         return 0, "Unknown"
 
 class GB:
-    def __init__(self, api_id, api_hash, headers, dw_folder):
+    def __init__(self, api_id: str, api_hash: str, headers: dict[str, str], dw_folder: str):
         self.api_id = api_id
         self.api_hash = api_hash
         self.dw_folder = dw_folder
         self.root_url = "https://gelbooru.com/index.php"
         self.HEADERS = headers
         
-    def get_from_id(self, id_post):
+    def get_from_id(self, id_post: int):
         params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "id": id_post, "api_key":self.api_hash, "user_id":self.api_id}
         response = requests.get(self.root_url, params=params, headers=self.HEADERS, timeout=15)
         print(response.url)
@@ -28,13 +30,10 @@ class GB:
         file_name = response.json()["post"][0]["image"]
         return response.json()["post"][0], url_file, file_name
     
-    def get_from_md5(self, md5):
+    def get_from_md5(self, md5: str):
         params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "tags":"md5:"+md5, "api_key":self.api_hash, "user_id":self.api_id}
         response = requests.get(self.root_url, params=params, headers=self.HEADERS, timeout=15)
         print(response.url)
         url_file = response.json()["post"][0]["file_url"]
         file_name = response.json()["post"][0]["image"]
         return response.json()["post"][0], url_file, file_name
-    
-    def get_binary_file(self, id_post):
-        pass

system_module/image_processor.py

@@ -1,6 +1,7 @@
-import subprocess
-from PIL import Image
 import os
+import subprocess
+
+from PIL import Image
 
 EXTENSIONS_IMG = ('jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp')
 EXTENSIONS_VID = ('mp4', 'webm')
@@ -15,7 +16,7 @@ def check_ffmpeg():
     except (FileNotFoundError, subprocess.CalledProcessError):
         return False
 
-def video_gen(input_path_file, output_path_file, TARGET_MIN_SIZE):
+def video_gen(input_path_file: str, output_path_file: str, TARGET_MIN_SIZE: int):
     vf_scale = f"scale='if(gt(iw,ih),-1,{TARGET_MIN_SIZE})':'if(gt(iw,ih),{TARGET_MIN_SIZE},-1)'"
     cmd = ["ffmpeg", "-y", "-i", input_path_file, "-ss", "00:00:00", "-vframes", "1", "-vf", vf_scale, "-q:v", "2", output_path_file]
     if os.name == 'nt': creationflags = subprocess.CREATE_NO_WINDOW
@@ -24,7 +25,7 @@ def video_gen(input_path_file, output_path_file, TARGET_MIN_SIZE):
     if result.returncode != 0:
         raise Exception(f"FFmpeg error: {result.stderr.decode('utf-8', errors='ignore')}")
 
-def image_and_gif_gen(input_path_file, output_path_file, TARGET_MIN_SIZE):
+def image_and_gif_gen(input_path_file: str, output_path_file: str, TARGET_MIN_SIZE: int):
     with Image.open(input_path_file) as img:
         img_rgba = img.convert("RGBA")
         background = Image.new("RGB", img_rgba.size, (255, 255, 255))
@@ -43,7 +44,7 @@ def image_and_gif_gen(input_path_file, output_path_file, TARGET_MIN_SIZE):
             resized_img = final_img.resize((new_width, new_height), Image.Resampling.LANCZOS)
             resized_img.save(output_path_file, "JPEG", quality=85)
 
-def generate_thumb(input_path_file, THUMB_FOLDER, TARGET_MIN_SIZE):
+def generate_thumb(input_path_file: str, THUMB_FOLDER: str, TARGET_MIN_SIZE: int):
     ext_file = input_path_file.split(".")[-1]
     if ext_file not in VALID_EXTENSIONS:
         return {"status":False, "message":"Расширение не поддерживаеться!"}

system_module/logger.py

@@ -1,28 +1,34 @@
+import sys
 import logging
 
 class Logger:
     def __init__(self):
-        self.logger_file = logging.getLogger(__name__)
+        self.logger = logging.getLogger(__name__)
-        self.logger_cli = logging.getLogger(__name__)
+        self.logger.setLevel(logging.DEBUG)
         formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
         
         file_handler = logging.FileHandler("logs.log", encoding="utf-8")
+        file_handler.setLevel(logging.DEBUG)
         file_handler.setFormatter(formatter)
         
         console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.DEBUG)
         console_handler.setFormatter(formatter)
         
-        self.logger_file.setLevel(logging.DEBUG)
+        self.logger.addHandler(file_handler)
-        self.logger_file.addHandler(file_handler)
+        self.logger.addHandler(console_handler)
     
-        self.logger_cli.setLevel(logging.INFO)
+    def init_global_exception_hook(self):
-        self.logger_cli.addHandler(console_handler)
+        def handle_exception(exc_type, exc_value, exc_traceback):
+            if issubclass(exc_type, KeyboardInterrupt):
+                sys.__excepthook__(
+                    exc_type,
+                    exc_value,
+                    exc_traceback
+                )
+                return
+            self.logger.critical("ERROR:", exc_info=(exc_type, exc_value, exc_traceback))
+        sys.excepthook = handle_exception
     
-    def send(self, level_log, type_log, message):
+    def send(self, level_log: str, message: str):
-        if type_log == "file":
+        getattr(self.logger, level_log)(message)
-            getattr(self.logger_file, level_log)(message)
-        elif type_log == "cli":
-            getattr(self.logger_cli, level_log)(message)
-        else:
-            getattr(self.logger_file, level_log)(message)
-            getattr(self.logger_cli, level_log)(message)

system_module/tasks.py

@@ -0,0 +1,224 @@
+import os
+import json
+from pathlib import Path
+
+def find_file(filename: str, file_folders: list[str], is_folder: bool = True):
+    for folder in file_folders:
+        folder_path = Path(folder)
+        if not folder_path.exists():
+            continue
+        for file_path in folder_path.rglob(filename):
+            return str(folder_path) if is_folder else str(file_path)
+    return None
+
+def get_size_format(b: int, factor=1024, suffix="B"):
+    for unit in ["", "K", "M", "G", "T"]:
+        if b < factor:
+            return f"{b:.2f} {unit}{suffix}"
+        b /= factor
+    return f"{b:.2f} P{suffix}"
+
+def get_dir_size(path: str):
+    total = 0
+    if os.path.exists(path):
+        for f in os.listdir(path):
+            fp = os.path.join(path, f)
+            if os.path.isfile(fp):
+                total += os.path.getsize(fp)
+    return total
+
+def get_file_list_size(file_list: list[str]):
+    total = 0
+    for fp in file_list:
+        if os.path.isfile(fp):
+            total += os.path.getsize(fp)
+    return total
+
+def get_size_edge(type_searсh: bool, file_list: list[str]):
+    if type_searсh:
+        result = min(file_list, key=lambda f: Path(f).stat().st_size)
+    else:
+        result = max(file_list, key=lambda f: Path(f).stat().st_size)
+    return str(result)
+
+def get_files_not_found(database, config):
+    all_db = database.get_all()
+    list_md5 = []
+    for one_post in all_db:
+        file_path = find_file(one_post["local_filename"], config["file"]["repos"], is_folder=False)
+        if file_path == None:
+            list_md5.append(one_post["md5"])
+    return list_md5
+
+def calculate_analytic(database, config):
+    all_db = database.get_all()
+    explicit_db = database.get_search(["e"], [])
+    sensitive_db = database.get_search(["s"], [])
+    questionable_db = database.get_search(["q"], [])
+    general_db = database.get_search(["g"], [])
+    rating_list_file_path = {"explicit":[], "sensitive":[], "questionable":[], "general":[]}
+    type_list_file_path = {"images":[], "videos":[], "gifs":[]}
+    files_not_found = 0
+    for one_post in all_db:
+        file_path = find_file(one_post["local_filename"], config["file"]["repos"], is_folder=False)
+        if file_path != None:
+            file_path = str(file_path)
+            rating_list_file_path[one_post["rating"]].append(file_path)
+            if one_post["is_gif"] == 1:
+                type_list_file_path["gifs"].append(file_path)
+            elif one_post["is_video"] == 1:
+                type_list_file_path["videos"].append(file_path)
+            else:
+                type_list_file_path["images"].append(file_path)
+        else:
+            files_not_found += 1
+    file_size_dict = {"explicit":get_file_list_size(rating_list_file_path["explicit"]), 
+                        "sensitive":get_file_list_size(rating_list_file_path["sensitive"]), 
+                        "questionable":get_file_list_size(rating_list_file_path["questionable"]), 
+                        "general":get_file_list_size(rating_list_file_path["general"])
+                    }
+    type_size_dict = {"images":get_file_list_size(type_list_file_path["images"]),
+                        "videos":get_file_list_size(type_list_file_path["videos"]), 
+                        "gifs":get_file_list_size(type_list_file_path["gifs"])}
+    ratings_stats = {
+        "explicit": {
+            "count": len(explicit_db),
+            "size_formatted": get_size_format(file_size_dict["explicit"])
+        },
+        "sensitive": {
+            "count": len(sensitive_db),
+            "size_formatted": get_size_format(file_size_dict["sensitive"])
+        },
+        "questionable": {
+            "count": len(questionable_db),
+            "size_formatted": get_size_format(file_size_dict["questionable"])
+        },
+        "general": {
+            "count": len(general_db),
+            "size_formatted": get_size_format(file_size_dict["general"])
+        }
+    }
+    total_file_data_size = file_size_dict["general"]+file_size_dict["questionable"]+file_size_dict["sensitive"]+file_size_dict["explicit"]
+    total_thumb_size = get_dir_size(config["file"]["thumb"])
+    total_log_size = get_file_list_size(["logs.log"])
+    total_db_size = get_file_list_size([os.path.join(config["file"]["data"], "db.sqlite")])
+    weight_stats_smallest_file = get_size_edge(True, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"])
+    weight_stats_largest_file = get_size_edge(False, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) 
+    weight_stats_data = {
+        "smallest":{
+            "file":"",
+            "md5":"",
+            "size":0
+        },
+        "largest":{
+            "file":"",
+            "md5":"",
+            "size":0
+        }
+    }
+    resolution_stats_data = {
+        "smallest":{
+            "file":"",
+            "md5":"",
+            "res":{
+                "width":0,
+                "height":0,
+                "total":99999999999999999999999999
+            }
+        },
+        "largest":{
+            "file":"",
+            "md5":"",
+            "res":{
+                "width":0,
+                "height":0,
+                "total":0
+            }
+        }
+    }
+    for one_post_scan in all_db:
+        if one_post_scan["local_filename"] == os.path.basename(weight_stats_smallest_file):
+            weight_stats_data["smallest"] = {
+                "file":os.path.basename(weight_stats_smallest_file),
+                "md5":one_post_scan["md5"],
+                "size":get_file_list_size([weight_stats_smallest_file])
+            }
+        if one_post_scan["local_filename"] == os.path.basename(weight_stats_largest_file):
+            weight_stats_data["largest"] = {
+                "file":os.path.basename(weight_stats_largest_file),
+                "md5":one_post_scan["md5"],
+                "size":get_file_list_size([weight_stats_largest_file])
+            }
+            print(get_file_list_size([os.path.basename(weight_stats_largest_file)]))
+        one_post_res_total = one_post_scan["width"]*one_post_scan["height"]
+        if resolution_stats_data["largest"]["res"]["total"] < one_post_res_total:
+            resolution_stats_data["largest"] = {
+                "file":one_post_scan["local_filename"],
+                "md5":one_post_scan["md5"],
+                "res":{
+                    "width":one_post_scan["width"],
+                    "height":one_post_scan["height"],
+                    "total":one_post_res_total
+                }
+            }
+        if resolution_stats_data["smallest"]["res"]["total"] > one_post_res_total:
+            resolution_stats_data["smallest"] = {
+                "file":one_post_scan["local_filename"],
+                "md5":one_post_scan["md5"],
+                "res":{
+                    "width":one_post_scan["width"],
+                    "height":one_post_scan["height"],
+                    "total":one_post_res_total
+                }
+            }
+    result = {
+        "total_images_analyzed": len(all_db),
+        "files_not_found":files_not_found,
+        "average_weight": {
+            "bytes": round(total_file_data_size/(len(rating_list_file_path["explicit"])+len(rating_list_file_path["sensitive"])+len(rating_list_file_path["questionable"])+len(rating_list_file_path["general"])), 2),
+            "formatted": get_size_format(round(total_file_data_size/(len(rating_list_file_path["explicit"])+len(rating_list_file_path["sensitive"])+len(rating_list_file_path["questionable"])+len(rating_list_file_path["general"])), 2))
+        },
+        "storage_stats": {
+            "folders": {
+                "data": get_size_format(total_file_data_size),
+                "db": get_size_format(total_db_size),
+                "thumb": get_size_format(total_thumb_size),
+                "logs": get_size_format(total_log_size)
+            },
+            "types": {
+                "images": get_size_format(type_size_dict["images"]),
+                "videos": get_size_format(type_size_dict["videos"]),
+                "gifs": get_size_format(type_size_dict["gifs"])
+            },
+            "grand_total": get_size_format(total_file_data_size+total_thumb_size+total_log_size+total_db_size)
+        },
+        "ratings_stats": ratings_stats,
+        "resolution_stats": {
+            "largest": {
+                "filename": resolution_stats_data["largest"]["file"],
+                "md5": resolution_stats_data["largest"]["md5"],
+                "resolution": str(resolution_stats_data["largest"]["res"]["width"]) + " x " + str(resolution_stats_data["largest"]["res"]["height"])
+            },
+            "smallest": {
+                "filename": resolution_stats_data["smallest"]["file"],
+                "md5": resolution_stats_data["smallest"]["md5"],
+                "resolution": str(resolution_stats_data["smallest"]["res"]["width"]) + " x " + str(resolution_stats_data["smallest"]["res"]["height"])
+            }
+        },
+        "weight_stats": {
+            "largest": {
+                "filename": os.path.basename(weight_stats_data["largest"]["file"]),
+                "md5": weight_stats_data["largest"]["md5"],
+                "size_formatted": get_size_format(weight_stats_data["largest"]["size"])
+            },
+            "smallest": {
+                "filename": os.path.basename(weight_stats_data["smallest"]["file"]),
+                "md5": weight_stats_data["smallest"]["md5"],
+                "size_formatted": get_size_format(weight_stats_data["smallest"]["size"])
+            }
+        },
+        "top_tag":database.get_top_tag("")
+    }
+    with open(os.path.join(config["file"]["data"], "analytic.json"), 'w', encoding='utf-8') as out_f:
+        json.dump(result, out_f, indent=4, ensure_ascii=False)
+    return True

templates/analytics.html

@@ -41,22 +41,21 @@
         .ext-block a { color: var(--link-color); text-decoration: none; word-break: break-all; }
         .ext-block a:hover { text-decoration: underline; }
 
+        .loader { color: var(--text-muted); font-size: 16px; }
+
         @media (max-width: 1000px) { .dashboard-grid { grid-template-columns: 1fr 1fr; } .left-col { grid-column: 1/-1; display: grid; grid-template-columns: 1fr 1fr; } }
         @media (max-width: 700px) { .dashboard-grid, .left-col { grid-template-columns: 1fr; } header { flex-wrap: wrap; } .search-wrapper { order: 3; min-width: 100%; margin-top: 10px;} }
     </style>
 </head>
 <body>
     <header>
-        <a href="index.html" class="logo">LocalBooru</a>
+        <a href="/" class="logo">LocalBooru</a>
-        <a href="analytics.html" class="analytics-link">📊 Analytics</a>
-        <div class="search-wrapper">
-            <input type="text" id="searchInput" placeholder="Search tags...">
-            <ul id="autocompleteList" class="autocomplete-list"></ul>
-        </div>
     </header>
 
     <main>
         <h1>Database Analytics</h1>
+        <div id="loadingIndicator" class="loader">Fetching data from API...</div>
+        
         <div class="dashboard-grid" id="analyticsGrid" style="display: none;">
             <div class="left-col">
                 <div class="panel">
@@ -68,7 +67,7 @@
                     <div class="data-list" id="storageContainer"></div>
                 </div>
                 <div class="panel">
-                    <div class="panel-header">🎨 Media Types (in img/)</div>
+                    <div class="panel-header">🎨 Media Types</div>
                     <div class="data-list" id="mediaContainer"></div>
                 </div>
                 <div class="panel">
@@ -78,125 +77,119 @@
             </div>
             
             <div class="panel">
-                <div class="panel-header">🏷️ All Tags List</div>
+                <div class="panel-header">🏷️ Top Tags List</div>
                 <div class="table-header"><span>Tag</span><span>Count</span></div>
                 <div class="data-list" id="tagsListContainer"></div>
-                <div class="pagination"><button class="page-btn" onclick="prevTagPage()">&lsaquo;</button><span id="tagPageInfo" style="font-weight:bold; padding:0 10px;"></span><button class="page-btn" onclick="nextTagPage()">&rsaquo;</button></div>
+                <div class="pagination">
+                    <button class="page-btn" onclick="prevTagPage()">&lsaquo;</button>
+                    <span id="tagPageInfo" style="font-weight:bold; padding:0 10px;"></span>
+                    <button class="page-btn" onclick="nextTagPage()">&rsaquo;</button>
+                </div>
             </div>
             
+            <!-- Заменено, так как в новом JSON нет top_pairs -->
             <div class="panel">
-                <div class="panel-header">🔗 Top 30 Tag Pairs</div>
+                <div class="panel-header">ℹ️ System Overview</div>
-                <div class="table-header"><span>Pair</span><span>Count</span></div>
+                <div class="data-list" id="overviewContainer"></div>
-                <div class="data-list" id="pairsContainer"></div>
             </div>
         </div>
     </main>
 
     <script>
-        const searchInput = document.getElementById('searchInput');
-        const list = document.getElementById('autocompleteList');
         let debounceTimer;
-
-        searchInput.addEventListener('input', () => {
-            clearTimeout(debounceTimer);
-            const words = searchInput.value.split(' ');
-            const currentWord = words[words.length - 1];
-            if (currentWord.length < 2) { list.style.display = 'none'; return; }
-
-            debounceTimer = setTimeout(async () => {
-                const req = { type: "autocomplete", q: currentWord };
-                const data = await mockAutocompleteAPI(req); 
-                
-                if (data.length === 0) { list.style.display = 'none'; return; }
-
-                list.innerHTML = data.map(item => `<li class="autocomplete-item" data-tag="${item.tag}"><span>${item.tag}</span><span class="autocomplete-count">${item.count}</span></li>`).join('');
-                list.style.display = 'block';
-
-                list.querySelectorAll('.autocomplete-item').forEach(li => {
-                    li.addEventListener('click', () => {
-                        words[words.length - 1] = li.getAttribute('data-tag');
-                        searchInput.value = words.join(' ') + ' ';
-                        list.style.display = 'none';
-                        searchInput.focus();
-                    });
-                });
-            }, 200);
-        });
-
-        document.addEventListener('click', (e) => { if (e.target !== searchInput && e.target !== list) list.style.display = 'none'; });
-        searchInput.addEventListener('keypress', (e) => { if(e.key === 'Enter') window.location.href = `index.html?query=${searchInput.value.trim()}`; });
-
         let allTags = [], currentTagPage = 1, tagsPerPage = 22;
 
+        // Укорачиваем длинные имена файлов (хэши) для красивого отображения
+        const shortenName = (name) => name.length > 20 ? name.substring(0, 10) + '...' + name.substring(name.length - 6) : name;
+
         async function fetchAnalytics() {
-            const data = await mockAnalyticsAPI({ type: "analytics" });
+            try {
+                const response = await fetch('/api', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ type: "analytic" })
+                });
 
-            document.getElementById('ratingContainer').innerHTML = Object.keys(data.ratings).map(k => `
+                if (!response.ok) throw new Error("Network response was not ok");
-                <div class="rating-box" style="border-top: 2px solid var(--color-${k[0]})">
-                    <div style="font-weight:bold; color:var(--color-${k[0]}); margin-bottom:5px;">${k.charAt(0).toUpperCase() + k.slice(1)}</div>
-                    <div style="color:var(--text-muted); margin-bottom:3px;">${data.ratings[k].count} items</div>
-                    <div style="font-weight:bold;">${data.ratings[k].size}</div>
-                </div>
-            `).join('');
                 
-            const s = data.storage;
+                const data = await response.json();
-            document.getElementById('storageContainer').innerHTML = `
-                <div class="data-row"><span>Logs:</span><span class="val">${s.logs}</span></div>
-                <div class="data-row"><span>DB:</span><span class="val">${s.db}</span></div>
-                <div class="data-row"><span>Thumb:</span><span class="val">${s.thumb}</span></div>
-                <div class="data-row"><span>Img:</span><span class="val">${s.main}</span></div>
-                <div class="data-row" style="color:var(--color-g); font-weight:bold; border-top:1px solid #444; border-bottom:none; padding-top:8px;"><span>GRAND TOTAL:</span><span>${s.total}</span></div>
-            `;
                 
-            const m = data.media;
+                // Скрываем лоадер, показываем сетку
-            document.getElementById('mediaContainer').innerHTML = `
+                document.getElementById('loadingIndicator').style.display = 'none';
-                <div class="data-row"><span>IMG:</span><span class="val">${m.img_size}</span></div>
+                document.getElementById('analyticsGrid').style.display = 'grid';
-                <div class="data-row"><span>VID:</span><span class="val">${m.vid_size}</span></div>
-                <div class="data-row"><span>GIF:</span><span class="val">${m.gif_size}</span></div>
-                <div style="font-size:11px; color:var(--text-muted); margin-top:5px;">Total: <b>${m.total_items}</b> | Avg: <b>${m.avg_weight}</b></div>
-            `;
 
-            const e = data.extremes;
+                // 1. Рейтинги
-            document.getElementById('extremesContainer').innerHTML = `
+                const rStats = data.ratings_stats;
-                <div class="ext-block"><b>Largest:</b> ${e.largest.val}<br>(<a href="post.html?id=${e.largest.id}">${e.largest.file}</a>)</div>
+                document.getElementById('ratingContainer').innerHTML = Object.keys(rStats).map(k => `
-                <div class="ext-block"><b>Smallest:</b> ${e.smallest.val}<br>(<a href="post.html?id=${e.smallest.id}">${e.smallest.file}</a>)</div>
+                    <div class="rating-box" style="border-top: 2px solid var(--color-${k[0]})">
-                <div class="ext-block"><b>Heaviest:</b> ${e.heaviest.val}<br>(<a href="post.html?id=${e.heaviest.id}">${e.heaviest.file}</a>)</div>
+                        <div style="font-weight:bold; color:var(--color-${k[0]}); margin-bottom:5px;">${k.charAt(0).toUpperCase() + k.slice(1)}</div>
-                <div class="ext-block"><b>Lightest:</b> ${e.lightest.val}<br>(<a href="post.html?id=${e.lightest.id}">${e.lightest.file}</a>)</div>
+                        <div style="color:var(--text-muted); margin-bottom:3px;">${rStats[k].count} items</div>
-            `;
+                        <div style="font-weight:bold;">${rStats[k].size_formatted}</div>
+                    </div>
+                `).join('');
 
-            document.getElementById('pairsContainer').innerHTML = data.top_pairs.map(p => `<div class="tag-row"><a href="index.html?query=${p.pair.replace(/ \+ /g, '+')}">${p.pair}</a> <span>${p.count}</span></div>`).join('');
+                // 2. Хранилище (Storage)
+                const s = data.storage_stats.folders;
+                document.getElementById('storageContainer').innerHTML = `
+                    <div class="data-row"><span>Logs:</span><span class="val">${s.logs}</span></div>
+                    <div class="data-row"><span>DB:</span><span class="val">${s.db}</span></div>
+                    <div class="data-row"><span>Thumb:</span><span class="val">${s.thumb}</span></div>
+                    <div class="data-row"><span>Data (Img):</span><span class="val">${s.data}</span></div>
+                    <div class="data-row" style="color:var(--color-g); font-weight:bold; border-top:1px solid #444; border-bottom:none; padding-top:8px;"><span>GRAND TOTAL:</span><span>${data.storage_stats.grand_total}</span></div>
+                `;
 
-            allTags = data.all_tags; renderTagsPage();
+                // 3. Медиа типы
-            document.getElementById('analyticsGrid').style.display = 'grid';
+                const m = data.storage_stats.types;
+                document.getElementById('mediaContainer').innerHTML = `
+                    <div class="data-row"><span>IMG:</span><span class="val">${m.images}</span></div>
+                    <div class="data-row"><span>VID:</span><span class="val">${m.videos}</span></div>
+                    <div class="data-row"><span>GIF:</span><span class="val">${m.gifs}</span></div>
+                    <div style="font-size:11px; color:var(--text-muted); margin-top:5px;">Total: <b>${data.total_images_analyzed}</b> | Avg: <b>${data.average_weight.formatted}</b></div>
+                `;
+
+                // 4. Рекорды (Extremes)
+                const res = data.resolution_stats;
+                const weight = data.weight_stats;
+                document.getElementById('extremesContainer').innerHTML = `
+                    <div class="ext-block"><b>Largest Res:</b> ${res.largest.resolution}<br>(<a href="post/${res.largest.md5}">${shortenName(res.largest.filename)}</a>)</div>
+                    <div class="ext-block"><b>Smallest Res:</b> ${res.smallest.resolution}<br>(<a href="post/${res.smallest.md5}">${shortenName(res.smallest.filename)}</a>)</div>
+                    <div class="ext-block"><b>Heaviest:</b> ${weight.largest.size_formatted}<br>(<a href="post/${weight.largest.md5}">${shortenName(weight.largest.filename)}</a>)</div>
+                    <div class="ext-block"><b>Lightest:</b> ${weight.smallest.size_formatted}<br>(<a href="post/${weight.smallest.md5}">${shortenName(weight.smallest.filename)}</a>)</div>
+                `;
+
+                // 5. System Overview (вместо Top Pairs)
+                document.getElementById('overviewContainer').innerHTML = `
+                    <div class="data-row"><span>Images Analyzed:</span><span class="val" style="font-weight:bold;">${data.total_images_analyzed}</span></div>
+                    <div class="data-row"><span>Files Not Found:</span><span class="val" style="font-weight:bold; color: ${data.files_not_found > 0 ? 'var(--color-e)' : 'inherit'};">${data.files_not_found}</span></div>
+                `;
+
+                // 6. Теги
+                allTags = data.top_tag; 
+                renderTagsPage();
+
+            } catch (error) {
+                console.error('Error fetching analytics:', error);
+                document.getElementById('loadingIndicator').innerHTML = `<span style="color: var(--color-e)">Failed to load data. See console for details.</span>`;
+            }
         }
 
+        // Пагинация тегов
         function renderTagsPage() {
-            document.getElementById('tagPageInfo').innerText = `${currentTagPage} / ${Math.ceil(allTags.length / tagsPerPage)}`;
+            if (!allTags || allTags.length === 0) return;
+            
+            const totalPages = Math.ceil(allTags.length / tagsPerPage) || 1;
+            document.getElementById('tagPageInfo').innerText = `${currentTagPage} / ${totalPages}`;
+            
             const start = (currentTagPage - 1) * tagsPerPage;
-            document.getElementById('tagsListContainer').innerHTML = allTags.slice(start, start + tagsPerPage).map(t => `<div class="tag-row"><a href="index.html?query=${t.name}">${t.name}</a> <span>${t.count}</span></div>`).join('');
+            document.getElementById('tagsListContainer').innerHTML = allTags.slice(start, start + tagsPerPage).map(t => 
+                `<div class="tag-row"><a href="index.html?query=${encodeURIComponent(t.tag)}">${t.tag}</a> <span>${t.count}</span></div>`
+            ).join('');
         }
+
         window.prevTagPage = () => { if(currentTagPage > 1) { currentTagPage--; renderTagsPage(); }};
         window.nextTagPage = () => { if(currentTagPage < Math.ceil(allTags.length/tagsPerPage)) { currentTagPage++; renderTagsPage(); }};
 
-        // --- ЗАГЛУШКИ ---
+        // Запуск
-        function mockAutocompleteAPI(req) { return new Promise(res => { const db = [{tag: "long_hair", count: 8184}, {tag: "1girl", count: 11061}]; res(db.filter(t => t.tag.includes(req.q.toLowerCase()))); }); }
-        
-        function mockAnalyticsAPI() { 
-            return new Promise(res => { setTimeout(() => { res({
-                ratings: { explicit: { count: 1605, size: "4.52 GB" }, sensitive: { count: 6623, size: "19.35 GB" }, questionable: { count: 1591, size: "4.72 GB" }, general: { count: 4184, size: "10.29 GB" } },
-                storage: { logs: "14.44 MB", db: "24.39 MB", thumb: "331.45 MB", main: "38.88 GB", total: "39.24 GB" },
-                media: { img_size: "38.56 GB", vid_size: "140.80 MB", gif_size: "181.30 MB", total_items: 14003, avg_weight: "2.84 MB" },
-                extremes: {
-                    largest: { val: "9921x14031", id: "5a63c", file: "5a63c28906dc7e0e...png" },
-                    smallest: { val: "240x280", id: "8fc61", file: "8fc614e712ae2ad...gif" },
-                    heaviest: { val: "65.21 MB", id: "4ab6a", file: "4ab6acf0a32c61...png" },
-                    lightest: { val: "6.49 KB", id: "28c67", file: "28c679ea3483e1...png" }
-                },
-                all_tags: Array.from({length: 100}, (_,i) => ({name:`tag_${i}`, count:100-i})),
-                top_pairs: [{ pair: "1girl + highres", count: 10236 }, { pair: "1girl + solo", count: 8902 }]
-            }); }, 300);}); 
-        }
-
         fetchAnalytics();
     </script>
 </body>

templates/index.html

@@ -18,6 +18,7 @@
         header { display: flex; align-items: center; background-color: var(--bg-header); padding: 10px 20px; border-bottom: 1px solid var(--border-color); gap: 15px; flex-wrap: wrap; }
         .logo { font-weight: bold; font-size: 18px; color: white; text-decoration: none; }
         .analytics { color: #88bece; text-decoration: none; font-size: 12px; }
+        .tasks { color: #88bece; text-decoration: none; font-size: 12px; }
 
         .search-wrapper { flex-grow: 1; max-width: 400px; position: relative; }
         .search-wrapper input { width: 100%; background-color: var(--bg-input); border: 1px solid var(--border-color); color: white; padding: 6px 10px; border-radius: 3px; outline: none; }
@@ -63,9 +64,8 @@
 
     <header>
         <a href="/" class="logo">LocalBooru</a>
-        <!--
         <a class="analytics" href="/analytics">📊 Analytics</a>
-        -->
+        <a class="tasks" href="/task">⚙️ Tasks</a>
         <div class="search-wrapper">
             <input type="text" id="searchInput" placeholder="Search tags...">
             <ul id="autocompleteList" class="autocomplete-list"></ul>

templates/task.html

@@ -0,0 +1,224 @@
+<!DOCTYPE html>
+<html lang="ru">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Tasks - LocalBooru</title>
+    <style>
+        :root { --bg-main: #1e1e1e; --bg-header: #252525; --bg-panel: #222; --bg-box: #2a2a2a; --bg-input: #333; --text-main: #eee; --text-muted: #aaa; --border-color: #444; --link-color: #4da8da; --color-g: #4caf50; --color-s: #ff9800; --color-q: #ffeb3b; --color-e: #f44336; }
+        * { box-sizing: border-box; margin: 0; padding: 0; }
+        body { background-color: var(--bg-main); color: var(--text-main); font-family: sans-serif; font-size: 13px; }
+
+        /* Header Styles */
+        header { display: flex; align-items: center; background-color: var(--bg-header); padding: 10px 20px; border-bottom: 1px solid var(--border-color); gap: 15px; flex-wrap: wrap; }
+        .logo { font-weight: bold; font-size: 18px; color: white; text-decoration: none; }
+        .nav-link { color: var(--link-color); text-decoration: none; font-size: 12px; font-weight: bold;}
+
+        /* Search Autocomplete Styles */
+        .search-wrapper { flex-grow: 1; max-width: 400px; position: relative; }
+        .search-wrapper input { width: 100%; background-color: var(--bg-input); border: 1px solid var(--border-color); color: white; padding: 6px 10px; border-radius: 3px; outline: none; }
+        
+        /* Main Content & Tasks Styles */
+        main { padding: 20px; max-width: 1000px; margin: 0 auto; }
+        h1 { font-size: 20px; margin-bottom: 20px; color: white; border-bottom: 1px solid var(--border-color); padding-bottom: 10px; }
+        
+        .task-list { display: flex; flex-direction: column; gap: 15px; }
+        
+        /* Добавил flex-wrap: wrap чтобы результаты могли падать на новую строку */
+        .task-card { background-color: var(--bg-panel); border: 1px solid var(--border-color); border-radius: 4px; padding: 15px 20px; display: flex; justify-content: space-between; align-items: center; gap: 20px; flex-wrap: wrap; transition: background-color 0.2s; }
+        .task-card:hover { background-color: var(--bg-box); }
+        
+        .task-info { flex: 1; min-width: 250px; }
+        .task-info h3 { font-size: 16px; color: var(--link-color); margin-bottom: 5px; }
+        .task-info p { color: var(--text-muted); font-size: 13px; line-height: 1.4; }
+        
+        .task-actions { display: flex; align-items: center; gap: 15px; min-width: 200px; justify-content: flex-end; }
+        
+        .btn { background-color: var(--bg-input); border: 1px solid var(--border-color); color: white; padding: 8px 16px; border-radius: 3px; cursor: pointer; font-weight: bold; transition: all 0.2s; }
+        .btn:hover:not(:disabled) { background-color: #444; border-color: #555; }
+        .btn:active:not(:disabled) { background-color: #222; }
+        .btn:disabled { opacity: 0.5; cursor: not-allowed; }
+
+        .task-status { font-weight: bold; font-size: 13px; min-width: 90px; text-align: right; }
+        .status-success { color: var(--color-g); }
+        .status-error { color: var(--color-e); }
+        
+        /* Стили для вывода результатов */
+        .task-result-box { width: 100%; margin-top: 10px; border-top: 1px solid var(--border-color); padding-top: 15px; display: none; }
+        .task-result-box textarea { width: 100%; height: 150px; background-color: var(--bg-input); color: var(--text-main); border: 1px solid var(--border-color); border-radius: 3px; padding: 10px; margin-top: 10px; resize: vertical; outline: none; font-family: monospace; }
+
+        @media (max-width: 700px) { 
+            header { flex-wrap: wrap; } 
+            .search-wrapper { order: 3; min-width: 100%; margin-top: 10px;} 
+            .task-card { flex-direction: column; align-items: flex-start; }
+            .task-actions { justify-content: flex-start; width: 100%; margin-top: 10px; }
+        }
+    </style>
+</head>
+<body>
+    <header>
+        <a href="/" class="logo">LocalBooru</a>
+    </header>
+
+    <main>
+        <h1>System Tasks</h1>
+        
+        <div class="task-list">
+            
+            <!-- ЗАДАЧА 1: Пересчёт аналитики -->
+            <div class="task-card">
+                <div class="task-info">
+                    <h3>Пересчёт аналитики</h3>
+                    <p>Обновляет статистику (количество файлов, занимаемое место). Рекомендуется запускать после массового добавления или удаления постов.</p>
+                </div>
+                <div class="task-actions">
+                    <span class="task-status"></span>
+                    <button class="btn" onclick="executeTask('analytic', this)">Запустить</button>
+                </div>
+            </div>
+
+            <!-- ЗАДАЧА 2: Поиск недоступных файлов -->
+            <div class="task-card">
+                <div class="task-info">
+                    <h3>Поиск недоступных файлов</h3>
+                    <p>Ищет посты, для которых отсутствуют физические файлы на диске. Выводит список ссылок на экран и предлагает скачать их в формате .txt.</p>
+                </div>
+                <div class="task-actions">
+                    <span class="task-status"></span>
+                    <button class="btn" onclick="executeNotFoundFilesTask(this)">Запустить</button>
+                </div>
+                <!-- Блок вывода результатов (изначально скрыт) -->
+                <div class="task-result-box" id="result-box-not-found">
+                    <div style="display: flex; justify-content: space-between; align-items: center;">
+                        <span>Найдено битых ссылок: <b id="not-found-count" style="color: var(--color-e);">0</b></span>
+                        <button class="btn" id="btn-download-txt" style="display: none;">Скачать .txt</button>
+                    </div>
+                    <textarea id="not-found-textarea" readonly></textarea>
+                </div>
+            </div>
+
+        </div>
+    </main>
+
+    <script>
+        // === БАЗОВАЯ ЛОГИКА ЗАДАЧ (БЕЗ ВОЗВРАТА ДАННЫХ) ===
+        async function executeTask(taskName, btn) {
+            const statusSpan = btn.previousElementSibling;
+            
+            btn.disabled = true;
+            btn.innerText = "Выполнение...";
+            statusSpan.innerText = "В процессе ⏳";
+            statusSpan.className = "task-status"; 
+
+            try {
+                const response = await fetch('/api', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ type: "task", task: taskName })
+                });
+
+                if (response.ok) {
+                    statusSpan.innerText = "Успешно ✔";
+                    statusSpan.classList.add('status-success');
+                } else {
+                    statusSpan.innerText = `Ошибка (${response.status}) ✖`;
+                    statusSpan.classList.add('status-error');
+                }
+            } catch (error) {
+                statusSpan.innerText = "Сбой сети ✖";
+                statusSpan.classList.add('status-error');
+                console.error("Task execution error:", error);
+            } finally {
+                btn.disabled = false;
+                btn.innerText = "Запустить";
+            }
+        }
+
+        // === ЛОГИКА ЗАДАЧИ: Поиск недоступных файлов ===
+        async function executeNotFoundFilesTask(btn) {
+            const statusSpan = btn.previousElementSibling;
+            
+            // Элементы интерфейса для вывода
+            const resultBox = document.getElementById('result-box-not-found');
+            const textarea = document.getElementById('not-found-textarea');
+            const countSpan = document.getElementById('not-found-count');
+            const downloadBtn = document.getElementById('btn-download-txt');
+            
+            btn.disabled = true;
+            btn.innerText = "Выполнение...";
+            statusSpan.innerText = "В процессе ⏳";
+            statusSpan.className = "task-status"; 
+            
+            // Скрываем прошлый результат перед новым запуском
+            resultBox.style.display = 'none';
+
+            try {
+                const response = await fetch('/api', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ type: "task", task: "not_found_file" })
+                });
+
+                if (response.ok) {
+                    const data = await response.json();
+                    const md5List = data.list || [];
+                    
+                    statusSpan.innerText = "Успешно ✔";
+                    statusSpan.classList.add('status-success');
+
+                    if (md5List.length > 0) {
+                        // Формируем ссылки
+                        const linksArray = md5List.map(md5 => `/post/${md5}`);
+                        const linksText = linksArray.join('\n');
+
+                        // Выводим на экран
+                        countSpan.innerText = md5List.length;
+                        textarea.value = linksText;
+                        resultBox.style.display = 'block';
+                        
+                        // Настраиваем скачивание
+                        downloadBtn.style.display = 'block';
+                        downloadBtn.onclick = () => downloadFileAsTxt(linksText, 'not_found_files.txt');
+                        
+                        // Сразу инициируем скачивание
+                        downloadFileAsTxt(linksText, 'not_found_files.txt');
+                    } else {
+                        // Если битых файлов нет
+                        countSpan.innerText = '0';
+                        textarea.value = "Недоступных файлов не обнаружено.";
+                        downloadBtn.style.display = 'none';
+                        resultBox.style.display = 'block';
+                    }
+
+                } else {
+                    statusSpan.innerText = `Ошибка (${response.status}) ✖`;
+                    statusSpan.classList.add('status-error');
+                }
+            } catch (error) {
+                statusSpan.innerText = "Сбой сети ✖";
+                statusSpan.classList.add('status-error');
+                console.error("Task execution error:", error);
+            } finally {
+                btn.disabled = false;
+                btn.innerText = "Запустить";
+            }
+        }
+
+        // === ВСПОМОГАТЕЛЬНАЯ ФУНКЦИЯ ДЛЯ СКАЧИВАНИЯ TXT ===
+        function downloadFileAsTxt(content, filename) {
+            const blob = new Blob([content], { type: 'text/plain' });
+            const url = URL.createObjectURL(blob);
+            
+            const a = document.createElement('a');
+            a.href = url;
+            a.download = filename;
+            
+            document.body.appendChild(a);
+            a.click();
+            
+            document.body.removeChild(a);
+            URL.revokeObjectURL(url);
+        }
+    </script>
+</body>
+</html>