Merge pull request '0.2.0b' (#1) from dev into main

Reviewed-on: #1

.gitignore

@@ -7,6 +7,7 @@ tmp/
 config_gb.json
 migration.py
 change_log.txt
+up/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

button_script.user.js

@@ -95,7 +95,7 @@ if (originalImageLinkElement) {
             // Получаем настроенный пользователем корневой путь
             const apiRoot = GM_getValue('nyakonya_gb_api_root', 'http://127.0.2.1');
             // Формируем корректный URL для запроса к dw_api
-            const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/dw_api`;
+            const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/api`;
 
             console.log(`Отправка URL на локальный API: ${imageUrl} (Endpoint: ${apiUrl})`);
             const originalOutline = newDownloadButton.style.outline;

config.json

@@ -16,6 +16,7 @@
     "file":{
         "repos":["data/files"],
         "dw":"data/files",
-        "thumb":"data/thumb"
+        "thumb":"data/thumb",
+        "data":"data"
     }
 }

doc/change_log/2b_en.md

@@ -0,0 +1,29 @@
+### 0.2.0b
+
+> **Fixes**
+
+1. Added 404 error handling when the source file is not found.
+
+> **Changes**
+
+1. Merged `/dw_api` and `/api` into a single `/api` endpoint.
+
+> **New**
+
+1. Added a system module for background tasks.
+2. Added an analytics page.
+
+> **Code**
+
+1. Deleted the legacy `db_old.py` file.
+2. Sorted and refactored imports.
+3. Added type hinting for input data in `system_module` functions.
+4. Removed `data/analytic.json`.
+5. Updated the logic for retrieving popular tags to support analytics.
+    - 5.1. Renamed `get_autocomplete` function to `get_top_tags`.
+    - 5.2. Added an optional `limit` argument to the `get_top_tags` function.
+6. Removed the `get_binary_file()` function from the `gelbooru` module as it was unused.
+7. Refactored the logging module.
+8. Added comprehensive error logging.
+9. Added the `/task` endpoint.
+10. Implemented new logic for passing the database path via the configuration file.

doc/change_log/2b_ru.md

@@ -0,0 +1,29 @@
+### 0.2.0b
+
+> **Иправления**
+
+1. Добавлен вывод 404 если исходный файл не найден
+
+> **Изменения**
+
+1. /dw_api и /api объединены в /api
+
+> **Новое**
+
+1. Системный модуль с задачами
+2. Страница с аналитикой
+
+> **Код**
+
+1. Удалён старый файл db_old.py
+2. Отсортированы и изменены импорты
+3. Добавлена типизация входных данных в функции расположенные в system_module
+4. Удалён data/analytic.json
+5. Изменена логика получения списка популярных тегов для адаптации к аналитике
+    - 5.1. Функция get_autocomplete переименована в get_top_tag
+    - 5.1. Для функции get_top_tag добален необязательный аргумент для лимита выдачи
+6. Функция get_binary_file() из модуля gelbooru удалена из-за отсусвия факта использования
+7. Переписан модуль логов
+8. Добавлено полное логирование ошибок
+9. Добавлен эндпойнт /task
+10. Новая логика передачи пути к базе данных через конфиг

doc/install_en.md

@@ -53,6 +53,7 @@ Config explanation:
 | `file`-`repos` | `["data/files"]` | List of directories where source files are stored |
 | `file`-`dw` | `"data/files"` | Directory for new downloads |
 | `file`-`thumb` | `"data/thumb"` | Directory for thumbnails |
+| `file`-`data` | `"data"` | Directory for database |
 
 1. You must obtain your Gelbooru User ID and Hash and add them to the config file for the system to function correctly.
 2. Install the script in [VioletMonkey](https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js).

doc/install_ru.md

@@ -54,6 +54,7 @@
 | `file`-`repos` | `["data/files"]` | Список каталогов где храняться полные файлы |
 | `file`-`dw` | `"data/files"` | Каталог хранения загрузок |
 | `file`-`thumb` | `"data/thumb"` | Каталог обложек |
+| `file`-`data` | `"data"` | Каталог базы данных |
 
 1. Обязательно надо получить ID и Hash пользователя и внести в конфиг для работы!
 

run.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 import requests
 from flask import (Flask, jsonify, render_template, render_template_string,
-                    request, send_from_directory, make_response)
+                    request, send_from_directory, make_response, send_file)
 
 from system_module import db, gelbooru, image_processor, logger, tasks
 
@@ -26,23 +26,15 @@ GB_HEADERS = CONFIG_ROOT["gb"]["headers"]
 FILE_FOLDERS = CONFIG_ROOT["file"]["repos"]
 FILE_DW_FOLDERS = CONFIG_ROOT["file"]["dw"]
 THUMB_FOLDER = CONFIG_ROOT["file"]["thumb"]
+DATA_FOLDER = CONFIG_ROOT["file"]["data"]
 
 
 gb = gelbooru.GB(GB_ID, GB_HASH, GB_HEADERS, FILE_DW_FOLDERS)
 log = logger.Logger()
 log.init_global_exception_hook()
-database = db.DB(True)
+database = db.DB(DATA_FOLDER)
 app = Flask(__name__)
 
-def find_file(filename, file_folders):
-    for folder in file_folders:
-        folder_path = Path(folder)
-        if not folder_path.exists():
-            continue
-        for file_path in folder_path.rglob(filename):
-            return folder_path
-    return None
-
 if not image_processor.check_ffmpeg(): print("WARNING!\n\nУ вас не установлен ffmpeg. Генерация превью для видео не доступна!")
 
 @app.route("/")
@@ -63,7 +55,8 @@ def post(post_id):
 
 @app.route('/file/<path:filename>')
 def serve_image(filename):
-    folder_path = find_file(filename, FILE_FOLDERS)
+    folder_path = tasks.find_file(filename, FILE_FOLDERS)
+    print(filename, FILE_FOLDERS)
     if folder_path == None:
         return make_response("Not found", 404)
     else:
@@ -100,12 +93,17 @@ def api():
         post_id = data["post_id"]
         result_post = database.get_from_id(post_id)
         return jsonify(result_post)
+    elif data["type"] == "analytic":
+        return send_file(os.path.join(DATA_FOLDER, "analytic.json"), mimetype="application/json")
     elif data["type"] == "task":
         if data["task"] == "analytic":
-            if tasks.calculate_analytic():
+            if tasks.calculate_analytic(database, CONFIG_ROOT):
                 return "", 200
             else:
                 return "", 500
+        if data["task"] == "not_found_file":
+            list_md5_not_found = tasks.get_files_not_found(database, CONFIG_ROOT)
+            return jsonify({'list': list_md5_not_found}), 200
     elif data["type"] == "download":
         if not data or 'id' not in data:
             return jsonify({'status': 'error', 'message': 'URL not get'}), 400

system_module/db.py

@@ -7,11 +7,7 @@ from system_module.gelbooru import gelbooru_date_parse
 
 
 class DB:
-    def __init__(self, debug: bool):
+    def __init__(self, db_path: str):
-        if debug: 
-            db_path = "data_test" 
-        else: 
-            db_path = "data"
             
         if not os.path.exists(db_path):
             os.makedirs(db_path)

system_module/tasks.py

@@ -1,2 +1,224 @@
-def calculate_analytic():
+import os
-    pass
+import json
+from pathlib import Path
+
+def find_file(filename: str, file_folders: list[str], is_folder: bool = True):
+    for folder in file_folders:
+        folder_path = Path(folder)
+        if not folder_path.exists():
+            continue
+        for file_path in folder_path.rglob(filename):
+            return str(folder_path) if is_folder else str(file_path)
+    return None
+
+def get_size_format(b: int, factor=1024, suffix="B"):
+    for unit in ["", "K", "M", "G", "T"]:
+        if b < factor:
+            return f"{b:.2f} {unit}{suffix}"
+        b /= factor
+    return f"{b:.2f} P{suffix}"
+
+def get_dir_size(path: str):
+    total = 0
+    if os.path.exists(path):
+        for f in os.listdir(path):
+            fp = os.path.join(path, f)
+            if os.path.isfile(fp):
+                total += os.path.getsize(fp)
+    return total
+
+def get_file_list_size(file_list: list[str]):
+    total = 0
+    for fp in file_list:
+        if os.path.isfile(fp):
+            total += os.path.getsize(fp)
+    return total
+
+def get_size_edge(type_searсh: bool, file_list: list[str]):
+    if type_searсh:
+        result = min(file_list, key=lambda f: Path(f).stat().st_size)
+    else:
+        result = max(file_list, key=lambda f: Path(f).stat().st_size)
+    return str(result)
+
+def get_files_not_found(database, config):
+    all_db = database.get_all()
+    list_md5 = []
+    for one_post in all_db:
+        file_path = find_file(one_post["local_filename"], config["file"]["repos"], is_folder=False)
+        if file_path == None:
+            list_md5.append(one_post["md5"])
+    return list_md5
+
+def calculate_analytic(database, config):
+    all_db = database.get_all()
+    explicit_db = database.get_search(["e"], [])
+    sensitive_db = database.get_search(["s"], [])
+    questionable_db = database.get_search(["q"], [])
+    general_db = database.get_search(["g"], [])
+    rating_list_file_path = {"explicit":[], "sensitive":[], "questionable":[], "general":[]}
+    type_list_file_path = {"images":[], "videos":[], "gifs":[]}
+    files_not_found = 0
+    for one_post in all_db:
+        file_path = find_file(one_post["local_filename"], config["file"]["repos"], is_folder=False)
+        if file_path != None:
+            file_path = str(file_path)
+            rating_list_file_path[one_post["rating"]].append(file_path)
+            if one_post["is_gif"] == 1:
+                type_list_file_path["gifs"].append(file_path)
+            elif one_post["is_video"] == 1:
+                type_list_file_path["videos"].append(file_path)
+            else:
+                type_list_file_path["images"].append(file_path)
+        else:
+            files_not_found += 1
+    file_size_dict = {"explicit":get_file_list_size(rating_list_file_path["explicit"]), 
+                        "sensitive":get_file_list_size(rating_list_file_path["sensitive"]), 
+                        "questionable":get_file_list_size(rating_list_file_path["questionable"]), 
+                        "general":get_file_list_size(rating_list_file_path["general"])
+                    }
+    type_size_dict = {"images":get_file_list_size(type_list_file_path["images"]),
+                        "videos":get_file_list_size(type_list_file_path["videos"]), 
+                        "gifs":get_file_list_size(type_list_file_path["gifs"])}
+    ratings_stats = {
+        "explicit": {
+            "count": len(explicit_db),
+            "size_formatted": get_size_format(file_size_dict["explicit"])
+        },
+        "sensitive": {
+            "count": len(sensitive_db),
+            "size_formatted": get_size_format(file_size_dict["sensitive"])
+        },
+        "questionable": {
+            "count": len(questionable_db),
+            "size_formatted": get_size_format(file_size_dict["questionable"])
+        },
+        "general": {
+            "count": len(general_db),
+            "size_formatted": get_size_format(file_size_dict["general"])
+        }
+    }
+    total_file_data_size = file_size_dict["general"]+file_size_dict["questionable"]+file_size_dict["sensitive"]+file_size_dict["explicit"]
+    total_thumb_size = get_dir_size(config["file"]["thumb"])
+    total_log_size = get_file_list_size(["logs.log"])
+    total_db_size = get_file_list_size([os.path.join(config["file"]["data"], "db.sqlite")])
+    weight_stats_smallest_file = get_size_edge(True, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"])
+    weight_stats_largest_file = get_size_edge(False, type_list_file_path["gifs"]+type_list_file_path["images"]+type_list_file_path["videos"]) 
+    weight_stats_data = {
+        "smallest":{
+            "file":"",
+            "md5":"",
+            "size":0
+        },
+        "largest":{
+            "file":"",
+            "md5":"",
+            "size":0
+        }
+    }
+    resolution_stats_data = {
+        "smallest":{
+            "file":"",
+            "md5":"",
+            "res":{
+                "width":0,
+                "height":0,
+                "total":99999999999999999999999999
+            }
+        },
+        "largest":{
+            "file":"",
+            "md5":"",
+            "res":{
+                "width":0,
+                "height":0,
+                "total":0
+            }
+        }
+    }
+    for one_post_scan in all_db:
+        if one_post_scan["local_filename"] == os.path.basename(weight_stats_smallest_file):
+            weight_stats_data["smallest"] = {
+                "file":os.path.basename(weight_stats_smallest_file),
+                "md5":one_post_scan["md5"],
+                "size":get_file_list_size([weight_stats_smallest_file])
+            }
+        if one_post_scan["local_filename"] == os.path.basename(weight_stats_largest_file):
+            weight_stats_data["largest"] = {
+                "file":os.path.basename(weight_stats_largest_file),
+                "md5":one_post_scan["md5"],
+                "size":get_file_list_size([weight_stats_largest_file])
+            }
+            print(get_file_list_size([os.path.basename(weight_stats_largest_file)]))
+        one_post_res_total = one_post_scan["width"]*one_post_scan["height"]
+        if resolution_stats_data["largest"]["res"]["total"] < one_post_res_total:
+            resolution_stats_data["largest"] = {
+                "file":one_post_scan["local_filename"],
+                "md5":one_post_scan["md5"],
+                "res":{
+                    "width":one_post_scan["width"],
+                    "height":one_post_scan["height"],
+                    "total":one_post_res_total
+                }
+            }
+        if resolution_stats_data["smallest"]["res"]["total"] > one_post_res_total:
+            resolution_stats_data["smallest"] = {
+                "file":one_post_scan["local_filename"],
+                "md5":one_post_scan["md5"],
+                "res":{
+                    "width":one_post_scan["width"],
+                    "height":one_post_scan["height"],
+                    "total":one_post_res_total
+                }
+            }
+    result = {
+        "total_images_analyzed": len(all_db),
+        "files_not_found":files_not_found,
+        "average_weight": {
+            "bytes": round(total_file_data_size/(len(rating_list_file_path["explicit"])+len(rating_list_file_path["sensitive"])+len(rating_list_file_path["questionable"])+len(rating_list_file_path["general"])), 2),
+            "formatted": get_size_format(round(total_file_data_size/(len(rating_list_file_path["explicit"])+len(rating_list_file_path["sensitive"])+len(rating_list_file_path["questionable"])+len(rating_list_file_path["general"])), 2))
+        },
+        "storage_stats": {
+            "folders": {
+                "data": get_size_format(total_file_data_size),
+                "db": get_size_format(total_db_size),
+                "thumb": get_size_format(total_thumb_size),
+                "logs": get_size_format(total_log_size)
+            },
+            "types": {
+                "images": get_size_format(type_size_dict["images"]),
+                "videos": get_size_format(type_size_dict["videos"]),
+                "gifs": get_size_format(type_size_dict["gifs"])
+            },
+            "grand_total": get_size_format(total_file_data_size+total_thumb_size+total_log_size+total_db_size)
+        },
+        "ratings_stats": ratings_stats,
+        "resolution_stats": {
+            "largest": {
+                "filename": resolution_stats_data["largest"]["file"],
+                "md5": resolution_stats_data["largest"]["md5"],
+                "resolution": str(resolution_stats_data["largest"]["res"]["width"]) + " x " + str(resolution_stats_data["largest"]["res"]["height"])
+            },
+            "smallest": {
+                "filename": resolution_stats_data["smallest"]["file"],
+                "md5": resolution_stats_data["smallest"]["md5"],
+                "resolution": str(resolution_stats_data["smallest"]["res"]["width"]) + " x " + str(resolution_stats_data["smallest"]["res"]["height"])
+            }
+        },
+        "weight_stats": {
+            "largest": {
+                "filename": os.path.basename(weight_stats_data["largest"]["file"]),
+                "md5": weight_stats_data["largest"]["md5"],
+                "size_formatted": get_size_format(weight_stats_data["largest"]["size"])
+            },
+            "smallest": {
+                "filename": os.path.basename(weight_stats_data["smallest"]["file"]),
+                "md5": weight_stats_data["smallest"]["md5"],
+                "size_formatted": get_size_format(weight_stats_data["smallest"]["size"])
+            }
+        },
+        "top_tag":database.get_top_tag("")
+    }
+    with open(os.path.join(config["file"]["data"], "analytic.json"), 'w', encoding='utf-8') as out_f:
+        json.dump(result, out_f, indent=4, ensure_ascii=False)
+    return True

templates/analytics.html

@@ -41,22 +41,21 @@
         .ext-block a { color: var(--link-color); text-decoration: none; word-break: break-all; }
         .ext-block a:hover { text-decoration: underline; }
 
+        .loader { color: var(--text-muted); font-size: 16px; }
+
         @media (max-width: 1000px) { .dashboard-grid { grid-template-columns: 1fr 1fr; } .left-col { grid-column: 1/-1; display: grid; grid-template-columns: 1fr 1fr; } }
         @media (max-width: 700px) { .dashboard-grid, .left-col { grid-template-columns: 1fr; } header { flex-wrap: wrap; } .search-wrapper { order: 3; min-width: 100%; margin-top: 10px;} }
     </style>
 </head>
 <body>
     <header>
-        <a href="index.html" class="logo">LocalBooru</a>
+        <a href="/" class="logo">LocalBooru</a>
-        <a href="analytics.html" class="analytics-link">📊 Analytics</a>
-        <div class="search-wrapper">
-            <input type="text" id="searchInput" placeholder="Search tags...">
-            <ul id="autocompleteList" class="autocomplete-list"></ul>
-        </div>
     </header>
 
     <main>
         <h1>Database Analytics</h1>
+        <div id="loadingIndicator" class="loader">Fetching data from API...</div>
+        
         <div class="dashboard-grid" id="analyticsGrid" style="display: none;">
             <div class="left-col">
                 <div class="panel">
@@ -68,7 +67,7 @@
                     <div class="data-list" id="storageContainer"></div>
                 </div>
                 <div class="panel">
-                    <div class="panel-header">🎨 Media Types (in img/)</div>
+                    <div class="panel-header">🎨 Media Types</div>
                     <div class="data-list" id="mediaContainer"></div>
                 </div>
                 <div class="panel">
@@ -78,125 +77,119 @@
             </div>
             
             <div class="panel">
-                <div class="panel-header">🏷️ All Tags List</div>
+                <div class="panel-header">🏷️ Top Tags List</div>
                 <div class="table-header"><span>Tag</span><span>Count</span></div>
                 <div class="data-list" id="tagsListContainer"></div>
-                <div class="pagination"><button class="page-btn" onclick="prevTagPage()">&lsaquo;</button><span id="tagPageInfo" style="font-weight:bold; padding:0 10px;"></span><button class="page-btn" onclick="nextTagPage()">&rsaquo;</button></div>
+                <div class="pagination">
+                    <button class="page-btn" onclick="prevTagPage()">&lsaquo;</button>
+                    <span id="tagPageInfo" style="font-weight:bold; padding:0 10px;"></span>
+                    <button class="page-btn" onclick="nextTagPage()">&rsaquo;</button>
+                </div>
             </div>
             
+            <!-- Заменено, так как в новом JSON нет top_pairs -->
             <div class="panel">
-                <div class="panel-header">🔗 Top 30 Tag Pairs</div>
+                <div class="panel-header">ℹ️ System Overview</div>
-                <div class="table-header"><span>Pair</span><span>Count</span></div>
+                <div class="data-list" id="overviewContainer"></div>
-                <div class="data-list" id="pairsContainer"></div>
             </div>
         </div>
     </main>
 
     <script>
-        const searchInput = document.getElementById('searchInput');
-        const list = document.getElementById('autocompleteList');
         let debounceTimer;
-
-        searchInput.addEventListener('input', () => {
-            clearTimeout(debounceTimer);
-            const words = searchInput.value.split(' ');
-            const currentWord = words[words.length - 1];
-            if (currentWord.length < 2) { list.style.display = 'none'; return; }
-
-            debounceTimer = setTimeout(async () => {
-                const req = { type: "autocomplete", q: currentWord };
-                const data = await mockAutocompleteAPI(req); 
-                
-                if (data.length === 0) { list.style.display = 'none'; return; }
-
-                list.innerHTML = data.map(item => `<li class="autocomplete-item" data-tag="${item.tag}"><span>${item.tag}</span><span class="autocomplete-count">${item.count}</span></li>`).join('');
-                list.style.display = 'block';
-
-                list.querySelectorAll('.autocomplete-item').forEach(li => {
-                    li.addEventListener('click', () => {
-                        words[words.length - 1] = li.getAttribute('data-tag');
-                        searchInput.value = words.join(' ') + ' ';
-                        list.style.display = 'none';
-                        searchInput.focus();
-                    });
-                });
-            }, 200);
-        });
-
-        document.addEventListener('click', (e) => { if (e.target !== searchInput && e.target !== list) list.style.display = 'none'; });
-        searchInput.addEventListener('keypress', (e) => { if(e.key === 'Enter') window.location.href = `index.html?query=${searchInput.value.trim()}`; });
-
         let allTags = [], currentTagPage = 1, tagsPerPage = 22;
 
+        // Укорачиваем длинные имена файлов (хэши) для красивого отображения
+        const shortenName = (name) => name.length > 20 ? name.substring(0, 10) + '...' + name.substring(name.length - 6) : name;
+
         async function fetchAnalytics() {
-            const data = await mockAnalyticsAPI({ type: "analytics" });
+            try {
-            
+                const response = await fetch('/api', {
-            document.getElementById('ratingContainer').innerHTML = Object.keys(data.ratings).map(k => `
+                    method: 'POST',
-                <div class="rating-box" style="border-top: 2px solid var(--color-${k[0]})">
+                    headers: { 'Content-Type': 'application/json' },
-                    <div style="font-weight:bold; color:var(--color-${k[0]}); margin-bottom:5px;">${k.charAt(0).toUpperCase() + k.slice(1)}</div>
+                    body: JSON.stringify({ type: "analytic" })
-                    <div style="color:var(--text-muted); margin-bottom:3px;">${data.ratings[k].count} items</div>
+                });
-                    <div style="font-weight:bold;">${data.ratings[k].size}</div>
-                </div>
-            `).join('');
 
-            const s = data.storage;
+                if (!response.ok) throw new Error("Network response was not ok");
-            document.getElementById('storageContainer').innerHTML = `
+                
-                <div class="data-row"><span>Logs:</span><span class="val">${s.logs}</span></div>
+                const data = await response.json();
-                <div class="data-row"><span>DB:</span><span class="val">${s.db}</span></div>
+                
-                <div class="data-row"><span>Thumb:</span><span class="val">${s.thumb}</span></div>
+                // Скрываем лоадер, показываем сетку
-                <div class="data-row"><span>Img:</span><span class="val">${s.main}</span></div>
+                document.getElementById('loadingIndicator').style.display = 'none';
-                <div class="data-row" style="color:var(--color-g); font-weight:bold; border-top:1px solid #444; border-bottom:none; padding-top:8px;"><span>GRAND TOTAL:</span><span>${s.total}</span></div>
+                document.getElementById('analyticsGrid').style.display = 'grid';
-            `;
 
-            const m = data.media;
+                // 1. Рейтинги
-            document.getElementById('mediaContainer').innerHTML = `
+                const rStats = data.ratings_stats;
-                <div class="data-row"><span>IMG:</span><span class="val">${m.img_size}</span></div>
+                document.getElementById('ratingContainer').innerHTML = Object.keys(rStats).map(k => `
-                <div class="data-row"><span>VID:</span><span class="val">${m.vid_size}</span></div>
+                    <div class="rating-box" style="border-top: 2px solid var(--color-${k[0]})">
-                <div class="data-row"><span>GIF:</span><span class="val">${m.gif_size}</span></div>
+                        <div style="font-weight:bold; color:var(--color-${k[0]}); margin-bottom:5px;">${k.charAt(0).toUpperCase() + k.slice(1)}</div>
-                <div style="font-size:11px; color:var(--text-muted); margin-top:5px;">Total: <b>${m.total_items}</b> | Avg: <b>${m.avg_weight}</b></div>
+                        <div style="color:var(--text-muted); margin-bottom:3px;">${rStats[k].count} items</div>
-            `;
+                        <div style="font-weight:bold;">${rStats[k].size_formatted}</div>
+                    </div>
+                `).join('');
 
-            const e = data.extremes;
+                // 2. Хранилище (Storage)
-            document.getElementById('extremesContainer').innerHTML = `
+                const s = data.storage_stats.folders;
-                <div class="ext-block"><b>Largest:</b> ${e.largest.val}<br>(<a href="post.html?id=${e.largest.id}">${e.largest.file}</a>)</div>
+                document.getElementById('storageContainer').innerHTML = `
-                <div class="ext-block"><b>Smallest:</b> ${e.smallest.val}<br>(<a href="post.html?id=${e.smallest.id}">${e.smallest.file}</a>)</div>
+                    <div class="data-row"><span>Logs:</span><span class="val">${s.logs}</span></div>
-                <div class="ext-block"><b>Heaviest:</b> ${e.heaviest.val}<br>(<a href="post.html?id=${e.heaviest.id}">${e.heaviest.file}</a>)</div>
+                    <div class="data-row"><span>DB:</span><span class="val">${s.db}</span></div>
-                <div class="ext-block"><b>Lightest:</b> ${e.lightest.val}<br>(<a href="post.html?id=${e.lightest.id}">${e.lightest.file}</a>)</div>
+                    <div class="data-row"><span>Thumb:</span><span class="val">${s.thumb}</span></div>
-            `;
+                    <div class="data-row"><span>Data (Img):</span><span class="val">${s.data}</span></div>
+                    <div class="data-row" style="color:var(--color-g); font-weight:bold; border-top:1px solid #444; border-bottom:none; padding-top:8px;"><span>GRAND TOTAL:</span><span>${data.storage_stats.grand_total}</span></div>
+                `;
 
-            document.getElementById('pairsContainer').innerHTML = data.top_pairs.map(p => `<div class="tag-row"><a href="index.html?query=${p.pair.replace(/ \+ /g, '+')}">${p.pair}</a> <span>${p.count}</span></div>`).join('');
+                // 3. Медиа типы
+                const m = data.storage_stats.types;
+                document.getElementById('mediaContainer').innerHTML = `
+                    <div class="data-row"><span>IMG:</span><span class="val">${m.images}</span></div>
+                    <div class="data-row"><span>VID:</span><span class="val">${m.videos}</span></div>
+                    <div class="data-row"><span>GIF:</span><span class="val">${m.gifs}</span></div>
+                    <div style="font-size:11px; color:var(--text-muted); margin-top:5px;">Total: <b>${data.total_images_analyzed}</b> | Avg: <b>${data.average_weight.formatted}</b></div>
+                `;
 
-            allTags = data.all_tags; renderTagsPage();
+                // 4. Рекорды (Extremes)
-            document.getElementById('analyticsGrid').style.display = 'grid';
+                const res = data.resolution_stats;
+                const weight = data.weight_stats;
+                document.getElementById('extremesContainer').innerHTML = `
+                    <div class="ext-block"><b>Largest Res:</b> ${res.largest.resolution}<br>(<a href="post/${res.largest.md5}">${shortenName(res.largest.filename)}</a>)</div>
+                    <div class="ext-block"><b>Smallest Res:</b> ${res.smallest.resolution}<br>(<a href="post/${res.smallest.md5}">${shortenName(res.smallest.filename)}</a>)</div>
+                    <div class="ext-block"><b>Heaviest:</b> ${weight.largest.size_formatted}<br>(<a href="post/${weight.largest.md5}">${shortenName(weight.largest.filename)}</a>)</div>
+                    <div class="ext-block"><b>Lightest:</b> ${weight.smallest.size_formatted}<br>(<a href="post/${weight.smallest.md5}">${shortenName(weight.smallest.filename)}</a>)</div>
+                `;
+
+                // 5. System Overview (вместо Top Pairs)
+                document.getElementById('overviewContainer').innerHTML = `
+                    <div class="data-row"><span>Images Analyzed:</span><span class="val" style="font-weight:bold;">${data.total_images_analyzed}</span></div>
+                    <div class="data-row"><span>Files Not Found:</span><span class="val" style="font-weight:bold; color: ${data.files_not_found > 0 ? 'var(--color-e)' : 'inherit'};">${data.files_not_found}</span></div>
+                `;
+
+                // 6. Теги
+                allTags = data.top_tag; 
+                renderTagsPage();
+
+            } catch (error) {
+                console.error('Error fetching analytics:', error);
+                document.getElementById('loadingIndicator').innerHTML = `<span style="color: var(--color-e)">Failed to load data. See console for details.</span>`;
+            }
         }
 
+        // Пагинация тегов
         function renderTagsPage() {
-            document.getElementById('tagPageInfo').innerText = `${currentTagPage} / ${Math.ceil(allTags.length / tagsPerPage)}`;
+            if (!allTags || allTags.length === 0) return;
+            
+            const totalPages = Math.ceil(allTags.length / tagsPerPage) || 1;
+            document.getElementById('tagPageInfo').innerText = `${currentTagPage} / ${totalPages}`;
+            
             const start = (currentTagPage - 1) * tagsPerPage;
-            document.getElementById('tagsListContainer').innerHTML = allTags.slice(start, start + tagsPerPage).map(t => `<div class="tag-row"><a href="index.html?query=${t.name}">${t.name}</a> <span>${t.count}</span></div>`).join('');
+            document.getElementById('tagsListContainer').innerHTML = allTags.slice(start, start + tagsPerPage).map(t => 
+                `<div class="tag-row"><a href="index.html?query=${encodeURIComponent(t.tag)}">${t.tag}</a> <span>${t.count}</span></div>`
+            ).join('');
         }
+
         window.prevTagPage = () => { if(currentTagPage > 1) { currentTagPage--; renderTagsPage(); }};
         window.nextTagPage = () => { if(currentTagPage < Math.ceil(allTags.length/tagsPerPage)) { currentTagPage++; renderTagsPage(); }};
 
-        // --- ЗАГЛУШКИ ---
+        // Запуск
-        function mockAutocompleteAPI(req) { return new Promise(res => { const db = [{tag: "long_hair", count: 8184}, {tag: "1girl", count: 11061}]; res(db.filter(t => t.tag.includes(req.q.toLowerCase()))); }); }
-        
-        function mockAnalyticsAPI() { 
-            return new Promise(res => { setTimeout(() => { res({
-                ratings: { explicit: { count: 1605, size: "4.52 GB" }, sensitive: { count: 6623, size: "19.35 GB" }, questionable: { count: 1591, size: "4.72 GB" }, general: { count: 4184, size: "10.29 GB" } },
-                storage: { logs: "14.44 MB", db: "24.39 MB", thumb: "331.45 MB", main: "38.88 GB", total: "39.24 GB" },
-                media: { img_size: "38.56 GB", vid_size: "140.80 MB", gif_size: "181.30 MB", total_items: 14003, avg_weight: "2.84 MB" },
-                extremes: {
-                    largest: { val: "9921x14031", id: "5a63c", file: "5a63c28906dc7e0e...png" },
-                    smallest: { val: "240x280", id: "8fc61", file: "8fc614e712ae2ad...gif" },
-                    heaviest: { val: "65.21 MB", id: "4ab6a", file: "4ab6acf0a32c61...png" },
-                    lightest: { val: "6.49 KB", id: "28c67", file: "28c679ea3483e1...png" }
-                },
-                all_tags: Array.from({length: 100}, (_,i) => ({name:`tag_${i}`, count:100-i})),
-                top_pairs: [{ pair: "1girl + highres", count: 10236 }, { pair: "1girl + solo", count: 8902 }]
-            }); }, 300);}); 
-        }
-
         fetchAnalytics();
     </script>
 </body>

templates/index.html

@@ -64,9 +64,7 @@
 
     <header>
         <a href="/" class="logo">LocalBooru</a>
-        <!--
         <a class="analytics" href="/analytics">📊 Analytics</a>
-        -->
         <a class="tasks" href="/task">⚙️ Tasks</a>
         <div class="search-wrapper">
             <input type="text" id="searchInput" placeholder="Search tags...">

templates/task.html

@@ -17,20 +17,18 @@
         /* Search Autocomplete Styles */
         .search-wrapper { flex-grow: 1; max-width: 400px; position: relative; }
         .search-wrapper input { width: 100%; background-color: var(--bg-input); border: 1px solid var(--border-color); color: white; padding: 6px 10px; border-radius: 3px; outline: none; }
-        .autocomplete-list { position: absolute; top: 100%; left: 0; right: 0; background-color: var(--bg-box); border: 1px solid var(--border-color); border-radius: 3px; margin-top: 4px; list-style: none; z-index: 1000; max-height: 250px; overflow-y: auto; box-shadow: 0 4px 10px rgba(0,0,0,0.5); display: none; }
+        
-        .autocomplete-item { padding: 8px 10px; cursor: pointer; display: flex; justify-content: space-between; border-bottom: 1px solid #333; }
-        .autocomplete-item:hover { background-color: var(--bg-input); }
-        .autocomplete-count { color: var(--text-muted); font-size: 12px; }
-
         /* Main Content & Tasks Styles */
         main { padding: 20px; max-width: 1000px; margin: 0 auto; }
         h1 { font-size: 20px; margin-bottom: 20px; color: white; border-bottom: 1px solid var(--border-color); padding-bottom: 10px; }
         
         .task-list { display: flex; flex-direction: column; gap: 15px; }
         
-        .task-card { background-color: var(--bg-panel); border: 1px solid var(--border-color); border-radius: 4px; padding: 15px 20px; display: flex; justify-content: space-between; align-items: center; gap: 20px; transition: background-color 0.2s; }
+        /* Добавил flex-wrap: wrap чтобы результаты могли падать на новую строку */
+        .task-card { background-color: var(--bg-panel); border: 1px solid var(--border-color); border-radius: 4px; padding: 15px 20px; display: flex; justify-content: space-between; align-items: center; gap: 20px; flex-wrap: wrap; transition: background-color 0.2s; }
         .task-card:hover { background-color: var(--bg-box); }
         
+        .task-info { flex: 1; min-width: 250px; }
         .task-info h3 { font-size: 16px; color: var(--link-color); margin-bottom: 5px; }
         .task-info p { color: var(--text-muted); font-size: 13px; line-height: 1.4; }
         
@@ -44,6 +42,10 @@
         .task-status { font-weight: bold; font-size: 13px; min-width: 90px; text-align: right; }
         .status-success { color: var(--color-g); }
         .status-error { color: var(--color-e); }
+        
+        /* Стили для вывода результатов */
+        .task-result-box { width: 100%; margin-top: 10px; border-top: 1px solid var(--border-color); padding-top: 15px; display: none; }
+        .task-result-box textarea { width: 100%; height: 150px; background-color: var(--bg-input); color: var(--text-main); border: 1px solid var(--border-color); border-radius: 3px; padding: 10px; margin-top: 10px; resize: vertical; outline: none; font-family: monospace; }
 
         @media (max-width: 700px) { 
             header { flex-wrap: wrap; } 
@@ -56,10 +58,6 @@
 <body>
     <header>
         <a href="/" class="logo">LocalBooru</a>
-        <div class="search-wrapper">
-            <input type="text" id="searchInput" placeholder="Search tags...">
-            <ul id="autocompleteList" class="autocomplete-list"></ul>
-        </div>
     </header>
 
     <main>
@@ -79,48 +77,46 @@
                 </div>
             </div>
 
-            <!-- ШАБЛОН ДЛЯ БУДУЩИХ ЗАДАЧ (просто скопируйте блок ниже и измените 'task_name') -->
+            <!-- ЗАДАЧА 2: Поиск недоступных файлов -->
-            <!--
             <div class="task-card">
                 <div class="task-info">
-                    <h3>Новая задача</h3>
+                    <h3>Поиск недоступных файлов</h3>
-                    <p>Описание новой задачи.</p>
+                    <p>Ищет посты, для которых отсутствуют физические файлы на диске. Выводит список ссылок на экран и предлагает скачать их в формате .txt.</p>
                 </div>
                 <div class="task-actions">
                     <span class="task-status"></span>
-                    <button class="btn" onclick="executeTask('new_task_name', this)">Запустить</button>
+                    <button class="btn" onclick="executeNotFoundFilesTask(this)">Запустить</button>
+                </div>
+                <!-- Блок вывода результатов (изначально скрыт) -->
+                <div class="task-result-box" id="result-box-not-found">
+                    <div style="display: flex; justify-content: space-between; align-items: center;">
+                        <span>Найдено битых ссылок: <b id="not-found-count" style="color: var(--color-e);">0</b></span>
+                        <button class="btn" id="btn-download-txt" style="display: none;">Скачать .txt</button>
+                    </div>
+                    <textarea id="not-found-textarea" readonly></textarea>
                 </div>
             </div>
-            -->
 
         </div>
     </main>
 
     <script>
-        // === ЛОГИКА ЗАДАЧ ===
+        // === БАЗОВАЯ ЛОГИКА ЗАДАЧ (БЕЗ ВОЗВРАТА ДАННЫХ) ===
         async function executeTask(taskName, btn) {
             const statusSpan = btn.previousElementSibling;
             
-            // Состояние загрузки
             btn.disabled = true;
             btn.innerText = "Выполнение...";
             statusSpan.innerText = "В процессе ⏳";
-            statusSpan.className = "task-status"; // сброс классов успеха/ошибки
+            statusSpan.className = "task-status"; 
 
             try {
-                // Отправляем POST запрос на API
                 const response = await fetch('/api', {
                     method: 'POST',
-                    headers: {
+                    headers: { 'Content-Type': 'application/json' },
-                        'Content-Type': 'application/json'
+                    body: JSON.stringify({ type: "task", task: taskName })
-                    },
-                    body: JSON.stringify({
-                        type: "task",
-                        task: taskName
-                    })
                 });
 
-                // Обработка ответа (200 - успех, 500+ - ошибка)
                 if (response.ok) {
                     statusSpan.innerText = "Успешно ✔";
                     statusSpan.classList.add('status-success');
@@ -129,54 +125,100 @@
                     statusSpan.classList.add('status-error');
                 }
             } catch (error) {
-                // Ошибка сети или сервер недоступен
                 statusSpan.innerText = "Сбой сети ✖";
                 statusSpan.classList.add('status-error');
                 console.error("Task execution error:", error);
             } finally {
-                // Возвращаем кнопку в исходное состояние
                 btn.disabled = false;
                 btn.innerText = "Запустить";
             }
         }
 
+        // === ЛОГИКА ЗАДАЧИ: Поиск недоступных файлов ===
+        async function executeNotFoundFilesTask(btn) {
+            const statusSpan = btn.previousElementSibling;
+            
+            // Элементы интерфейса для вывода
+            const resultBox = document.getElementById('result-box-not-found');
+            const textarea = document.getElementById('not-found-textarea');
+            const countSpan = document.getElementById('not-found-count');
+            const downloadBtn = document.getElementById('btn-download-txt');
+            
+            btn.disabled = true;
+            btn.innerText = "Выполнение...";
+            statusSpan.innerText = "В процессе ⏳";
+            statusSpan.className = "task-status"; 
+            
+            // Скрываем прошлый результат перед новым запуском
+            resultBox.style.display = 'none';
 
-        // === ЛОГИКА ПОИСКА (оставлена из предыдущей версии) ===
+            try {
-        const searchInput = document.getElementById('searchInput');
+                const response = await fetch('/api', {
-        const list = document.getElementById('autocompleteList');
+                    method: 'POST',
-        let debounceTimer;
+                    headers: { 'Content-Type': 'application/json' },
-
+                    body: JSON.stringify({ type: "task", task: "not_found_file" })
-        searchInput.addEventListener('input', () => {
-            clearTimeout(debounceTimer);
-            const words = searchInput.value.split(' ');
-            const currentWord = words[words.length - 1];
-            if (currentWord.length < 2) { list.style.display = 'none'; return; }
-
-            debounceTimer = setTimeout(async () => {
-                const req = { type: "autocomplete", q: currentWord };
-                const data = await mockAutocompleteAPI(req); 
-                
-                if (data.length === 0) { list.style.display = 'none'; return; }
-
-                list.innerHTML = data.map(item => `<li class="autocomplete-item" data-tag="${item.tag}"><span>${item.tag}</span><span class="autocomplete-count">${item.count}</span></li>`).join('');
-                list.style.display = 'block';
-
-                list.querySelectorAll('.autocomplete-item').forEach(li => {
-                    li.addEventListener('click', () => {
-                        words[words.length - 1] = li.getAttribute('data-tag');
-                        searchInput.value = words.join(' ') + ' ';
-                        list.style.display = 'none';
-                        searchInput.focus();
-                    });
                 });
-            }, 200);
-        });
 
-        document.addEventListener('click', (e) => { if (e.target !== searchInput && e.target !== list) list.style.display = 'none'; });
+                if (response.ok) {
-        searchInput.addEventListener('keypress', (e) => { if(e.key === 'Enter') window.location.href = `index.html?query=${searchInput.value.trim()}`; });
+                    const data = await response.json();
+                    const md5List = data.list || [];
+                    
+                    statusSpan.innerText = "Успешно ✔";
+                    statusSpan.classList.add('status-success');
 
-        // Заглушка для поиска (оставьте или замените на ваш fetch)
+                    if (md5List.length > 0) {
-        function mockAutocompleteAPI(req) { return new Promise(res => { const db = [{tag: "long_hair", count: 8184}, {tag: "1girl", count: 11061}]; res(db.filter(t => t.tag.includes(req.q.toLowerCase()))); }); }
+                        // Формируем ссылки
+                        const linksArray = md5List.map(md5 => `/post/${md5}`);
+                        const linksText = linksArray.join('\n');
+
+                        // Выводим на экран
+                        countSpan.innerText = md5List.length;
+                        textarea.value = linksText;
+                        resultBox.style.display = 'block';
+                        
+                        // Настраиваем скачивание
+                        downloadBtn.style.display = 'block';
+                        downloadBtn.onclick = () => downloadFileAsTxt(linksText, 'not_found_files.txt');
+                        
+                        // Сразу инициируем скачивание
+                        downloadFileAsTxt(linksText, 'not_found_files.txt');
+                    } else {
+                        // Если битых файлов нет
+                        countSpan.innerText = '0';
+                        textarea.value = "Недоступных файлов не обнаружено.";
+                        downloadBtn.style.display = 'none';
+                        resultBox.style.display = 'block';
+                    }
+
+                } else {
+                    statusSpan.innerText = `Ошибка (${response.status}) ✖`;
+                    statusSpan.classList.add('status-error');
+                }
+            } catch (error) {
+                statusSpan.innerText = "Сбой сети ✖";
+                statusSpan.classList.add('status-error');
+                console.error("Task execution error:", error);
+            } finally {
+                btn.disabled = false;
+                btn.innerText = "Запустить";
+            }
+        }
+
+        // === ВСПОМОГАТЕЛЬНАЯ ФУНКЦИЯ ДЛЯ СКАЧИВАНИЯ TXT ===
+        function downloadFileAsTxt(content, filename) {
+            const blob = new Blob([content], { type: 'text/plain' });
+            const url = URL.createObjectURL(blob);
+            
+            const a = document.createElement('a');
+            a.href = url;
+            a.download = filename;
+            
+            document.body.appendChild(a);
+            a.click();
+            
+            document.body.removeChild(a);
+            URL.revokeObjectURL(url);
+        }
     </script>
 </body>
 </html>

updater/config.json

@@ -1,5 +0,0 @@
-{
-    "host":"https://git.nekono.su",
-    "user":"Nyako",
-    "repo":"Local-Booru"
-}

updater/run.py

@@ -1,59 +0,0 @@
-import os
-import json
-import shutil
-import tempfile
-import zipfile
-import urllib.request
-
-CONFIG_PATH = os.path.join(os.path.dirname(__file__), "config.json")
-
-EXCLUDE_PATHS = {
-    "data",
-    "config.json",
-    "logs.log",
-    "updater"
-}
-
-def load_config():
-    with open(CONFIG_PATH, "r", encoding="utf-8") as f:
-        return json.load(f)
-
-def get_latest_release(cfg):
-    url = f"{cfg['host']}/api/v1/repos/{cfg['user']}/{cfg['repo']}/releases/latest"
-    with urllib.request.urlopen(url) as r:
-        return json.loads(r.read().decode("utf-8"))
-
-def download(url, path):
-    urllib.request.urlretrieve(url, path)
-
-def should_exclude(relative_path: str) -> bool:
-    parts = set(relative_path.split(os.sep))
-    return any(ex in parts for ex in EXCLUDE_PATHS)
-
-def apply_update(src_root, target_root):
-    for root, dirs, files in os.walk(src_root):
-        rel_dir = os.path.relpath(root, src_root)
-        if rel_dir == ".":
-            rel_dir = ""
-        for file in files:
-            rel_path = os.path.normpath(os.path.join(rel_dir, file))
-            if should_exclude(rel_path):
-                continue
-            src_file = os.path.join(root, file)
-            dst_file = os.path.join(target_root, rel_path)
-            os.makedirs(os.path.dirname(dst_file), exist_ok=True)
-            shutil.copy2(src_file, dst_file)
-
-cfg = load_config()
-release = get_latest_release(cfg)
-zip_url = release["zipball_url"]
-tmp = tempfile.mkdtemp()
-zip_path = os.path.join(tmp, "release.zip")
-print("Downloading:", release["tag_name"])
-download(zip_url, zip_path)
-with zipfile.ZipFile(zip_path, "r") as z:
-    z.extractall(tmp)
-extracted_root = os.path.join(tmp, os.listdir(tmp)[0])
-project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
-apply_update(extracted_root, project_root)
-print("Update success!")