diff --git a/button_script.user.js b/button_script.user.js new file mode 100644 index 0000000..90f5880 --- /dev/null +++ b/button_script.user.js @@ -0,0 +1,128 @@ +// ==UserScript== +// @name gelbooru.com Button DW ADD +// @namespace nyakonya_gelbooruocdw_dev +// @match https://gelbooru.com/index.php* +// @grant GM_xmlhttpRequest +// @grant GM_registerMenuCommand +// @grant GM_getValue +// @grant GM_setValue +// @version 1.0r +// @author https://t.me/Nyako_TW +// @downloadURL https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js +// @updateURL https://git.nekono.su/Nyako/Local-Booru/raw/branch/main/button_script.user.js +// @description Adds a one-click button to download the original image on Gelbooru post pages. +// ==/UserScript== + +'use strict'; // Рекомендуется оставить для более строгого и безопасного выполнения кода. + +// Функция безопасного запуска функций +function safeRun(fn, name) { + try { + fn(); + } catch (e) { + console.error(`Ошибка в ${name}:`, e); + } +} + +// Функция генерации текстовой конфигурации +function registerTextInput(key, defaultValue, title, promptMessage) { + const currentValue = GM_getValue(key, defaultValue); + const menuText = `⚙️ ${title}: ${currentValue}`; + + GM_registerMenuCommand(menuText, () => { + const newValue = prompt(promptMessage, currentValue); + if (newValue !== null) { + GM_setValue(key, newValue.trim()); + alert( + `Настройка изменена!\n${title}: ${newValue.trim()}\nОбновите страницу для применения настроек.` + ); + location.reload(); + } + }); +} + +// Генерация конфигураций +function config_runner() { + registerTextInput( + 'nyakonya_gb_api_root', + 'http://127.0.2.1', + 'Адрес API', + 'Введите базовый URL локального API (например, http://127.0.2.1 или https://booru.loc):' + ); +} + +// Запуск конфигуратора +safeRun(config_runner, "config_runner"); + +// 1. Находим ссылку на оригинальное изображение по тексту +const params = new URLSearchParams(window.location.search); +const imageID = params.get("id"); +const xpathResult = document.evaluate( + '//a[text()="Original image"]', + document, + null, + XPathResult.FIRST_ORDERED_NODE_TYPE, + null +); +const originalImageLinkElement = xpathResult.singleNodeValue; + +if (originalImageLinkElement) { + const imageUrl = originalImageLinkElement.href; + const existingAnchor = document.querySelector('a[onclick*="saveTagSearch"]'); + + if (existingAnchor) { + const downloadSvgIcon = ` + + `; + + const newDownloadButton = document.createElement('a'); + newDownloadButton.id = "nyakonya-gelbooruocdw-geter"; + newDownloadButton.title = "Download original image"; + newDownloadButton.innerHTML = downloadSvgIcon; + newDownloadButton.className = existingAnchor.className; + newDownloadButton.style.cssText = existingAnchor.style.cssText; + existingAnchor.parentNode.insertBefore(newDownloadButton, existingAnchor.nextSibling); + + const searchBox = document.getElementById("tags-search"); + if (searchBox) { + searchBox.style.width = "calc(100% - 380px)"; + } + + console.log('Кнопка для скачивания добавлена:', newDownloadButton); + newDownloadButton.addEventListener('click', (event) => { + event.preventDefault(); // Отменяем стандартное действие ссылки + + // Получаем настроенный пользователем корневой путь + const apiRoot = GM_getValue('nyakonya_gb_api_root', 'http://127.0.2.1'); + // Формируем корректный URL для запроса к dw_api + const apiUrl = apiRoot.endsWith('/') ? `${apiRoot}dw_api` : `${apiRoot}/dw_api`; + + console.log(`Отправка URL на локальный API: ${imageUrl} (Endpoint: ${apiUrl})`); + const originalOutline = newDownloadButton.style.outline; + newDownloadButton.style.outline = "2px solid yellow"; // Статус "в процессе" + + GM_xmlhttpRequest({ + method: "POST", + url: apiUrl, + headers: { + "Content-Type": "application/json" + }, + data: JSON.stringify({ id: imageID }), + onload: function(response) { + console.log('Ответ от API:', response.responseText); + newDownloadButton.style.outline = "2px solid green"; // Успех + setTimeout(() => { newDownloadButton.style.outline = originalOutline; }, 2000); + }, + onerror: function(response) { + console.error('Ошибка при отправке на API:', response); + newDownloadButton.style.outline = "2px solid red"; // Ошибка + alert('Не удалось отправить на локальный загрузчик. Убедитесь, что Python-скрипт запущен и вы приняли самоподписанный сертификат в браузере. Подробности в консоли (F12).'); + setTimeout(() => { newDownloadButton.style.outline = originalOutline; }, 5000); + } + }); + }); + + } else { + console.warn('Исходный элемент для вставки кнопки (saveTagSearch) не найден.'); + } +} \ No newline at end of file diff --git a/config.json b/config.json index c01c85b..c36c0c1 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,7 @@ { "server":{ - "port":8084, + "host":"127.0.2.1", + "port":80, "count_page":25 }, "gb":{ @@ -12,6 +13,7 @@ } }, "file":{ - "repos":["data/files", "E:/local_booru/img"] + "repos":["data_test/files", "E:/local_booru/img"], + "dw":"data_test/files" } } \ No newline at end of file diff --git a/run.py b/run.py index 582b9f5..cdb2a57 100644 --- a/run.py +++ b/run.py @@ -3,19 +3,22 @@ from flask import Flask, render_template, jsonify, send_from_directory, render_t import json import requests from pathlib import Path +import traceback import os CONFIG_ROOT = json.load(open("config.json", "r", encoding="utf-8")) CONFIG_ROOT_S = json.load(open("config_gb.json", "r", encoding="utf-8")) +CONFIG_HOST = CONFIG_ROOT["server"]["host"] CONFIG_PORT = CONFIG_ROOT["server"]["port"] PAGINATION_COUNT = CONFIG_ROOT["server"]["count_page"] GB_ID = CONFIG_ROOT_S["gb"]["id"] GB_HASH = CONFIG_ROOT_S["gb"]["hash"] GB_HEADERS = CONFIG_ROOT["gb"]["headers"] FILE_FOLDERS = CONFIG_ROOT["file"]["repos"] +FILE_DW_FOLDERS = CONFIG_ROOT["file"]["dw"] THUMB_FOLDER = "data_test/thumb" -gb = gelbooru.GB(GB_ID, GB_HASH, GB_HEADERS) +gb = gelbooru.GB(GB_ID, GB_HASH, GB_HEADERS, FILE_DW_FOLDERS) log = logger.Logger() database = db.DB(True) app = Flask(__name__) @@ -58,7 +61,7 @@ def dw_api(): image_id = data["id"] try: data_get, url_file, file_name = gb.get_from_id(image_id) - if os.path.exists("dw/"+file_name): + if database.check_uni(data_get["md5"]): message = f"File '{file_name}' already exists. Skipping." log.send("warning", "all", message) return jsonify({'status': 'skipped', 'message': message}) @@ -68,7 +71,7 @@ def dw_api(): if 'text/html' in content_type: log.send("error", "all", "The server returned HTML instead of an image. Bot protection.") return False - with open("dw/"+file_name, 'wb') as f: + with open(FILE_DW_FOLDERS+"/"+file_name, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) database.add_raw(data_get) @@ -78,6 +81,7 @@ def dw_api(): log.send("error", "all", error_message) return jsonify({'status': 'error', 'message': error_message}), 500 except Exception as e: + traceback.print_exc() error_message = f"Unknown error: {e}" log.send("error", "all", error_message) return jsonify({'status': 'error', 'message': error_message}), 500 @@ -115,4 +119,4 @@ def api(): if __name__ == "__main__": - app.run(host="0.0.0.0", port=CONFIG_PORT) \ No newline at end of file + app.run(host=CONFIG_HOST, port=CONFIG_PORT) \ No newline at end of file diff --git a/system_module/db.py b/system_module/db.py index bfc800d..b675716 100644 --- a/system_module/db.py +++ b/system_module/db.py @@ -1,31 +1,112 @@ -from tinydb import TinyDB, Query +import sqlite3 import os import json -from collections import Counter -from system_module import gelbooru from datetime import datetime +from system_module import gelbooru class DB: - def __init__(self, debug): + def __init__(self, debug: bool): if debug: db_path = "data_test" else: db_path = "data" - if not os.path.exists(os.path.join(db_path, "db.json")): - json.dump({}, open(os.path.join(db_path, "db.json"), "w", encoding="utf-8")) - self.db = TinyDB(db_path+"/db.json") - self.File = Query() + + if not os.path.exists(db_path): + os.makedirs(db_path) + + self.conn = sqlite3.connect(os.path.join(db_path, "db.sqlite"), check_same_thread=False) + self.conn.row_factory = sqlite3.Row + self._create_table() + + def _create_table(self): + with self.conn: + self.conn.execute(""" + CREATE TABLE IF NOT EXISTS files ( + md5 TEXT PRIMARY KEY, + local_filename TEXT, + is_video INTEGER, + is_gif INTEGER, + tags TEXT, + rating TEXT, + score INTEGER, + timestamp TEXT, + parsed_time INTEGER, + display_date TEXT, + source TEXT, + width INTEGER, + height INTEGER + ) + """) + + def _row_to_dict(self, row: sqlite3.Row) -> dict: + if not row: + return None + d = dict(row) + d["is_video"] = bool(d["is_video"]) + d["is_gif"] = bool(d["is_gif"]) - def add(self, data_add): - self.db.insert(data_add) + # Парсинг тегов из JSON-строки обратно в массив + d["tags"] = json.loads(d["tags"]) if d["tags"] else [] + + # Восстанавливаем другие поля (например, display_date и source), + # если в них были переданы кортежи или списки + for field in ["display_date", "source"]: + val = d.get(field) + if isinstance(val, str) and (val.startswith('[') or val.startswith('{')): + try: + d[field] = json.loads(val) + except ValueError: + pass + + return d + + def add(self, data_add: dict): + parsed_time = 0 + if "timestamp" in data_add: + try: + parsed_time = int(datetime.strptime(data_add["timestamp"], "%a %b %d %H:%M:%S %z %Y").timestamp()) + except ValueError: + pass + + row_data = { + "md5": data_add.get("md5"), + "local_filename": data_add.get("local_filename", data_add.get("image")), + "is_video": int(data_add.get("is_video", False)), + "is_gif": int(data_add.get("is_gif", False)), + "tags": data_add.get("tags", []), + "rating": data_add.get("rating"), + "score": data_add.get("score"), + "timestamp": data_add.get("timestamp", data_add.get("created_at")), + "parsed_time": parsed_time, + "display_date": data_add.get("display_date"), + "source": data_add.get("source"), + "width": data_add.get("width"), + "height": data_add.get("height") + } + + # БЕЗОПАСНАЯ СЕРИАЛИЗАЦИЯ ДЛЯ SQLITE: + # SQLite принимает только числа, строки и байты. + # Любые списки (tags) или кортежи (display_date) мы перегоняем в JSON. + for key, value in row_data.items(): + if isinstance(value, (list, tuple, dict)): + row_data[key] = json.dumps(value) + + columns = ", ".join(row_data.keys()) + placeholders = ", ".join(["?"] * len(row_data)) + + with self.conn: + self.conn.execute( + f"INSERT OR REPLACE INTO files ({columns}) VALUES ({placeholders})", + tuple(row_data.values()) + ) - def add_raw(self, data_add): - is_video = False - is_gif = False - if data_add["image"].split(".")[-1] in ["webm", "mp4", "avi"]: - is_video = True - if data_add["image"].split(".")[-1] == "gif": - is_gif = True + def add_raw(self, data_add: dict): + is_video = data_add["image"].split(".")[-1] in ["webm", "mp4", "avi"] + is_gif = data_add["image"].split(".")[-1] == "gif" + + dt_obj = datetime.strptime(data_add["created_at"], "%a %b %d %H:%M:%S %z %Y") + parsed_time = int(dt_obj.timestamp()) + add_data = { "md5": data_add["md5"], "local_filename": data_add["image"], @@ -35,70 +116,100 @@ class DB: "rating": data_add["rating"], "score": data_add["score"], "timestamp": data_add["created_at"], + "parsed_time": parsed_time, "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]), "source": data_add["source"], "width": data_add["width"], "height": data_add["height"] } - self.db.insert(add_data) + self.add(add_data) def get_all(self): - sorted_records = sorted( - self.db.all(), - key=lambda x: datetime.strptime( - x["timestamp"], - "%a %b %d %H:%M:%S %z %Y" - ), - reverse=True - ) - return sorted_records + cursor = self.conn.execute("SELECT * FROM files ORDER BY parsed_time DESC") + return [self._row_to_dict(row) for row in cursor.fetchall()] - def get_search(self, rating_tmp, tags): + def get_search(self, rating_tmp: list[str], tags: list[str]): mapping_rating = { "e": "explicit", "g": "general", "s": "sensitive", "q": "questionable" } - rating = [mapping_rating.get(x, x) for x in rating_tmp] - if rating: - query = self.File.rating.one_of(rating) - if tags != [""]: - tag_query = self.File.tags.test(lambda t: set(tags).issubset(set(t or []))) - query = tag_query if query is None else query & tag_query - results = self.db.search(query) if query else self.db.all() - sorted_records = sorted( - results, - key=lambda x: datetime.strptime( - x["timestamp"], - "%a %b %d %H:%M:%S %z %Y" - ), - reverse=True - ) - return sorted_records + + query = "SELECT * FROM files WHERE 1=1" + params = [] + + ratings = [mapping_rating.get(x, x) for x in rating_tmp if x] + if ratings: + placeholders = ",".join("?" * len(ratings)) + query += f" AND rating IN ({placeholders})" + params.extend(ratings) + + tags = [t for t in tags if t] + if tags: + for tag in tags: + query += " AND EXISTS (SELECT 1 FROM json_each(files.tags) WHERE value = ?)" + params.append(tag) + + query += " ORDER BY parsed_time DESC" + + cursor = self.conn.execute(query, tuple(params)) + return [self._row_to_dict(row) for row in cursor.fetchall()] def get_unknow(self): - results = self.db.search(self.File.tags == []) - return results + cursor = self.conn.execute("SELECT * FROM files WHERE tags = '[]' OR tags IS NULL") + return [self._row_to_dict(row) for row in cursor.fetchall()] - def get_autocomplete(self, tag_slise): - counter = Counter() - for record in self.db.all(): - counter.update(record.get("tags", [])) - results = [ - { - "tag": tag, - "count": count - } - for tag, count in counter.items() - if tag.lower().startswith(tag_slise) - ] - results.sort(key=lambda x: x["count"], reverse=True) - top_10 = results[:10] - return top_10 + def check_uni(self, md5: str): + query = "SELECT 1 FROM files WHERE md5 = ? LIMIT 1" + cursor = self.conn.execute(query, (md5,)) + return cursor.fetchone() is not None - def edit_data(self, id_file, edit_data): - self.db.update(self.edit_data, self.File.md5 == self.id_file) + def get_autocomplete(self, tag_slise: str): + query = """ + SELECT value AS tag, COUNT(*) AS count + FROM files, json_each(files.tags) + WHERE value LIKE ? + GROUP BY value + ORDER BY count DESC + LIMIT 10 + """ + cursor = self.conn.execute(query, (f"{tag_slise.lower()}%",)) + return [{"tag": row["tag"], "count": row["count"]} for row in cursor.fetchall()] - def get_from_id(self, id_file): - return self.db.get(self.File.md5 == id_file) \ No newline at end of file + def edit_data(self, id_file: str, edit_data: dict): + if not edit_data: + return + + set_clauses = [] + params = [] + + for key, value in edit_data.items(): + # Если передаем списки/кортежи, пакуем в JSON + if isinstance(value, (list, tuple, dict)): + value = json.dumps(value) + + set_clauses.append(f"{key} = ?") + + if key == "timestamp": + params.append(value) + try: + set_clauses.append("parsed_time = ?") + params.append(int(datetime.strptime(value, "%a %b %d %H:%M:%S %z %Y").timestamp())) + except ValueError: + pass + else: + params.append(value) + + params.append(id_file) + + query = f"UPDATE files SET {', '.join(set_clauses)} WHERE md5 = ?" + with self.conn: + self.conn.execute(query, tuple(params)) + + def get_from_id(self, id_file: str): + cursor = self.conn.execute("SELECT * FROM files WHERE md5 = ?", (id_file,)) + return self._row_to_dict(cursor.fetchone()) + + def close(self): + self.conn.close() \ No newline at end of file diff --git a/system_module/db_old.py b/system_module/db_old.py new file mode 100644 index 0000000..ddcc998 --- /dev/null +++ b/system_module/db_old.py @@ -0,0 +1,104 @@ +from tinydb import TinyDB, Query +import os +import json +from collections import Counter +from system_module import gelbooru +from datetime import datetime + +class DB: + def __init__(self, debug: bool): + if debug: + db_path = "data_test" + else: + db_path = "data" + if not os.path.exists(os.path.join(db_path, "db.json")): + json.dump({}, open(os.path.join(db_path, "db.json"), "w", encoding="utf-8")) + self.db = TinyDB(db_path+"/db.json") + self.File = Query() + + def add(self, data_add: dict): + self.db.insert(data_add) + + def add_raw(self, data_add: dict): + is_video = False + is_gif = False + if data_add["image"].split(".")[-1] in ["webm", "mp4", "avi"]: + is_video = True + if data_add["image"].split(".")[-1] == "gif": + is_gif = True + add_data = { + "md5": data_add["md5"], + "local_filename": data_add["image"], + "is_video": is_video, + "is_gif": is_gif, + "tags": data_add["tags"].split(" "), + "rating": data_add["rating"], + "score": data_add["score"], + "timestamp": data_add["created_at"], + "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]), + "source": data_add["source"], + "width": data_add["width"], + "height": data_add["height"] + } + self.db.insert(add_data) + + def get_all(self): + sorted_records = sorted( + self.db.all(), + key=lambda x: datetime.strptime( + x["timestamp"], + "%a %b %d %H:%M:%S %z %Y" + ), + reverse=True + ) + return sorted_records + + def get_search(self, rating_tmp: list[str], tags: list[str]): + mapping_rating = { + "e": "explicit", + "g": "general", + "s": "sensitive", + "q": "questionable" + } + rating = [mapping_rating.get(x, x) for x in rating_tmp] + if rating: + query = self.File.rating.one_of(rating) + if tags != [""]: + tag_query = self.File.tags.test(lambda t: set(tags).issubset(set(t or []))) + query = tag_query if query is None else query & tag_query + results = self.db.search(query) if query else self.db.all() + sorted_records = sorted( + results, + key=lambda x: datetime.strptime( + x["timestamp"], + "%a %b %d %H:%M:%S %z %Y" + ), + reverse=True + ) + return sorted_records + + def get_unknow(self): + results = self.db.search(self.File.tags == []) + return results + + def get_autocomplete(self, tag_slise: str): + counter = Counter() + for record in self.db.all(): + counter.update(record.get("tags", [])) + results = [ + { + "tag": tag, + "count": count + } + for tag, count in counter.items() + if tag.lower().startswith(tag_slise) + ] + results.sort(key=lambda x: x["count"], reverse=True) + top_10 = results[:10] + return top_10 + + def edit_data(self, id_file: str, edit_data: dict): + self.db.update(edit_data, self.File.md5 == id_file) + + def get_from_id(self, id_file: str): + return self.db.get(self.File.md5 == id_file) \ No newline at end of file diff --git a/system_module/gelbooru.py b/system_module/gelbooru.py index d3010f9..4eae091 100644 --- a/system_module/gelbooru.py +++ b/system_module/gelbooru.py @@ -1,5 +1,6 @@ import requests from datetime import datetime +import os def gelbooru_date_parse(date_str): if not date_str: @@ -12,22 +13,20 @@ def gelbooru_date_parse(date_str): return 0, "Unknown" class GB: - def __init__(self, api_id, api_hash, headers): + def __init__(self, api_id, api_hash, headers, dw_folder): self.api_id = api_id self.api_hash = api_hash + self.dw_folder = dw_folder self.root_url = "https://gelbooru.com/index.php" self.HEADERS = headers def get_from_id(self, id_post): - params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "id": id_post, "api_key":self.api_id, "user_id":self.api_hash} + params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "id": id_post, "api_key":self.api_hash, "user_id":self.api_id} response = requests.get(self.root_url, params=params, headers=self.HEADERS, timeout=15) + print(response.url) url_file = response.json()["post"][0]["file_url"] file_name = response.json()["post"][0]["image"] - if os.path.exists("dw/"+file_name): - message = f"Файл '{file_name}' уже существует. Пропускаем." - print(message) - return response.json()["post"][0], url_file, file_name - pass + return response.json()["post"][0], url_file, file_name def get_from_md5(self, md5): pass diff --git a/system_module/image_processor.py b/system_module/image_processor.py new file mode 100644 index 0000000..107e8e4 --- /dev/null +++ b/system_module/image_processor.py @@ -0,0 +1,4 @@ +import PIL + +def generate_thumb(): + pass \ No newline at end of file diff --git a/system_module/logger.py b/system_module/logger.py index ca3e682..32ce271 100644 --- a/system_module/logger.py +++ b/system_module/logger.py @@ -19,10 +19,10 @@ class Logger: self.logger_cli.addHandler(console_handler) def send(self, level_log, type_log, message): - if self.type_log == "file": - getattr(self.logger_file, level_log)(self.message) - elif self.type_log == "cli": - getattr(self.logger_cli, level_log)(self.message) + if type_log == "file": + getattr(self.logger_file, level_log)(message) + elif type_log == "cli": + getattr(self.logger_cli, level_log)(message) else: - getattr(self.logger_file, level_log)(self.message) - getattr(self.logger_cli, level_log)(self.message) \ No newline at end of file + getattr(self.logger_file, level_log)(message) + getattr(self.logger_cli, level_log)(message) \ No newline at end of file