0.0.2s

.gitignore

@@ -1,4 +1,9 @@
 # ---> Python
+# Secure data
+old/
+config_gb.json
+migration.py
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -6,7 +6,7 @@ Version Naming Rules:
 
 All technical versions are designated as snapshots (s)
 
-All test versions are designated as alpha and beta (a, b)
+All test versions are designated as beta (b)
 
 All pre-release and release versions are designated as release candidate and release (rc, r)
 

README_ru.md

@@ -6,7 +6,7 @@
 
 Все технические версии обозначаються как снапшоты (s)
 
-Все тестовые версии обозначаються как альфа и бета (a, b)
+Все тестовые версии обозначаються как бета (b)
 
 Все предрелизны и релизные версии обозначаються как кандидат в релиз и релиз (rc, r)
 

config.json

@@ -1,3 +1,17 @@
 {
-	"port":8084
+	"server":{
+        "port":8084,
+        "count_page":25
+    },
+	"gb":{
+        "id":"",
+        "hash":"",
+        "headers":{
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            "Referer": "https://gelbooru.com/"
+        }
+    },
+    "file":{
+        "repos":["data/files"]
+    }
 }

data/analytic.json

@@ -0,0 +1,64 @@
+{
+    "total_images_analyzed": 14003,
+    "average_weight": {
+        "bytes": 2980950.54,
+        "formatted": "2.84 MB"
+    },
+    "storage_stats": {
+        "folders": {
+            "img": "38.88 GB",
+            "json_db": "24.39 MB",
+            "thumb": "331.45 MB",
+            "logs": "14.44 MB"
+        },
+        "types_in_img": {
+            "images": "38.56 GB",
+            "videos": "140.80 MB",
+            "gifs": "181.30 MB",
+            "logs": "14.44 MB"
+        },
+        "grand_total": "39.24 GB"
+    },
+    "ratings_stats": {
+        "explicit": {
+            "count": 1605,
+            "size_formatted": "4.52 GB"
+        },
+        "sensitive": {
+            "count": 6623,
+            "size_formatted": "19.35 GB"
+        },
+        "questionable": {
+            "count": 1591,
+            "size_formatted": "4.72 GB"
+        },
+        "general": {
+            "count": 4184,
+            "size_formatted": "10.29 GB"
+        }
+    },
+    "resolution_stats": {
+        "largest": {
+            "filename": "5adc7e0e773417f9c379accf.png",
+            "md5": "5a63c28906dc7e0e7734c379accf",
+            "resolution": "9921x14031"
+        },
+        "smallest": {
+            "filename": "8fc614e712ad21d2151ccde8a3c87.gif",
+            "md5": "8fc614e712ae22151ccde8a3c87",
+            "resolution": "240x280"
+        }
+    },
+    "weight_stats": {
+        "largest": {
+            "filename": "4cf0a32c61c7e7914095d1fe.png",
+            "md5": "4ab6acf0a32c61c7e79fe",
+            "size_formatted": "65.21 MB"
+        },
+        "smallest": {
+            "filename": "2e1c8a5870b90c3f538d7.png",
+            "md5": "28c679ea3470b90c3f538d7",
+            "size_formatted": "6.49 KB"
+        }
+    }
+}

db.json

@@ -1 +0,0 @@
-{"_default": {"1": {"name": "Alex", "age": 25}}}

run.py

@@ -1,29 +1,107 @@
 from system_module import db, gelbooru, logger
-from flask import Flask, render_template
+from flask import Flask, render_template, jsonify, send_from_directory, render_template_string
 import json
+import requests
+from pathlib import Path
+import os
 
 CONFIG_ROOT = json.load(open("config.json", "r", encoding="utf-8"))
-CONFIG_PORT = CONFIG_ROOT["port"]
+CONFIG_ROOT_S = json.load(open("config_gb.json", "r", encoding="utf-8"))
+CONFIG_PORT = CONFIG_ROOT["server"]["port"]
+PAGINATION_COUNT = CONFIG_ROOT["server"]["count_page"]
+GB_ID = CONFIG_ROOT_S["gb"]["id"]
+GB_HASH = CONFIG_ROOT_S["gb"]["hash"]
+GB_HEADERS = CONFIG_ROOT["gb"]["headers"]
+FILE_FOLDERS = CONFIG_ROOT["file"]["repos"]
+THUMB_FOLDER = "data/thumb"
 
+gb = gelbooru.GB(GB_ID, GB_HASH, GB_HEADERS)
+log = logger.Logger()
+database = db.DB()
 app = Flask(__name__)
 
+def find_file(filename, file_folders):
+    for folder in file_folders:
+        folder_path = Path(folder)
+        if not folder_path.exists():
+            continue
+        for file_path in folder_path.rglob(filename):
+            return folder_path
+    return None
+
 @app.route("/")
 def index():
     return render_template("index.html")
 
-
 @app.route("/analytics")
 def analytics():
     return render_template("analytics.html")
 
-
 @app.route("/post/<path:post_id>")
 def post(post_id):
     return render_template("post.html")
 
+@app.route('/file/<path:filename>')
+def serve_image(filename):
+    folder_path = find_file(filename, FILE_FOLDERS)
+    return send_from_directory(folder_path, filename)
+
+@app.route('/thumb/<path:filename>')
+def serve_thumb(filename):
+    return send_from_directory(THUMB_FOLDER, filename)
+
+@app.route("/dw_api", methods=['POST'])
+def dw_api():
+    data = request.get_json()
+    if not data or 'id' not in data:
+        return jsonify({'status': 'error', 'message': 'URL not get'}), 400
+    image_id = data["id"]
+    try:
+        data_get, url_file, file_name = gb.get_from_id(image_id)
+        if os.path.exists("dw/"+file_name):
+            message = f"File '{file_name}' already exists. Skipping."
+            log.send("warning", "all", message)
+            return jsonify({'status': 'skipped', 'message': message})
+        with requests.get(url_file, stream=True, headers=GB_HEADERS, timeout=30) as r:
+            r.raise_for_status()
+            content_type = r.headers.get('Content-Type', '')
+            if 'text/html' in content_type:
+                log.send("error", "all", "The server returned HTML instead of an image. Bot protection.")
+                return False
+            with open("dw/"+file_name, 'wb') as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        database.add_raw(data_get)
+        return jsonify({'status': 'success'})
+    except requests.exceptions.RequestException as e:
+        error_message = f"Network error {image_id}: {e}"
+        log.send("error", "all", error_message)
+        return jsonify({'status': 'error', 'message': error_message}), 500
+    except Exception as e:
+        error_message = f"Unknown error: {e}"
+        log.send("error", "all", error_message)
+        return jsonify({'status': 'error', 'message': error_message}), 500
 
 @app.route("/api", methods=["GET", "POST"])
 def api():
+    data = request.get_json()
+    if data["type"] == "post":
+        search_query = data["query"].split(" ")
+        search_rating = data["rating"].split("+")
+        results = database.get_search(search_query, search_rating)
+        if len(results) <= PAGINATION_COUNT:
+            results_send = {"results_info":{"total":len(results), "page":int(data["page"]), "count_page":PAGINATION_COUNT}, "results":results}
+            return jsonify(results_send)
+        else:
+            results_slile = results[int(data["page"])*PAGINATION_COUNT:(int(data["page"])+1)*PAGINATION_COUNT]
+            results_send = {"results_info":{"total":len(results), "page":int(data["page"]), "count_page":PAGINATION_COUNT}, "results":results_slile}
+            return jsonify(results_send)
+    elif data["type"] == "autocomplete":
+        tag_slise = data["q"]
+        result_autocomplete = database.get_autocomplete(tag_slise)
+        return jsonify(result_autocomplete)
+    else:
+        pass
     return "", 204
 
 

system_module/db.py

@@ -1,7 +1,75 @@
-# Образец для расширения!
+from tinydb import TinyDB, Query
+import os
+import json
+from collections import Counter
+import gelbooru
 
-from tinydb import TinyDB
+class DB:
-
+    def __init__(self):
-db = TinyDB("db.json")
+        db_path = "datas"
-
+        if not os.path.exists(db_path):
-db.insert({"name": "Alex", "age": 25})
+            os.makedirs(os.path.dirname(db_path), exist_ok=True)
+            json.dump({}, open(db_path+"/db.json", "w", encoding="utf-8"))
+        self.db = TinyDB(db_path+"/db.json")
+        self.File = Query()
+        
+    def add(self, data_add):
+        self.db.insert(data_add)
+    
+    def add_raw(self, data_add):
+        is_video = False
+        is_gif = False
+        if data_add["image"].split(".")[-1] in ["webm", "mp4", "avi"]:
+            is_video = True
+        if data_add["image"].split(".")[-1] == "gif":
+            is_gif = True
+        add_data = {
+            "md5": data_add["md5"],
+            "local_filename": data_add["image"],
+            "is_video": is_video,
+            "is_gif": is_gif,
+            "tags": data_add["tags"].split(" "),
+            "rating": data_add["rating"],
+            "score": data_add["score"],
+            "timestamp": data_add["created_at"],
+            "display_date": gelbooru.gelbooru_date_parse(data_add["created_at"]),
+            "source": data_add["source"],
+            "width": data_add["width"],
+            "height": data_add["height"]
+        }
+        self.db.insert(add_data)
+        pass
+    
+    def get_all(self):
+        return self.db.all()
+    
+    def get_search(self, ratings, tags):
+        ratings
+        results = self.db.search(
+            self.File.rating.one_of(self.ratings) &
+            self.File.tags.any(self.tags)
+        )
+        return results
+    
+    def get_unknow(self):
+        results = self.db.search(self.File.tags == [])
+        return results
+    
+    def get_autocomplete(self, tag_slise):
+        counter = Counter()
+        for record in self.db.all():
+            counter.update(record.get("tags", []))
+        results = [
+            {
+                "tag": tag,
+                "count": count
+            }
+            for tag, count in counter.items()
+            if tag.lower().startswith(prefix)
+        ]
+        results.sort(key=lambda x: x["count"], reverse=True)
+        top_10 = results[:10]
+        return top_10
+    
+    def edit_data(self, id_file, edit_data):
+        self.db.update(self.edit_data, self.File.md5 == self.id_file)

system_module/gelbooru.py

@@ -1,12 +1,32 @@
 import requests
+from datetime import datetime
+
+def gelbooru_date_parse(date_str):
+    if not self.date_str:
+        return 0, "Unknown"
+    try:
+        dt = datetime.strptime(date_str, "%a %b %d %H:%M:%S %z %Y")
+        return dt.timestamp(), dt.strftime("%Y-%m-%d")
+    except Exception as e:
+        print(f"Ошибка парсинга даты '{date_str}': {e}")
+        return 0, "Unknown"
 
 class GB:
-    def __init__(self, api_id, api_hash):
+    def __init__(self, api_id, api_hash, headers):
         self.api_id = api_id
         self.api_hash = api_hash
         self.root_url = "https://gelbooru.com/index.php"
+        self.HEADERS = headers
         
     def get_from_id(self, id_post):
+        params = {"page": "dapi", "s": "post", "q": "index", "json": 1, "id": id_post, "api_key":self.api_id, "user_id":self.api_hash}
+        response = requests.get(self.root_url, params=params, headers=self.HEADERS, timeout=15)
+        url_file = response.json()["post"][0]["file_url"]
+        file_name = response.json()["post"][0]["image"]
+        if os.path.exists("dw/"+file_name):
+            message = f"Файл '{file_name}' уже существует. Пропускаем."
+            print(message)
+            return response.json()["post"][0], url_file, file_name
         pass
     
     def get_from_md5(self, md5):

system_module/logger.py

@@ -1,13 +1,12 @@
 import logging
 
 class Logger:
-    def __init__(self, name):
+    def __init__(self):
-        self.name = name
         self.logger_file = logging.getLogger(__name__)
         self.logger_cli = logging.getLogger(__name__)
         formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
         
-        file_handler = logging.FileHandler("logs/"+self.name+".log", encoding="utf-8")
+        file_handler = logging.FileHandler("logs.log", encoding="utf-8")
         file_handler.setFormatter(formatter)
         
         console_handler = logging.StreamHandler()