diff --git a/db/pages.pk b/db/pages.pk new file mode 100644 index 0000000..53de275 Binary files /dev/null and b/db/pages.pk differ diff --git a/db/pages.pkl.pk b/db/pages.pkl.pk new file mode 100644 index 0000000..6a7d4b7 Binary files /dev/null and b/db/pages.pkl.pk differ diff --git a/extf.py b/extf.py new file mode 100644 index 0000000..08c6a94 --- /dev/null +++ b/extf.py @@ -0,0 +1,18 @@ +import random +import db + +def whatisthis(s): + if isinstance(s, str): + print("ordinary string") + elif isinstance(s, unicode): + print("unicode string") + else: + print("not a string") + print(type(s)) + +def gen_id(): + symbols = list("qwertyuiopasdfghjklzxcvbnm1234567890") + text = "" + for ii in range(30): + text += random.choice(symbols) + return text diff --git a/main.py b/main.py index e11627d..b5e2451 100644 --- a/main.py +++ b/main.py @@ -1,18 +1,9 @@ import db -import requests from bs4 import BeautifulSoup import wget import os import cfscrape - -def whatisthis(s): - if isinstance(s, str): - print("ordinary string") - elif isinstance(s, unicode): - print("unicode string") - else: - print("not a string") - print(type(s)) +import extf def get_data(url): datas = {} @@ -22,5 +13,21 @@ def get_data(url): datas["url"] = url datas["title"] = soup.find("title").text datas["text"] = (soup.find("body").text).replace("\\n", " ") - whatisthis(datas["text"]) - return datas \ No newline at end of file + return datas + +def save_data(datas): + try: + data = db.load("pages") + except: + data = {"0":{"url":"0", "title":"0", "text":""}} + for one_page in data.keys(): + if datas["url"] == data[one_page]["url"]: + return {"code":1} + add_page = 1 + while add_page == 1: + id_page = extf.gen_id() + if id_page not in data: + add_page = 0 + data[id_page] = datas + print(data) + db.save("pages", data)