build0001

.gitignore

@@ -26,6 +26,8 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+test.py
+tmp.txt
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

db.py

@@ -0,0 +1,15 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import pickle, os
+
+def load(name):
+    file = open(os.getcwd() + "/db/" + name + ".pk", "rb")
+    res = pickle.load(file)
+    file.close()
+    return res
+
+def save(name, setting):
+    file = open(os.getcwd() + "/db/" + name + ".pk", "wb")
+    pickle.dump(setting, file)
+    file.close()

main.py

@@ -0,0 +1,26 @@
+import db
+import requests
+from bs4 import BeautifulSoup
+import wget
+import os
+import cfscrape
+
+def whatisthis(s):
+    if isinstance(s, str):
+        print("ordinary string")
+    elif isinstance(s, unicode):
+        print("unicode string")
+    else:
+        print("not a string")
+    print(type(s))
+
+def get_data(url):
+    datas = {}
+    scraper = cfscrape.CloudflareScraper()
+    req = scraper.get(url)
+    soup = BeautifulSoup(req.text, 'lxml')
+    datas["url"] = url
+    datas["title"] = soup.find("title").text
+    datas["text"] = (soup.find("body").text).replace("\\n", " ")
+    whatisthis(datas["text"])
+    return datas