build0001

2021-12-28 23:38:27 +03:00
parent e0e3c14df4
commit 9f64e1eaca
3 changed files with 43 additions and 0 deletions
@@ -0,0 +1,26 @@
+import db
+import requests
+from bs4 import BeautifulSoup
+import wget
+import os
+import cfscrape
+
+def whatisthis(s):
+    if isinstance(s, str):
+        print("ordinary string")
+    elif isinstance(s, unicode):
+        print("unicode string")
+    else:
+        print("not a string")
+    print(type(s))
+
+def get_data(url):
+    datas = {}
+    scraper = cfscrape.CloudflareScraper()
+    req = scraper.get(url)
+    soup = BeautifulSoup(req.text, 'lxml')
+    datas["url"] = url
+    datas["title"] = soup.find("title").text
+    datas["text"] = (soup.find("body").text).replace("\\n", " ")
+    whatisthis(datas["text"])
+    return datas