diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/__pycache__/__init__.cpython-39.pyc b/config/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..e9825a5 Binary files /dev/null and b/config/__pycache__/__init__.cpython-39.pyc differ diff --git a/config/__pycache__/helper.cpython-39.pyc b/config/__pycache__/helper.cpython-39.pyc new file mode 100644 index 0000000..c018293 Binary files /dev/null and b/config/__pycache__/helper.cpython-39.pyc differ diff --git a/config/helper.py b/config/helper.py new file mode 100644 index 0000000..3a737e5 --- /dev/null +++ b/config/helper.py @@ -0,0 +1,9 @@ +from pathlib import Path +import yaml + +def config(): + settings_file = str(Path(__file__).parent.absolute()) + '/settings.yml' + + with open(settings_file, 'r') as f: + return yaml.load(f, Loader=yaml.FullLoader) + diff --git a/config/settings.yml b/config/settings.yml new file mode 100644 index 0000000..1a903ff --- /dev/null +++ b/config/settings.yml @@ -0,0 +1,11 @@ +watchdog: + dir: '/Users/geng/douyin_live/' + +webdriver: + bin: '/usr/local/bin/chromedriver' + proxy: '127.0.0.1:8080' + +mongo: + uri : 'mongodb://localhost:27017/' + dbname: 'tiktok' + enabled: 0 diff --git a/main.py b/main.py index dc42b36..ce2e464 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,15 @@ import sys +from urllib.parse import urlparse from scripts import watcher, webdriver +from config.helper import config if __name__ == '__main__': - webdriver.lunch(sys.argv[1]) + if len(sys.argv) == 1 or not urlparse(sys.argv[1]).scheme: + print('Invalid url provided, please check...') + sys.exit(1) - w = watcher.Watcher(directory='/Users/geng/douyin_live') + webdriver.go(sys.argv[1]) + + w = watcher.Watcher(directory=config()['watchdog']['dir']) w.run() diff --git a/messages/__pycache__/base.cpython-39.pyc b/messages/__pycache__/base.cpython-39.pyc index 0207048..bbbef2f 100644 Binary files a/messages/__pycache__/base.cpython-39.pyc and b/messages/__pycache__/base.cpython-39.pyc differ diff --git a/messages/base.py b/messages/base.py index f95beb7..c91e9d2 100644 --- a/messages/base.py +++ b/messages/base.py @@ -1,5 +1,7 @@ from store.mongo import MongoStore +from config.helper import config + class Base: instance = None @@ -11,6 +13,9 @@ class Base: return self.instance.user def persists(self): + if config()['mongo']['enabled'] != 'on': + return + try: store = MongoStore() store.set_collection('user') diff --git a/requirements.txt b/requirements.txt index f1d62a5..bdd32eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ protobuf==3.19.1 pycparser==2.21 pymongo==3.12.1 pyOpenSSL==21.0.0 +PyYAML==6.0 selenium==4.1.0 six==1.16.0 sniffio==1.2.0 diff --git a/scripts/__pycache__/mitmproxy.cpython-39.pyc b/scripts/__pycache__/mitmproxy.cpython-39.pyc new file mode 100644 index 0000000..abd43d5 Binary files /dev/null and b/scripts/__pycache__/mitmproxy.cpython-39.pyc differ diff --git a/scripts/__pycache__/webdriver.cpython-39.pyc b/scripts/__pycache__/webdriver.cpython-39.pyc index d13bd27..9710a7b 100644 Binary files a/scripts/__pycache__/webdriver.cpython-39.pyc and b/scripts/__pycache__/webdriver.cpython-39.pyc differ diff --git a/scripts/mitmproxy.py b/scripts/mitmproxy.py index 4ac654a..41ccf12 100644 --- a/scripts/mitmproxy.py +++ b/scripts/mitmproxy.py @@ -4,10 +4,12 @@ import uuid from mitmproxy import http +from config.helper import config + class Writer: def response(self, flow: http.HTTPFlow) -> None: if flow.request.host == 'live.douyin.com': - with open('/Users/geng/douyin_live/' + uuid.uuid4().hex, 'wb') as f: + with open(config().mitmproxy.log_dir + uuid.uuid4().hex, 'wb') as f: f.write(bytes(flow.response.content)) addons = [Writer()] diff --git a/scripts/webdriver.py b/scripts/webdriver.py index 82ff276..7ae1ed2 100644 --- a/scripts/webdriver.py +++ b/scripts/webdriver.py @@ -1,13 +1,24 @@ from selenium import webdriver from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver.common.proxy import Proxy, ProxyType -def lunch(url): +from config.helper import config + +def go(url): chrome_options = Options() - chrome_options.add_argument('--proxy-server=127.0.0.1:8080') - chrome_options.add_argument('--headless') + chrome_options.add_argument('--proxy-server=%s' % config()['webdriver']['proxy']) + # chrome_options.add_argument('--headless') chrome_options.add_experimental_option('detach', True) - driver = webdriver.Chrome(options=chrome_options) + proxy = Proxy() + proxy.proxy_type = ProxyType.MANUAL + proxy.http_proxy = config()['webdriver']['proxy'] + proxy.ssl_proxy = config()['webdriver']['proxy'] + + capabilities = DesiredCapabilities.CHROME + proxy.add_to_capabilities(capabilities) + + driver = webdriver.Chrome(options=chrome_options, desired_capabilities=capabilities, executable_path=config()['webdriver']['bin']) driver.get(url) - diff --git a/store/__pycache__/mongo.cpython-39.pyc b/store/__pycache__/mongo.cpython-39.pyc index 2bf8062..aaad988 100644 Binary files a/store/__pycache__/mongo.cpython-39.pyc and b/store/__pycache__/mongo.cpython-39.pyc differ diff --git a/store/mongo.py b/store/mongo.py index 000ba84..1d19eb6 100644 --- a/store/mongo.py +++ b/store/mongo.py @@ -1,9 +1,11 @@ import pymongo +from config.helper import config + class MongoStore: def __init__(self): - self.client = pymongo.MongoClient("mongodb://localhost:27017/") - self.db = self.client['tiktok'] + self.client = pymongo.MongoClient(config()['mongo']['uri']) + self.db = self.client[config()['mongo']['dbname']] def set_collection(self, collection): self.collection = self.db[collection]