From 93ff79125f68e58bd57a6fd29a97a54ba6fcbe32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=80=BF=E4=BC=A6=E4=BC=A6?= Date: Tue, 30 Nov 2021 20:33:32 +0800 Subject: [PATCH] intro pyyml --- config/__init__.py | 0 config/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 141 bytes config/__pycache__/helper.cpython-39.pyc | Bin 0 -> 483 bytes config/helper.py | 9 ++++++++ config/settings.yml | 11 ++++++++++ main.py | 10 +++++++-- messages/__pycache__/base.cpython-39.pyc | Bin 1420 -> 1515 bytes messages/base.py | 5 +++++ requirements.txt | 1 + scripts/__pycache__/mitmproxy.cpython-39.pyc | Bin 0 -> 740 bytes scripts/__pycache__/webdriver.cpython-39.pyc | Bin 601 -> 1013 bytes scripts/mitmproxy.py | 4 +++- scripts/webdriver.py | 21 ++++++++++++++----- store/__pycache__/mongo.cpython-39.pyc | Bin 976 -> 1029 bytes store/mongo.py | 6 ++++-- 15 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 config/__init__.py create mode 100644 config/__pycache__/__init__.cpython-39.pyc create mode 100644 config/__pycache__/helper.cpython-39.pyc create mode 100644 config/helper.py create mode 100644 config/settings.yml create mode 100644 scripts/__pycache__/mitmproxy.cpython-39.pyc diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/__pycache__/__init__.cpython-39.pyc b/config/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9825a5c9d7656e1769e1a9c3d0021b5ae24cbe5 GIT binary patch literal 141 zcmYe~<>g`kf>mtG5<&E15P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HKerR!OQL%n{ zYF@g&OKNd;Nq&KTNhXlWj?c*~OVv-#&r8cp*N>0S%*!l^kJl@xyv1RYo1apelWGSt J>oX8D007c$AQS)q literal 0 HcmV?d00001 diff --git a/config/__pycache__/helper.cpython-39.pyc b/config/__pycache__/helper.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c018293fa6d49292a7a7f5a333f4f5dadafd7af0 GIT binary patch literal 483 zcmYjNy-ve05WaKVG_4v&SXr|W*bzbkF(HIh5F$&IVz*7}CXwwT6~Po%#4Esp#4CAV zVd52-xCAP3r~B^D{_b>H81{jr8@y#7u^0D%e?m>@PWQ!HV^6Z{bp zPp~0GoiBO?5juXJ+3DF$ldYv;3doMg%msl3i8|)iMeLa+Ye~{ zS@*udKzL8=3K&i0q_u}Iggr1PyMV1;6AG$D{Om4_jkGo|CT3@~P!2Wih=q0u5|W2nZtiGe{HHoQm8cSB z2m=F4<$)zbGl0ZLU}R-rVzR^sKzs?_v)dwJ>;3%h{r&m9`(XdEo4(^%1fJv0XaAXg z+PuDd7fyl*8j^^*l%mc;7HO_V$#Wt!!QTq4j*9KHbzYmMH^11s|nSeWA*PgF+G&*r+)Gjr<3_H!VQR$P5T| zfCz~8MpJD;dgSDNMrUL~$MlQ|z5uFYa;7cFgn@;RS)hX$ATg8Y*9Tc^Z3SSJl(T%B@88DFRte2Ztz0VZ zv#5C^fIp~iBHTjwcMC346$b|vPtjyD&P)y7jncl?)m^*fyj9M( ooMPcrcW-MTLttEu7pq*9hgOwZQZu)sM2tfDm|_!*Yp@3U3z6%r7XSbN delta 680 zcmZuuy=xRf6rVRU`*FLsn{PppgQU?@jMx|pQP9H5G%18FT*A$g+;Y1kvlBwX9Vwz< zVH38nxgsZ6_$Sy{`48A`k;+E=Cwy-`6@oLo-@G^P`@NsxyD;93qJZJq|L}45LY~Ff zw|XckreMShNC4O~rnnNXnG&Zw;Y#kXzVm}z`p^W=b-AZ^tTx%%;b-X7_uvV5wS}j5 zBw29d;M2&1Dl?zJcqo|8U!cUMixAY`;qJ*-Uw};nu;ZRx{S!vH^*6pQy=~L21M&XBn&DV@5?7f`2YNKh26m{+wyz`@ zme{#Xb(xt(>UN1>nP8>95Z7+45*rgN5VX^B2f8S9*|q`6XW$eUXJ~7}Rg%%Nw+i-^Qf5?B=CuWlfsit^?=s3Dsdj0$Pt* zC)gnPx4f2XHz-g+RFUL!$$f#hXXtOcFOsmWcN1IM)9tKU&Hq9*{=-76gHe<{0t!fj G?ebrMo{H=M diff --git a/messages/base.py b/messages/base.py index f95beb7..c91e9d2 100644 --- a/messages/base.py +++ b/messages/base.py @@ -1,5 +1,7 @@ from store.mongo import MongoStore +from config.helper import config + class Base: instance = None @@ -11,6 +13,9 @@ class Base: return self.instance.user def persists(self): + if config()['mongo']['enabled'] != 'on': + return + try: store = MongoStore() store.set_collection('user') diff --git a/requirements.txt b/requirements.txt index f1d62a5..bdd32eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ protobuf==3.19.1 pycparser==2.21 pymongo==3.12.1 pyOpenSSL==21.0.0 +PyYAML==6.0 selenium==4.1.0 six==1.16.0 sniffio==1.2.0 diff --git a/scripts/__pycache__/mitmproxy.cpython-39.pyc b/scripts/__pycache__/mitmproxy.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..abd43d537ec7e6f493727d86a955e5879f8a2592 GIT binary patch literal 740 zcmYjPJ8u**5T5aRAGz!ika#qRf^M4wB&rA@BqX3ZArUFCinO|%eV1MK72C_@qEKym z{s2YNAn}(hIx7AG72`b-ovrz7&)DDi8+$e$kAT{*(T~{?0{nK1?dH+q6`9^A5FoIG z)Qzmfilq$Q+$+6`rySue2rm3f5Pr{6U-~^PF%1NMfF$@s=V6+lZxlU~6E3ih1|Pw-2g`U9(PgF}@Ysiq0Dd{H(l8z^ZyRnOTLr=yI> zUX$q~f(tC5K!Kl``e>N(dMJ3|o8T(Eh9Y`E@_P2|@mF@iu6E3bcJCH(kJpHBNP;=_ z-b1KwKLb5|3OMvngaSyaH$C4g^DlBDnr@xflX+8FwmMG6HdOMoliE-X8i!3QYvnFy zcdFb}t!mEJX;?PPOytV)uFHk>9X_|-i9ECZ%89l9@!CjjBPDg))LPn*N@-+m63=oi z%Z2hCCoX=1x<|9WclBK5tRHz tc=IzRC$em%8VWS*w82aW61>Ig#13}WbS?#UO-6s2z?Slkus?Z|A zKS&xR{?fKcNc;sV#@>(|D0t<0d3HSW%{cD$LImr_`q%giMd-Ux78e2K8Emx;gd>hK zlw*dmkx52!%BYoT=HxDOt?Xo8?lV6RSYUNG>*OH|LH4-+5wRs6yg`G`Pl$*HxaqtX z6E4$tLW1J&RXQyT1>)*+p;9UMIIiMJnx#4wYCszIl`M}>nvP}rCzSwKFDZ++=`_J6 z~iDJ_FI{44vW{pJR?SIfMK+5T_ex4P8JzV5i*S?gha}JK#yQd+609@)vzU z5B||RIy_uDr8PP4*3?AOy~A5IuIZJZ2lx1&=&4gX=gYOdjUzxH*!{4wxmlTVHkFV7 zz>|kc8pIpdS{siG9VfH>hECFAKpVeo(P4LZi%pa)fWwx>u}J1Ro@64bVm%x5CG-?E z-I((zmeYAI3N3vwv2|k-$CZ$&F~(WchL#30=@}PqXKVLhYnz2Omq-KDESzawMb^MN zN@cAm9qT`S`Fd~X1zR zzDv5q!vXH&KJbv-Bl0$!><+ptP$CmWI?vmzAN)~=*8$t4hnMZJD6`>Ber2AQ#qg$X zwV>Vq|4$a>>wYx7cF2b_kwM#R8*R!Kl?Ac_Wi-L#yyWvtJT*^3nK$0W=IcY*zX3k+ B6CeNp literal 601 zcmYjPJ#X7E5Is_slDJCOP8|Yd(IC-I;?^+QAb>NcLy|>{p^Aj1MqePQG&Y>6S@R$4 zAw&L>fJ~Y47dquAJ8h1*<2~?r@A1HBcee{@U!ESK6#;lQ;Jr}K_8>56AT*u zz%49bNjO;{Nt@A4Mh(r1642JK&H}i7Pj1PAEN$+X3JyrVbvFO(`;{zjX1)>q?Cn%TRg1%X*d;L0sl+&>sws zh6nw@Veim$q=j-DH{yH}$!VR7LWiFW5;v8Qslib;SvQsDouM;MWSNV`nhNdF@MGM< zO^MOcj_MKQ4p#p?Vwb32v8gDg>{O^LT~+%6Kn0wu+o9RL6T diff --git a/scripts/mitmproxy.py b/scripts/mitmproxy.py index 4ac654a..41ccf12 100644 --- a/scripts/mitmproxy.py +++ b/scripts/mitmproxy.py @@ -4,10 +4,12 @@ import uuid from mitmproxy import http +from config.helper import config + class Writer: def response(self, flow: http.HTTPFlow) -> None: if flow.request.host == 'live.douyin.com': - with open('/Users/geng/douyin_live/' + uuid.uuid4().hex, 'wb') as f: + with open(config().mitmproxy.log_dir + uuid.uuid4().hex, 'wb') as f: f.write(bytes(flow.response.content)) addons = [Writer()] diff --git a/scripts/webdriver.py b/scripts/webdriver.py index 82ff276..7ae1ed2 100644 --- a/scripts/webdriver.py +++ b/scripts/webdriver.py @@ -1,13 +1,24 @@ from selenium import webdriver from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver.common.proxy import Proxy, ProxyType -def lunch(url): +from config.helper import config + +def go(url): chrome_options = Options() - chrome_options.add_argument('--proxy-server=127.0.0.1:8080') - chrome_options.add_argument('--headless') + chrome_options.add_argument('--proxy-server=%s' % config()['webdriver']['proxy']) + # chrome_options.add_argument('--headless') chrome_options.add_experimental_option('detach', True) - driver = webdriver.Chrome(options=chrome_options) + proxy = Proxy() + proxy.proxy_type = ProxyType.MANUAL + proxy.http_proxy = config()['webdriver']['proxy'] + proxy.ssl_proxy = config()['webdriver']['proxy'] + + capabilities = DesiredCapabilities.CHROME + proxy.add_to_capabilities(capabilities) + + driver = webdriver.Chrome(options=chrome_options, desired_capabilities=capabilities, executable_path=config()['webdriver']['bin']) driver.get(url) - diff --git a/store/__pycache__/mongo.cpython-39.pyc b/store/__pycache__/mongo.cpython-39.pyc index 2bf8062e9ac94edc83719b5b0f9e7d34a3d6d679..aaad988fa5e97578efe0c52cfca37a164c4b63c4 100644 GIT binary patch literal 1029 zcmbVLL2uJA6n=J+HcQ(vgqYMjLW+bMegHy~hQx_pDlWNLS?ratG%2==)Ty`ahin(5 z{w1G~IPn)a@t#Y!uyMlDv;Cf5-uHd>OEDTHfaAr{x8gek_(4v01n8WgnU^?8kgSHB zR$j9O&pCtlAOjg*gAA{D9!NHWH2R6TFin~3m2Gog%}esTgQIhTW?tc>AZHSCE}7)l zfHi!*Jd}~_p&!YeAFGmq)@PWBlkVebd+$-M9*+@M6*plB3^7dwM3o2Fb-RhN`)U8~YoZG)jc zhV+moBD|XpDDBWFEACUNMU01Jm*Wndzq1~ZqTcbO2!cr6unoJ#4$%V)AU?%iQZ;x@ zRBVX<#nOter8o5fN_+qd zUOefec+MO67S2S`Qe4=N*_q!rJG+nKBky>QQy{QfPcPov)Y;f9!$A^BL*kM{BAMh5 zMDly)Qb|uq#rj0LJG}@79D+9Ncpb!$X8>k&LV9dMu9#$N1Y#};n0WWDJW|o6lIM+T zHB@~s9H`hdYW1DndezODU})5^Vx{@(amKm!G2-no@FSBP&{djC&S;dLm47Lyt6 zG%-#KeHDg&-vlbEFuepRjc?MqU&$j@iha}jLq)R1PWG0e0w(0sLJ-BiHa!*juit%A zOM1pzXrzk(+sQjSs?EFYANR