From 3a6d19083f49c07e39b422cbc2161dcfad104dc7 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Thu, 2 Jul 2020 10:35:41 +0530 Subject: [PATCH 01/17] lib files --- lib/__pycache__/PrettyPrint.cpython-38.pyc | Bin 0 -> 6761 bytes lib/__pycache__/config.cpython-38.pyc | Bin 0 -> 2051 bytes lib/__pycache__/utils.cpython-38.pyc | Bin 0 -> 3537 bytes lib/config.py | 42 ++++++ lib/git.py | 0 lib/google.py | 0 lib/prettyPrint.py | 146 +++++++++++++++++++++ lib/utils.py | 103 +++++++++++++++ 8 files changed, 291 insertions(+) create mode 100644 lib/__pycache__/PrettyPrint.cpython-38.pyc create mode 100644 lib/__pycache__/config.cpython-38.pyc create mode 100644 lib/__pycache__/utils.cpython-38.pyc create mode 100644 lib/config.py create mode 100644 lib/git.py create mode 100644 lib/google.py create mode 100644 lib/prettyPrint.py create mode 100644 lib/utils.py diff --git a/lib/__pycache__/PrettyPrint.cpython-38.pyc b/lib/__pycache__/PrettyPrint.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81b720afb735c56aec4391b5804ef07d789b67ff GIT binary patch literal 6761 zcmb`MS#R4$5XVVTlqg&BZOhK>+{$6)z7NZ`d^C!Z+78kPXjxgq*orAJT}g192IxbB zKJ=lVA$iC{9{Y9p+NXYnqCh*dltfx)fepG8IJ5Mu!=HCXe-1a#QmH-#kLln4{_fMT zqWnP@!9&A~Sv(te&}=Ft1r$(iwH7I9DvcvJj+S&j)^Hpv#Z_fVfhg$j70`cSsaHYR;ji=0NHmfgpE0n#?L1P`yc|03KXx=Fw zly@qAB5CDgWKH?37BL@jX1%t{6)U5;YSUGjt5!ZMtRLt9wV2&^^1HdUNLWu4|ZP#i=(FT4F=Hs-orZAeD(OvbWtF7i2 z82#Zag~rua^K1F_&scy>>rIpYVV+4+RAUwl;-Ev7>{4Ms0uUSE40DCt{39QVQ92lY z{k(+^@aP+ui|vsB(Lk4ohomt@=L*mAB1q88dK~=zEF7bTx79gyOwP`J0}BFgWa|AV$!fug_Wh{ zb*>3!{u>dxe!IH4)ik*RoFPR15LXA(=s!_B-;Q-r0~4k3CfmHv@`AX19g3E9K7@93 zAFV_g@nA_yz6_S6BP#$aRbsqK;5s8n+pYwzDpx~rdn-Y%{0#C+Xlsxlamp)(w%*(f zVV>xz2ielp&#AF35F}1`#MlAc4WS+Bp_SPvc=f=QRFJh@4_r4NhS2ubgIs|f;4AB? ztRQgGTU<|4d3V>NW#c{TD@}Y|b%Rl_CnC5pr*2|mAJK}o8&>~u;KgLf>lMNo6J#>* zzWfPL3{d%PbSEl}YeJAK&hOl^z7AmdeMUPdjfxsSkqi0O46|{Mk-Zh&I*$H`R8ilgrJ7xU#{|;6J#CgCbg|UjS$+Y{naY5 zN#ylp}FNpxr6O%o*B6u9s_;X}W!;S#zLe(>Q6GKeWoW7vVd;Y?PLSHY{11%oG{gn`4K` z93?}WPwWJlQ)FmMg=NUlb_KgghV}y3Wis@d&90HTPUZ#~dW~hb$lNAF|5?K5Ws%(@ zbDzuuGW1HuX36Bp%#m3nvqWZ@O#fa(+tPdy4IDIRh7mDzL+klOjo`=ViE4YIQ6o;* z{9i3*5^mfy%eHNrC8Ofl4lCEnjNWnhUNzHb#1%10BPu*Ery-oKanDP{1L_(YL>!GI IRs3lG0;bc_DgXcg literal 0 HcmV?d00001 diff --git a/lib/__pycache__/config.cpython-38.pyc b/lib/__pycache__/config.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1093f7ac827106f26adbc147d249f8528dc39ca GIT binary patch literal 2051 zcmb7F&2A$_5bl~Dd&XXSSpw`YWX-P_X=NiWTq3k+7jZyXks>Z5!DN_jCr-v5r+W+= zGr5qJ_BHHdUVw+`D<@t7AwgA-$FY-D5G{4Ndb+y$tE#Ws-?!UMhU4?!|Ne2`W$X`H ztPhThr#SUCiUSrh!33lb!Z%Dfg1=;fpF5!=V2=gvpJ>DS3CZ=b^3lbwl7%1`od|j9 zA#{Z+YA;z>L!Wiwp-&He)~!$9w4Y_;!SLX_cpM){b=|vzPM_h_cTwc*2=twt&7B46 z5zu;%T|htJiVRIXvEC}LQx)gKY<$5~9e1tx>#N>jHj+It97}!Jd!8lJksRl`_e-XZ zt$3fuXNp=oSMdg%uColAtT=m>d=JQy^UzPE4u{9W;w7B zj1Jncc2U8AV8PCyfVoq!1pq^hFW7VT!;>?ofWmn|`t}*m-Fa=n=JkS?U*}%I7qD=E zoqTfgu-x5aib@jrrd2|o#7avQwn}Wfc4b^8V-flqL&OYen?6OT4A(L@9H9qpSW5=O zRE91&GN26{UuIHyWbLL|EZV5)777MFG@TZ-VB48(Z6<>Z6S7$!l#8c0^$vC{usJX=UAS0_J2Lw?+mi( zSOl(m6TKN1KWnqq&-T#fTY38Ri$qSS=B7T#)F{paN4-Uc91X9;sHmooaBBOEz-=t9 z56(Zc$2WzE)Jcee)gYawjqo-q1+tu?+PO@*x`W@biwTLMgqqsVaXOV(0o}r_4{%ob zz7Yr#`R2M;iLP44u_;sU5Qr@jt7oGmyN97+3iZcp2tNFeRFHjw_8WuJK|{ZbqN4pP zHUI%geLv){n2W^60`pn-rVFl@^%}Y8E}EKBoU?PJ7EuT;lOH&yrl&}E^){MReDFq6 zYkZWE|BWb0(pc*#Dlhj&5Y!B|(l{Q;C^F3`8f9XdlHQ7}fD#3SVZLcq&3aPes_RP$g+3!1ZF7se&%4hif@z1{xuTL`e zCv^sY7COs#q7^h-tjr{nyw1zqyhT}x+b-L8nH17`&ZKoD$_{$AbkK9qb0uGA zMfY#)#MWxm4*vRI+Qw+W=nL4LOYxjx_p@dnUbf{q>B`)5UUuYp>B$L5uDl?9ISDBz z^KuIMp1dfh;ZMIt&Nf6Ud{L?rwsMw@rpW9smP^~MMo^MrGl;fJE3Mi=1A0;EaZBx*Cfcjkf~a&~wRVCUCN{4< zDb>SAr2_ztmiC@%f2S2TE6r*nh$aZDIZbYJpZg-u+jCuVOPQKxLriU~qD6-3FA!qZ z;R#Flu{ad5*s(fxB93{q(JsWI;4L+f{IgZ<2!VuKm$@U`NEB15Y@8ZBCpw)d*>wswOjVH8K&YgBgw8LCL*I^rIOxjOXze zyufGho*ytzraWXEase`sG0q1N-e8=?*pd8*KjTEQDAwAiVRsx@OFdm|w&KN<+)~la zxT|SwrwEmH!U!nUDab|3uu~q5b2&;Xw-v<1=mMrA(n%S4ygiK%k{2&^HHK)A!AaZD zScK4{5sTQ0?T&p6zn*7Dps>J>Q!ftx0{Nno0$#uxZAG!tRtP_~t8wfxzRTzynxvR1rxZ8# zO0b~uFZ!Ay-0B@_$i)Y|W4imzIB4tzeOq z9*@%SXa(UfVtqgM3&zdcGd=u0rPU$34k5plA{Q|d-4WrwdT7Obhmc!Hct~?y+^{$G zVmkqsBSC3zAN?K^M^UoOd|RC*V)?F9RtU*S8o=co0^~qNfw8^j;RrWF=^D`@6EUPT zid`S?8Qz}nQ5)R&&Fq%30nR03w=~4=O2!Uc-in`g!)PMBVltGm%Q)u*!EX#EywA6a_K{L{fB;rOY1WJK$xa zk)t5>cnC$YT6R?NL6|NVKCG2(9lh6PddKZ%_43$;_-_-4td=UcwA^XI6`R zwK%1{imB0+iuNm&MoS*l1IYPGW&faBr$Sb#7~7>8>pX!m3G986Oh{3LEAk*?AoVFp zndOW`bUSsEhqbT zuYCXR<)h+(;3|Co|GP#h!lNj$b}OQGHv9E{fVZJcPIn(qKLK_xKc{+_25vuN0Yz!> zHZ3MkPF-c}?lV^2r)lSb%8tVRIhLnK*e~p9NDP+sn^4kxbA*8QddXvhfY9mn>-ub; zLr=#z))0-iF7M|4fZCp6zKBn%OVnJZ=5-}=Zn#Wu!U7MWJ6O>#^udnnjv@Zfny%i% z_Bein@=Jn62VE3ttS-_(769hXH#1n(3E(3rjh6LuqaaOPC+&Vb4@PiKeMVopPR$K! z2!x_Et_-HHNXl?zfk?3L;W6h)f#@L&4Vhw2=PH$Iy^f$a&SGp%@oKVCnqm#>*X*mk VuhaZ-ixOQ#Xl#M)=`RX*UjcqXzWD$E literal 0 HcmV?d00001 diff --git a/lib/config.py b/lib/config.py new file mode 100644 index 0000000..3977c0a --- /dev/null +++ b/lib/config.py @@ -0,0 +1,42 @@ +__all__ = [] + +import configparser + +class ConfigManager: + config = None + + def getConfig(): + if not ConfigManager.config: + ConfigManager.config = ConfigManager.Configuration() + return ConfigManager.config + + def load(cfile): + conf = configparser.ConfigParser() + conf.read(cfile) + + for s in conf.sections(): + for k in conf[s]: + ConfigManager.getConfig().set(k, conf[s].get(k)) + + return ConfigManager.getConfig() + + class Configuration(): + def __init__(self): + self.properties = {} + + def get(self, key): + if key in self.properties.keys(): + return self.properties[key] + else: + raise self.NoSuchKeyException("No such key `{}` found"\ + .format(key)) + + def set(self, key, value): + self.properties[key] = value + + def getKeys(self): + return self.properties.keys() + + class NoSuchKeyException(Exception): + def __init__(self, message): + super().__init__(message) diff --git a/lib/git.py b/lib/git.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/google.py b/lib/google.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/prettyPrint.py b/lib/prettyPrint.py new file mode 100644 index 0000000..ffdcf38 --- /dev/null +++ b/lib/prettyPrint.py @@ -0,0 +1,146 @@ +__all__ = [] + +from colorama import Fore as f +from colorama import Back as b +from colorama import Style as s + + +class Prettify: + + def __cont__(t, r, c): + return t.replace(r, r + c) + + # *** text *** + + def red(text): + return f.RED + Prettify.__cont__(text, f.RESET, f.RED) + f.RESET + + def black(text): + return f.BLACK + Prettify.__cont__(text, f.RESET, f.BLACK) + f.RESET + + def blue(text): + return f.BLUE + Prettify.__cont__(text, f.RESET, f.BLUE) + f.RESET + + def cyan(text): + return f.CYAN + Prettify.__cont__(text, f.RESET, f.CYAN) + f.RESET + + def green(text): + return f.GREEN + Prettify.__cont__(text, f.RESET, f.GREEN) + f.RESET + + def magenta(text): + return f.MAGENTA + Prettify.__cont__(text, f.RESET, f.MAGENTA) + \ + f.RESET + + def white(text): + return f.WHITE + Prettify.__cont__(text, f.RESET, f.WHITE) + f.RESET + + def yellow(text): + return f.YELLOW + Prettify.__cont__(text, f.RESET, f.YELLOW) + f.RESET + + def light_black(text): + return f.LIGHTBLACK_EX + Prettify.__cont__(text, f.RESET, \ + f.LIGHTBLACK_EX) + f.RESET + + def light_blue(text): + return f.LIGHTBLUE_EX + Prettify.__cont__(text, f.RESET, \ + f.LIGHTBLUE_EX) + f.RESET + + def light_cyan(text): + return f.LIGHTCYAN_EX + Prettify.__cont__(text, f.RESET, \ + f.LIGHTCYAN_EX ) + f.RESET + + def light_green(text): + return f.LIGHTGREEN_EX + Prettify.__cont__(text, f.RESET, \ + f.LIGHTGREEN_EX) + f.RESET + + def light_magenta(text): + return f.LIGHTMAGENTA_EX + Prettify.__cont__(text, f.RESET, \ + f.LIGHTMAGENTA_EX) + f.RESET + + def light_red(text): + return f.LIGHTRED_EX + Prettify.__cont__(text, f.RESET, \ + f.LIGHTRED_EX) + f.RESET + + def light_white(text): + return f.LIGHTWHITE_EX + Prettify.__cont__(text, f.RESET, \ + f.LIGHTWHITE_EX) + f.RESET + + def light_yellow(text): + return f.LIGHTYELLOW_EX + Prettify.__cont__(text, f.RESET, \ + f.LIGHTYELLOW_EX) + f.RESET + + + + # *** background *** + + def on_red(text): + return b.RED + Prettify.__cont__(text, b.RESET, b.RED) + b.RESET + + def on_black(text): + return b.BLACK + Prettify.__cont__(text, b.RESET, b.BLACK) + b.RESET + + def on_blue(text): + return b.BLUE + Prettify.__cont__(text, b.RESET, b.BLUE) + b.RESET + + def on_cyan(text): + return b.CYAN + Prettify.__cont__(text, b.RESET, b.CYAN) + b.RESET + + def on_green(text): + return b.GREEN + Prettify.__cont__(text, b.RESET, b.GREEN) + b.RESET + + def on_magenta(text): + return b.MAGENTA + Prettify.__cont__(text, b.RESET, b.MAGENTA) + \ + b.RESET + + def on_white(text): + return b.WHITE + Prettify.__cont__(text, b.RESET, b.WHITE) + b.RESET + + def on_yellow(text): + return b.YELLOW + Prettify.__cont__(text, b.RESET, b.YELLOW) + b.RESET + + def on_light_black(text): + return b.LIGHTBLACK_EX + Prettify.__cont__(text, b.RESET, \ + b.LIGHTBLACK_EX) + b.RESET + + def on_light_blue(text): + return b.LIGHTBLUE_EX + Prettify.__cont__(text, b.RESET, \ + b.LIGHTBLUE_EX) + b.RESET + + def on_light_cyan(text): + return b.LIGHTCYAN_EX + Prettify.__cont__(text, b.RESET, \ + b.LIGHTCYAN_EX) + b.RESET + + def on_light_green(text): + return b.LIGHTGREEN_EX + Prettify.__cont__(text, b.RESET, \ + b.LIGHTGREEN_EX) + b.RESET + + def on_light_magenta(text): + return b.LIGHTMAGENTA_EX + Prettify.__cont__(text, b.RESET, \ + b.LIGHTMAGENTA_EX) + b.RESET + + def on_light_red(text): + return b.LIGHTRED_EX + Prettify.__cont__(text, b.RESET, \ + b.LIGHTRED_EX) + b.RESET + + def on_light_white(text): + return b.LIGHTWHITE_EX + Prettify.__cont__(text, b.RESET, \ + b.LIGHTWHITE_EX) + b.RESET + + def on_light_yellow(text): + return b.LIGHTYELLOW_EX + Prettify.__cont__(text, b.RESET, \ + b.LIGHTYELLOW_EX) + b.RESET + + + # *** sytle *** + + def as_bold(text): + return s.BRIGHT + Prettify.__cont__(text, s.RESET_ALL, s.BRIGHT) + \ + s.RESET_ALL + + def as_dim(text): + return s.DIM + Prettify.__cont__(text, s.RESET_ALL, s.BRIGHT) + \ + s.RESET_ALL + + def as_normal(text): + return s.NORMAL + Prettify.__cont__(text, s.RESET_ALL, s.BRIGHT) + \ + s.RESET_ALL diff --git a/lib/utils.py b/lib/utils.py new file mode 100644 index 0000000..a0243d8 --- /dev/null +++ b/lib/utils.py @@ -0,0 +1,103 @@ +__all__ = [] + +import os +import time +import errno +import shutil + +class DirUtil: + def join_names(parent, child): + o = parent + if o.endswith('/'): + o += child + else: + o += '/' + child + return o + + def create_dir(parent, dname = ''): + o = DirUtil.join_names(parent, dname) + if not os.path.exists(o): + try: + os.makedirs(o) + except OSError as err: + if err.errno != errno.EEXIST: + raise + return o + + def get_dir(parent, child = ''): + d = DirUtil.join_names(parent, child) + if not os.path.isdir(d): + raise OSError('Directory not exists.') + return d + + def create_random_dir(parent, prefix = ''): + return DirUtil.create_dir(parent, prefix + str(int(time.time() * \ + 1000))) + + def create_temp_dir(parent = '', prefix = ''): + return DirUtil.create_random_dir(DirUtil.create_dir('/tmp/', \ + parent), prefix) + + def list_dir(parent): + l = [] + for j in os.listdir(parent): + t = DirUtil.join_names(parent, j) + l += [t] + return l + + def get_dir_list(parent, recurse = False): + l = [] + for i in list_dir(parent): + if os.path.isdir(i): + l += [i] + if recurse: + l += get_dir_list(i, recurse) + return l + + def get_files_list(parent, recurse = False): + l = [] + for i in list_dir(parent): + if os.path.isfile(i): + l += [i] + else: + if recurse: + l += get_files_list(i, recurse) + return l + + def merge_dirs(source, dest): + DirUtil.create_dir(dest) + for i in os.listdir(source): + if os.path.isfile(DirUtil.join_names(source, i)): + shutil.move(DirUtil.join_names(source, i), \ + DirUtil.join_names(dest, i)) + else: + DirUtil.merge_dirs(DirUtil.join_names(source, i), \ + DirUtil.create_dir(dest, i)) + def rmdir(dname): + shutil.rmtree(dname) + +class FileUtil: + def join_names(parent, fname): + o = parent + if o.endswith('/'): + o += fname + else: + o += '/' + fname + return o + + def get_file(parent, fname): + f = FileUtil.join_names(parent, fname) + if not os.path.isfile(f): + raise OSError('File not exists') + return f + + def create_random_file(parent, prefix = ''): + return FileUtil.join_names(parent, prefix + str(int(time.time() * \ + 1000))) + + def create_temp_file(dname = '', prefix = ''): + return FileUtil.create_random_file(FileUtil.join_names('/tmp/', \ + dname), prefix) + + + From 6648548e83a2fa4a66432a29cbcfbf4b0f562afd Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Thu, 2 Jul 2020 12:16:54 +0530 Subject: [PATCH 02/17] updated google.py --- f0x.config | 3 ++ f0x.py | 2 +- f0x2.py | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/google.py | 76 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 f0x.config create mode 100755 f0x2.py diff --git a/f0x.config b/f0x.config new file mode 100644 index 0000000..bb2a69c --- /dev/null +++ b/f0x.config @@ -0,0 +1,3 @@ +repo_url=https://github.com/em-corp/dorks.git +dork_path=/home/dinesh/.f0x/dorks +useragents=./user-agents diff --git a/f0x.py b/f0x.py index 2faa190..38a2929 100755 --- a/f0x.py +++ b/f0x.py @@ -1,4 +1,4 @@ -#!/usr/bin/env pyton3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse diff --git a/f0x2.py b/f0x2.py new file mode 100755 index 0000000..8abab30 --- /dev/null +++ b/f0x2.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +__NAME__ = "f0x.py" +__VERSION__ = "1.0" + +from lib.prettyPrint import Prettify as pp + +def banner(): + print(pp.green(' .o88o. .o o.')) + print(pp.green(' 888 `" .8\' `8.')) + print(pp.green(' o888oo .8\' ' + pp.yellow('oooo ooo') + ' `8.')) + print(pp.green(' 888 88 ' + pp.yellow('`88b..8P') + ' 88')) + print(pp.green(' 888 88 ' + pp.yellow('Y888') + ' 88')) + print(pp.green(' 888 `8. ' + pp.yellow('.o8"\'88b') + ' .8\'')) + print(pp.green(' o888o `8. ' + pp.yellow('o88\' 888o') + ' .8\'')) + + +import argparse + +parser = argparse.ArgumentParser() + +parser.add_argument('-s', '--site', help='Specify Target site', dest='site') + +parser.add_argument('-q', '--query', help='Dork to use. If specified, \ + other files will not be read.', dest='query') +parser.add_argument('-i', '--inclusive', help='This works with `query` option \ + only, if used, will also read dorks from file. ', dest='inc', + action="store_true") +parser.add_argument('-A', '--args', help='Specify extra query to supply with \ + each dorks.', dest='ex_query') + +parser.add_argument('-C', '--category', help='Use dorks from this category \ + only.', dest='category') +parser.add_argument('-S', '--severity', help='Specify minimum severity\ + (inclusive) dork file to read, range is [0, 10], defalut: 5.', + dest='severity', type=int, choices=range(1, 11)) +parser.add_argument( '--only', help='Use along with severity, to \ + select only a particular value.', dest='s_only', action='store_true') +parser.add_argument( '--upper', help='Use along with severity, to mark \ + provided value as upper limit (exclusive).', dest='s_upper', + action='store_true') +parser.add_argument('-a', '--all', help='Use all the dork files to fetch \ + result (overrides --only, --upper flags).', dest='s_all', + action='store_true') +parser.add_argument('-Q', '--quality', help='Use only top severity(>=8) dork \ + files (overrides --only, --upper flags). ', dest='s_qual', + action='store_true') + +parser.add_argument('-r', '--results', help='Total results to fetch in one \ + request, default is 30.', dest='page_size', type=int) +parser.add_argument('-t', '--total', help='Total results to fetch for each \ + dork, default is 150.', dest='dork_size', type=int) +parser.add_argument('-T', '--max', help='Maximum results to fetch for all the \ + dorks combined.', dest='max_results', type=int) + +#parser.add_argument('-P', '--proxy', help='proxy', dest='') +#parser.add_argument('-f', '--proxy-file', help='list of proxies', dest='') +#parser.add_argument('-c', '--conn', help='connections per proxy', dest='') + +parser.add_argument('-m', '--mintime', help='Specify minimum sec to wait \ + between requests, If not specified, default 5 sec range is assumed', + dest='min', type=int) +parser.add_argument('-M', '--maxtime', help='Specify maximum sec to wait \ + between requests, if not specified, default 5 sec range is assumed.', + dest='max', type=int) +parser.add_argument('-d', '--delay', help='Specify fix delay(in sec), if \ + specified, took priority over variable delay.', dest='delay', type=int) +parser.add_argument('-p', '--parallel', help='Specify total no of parallel \ + requests, default is 5.', dest='parallel', type=int) +parser.add_argument('-U', '--user-agent', help='Specify User Agent ', + dest='UA') + +parser.add_argument('-o', '--output', help='Specify output directory', + dest='output') +parser.add_argument('-j', '--json', help='Save output in JSON format only', + dest='json', action="store_true") +parser.add_argument('-R', '--report', help='Create Report along with JSON \ + format ouput, default', dest='report', action="store_true") + +parser.add_argument('--update', help='Update Dorks Repo, and exit', + dest='updaterepo', action="store_true") +parser.add_argument('-L', '--list', help='List Repo categories, total \ + dorks and exit', dest='listrepo', action="store_true") +parser.add_argument('-v', '--verbose', help='Be verbose.', dest='verbose', + action="store_true") + +args = parser.parse_args() + + + +banner() +print("\t{} v{}".format(pp.as_bold(pp.red(__NAME__)), pp.blue(__VERSION__))) diff --git a/lib/google.py b/lib/google.py index e69de29..543ffd8 100644 --- a/lib/google.py +++ b/lib/google.py @@ -0,0 +1,76 @@ +__all__ = [] + +import urllib.parse as urlparse +import requests + +class GoogleSearch: + + def __qencode__(q): + return urlparse.quote_plus(q) + + def prepare_URL(query, site = '', params = '', page_num = 0, page_size = 30): + u = 'https://www.google.com/search?gbv=1&q=' + if query == '': + raise Exception("Query cannot be empty") + u += GoogleSearch.__qencode__(query) + if params != '': + u += '+' + GoogleSearch.__qencode__(params) + if site != '': + u += '+' + GoogleSearch.__qencode__(site) + u += '&btnG=Google+Search' + + page_size = int(page_size) / 10 + # consider lower bound + if page_size >= 10: + page_size = 100 + elif page_size >= 5: + page_size = 50 + elif page_size > 0: + page_size = page_size * 10 + else: + page_size = 30 + + fmt = '{}&start={}&num={}' + + page_num = int(page_num) + if page_num <= 1: + return fmt.format(u, '', page_size) + else: + return fmt.format(u, (page_num - 1) * page_size, page_size) + + def fetch(url, UA = 'f0x.py/1.0 (Linux; python/requests)'): + hdrs = { + 'Host': 'www.google.com', + 'User-Agent': UA, + 'Accept': 'text/html,application/xhtml+xml,application/xml;\ + q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate', + 'Referer': 'https://www.google.com/', + 'DNT': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Cache-Control': 'max-age=0, no-cache', + 'Pragma': 'no-cache', + 'TE': 'Trailers' + } + return requests.get(u, headers=hdrs).text + + def has_next(text): + o = re.search('aria-label="Next page"[^>]*>((Next >)|\ + (]*>>))', text, re.I) + if o: + return True + else: + return False + + def extract_urls(text): + urls = [] + for pat in re.findall('\s+href="/url\?q=([^"&]*)[^"]*"[^>]*>', text, \ + re.M|re.I): + if not re.search('^http(s)?://(www\.)?[^.]*\.google\.com', pat, \ + re.I): + urls += [urlparse.unquote(pat)] + return urls + + From 0c510c7459bd986ebb0cdc64976c1d1c51f04fcf Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Thu, 2 Jul 2020 14:02:05 +0530 Subject: [PATCH 03/17] added useragent lib --- lib/__pycache__/useragents.cpython-38.pyc | Bin 0 -> 6800 bytes lib/useragents.py | 150 ++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 lib/__pycache__/useragents.cpython-38.pyc create mode 100644 lib/useragents.py diff --git a/lib/__pycache__/useragents.cpython-38.pyc b/lib/__pycache__/useragents.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f792fdbcb82afcdace4bd495eb73fe8bf7c92b4d GIT binary patch literal 6800 zcmc&(?Qh%09j0W*anq#h-e22Z*RHdcnB*N_L{-BOS+V87l5AMAop{&?Sw~w;BvB=) z+ToWDTY}L2g2RHZO*LxK%N5Z~91)6H7_ERN+rs>@K0-gIsdOuB5 zH6b(qFX$swo9sIL(ctDj{CdBEizV!*sKDM}@6&mDgD%jUbdkP5Z_yX&ZF+~kM87~^ zreCCAqF<(8pn5qTi<9p|8+a>1!0x*XeiZ8}v>3J^B{?K0JF1-d>^^ z`U83wp1nfv(Pb)9iONUX{v5@Wyf5s}Q-!MV_XgDfuYQ`|DFnZ&IDy@6HzZAz(Nfv& z_D}Mt(50?pQ)Gy;DCbeFvRumP@)}y{+ifasW4*eUL4{tgZS7eH8@4ZLN>)^Kw6w8a zuaxhhw*AyXC9C<=$)IA(bvl-$6BtV&G7$|GRh$F6ZJ}D@u;JSA%fZLfkYqUyBS#7` zj_US!4E$RsBk%4B4J zn#P$9Z{yf{idKuERVphaw{cb0G;kM@7=|L|@MLJeo)sFAcN5TNHfC%QyssP1! z4k((42P_feQ4FPAd8}q1T^5Q8ZYmOu=;s=27FMd(N#8^I)-Wqu*AL~d(+rf2SScAq z!7&aMi$pB21>)S5-6sd*C~_94C)#AAeJ~AP?Aoikt|>7*g>ZP9a#^x2cA>Qdnx@m~ zHGKP^ZRJs=_HY%!a`I@;?o#L2L)ALcnM>_$?&)eC)m@ufUC5E5oC&IlaUNmlwsX+$ z`u#i#b1ynEbd$a1zWX+2Z$xQSA~KfYA4Y4oYaKc#5^EIk^z;oZ<%2Swk}W({Ft%29EsG@9(eNQCKf7>(M*Lc()5WNJBf z8DL=*s|GJ5x~u{vMGk}|`K2*ag`^Cb3TDiswOSdm*o9xlb0)aY2LYPV(opx%vS6HP zwt+Sh{4O!V=fFqNwKr>$9!8RygMgKJB#qGmYyAB2ter4CSP{tq#)hsBY@`{RlZlf|ve-Bq<$lL*pqw^_<{~k#GV2r7fIkz9H;OE#Fd~fL=Lv`-TZ}NyVS`aO zNOoI|OH`!nM>)C(m42q=!IP4ki>YD&rBZPa%RwK6*@ zyZuxGb&ADoE-JvWV+|=*d4#f$reTUHE0>mcRV+*FLgeQ2QN|1V(56g*?;~Jm;0iN)m*r7pUs(SEFf|yXl621Jamj@V*@PkD)AS=)u&H(n$&!8#vol)z!t%Je7jalZb9#_OzH zmnxy)337u|6sf-7Cl(bI(cC=+)aW@PTI%-aSMM=hwbas~)q zKa-)OEJOT9@*Z;f`Sh}ckAump0{8xZt;$c(3@YC4`i|GiqlaDJY9lV|Ar?)H%@Hx6 z%b=CS+*DeboE>Dx_=TX8m~}`Mo_XwLgY7KwVT`%u;tccK#B^a0FGn7gEfNW{Lo8L- z2Sx}_yZFlU$m%Ass6@Uyb;)4rfg=?(b>K%T&WTD;N`M*O&K%g?qXWm6AQJQFZp-(3 zZ%fkg@v+DnMCd8SJ;avZ$pp1?2YTFSInL{-*6zE#JcDhPUZgK1B!~NE zEk8i@2Ao;=XoFj$5}pNI*$y!A;4m<0hgFVkiXDdH83r1vS*#vHrq`zV?n=X^eXj>c zEbfFrV2z>sa<2_LvUq629zQg*1gERXEq<7wA;Gdh-f}I^cVJ{l_r%DdLG}l8;IX6r z6rAuE*-`%jTzugP{I|fTscI(ers4kdmTC4IZQJur^HagS0guAVe^#XyJ0Yib*YaA@ z1E<;VzYD literal 0 HcmV?d00001 diff --git a/lib/useragents.py b/lib/useragents.py new file mode 100644 index 0000000..6fc4ac9 --- /dev/null +++ b/lib/useragents.py @@ -0,0 +1,150 @@ +__all__ = [] + +import random + +class UA: + __ualist__ = [ + 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.'+ + '84 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + + 'Chrome/60.0.3112.107 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 7.0; SM-G930VC Build/NRD90M; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Chrome/58.0.3029.83 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G935S Build/MMB29K; wv) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Chrome/55.0.2883.91 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G920V Build/MMB29K) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.'+ + '98 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.'+ + '83 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 6P Build/MMB29P) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.'+ + '83 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 7.1.1; G8231 Build/41.2.A.0.219; wv)'+ + ' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Chrome/59.0.3071.125 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; E6653 Build/32.2.A.0.253) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.'+ + '98 Mobile Safari/537.36', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; '+ + 'Trident/5.0)', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 '+ + 'Firefox/15.0.1', + 'Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.'+ + '98 Mobile Safari/537.3', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) '+ + 'AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 '+ + 'Mobile/15E148 Safari/604.1', + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FSL 7.0.7.'+ + '01001)', + 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.02', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) '+ + 'AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/'+ + '69.0.3497.105 Mobile/15E148 Safari/605.1', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) '+ + 'AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/13.'+ + '2b11866 Mobile/16A366 Safari/605.1.15', + 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; '+ + '.NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET '+ + 'CLR 3.5.30729)', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) '+ + 'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/'+ + '11.0 Mobile/15A372 Safari/604.1', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) '+ + 'AppleWebKit/604.1.34 (KHTML, like Gecko) Version/'+ + '11.0 Mobile/15A5341f Safari/604.1', + 'Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; '+ + 'RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/'+ + '52.0.2743.116 Mobile Safari/537.36 Edge/15.15254', + 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; '+ + 'RM-1127_16056) AppleWebKit/537.36(KHTML, like Gecko) '+ + 'Chrome/42.0.2311.135 Mobile Safari/537.36 Edge/12.10536', + 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like '+ + 'Gecko) Chrome/13.0.782.112 Safari/535.1', + 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; '+ + 'Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) '+ + 'Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.1058', + 'Mozilla/5.0 (Linux; Android 7.0; SM-T827R4 Build/NRD90M) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.'+ + '3112.116 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T550 Build/LRX22G)'+ + ' AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser'+ + '/3.3 Chrome/38.0.2125.102 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 4.4.3; KFTHWI Build/KTU84M) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Silk/47.1.79 like'+ + ' Chrome/47.0.2526.80 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 5.0.2; LG-V410/V41020c Build/LRX22G)'+ + ' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Chrome/34.0.1847.118 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '+ + '(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 '+ + 'Edge/12.246', + 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 '+ + '(KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36', + 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET '+ + 'CLR 2.0.50727)', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 '+ + 'Firefox/12.0', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) '+ + 'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 '+ + 'Mobile/15A5370a Safari/604.1', + 'Mozilla/5.0 (iPhone9,3; U; CPU iPhone OS 10_0_1 like Mac OS X) '+ + 'AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 '+ + 'Mobile/14A403 Safari/602.1', + 'Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/'+ + '13.0.1', + 'Mozilla/5.0 (iPhone9,4; U; CPU iPhone OS 10_0_1 like Mac OS X) '+ + 'AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 '+ + 'Mobile/14A403 Safari/602.1', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/'+ + '601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, '+ + 'like Gecko) Chrome/47.0.2526.111 Safari/537.36', + 'Mozilla/5.0 (Apple-iPhone7C2/1202.466; U; CPU like Mac OS X; en)'+ + ' AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 '+ + 'Mobile/1A543 Safari/419.3', + 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 '+ + '(KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, '+ + 'like Gecko) Chrome/47.0.2526.111 Safari/537.36', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 '+ + 'Firefox/15.0.1', + 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/'+ + 'bingbot.htm)', + 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/'+ + 'help/us/ysearch/slurp)', + 'Mozilla/5.0 (Linux; Android 6.0.1; SGP771 Build/32.2.A.0.253; '+ + 'wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/'+ + '52.0.2743.98 Safari/537.36', + 'Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) '+ + 'Opera 7.02 Bork-edition [en]', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/'+ + '601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '+ + '(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 '+ + 'Edge/12.246', + 'Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) '+ + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Chrome/52.0.2743.98 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; SHIELD Tablet K1 Build/MRA58K'+ + '; wv) AppleWebKit/537.36 (KHTML, like Gecko) '+ + 'Version/4.0 Chrome/55.0.2883.91 Safari/537.36', + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET '+ + 'CLR 1.0.3705)', + 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/'+ + 'search/spider.html)', + 'Opera/9.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.01' + ] + + def get_ua_list(): + return UA.__ualist__ + + def get_random_ua(): + return random.choice(UA.__ualist__) + From e1bc642a0c79589d4eaddb0ff8f736b2f157d8f4 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Thu, 2 Jul 2020 18:21:20 +0530 Subject: [PATCH 04/17] updated code --- f0x2.config | 4 + f0x2.py | 132 ++++++++++++++++++++- lib/__pycache__/prettyPrint.cpython-38.pyc | Bin 0 -> 7787 bytes lib/__pycache__/utils.cpython-38.pyc | Bin 3537 -> 4054 bytes lib/git.py | 0 lib/prettyPrint.py | 22 ++++ lib/utils.py | 13 ++ 7 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 f0x2.config create mode 100644 lib/__pycache__/prettyPrint.cpython-38.pyc delete mode 100644 lib/git.py diff --git a/f0x2.config b/f0x2.config new file mode 100644 index 0000000..730a669 --- /dev/null +++ b/f0x2.config @@ -0,0 +1,4 @@ +[defaults] + +repo_url=https://github.com/em-corp/dorks.git +dork_path=~/.f0x/dorks diff --git a/f0x2.py b/f0x2.py index 8abab30..e1a6294 100755 --- a/f0x2.py +++ b/f0x2.py @@ -4,7 +4,14 @@ __NAME__ = "f0x.py" __VERSION__ = "1.0" +import argparse +import re +import os +from git import Repo from lib.prettyPrint import Prettify as pp +from lib.config import ConfigManager as conf +from lib.utils import DirUtil as dutil +from lib.utils import FileUtil as futil def banner(): print(pp.green(' .o88o. .o o.')) @@ -15,9 +22,6 @@ def banner(): print(pp.green(' 888 `8. ' + pp.yellow('.o8"\'88b') + ' .8\'')) print(pp.green(' o888o `8. ' + pp.yellow('o88\' 888o') + ' .8\'')) - -import argparse - parser = argparse.ArgumentParser() parser.add_argument('-s', '--site', help='Specify Target site', dest='site') @@ -87,7 +91,125 @@ def banner(): args = parser.parse_args() - - banner() print("\t{} v{}".format(pp.as_bold(pp.red(__NAME__)), pp.blue(__VERSION__))) + +if not (args.site or \ + args.query or \ + args.category or \ + args.severity or \ + args.s_all or \ + args.s_qual or \ + args.page_size or \ + args.dork_size or \ + args.max_results or \ + args.min or \ + args.max or \ + args.delay or \ + args.parallel or \ + args.UA or \ + args.output or \ + args.updaterepo or \ + args.listrepo): + pp.p_error('No options are specified.') + print (parser.format_help()) + quit() + +#------------------------------------------------------------------------------ + +class Fox: + def _load_conf(): + conf.load('f0x2.config') + + def get_conf(): + if conf.getConfig is None: + Fox._load_conf() + return conf.getConfig() + + def get_dork_path(): + dp = '' + try: + dp = Fox.get_conf().get('dork_path') + except: + pp.p_error("Dorks path not exists, Check config file.") + return + else: + if dp = '': + pp.p_error("Dorks path not defined, Check config file.") + return + + if dp.startswith('~'): + dp = os.path.expanduser(dp) + elif dp.startswith('/'): + pass + elif dp.startswith('./'): + cwd = os.path.realpath('.') + dp = dutil.join_names(cwd, dp[2:]) + else: + cwd = os.path.realpath('.') + dp = dutil.join_names(cwd, dp) + + return dp + + def list_dorks_stats(): + dp = Fox.get_dork_path() + if dp is None or dp = '': + pp.p_error("Dorks path not defined, Check config file.") + return + + dl = dutil.get_dir_list(dp, True) + if len(dl) == 0: + pp.p_log("No Dorks available, Update dork repo.") + + for i in dl: + dc = re.sub('^{}[/]?'.format(dp), '', i) + dc = re.sub('/', '.', dc) + td = len (dutil.get_files_list(i, True)) + pp.p_log("Category: {}".format(dc)) + pp.p_log("Total Dorks: {}\n".format(td), '**') + + def update_dorks_repo(): + pp.p_log("Building Dork Repo.") + repo_url = Fox.get_conf().get('repo_url') + pp.p_log("Fetching from '{}'".format(repo_url)) + + tmpdir = dutil.create_temp_dir('f0x', 'repo_') + Repo.clone_from(repo_url, tmpdir) + pp.p_log("Done Fetching.") + + try: + g = dutil.get_dir(tmpdir, '.git') + except: + pass + else: + dutil.rmdir(g) + + try: + f = futil.get_file(tmpdir, 'README.md') + except: + pass + else: + os.remove(f) + + try: + f = futil.get_file(tmpdir, 'LICENSE') + except: + pass + else: + os.remove(f) + + dutil.merge_dirs(tmpdir, Fox.get_dork_path) + pp.p_log("Dork Repo updated.") + + + +#------------------------------------------------------------------------------ +verbose = args.verbose + +if args.listrepo: + Fox.list_dorks_stats() + quit() + +if args.updaterepo: + Fox.update_dorks_repo() + quit() diff --git a/lib/__pycache__/prettyPrint.cpython-38.pyc b/lib/__pycache__/prettyPrint.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..463cee8b7b6a327e674022d650ee78f579aeb4e4 GIT binary patch literal 7787 zcmb`MTW=ay5P)~tUBEW@7T41znB4C-Y~xFUv5W0GA!=5bS=%IBoC7Fv;yhH% zLm&Fkr%IJ7HBqHX{tf*n``V}eg-BI8a~9YQtfRGhK+GJN1LtFAc6RpcayZ;B;UoOl zzt6_{BG`Dm>BOqL`=O3NUneISGHy_8l! z0Tnf_0e;jP1R#jo4{gwnIshFILLG!KbfRv9F6c(x4n5F|x&!t>AL55sl1f%*vCgj=YO!fm*NdKm7)J=DkG zK0H8u9AYqz`UFhCBQPvM6fC}% z(`R7`mhqEw@GZvXu^C)oMx~Nl+scyP>kk@h_?p01xf{i{^g-H|@#!0sKKd4=9XW{+ z3@~FepUD~##gr?i%uG4EBe4#g|7j|^Q7RZw$Q6z9Ms%{2trm=8r5t@#VlTKTZ{ior z(FIm|Ze-E0cHwn2pIeP?63^BG%M~kQn_Fg3*R!Q!Mb{}=%RUs6{HCW4Jyy47F;^N7 z8vF6Z>5t%`B0ZNYz7NVajZ}K~Q5odpL1|H9G&iD|@|vln;**U2I18gNwN!jDzO;iW zTQOc&_#d-Gh^z`TpbH0es!)Rp4FZ5zCubN>#3mluP>jOCdPtC?9X#49#^OaUepn~! zY!GpiFeY_8u^jh;q^{>#Qd;(#We#FCzXvRj$rkSh>bgaK*Osd(HIk+#kZ zqj=DPWJisggfRu?V$<>DQp}5=0)@;v9 z2217Rf(KZkWyZ<`UTZwk)-r)tmMbo}&6yy!1aC$)eo3O&h{Wk28s|~rV0=4P@ zvriEXvoMcQ-|WR>ow#(p4a3VygYcTV#=%CV{c^4_){`Dt!;bZ&>NDz*wYw42S`DhY(C%-|R)LKoKkDgX#fxu4 z4PNXKwp>{49>zISShkLNe3KYAidFD^lt0SbxYhlGOgA`it6WKb3O;@N?z>yVZ{MZ0cq%oY!aZYc zlQybN{~A$LEbuA(Iku04+b(TjQ#TkZv0rdP{nEe=cg@(sO1Z&VsZq39iL;54&Ekx- zI-8uDkErY+zG06@@LlcwG;*WzZ|b??TIpA`*P-kvTekntv=U1tTbXwC6f(Wau;OpK zFp;f=X|FN0yroUX$CsxgTC=-&B#(W?XRumb|DE`(W4BNCeSYKg8`({HG|MjmaUMNr z{1wb?OuR3Hy5mc7%w6~f@%MIotQVE(-!L+O>D7E^U{lYR*8jj6_2Alj%-qxhNZG>k zjV>;&e8ZbRj^lxSgNBID;jt8c(7Z5)FpDeyiKhC>PWO-v?<27V`mOVHucw$P7`kq@ z>w2LC)jW+uy8g17$y*kFU&cDP=(Yl*KhJnp(+)a2OyVdB9<>uR8X<9-M1;gy66Z-o zNn9jxnFQ^Iv8yDmk+@ERcAMBu5_A)k-6lbMI_xfqdnE3Yct9dXVw?oen`s)&kf2=v zHb){!f-b+=5(&DtbhwO1EkvJ@`zc4tG%e^0YC)y(RQX5^I-kLYRl(0`uG6Xp1N^0q{$FR@G^!ze-dy(gy7h<7$BjVK|kI%4NngV_w;u|jrF O#%X1uSA8KFpUQvV?UW<{ literal 0 HcmV?d00001 diff --git a/lib/__pycache__/utils.cpython-38.pyc b/lib/__pycache__/utils.cpython-38.pyc index 2423329f4221c10143e75ff322bb5ab18f0fc130..deb9ff1ebe45dcbdc47d9eb1f515e21a206c790f 100644 GIT binary patch delta 754 zcmZ8e&1(};5P$PF`;{g!YNQc^2GPnY2;v7K#rjE42CaDL9$eFX3kH+j_U%?n2n1|z zUK*Z=&`a_U2wD%`Jb6&?kV}P}Jc(Yt3(jn;24|Vy&Tn?!yx+_Y5A7T&+{@>)4DI{i z*Zmi{UZDbR@6?Go!2~pQSD$Bs3vGi5ZJoOYvW(D?>BvlEhA@$t$SeVOnPdH+UNyhB z_u%X27Xh-{pP>w}PkrMQ6HqEfPR+E5-+tUld%4$Of)#WH|B#p;GHC$~i1=8Uo`{tS-EGLa992qhd zf=NA2>M|K=K`12sm@!OYG|Ewtw-h8#f*@>QCeE@hzQ7tkyuE)s&*{lRpJePB#eav( zAxT&t$LX*EN6mb!+2L{)Kgp{I4!B0EzUGC||7y(il5QBSU|F1Q5ZZ-F0bx{gba|Fq z(H`Y_0%|I+5&WAjjcCO5-iH{(BofE6(u`w|OwX$~o1Q24P}yOE5`o&x>&43ykpg*v WU{-DAXG%p3GyKnLz~Y7ZLisnQ%b}bA delta 230 zcmca6e^Ht*l$V!_0SH!q`x9rvJ&{j_(Pg5xx{^CX3R4Pm3quNXGgA~RkjIk33goc@ zd2A_+!3>&g8@nPH896sUW|UxL6rB8r=^~@_nLNy{~@+@FX zn*5EYUcw*fs$vcx!NJJE!NJDGQe+0?XtGbvgf- diff --git a/lib/git.py b/lib/git.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/prettyPrint.py b/lib/prettyPrint.py index ffdcf38..967a2fa 100644 --- a/lib/prettyPrint.py +++ b/lib/prettyPrint.py @@ -144,3 +144,25 @@ def as_dim(text): def as_normal(text): return s.NORMAL + Prettify.__cont__(text, s.RESET_ALL, s.BRIGHT) + \ s.RESET_ALL + + # *** logger *** + + def p_error(text): + print("[{}]: {}".format(Prettify.as_bold(Prettify.red('ERROR')), text)) + + def p_info(text): + print("[{}]: {}".format(Prettify.cyan('INFO'), text)) + + def p_warn(text): + print("[{}]: {}".format(Prettify.red('WARN'), text)) + + def p_dbug(text): + print("[{}]: {}".format(Prettify.as_bold(Prettify.yellow('DEBUG')), + text)) + + def p_log(text, header = '*'): + print("[{}]: {}".format(Prettify.as_bold(Prettify.blue(header)), text)) + + def p_clog(text, header): + print("[{}]: {}".format(header, text)) + diff --git a/lib/utils.py b/lib/utils.py index a0243d8..5a11a16 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -4,6 +4,7 @@ import time import errno import shutil +import random class DirUtil: def join_names(parent, child): @@ -100,4 +101,16 @@ def create_temp_file(dname = '', prefix = ''): dname), prefix) +class Random: + def rand_between(start, end): + if start < 0: + raise Exception('Require positive number') + if end < 0: + raise Exception('Require positive number') + + return start + ((end - start) * random.random()) + + def rand_no(max_no): + return Random.rand_between(0, max_no) + From 4ac653feef07192c3259878a50cc20eac6521e07 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Thu, 2 Jul 2020 22:17:21 +0530 Subject: [PATCH 05/17] moded --- f0x2.py | 162 ++++++++++++++++++++++++++++++++++++++++++++- lib/prettyPrint.py | 2 +- 2 files changed, 161 insertions(+), 3 deletions(-) diff --git a/f0x2.py b/f0x2.py index e1a6294..e8c8980 100755 --- a/f0x2.py +++ b/f0x2.py @@ -134,7 +134,7 @@ def get_dork_path(): pp.p_error("Dorks path not exists, Check config file.") return else: - if dp = '': + if dp == '': pp.p_error("Dorks path not defined, Check config file.") return @@ -153,7 +153,7 @@ def get_dork_path(): def list_dorks_stats(): dp = Fox.get_dork_path() - if dp is None or dp = '': + if dp is None or dp == '': pp.p_error("Dorks path not defined, Check config file.") return @@ -213,3 +213,161 @@ def update_dorks_repo(): if args.updaterepo: Fox.update_dorks_repo() quit() + +site = '' +if args.site: + site = args.site.strip() + site = re.sub(r'^http(s)?://(www\.)?', '', site) + site = re.sub('/.*(/)?', '', site) + +if verbose and args.site: + pp.p_debug("Using target ==> {}".format(site)) + +query_extra = '' +if args.ex_query: + query_extra = args.ex_query.strip() + +if verbose and args.ex_query: + pp.p_debug("Using extra query parameters ==> {}".format(query_extra)) + +r_query='' +# if raw query provided +if args.query: + r_query = args.query.strip() + +if verbose and args.query: + pp.p_debug("Using query ==> {}".format(r_query)) + +inclusive = args.inc + +if inclusive and not args.query: + pp.p_error("Query not found, but inclusive switch is on") + quit() + +if verbose and inclusive: + pp.p_debug("Including dorks results along with query results") + +category = '' +if args.category: + category = args.category.strip() + +if verbose and category: + pp.p_debug("Using category ==> {}".format(category)) + +severity = 5 +if args.severity: + severity = args.severity + +if verbose and args.severity: + pp.p_debug("Using severity ==> {}".format(severity)) + +severity_flag = 0 +# 0 for >= severity +# 1 for = severity +# 2 for < severity +if args.s_only: + severity_flag = 1 +if args.s_upper: + severity_flag = 2 + +if args.s_all: + severity = 0 + severity_flag = 0 + +if verbose and args.s_all: + if args.severity: + pp.p_debug("Severity is overridden by `--all` switch") + pp.p_debug(" Using severity ==> {}".format(severity)) + +if args.s_qual: + severity = 8 + severity_flag = 0 + +if verbose and args.s_qual: + if args.severity or args.s_all: + pp.p_debug("Severity is overridden by `--quality` switch") + pp.p_debug("Using severity ==> {}".format(severity)) + +if verbose: + s_m = '' + if severity_flag == 0: + s_m = 'min' + elif severity_flag == 1: + s_m = 'only' + elif severity_flag == 2: + s_m = 'max' + pp.p_debug("Using Severity as ==> {}".format(s_m)) + +page_size = args.page_size + +if verbose: + pp.p_debug("Using page size ==> {}".format(page_size)) + +dork_size = 150 +if args.dork_size and args.dork_size >= page_size: + dork_size = args.dork_size + +if verbose: + pp.p_debug("Max results per dork ==> {}".format(dork_size)) + +# defaults to 100 dorks +max_results = dork_size * 100 +if args.max_results and args.max_results >= 0: + max_results = args.max_results + +if verbose: + pp.p_debug("Total results limit to ==> {}".format(max_results)) + +s_delay = 2 +e_delay = 7 +if args.delay and args.delay >= 0: + s_delay = e_delay = args.delay +else: + if args.min and args.max: + s_delay = args.min + e_delay = args.max + elif args.min: + s_delay = args.min + e_delay = s_delay + 5 + elif args.max: + e_delay = args.max + s_delay = 0 + if e_delay - 5 > 0: + s_delay = e_delay - 5 + +if verbose: + pp.p_debug("Using delay range ==> [{}, {}] sec".format(s_delay, e_delay)) + +parallel_req = 5 +if args.parallel and args.parallel > 0: + parallel_req = args.parallel + +if verbose: + pp.p_debug("Using parallel requests ==> {}".format(parallel_req)) + +useragent = '' +if args.UA: + useragent = args.UA.strip() + +if verbose and args.UA: + pp.p_debug("Using User-Agent ==> {}".format(useragent)) + +out_dir = '' +if args.output: + out_dir = dutil.create_dir(args.output.strip()) +else: + pp.p_error("Output directory is not specified") + quit() + +buildReport = True +if args.json and not args.report: + buildReport = False + +if verbose: + pp.p_debug("Using output directory ==> {}".format(out_dir)) + if not buildReport: + pp.p_debug("Output will be saved in JSON format") + else: + pp.p_debug("Reporting is enabled, along with JSON format") + + diff --git a/lib/prettyPrint.py b/lib/prettyPrint.py index 967a2fa..bf36e58 100644 --- a/lib/prettyPrint.py +++ b/lib/prettyPrint.py @@ -156,7 +156,7 @@ def p_info(text): def p_warn(text): print("[{}]: {}".format(Prettify.red('WARN'), text)) - def p_dbug(text): + def p_debug(text): print("[{}]: {}".format(Prettify.as_bold(Prettify.yellow('DEBUG')), text)) From 5a8b96b52b57903975cca66b072fca83fa87ccd4 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Fri, 3 Jul 2020 03:08:47 +0530 Subject: [PATCH 06/17] updated --- f0x2.py | 423 ++++++++++++++++----- lib/__pycache__/prettyPrint.cpython-38.pyc | Bin 7787 -> 7789 bytes lib/google.py | 2 +- lib/utils.py | 5 + 4 files changed, 343 insertions(+), 87 deletions(-) diff --git a/f0x2.py b/f0x2.py index e8c8980..89ee1aa 100755 --- a/f0x2.py +++ b/f0x2.py @@ -7,11 +7,15 @@ import argparse import re import os +import time from git import Repo from lib.prettyPrint import Prettify as pp from lib.config import ConfigManager as conf from lib.utils import DirUtil as dutil from lib.utils import FileUtil as futil +from lib.google import GoogleSearch as gs +from lib.utils import Random as rand +from lib.useragents import UA def banner(): print(pp.green(' .o88o. .o o.')) @@ -118,18 +122,150 @@ def banner(): #------------------------------------------------------------------------------ class Fox: - def _load_conf(): + def __init__(self): + self.verbose = None + self.site = None + self.ex_q = None + self.raw_q = None + self.inc = None + self.cat = None + self.sev = None + self.sev_flag = None + self.psize = None + self.dsize = None + self.max_res = None + self.del_range = [None, None] + self.par = None + self.UA = None + self.out_dir = None + self.breport = None + self.mr_achived = 0 + + def set_site(self, site): + self.site = site + + def get_site(self): + return self.site + + def set_ex_query(self, q): + self.ex_q = q + + def get_ex_query(self): + return self.ex_q + + def set_raw_query(self, q): + self.raw_q = q + + def get_raw_query(self): + return self.raw_q + + def set_inc_q(self): + self.inc = True + + def unset_inc_q(self): + self.inc = False + + def get_inc_q(self): + return self.inc + + def set_category(self, c): + self.cat = c + + def get_category(self): + return self.cat + + def set_severity(self, s): + self.sev = s + + def get_severity(self): + return self.sev + + def set_severity_flag(self, sf): + self.sev_flag = sf + + def get_severity_flag(self): + return self.sev_flag + + def set_page_size(self, ps): + self.psize = ps + + def get_page_size(self): + return self.psize + + def set_dork_size(self, ds): + self.dsize = ds + + def get_dork_size(self): + return self.dsize + + def set_max_results(self, mr): + self.max_res = mr + + def get_max_results(self): + return self.max_res + + def set_delay_range(self, s, e): + self.del_range = [s, e] + + def get_delay_range(self): + return self.del_range + + def get_delay_start(self): + return self.get_delay_range()[0] + + def get_delay_end(self): + return self.get_delay_range()[1] + + def set_max_parallel(self, p): + self.par = p + + def get_max_parallel(self): + return self.par + + def set_ua(self, ua): + self.UA = ua + + def get_ua(self): + return self.UA + + def has_ua(self): + if self.UA and not (self.UA == ''): + return True + return False + + def set_out_dir(self, d): + self.out_dir = d + + def get_out_dir(self): + return self.out_dir + + def set_build_report(self): + self.breport = True + + def unset_build_report(self): + self.breport = False + + def get_build_report(self): + return self.breport + + def set_verbose(self): + self.verbose = True + + def is_verbose(self): + return self.verbose + + def _load_conf(self): conf.load('f0x2.config') - def get_conf(): + def get_conf(self): if conf.getConfig is None: - Fox._load_conf() + self._load_conf() return conf.getConfig() - def get_dork_path(): + def get_dork_path(self): dp = '' try: - dp = Fox.get_conf().get('dork_path') + dp = self.get_conf().get('dork_path') except: pp.p_error("Dorks path not exists, Check config file.") return @@ -151,8 +287,8 @@ def get_dork_path(): return dp - def list_dorks_stats(): - dp = Fox.get_dork_path() + def list_dorks_stats(self): + dp = self.get_dork_path() if dp is None or dp == '': pp.p_error("Dorks path not defined, Check config file.") return @@ -168,9 +304,9 @@ def list_dorks_stats(): pp.p_log("Category: {}".format(dc)) pp.p_log("Total Dorks: {}\n".format(td), '**') - def update_dorks_repo(): + def update_dorks_repo(self): pp.p_log("Building Dork Repo.") - repo_url = Fox.get_conf().get('repo_url') + repo_url = self.get_conf().get('repo_url') pp.p_log("Fetching from '{}'".format(repo_url)) tmpdir = dutil.create_temp_dir('f0x', 'repo_') @@ -198,125 +334,237 @@ def update_dorks_repo(): else: os.remove(f) - dutil.merge_dirs(tmpdir, Fox.get_dork_path) + dutil.merge_dirs(tmpdir, self.get_dork_path) pp.p_log("Dork Repo updated.") + def get_severity_list(self): + sev = [] + s = self.get_severity() + s_f = self.get_severity_flag() + + if s_f == 0: + sev = list(range(s, 11)) + elif s_f == 1: + sev = [s] + elif s_f == 2: + sev = list(range(1, s)) #if severity = 1, return empty set + + return sev + + def get_dorks(self, svr): + dorks = [] + if self.get_raw_query(): + if svr == 10: + dorks += [self.get_raw_query()] + if not self.get_inc_q(): + return dorks + + dpath = self.get_dork_path() + chome = '' + + if self.get_category() and self.get_category() != '': + chome = re.sub('\.', '/', self.get_category()) + + dpath = dutil.get_dir(dpath, chome) + + for i in dutil.get_files_list(dpath, True): + with open (i, 'r') as dfile: + d = '' + j = '' + for l in dfile: + if l.lstrip().lower().startswith('dork:'): + d = re.sub('^[dD][oO][rR][kK]:', '', l.lstrip()) + d = d.strip() + elif l.lstrip().lower().startswith('severity:'): + j = re.sub('^severity:', '', l.lstrip().lower()) + j = j.strip() + + if int(j) == svr: + dorks.append(d) + + return dorks + + def _get_count(self): + self.mr_achived + + def _set_count(self, c): + self.mr_achived = c + + def _update_results_count(self, c): + self._set_count(self._get_count() + c) + + def _can_fetch_more(self): + return (self.get_max_results() - self._get_count()) > 0 + + def persist(self, r, d, s, o, fc): + if fc == 1: + fd = open(futil.join_names(o, 'dork.info'), 'w') + fd.write("dork: {}\n".format(d)) + fd.write("severity: {}\n".format(s)) + fd.close() + fd = open(futil.join_names(o, 'dork_page_' + str(fc)), 'w') + fd.write(r) + fd.close() + + def process_dork(self, d, s): + if not self._can_fetch_more(): + return + + i = 0 + rFlag = True + dd = dutil.create_random_dir(dutil.create_dir(self.get_out_dir(), \ + 'dorks'), 'dork_') + pp.p_log("Processing dork: {}".format(d)) + + while rFlag and self._can_fetch_more() and \ + ((self.get_page_size() * i) <= self.get_dork_size()): + i += 1 + url = gsrch.prepare_URL(d, self.get_site(), \ + self.get_ex_query(), i, self.get_page_size()) + t = rand.rand_between(self.get_delay_start(), \ + self.get_delay_end()) + + pp.p_log("Sleeping for {} sec.".format(t)) + time.sleep(t) + pp.p_log("Processing now.") + pp.p_log("#Page to fetch: {}".format(i)) + + if self.has_ua(): + ua = self.get_ua() + else: + ua = UA.get_random_ua() + if self.is_verbose(): + pp.p_debug("Using UA ==> {}".format(ua)) + + response = gsrch.fetch(url, ua) + pp.p_log("Got Response.") + + self.persist(response, d, s, dd, i) + self._update_results_count(self.get_page_size()) + rFlag = gsrch.has_next(response) + futil.dump_list(futil.join_names(dd, 'urls.txt'), \ + gsrch.extract_urls(response)) + def dbBuilder(self): + for s in self.get_severity_list(): + for d in self.get_dorks(s): + self.process_dork(d, s) #------------------------------------------------------------------------------ -verbose = args.verbose +fox = Fox() + +if args.verbose: + fox.set_verbose() if args.listrepo: - Fox.list_dorks_stats() + fox.list_dorks_stats() quit() if args.updaterepo: - Fox.update_dorks_repo() + fox.update_dorks_repo() quit() -site = '' if args.site: + site = '' site = args.site.strip() site = re.sub(r'^http(s)?://(www\.)?', '', site) site = re.sub('/.*(/)?', '', site) -if verbose and args.site: - pp.p_debug("Using target ==> {}".format(site)) + fox.set_site(site) + + if fox.is_verbose(): + pp.p_debug("Using target ==> {}".format(fox.get_site())) -query_extra = '' if args.ex_query: - query_extra = args.ex_query.strip() + fox.set_ex_query(args.ex_query.strip()) -if verbose and args.ex_query: - pp.p_debug("Using extra query parameters ==> {}".format(query_extra)) + if fox.is_verbose(): + pp.p_debug("Using extra query parameters ==> {}".format(fox.\ + get_ex_query())) -r_query='' -# if raw query provided if args.query: - r_query = args.query.strip() + fox.set_raw_query(args.query.strip()) -if verbose and args.query: - pp.p_debug("Using query ==> {}".format(r_query)) + if fox.is_verbose(): + pp.p_debug("Using query ==> {}".format(fox.get_raw_query())) -inclusive = args.inc +if args.inc: + if not args.query: + pp.p_error("Query not found, but inclusive switch is on") + quit() -if inclusive and not args.query: - pp.p_error("Query not found, but inclusive switch is on") - quit() - -if verbose and inclusive: - pp.p_debug("Including dorks results along with query results") + fox.set_inc_q() + if fox.is_verbose(): + pp.p_debug("Including dorks results along with query results") -category = '' if args.category: - category = args.category.strip() + fox.set_category(args.category.strip()) -if verbose and category: - pp.p_debug("Using category ==> {}".format(category)) + if fox.is_verbose(): + pp.p_debug("Using category ==> {}".format(fox.get_category())) -severity = 5 -if args.severity: - severity = args.severity +fox.set_severity(5) +fox.set_severity_flag(0) -if verbose and args.severity: - pp.p_debug("Using severity ==> {}".format(severity)) +if args.severity: + fox.set_severity(args.severity) -severity_flag = 0 # 0 for >= severity # 1 for = severity # 2 for < severity if args.s_only: - severity_flag = 1 + fox.set_severity_flag(1) if args.s_upper: - severity_flag = 2 + fox.set_severity_flag(2) if args.s_all: - severity = 0 - severity_flag = 0 + fox.set_severity(0) + fox.set_severity_flag(0) -if verbose and args.s_all: +if fox.is_verbose() and args.s_all: if args.severity: pp.p_debug("Severity is overridden by `--all` switch") - pp.p_debug(" Using severity ==> {}".format(severity)) if args.s_qual: - severity = 8 - severity_flag = 0 + fox.set_severity(8) + fox.set_severity_flag(0) -if verbose and args.s_qual: +if fox.is_verbose() and args.s_qual: if args.severity or args.s_all: pp.p_debug("Severity is overridden by `--quality` switch") - pp.p_debug("Using severity ==> {}".format(severity)) -if verbose: +if fox.is_verbose(): s_m = '' - if severity_flag == 0: + if fox.get_severity_flag() == 0: s_m = 'min' - elif severity_flag == 1: + elif fox.get_severity_flag() == 1: s_m = 'only' - elif severity_flag == 2: + elif fox.get_severity_flag() == 2: s_m = 'max' + pp.p_debug("Using severity ==> {}".format(fox.get_severity())) pp.p_debug("Using Severity as ==> {}".format(s_m)) -page_size = args.page_size +fox.set_page_size(30) +if args.page_size: + fox.set_page_size(args.page_size) -if verbose: - pp.p_debug("Using page size ==> {}".format(page_size)) +if fox.is_verbose(): + pp.p_debug("Using page size ==> {}".format(fox.get_page_size())) -dork_size = 150 -if args.dork_size and args.dork_size >= page_size: - dork_size = args.dork_size +fox.set_dork_size(150) +if args.dork_size: + fox.set_dork_size(args.dork_size) -if verbose: - pp.p_debug("Max results per dork ==> {}".format(dork_size)) +if fox.is_verbose(): + pp.p_debug("Max results per dork ==> {}".format(fox.get_dork_size())) # defaults to 100 dorks -max_results = dork_size * 100 +fox.set_max_results(fox.get_dork_size() * 100) if args.max_results and args.max_results >= 0: - max_results = args.max_results + fox.set_max_results(args.max_results) -if verbose: - pp.p_debug("Total results limit to ==> {}".format(max_results)) +if fox.is_verbose(): + pp.p_debug("Total results limit to ==> {}".format(fox.get_max_results())) s_delay = 2 e_delay = 7 @@ -335,39 +583,42 @@ def update_dorks_repo(): if e_delay - 5 > 0: s_delay = e_delay - 5 -if verbose: - pp.p_debug("Using delay range ==> [{}, {}] sec".format(s_delay, e_delay)) +fox.set_delay_range(s_delay, e_delay) +if fox.is_verbose(): + pp.p_debug("Using delay range ==> [{}, {}] sec".format\ + (fox.get_delay_start(), fox.get_delay_end())) -parallel_req = 5 +fox.set_max_parallel(5) if args.parallel and args.parallel > 0: - parallel_req = args.parallel + fox.set_max_parallel(args.parallel) -if verbose: - pp.p_debug("Using parallel requests ==> {}".format(parallel_req)) +if fox.is_verbose(): + pp.p_debug("Using parallel requests ==> {}".format(\ + fox.get_max_parallel())) -useragent = '' if args.UA: - useragent = args.UA.strip() + fox.set_ua(args.UA.strip()) -if verbose and args.UA: - pp.p_debug("Using User-Agent ==> {}".format(useragent)) +if fox.is_verbose() and fox.has_ua(): + pp.p_debug("Using User-Agent ==> {}".format(fox.get_ua())) -out_dir = '' if args.output: - out_dir = dutil.create_dir(args.output.strip()) + fox.set_out_dir(dutil.create_dir(args.output.strip())) else: pp.p_error("Output directory is not specified") quit() -buildReport = True +fox.set_build_report() if args.json and not args.report: - buildReport = False + fox.unset_build_report() -if verbose: - pp.p_debug("Using output directory ==> {}".format(out_dir)) - if not buildReport: +if fox.is_verbose(): + pp.p_debug("Using output directory ==> {}".format(fox.get_out_dir())) + if fox.get_build_report(): pp.p_debug("Output will be saved in JSON format") else: pp.p_debug("Reporting is enabled, along with JSON format") - +pp.p_log("Building db.") +fox.dbBuilder() +pp.p_log("Finished building db") diff --git a/lib/__pycache__/prettyPrint.cpython-38.pyc b/lib/__pycache__/prettyPrint.cpython-38.pyc index 463cee8b7b6a327e674022d650ee78f579aeb4e4..488466a72048ef21e1edc52ab152a3008c6e2345 100644 GIT binary patch delta 207 zcmaED^VWtpl$V!_0SG$y|HbKV7EeDne@&JmZ=ht6&*V3ftw6DG z#~?q1Sc6ozz%5o6SEo>SO;(sfX=!ye_Ja77)TGk%`9QtJY#`$qMHrd>R|y0ZrIwUr zrd8^J6iiN&_Gg0{zgxOhC={qplL=}LkX;luSx`b`a+-`9W60!QnR`;cAVGf+5eOoJ OK|}IHPza~qO7f`aud-5B}R-jn8 zV~`(0tU;<#MyCH&`~gL&B_)|@ zm3lz=$r;lAY*54ZO1BCH0+nepK}`X&i-IN#N{CF(kWpg{nA|UOPs#@*=nEqJK!iVt N2mukHliOvV0RR{3GT;CJ diff --git a/lib/google.py b/lib/google.py index 543ffd8..49c47cf 100644 --- a/lib/google.py +++ b/lib/google.py @@ -54,7 +54,7 @@ def fetch(url, UA = 'f0x.py/1.0 (Linux; python/requests)'): 'Pragma': 'no-cache', 'TE': 'Trailers' } - return requests.get(u, headers=hdrs).text + return requests.get(url, headers=hdrs).text def has_next(text): o = re.search('aria-label="Next page"[^>]*>((Next >)|\ diff --git a/lib/utils.py b/lib/utils.py index 5a11a16..51d3400 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -100,6 +100,11 @@ def create_temp_file(dname = '', prefix = ''): return FileUtil.create_random_file(FileUtil.join_names('/tmp/', \ dname), prefix) + def dump_list(ofile, l): + if l: + with open(ofile, 'w') as f: + for i in l: + f.write("{}\n".format(i)) class Random: def rand_between(start, end): From 1d291a572dd6eb0a80c74dfaf27d9e4c12fec96a Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Fri, 3 Jul 2020 03:15:18 +0530 Subject: [PATCH 07/17] moded --- f0x2.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/f0x2.py b/f0x2.py index 89ee1aa..fc69e06 100755 --- a/f0x2.py +++ b/f0x2.py @@ -419,15 +419,19 @@ def process_dork(self, d, s): while rFlag and self._can_fetch_more() and \ ((self.get_page_size() * i) <= self.get_dork_size()): i += 1 + pp.p_log("Page request: {}".format(i)) url = gsrch.prepare_URL(d, self.get_site(), \ self.get_ex_query(), i, self.get_page_size()) t = rand.rand_between(self.get_delay_start(), \ self.get_delay_end()) - pp.p_log("Sleeping for {} sec.".format(t)) + if self.is_verbose(): + pp.p_debug("Sleeping for {} sec.".format(t)) + time.sleep(t) - pp.p_log("Processing now.") - pp.p_log("#Page to fetch: {}".format(i)) + if self.is_verbose(): + pp.p_bebug("Processing now.") + pp.p_debug("#Page to fetch: {}".format(i)) if self.has_ua(): ua = self.get_ua() @@ -437,11 +441,16 @@ def process_dork(self, d, s): pp.p_debug("Using UA ==> {}".format(ua)) response = gsrch.fetch(url, ua) - pp.p_log("Got Response.") - + if self.is_verbose(): + pp.p_debug("Got Response.") + + if self.is_verbose(): + pp.p_debug("Saving Response.") self.persist(response, d, s, dd, i) self._update_results_count(self.get_page_size()) rFlag = gsrch.has_next(response) + if self.is_verbose(): + pp.p_debug("Extracting URLs.") futil.dump_list(futil.join_names(dd, 'urls.txt'), \ gsrch.extract_urls(response)) From 357d8a8349b2d4356cbd6e10ed6e405f5f8e8b67 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Fri, 3 Jul 2020 11:19:11 +0530 Subject: [PATCH 08/17] updated code --- f0x.config | 5 +- f0x.py | 1200 ++++++++++---------- f0x2.config | 4 - f0x2.py | 633 ----------- lib/__pycache__/PrettyPrint.cpython-38.pyc | Bin 6761 -> 0 bytes lib/__pycache__/config.cpython-38.pyc | Bin 2051 -> 0 bytes lib/__pycache__/prettyPrint.cpython-38.pyc | Bin 7789 -> 0 bytes lib/__pycache__/useragents.cpython-38.pyc | Bin 6800 -> 0 bytes lib/__pycache__/utils.cpython-38.pyc | Bin 4054 -> 0 bytes user-agents | 53 - 10 files changed, 617 insertions(+), 1278 deletions(-) delete mode 100644 f0x2.config delete mode 100755 f0x2.py delete mode 100644 lib/__pycache__/PrettyPrint.cpython-38.pyc delete mode 100644 lib/__pycache__/config.cpython-38.pyc delete mode 100644 lib/__pycache__/prettyPrint.cpython-38.pyc delete mode 100644 lib/__pycache__/useragents.cpython-38.pyc delete mode 100644 lib/__pycache__/utils.cpython-38.pyc delete mode 100644 user-agents diff --git a/f0x.config b/f0x.config index bb2a69c..730a669 100644 --- a/f0x.config +++ b/f0x.config @@ -1,3 +1,4 @@ +[defaults] + repo_url=https://github.com/em-corp/dorks.git -dork_path=/home/dinesh/.f0x/dorks -useragents=./user-agents +dork_path=~/.f0x/dorks diff --git a/f0x.py b/f0x.py index 38a2929..b459bd2 100755 --- a/f0x.py +++ b/f0x.py @@ -1,71 +1,103 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +__NAME__ = "f0x.py" +__VERSION__ = "1.0" + import argparse import re import os -import errno -import urllib.parse as urlparse -import random import time -import mechanize -import requests import json from git import Repo -import shutil +from lib.prettyPrint import Prettify as pp +from lib.config import ConfigManager as conf +from lib.utils import DirUtil as dutil +from lib.utils import FileUtil as futil +from lib.google import GoogleSearch as gs +from lib.utils import Random as rand +from lib.useragents import UA def banner(): - print (''' - .o88o. .o o. - 888 `" .8' `8. - o888oo .8' oooo ooo `8. - 888 88 `88b..8P' 88 - 888 88 Y888' 88 - 888 `8. .o8"'88b .8' - o888o `8. o88' 888o .8' - - '''); + print(pp.green(' .o88o. .o o.')) + print(pp.green(' 888 `" .8\' `8.')) + print(pp.green(' o888oo .8\' ' + pp.yellow('oooo ooo') + ' `8.')) + print(pp.green(' 888 88 ' + pp.yellow('`88b..8P') + ' 88')) + print(pp.green(' 888 88 ' + pp.yellow('Y888') + ' 88')) + print(pp.green(' 888 `8. ' + pp.yellow('.o8"\'88b') + ' .8\'')) + print(pp.green(' o888o `8. ' + pp.yellow('o88\' 888o') + ' .8\'')) parser = argparse.ArgumentParser() -parser.add_argument('-s', '--site', help='Specify target site.', dest='site') - -parser.add_argument('-q', '--query', help='Dork to use. If specified, other files will not be read.', dest='query') -parser.add_argument('-i', '--inclusive', help='This works with `query` option only, if used, will also read dorks from file. ', dest='inc', action="store_true") -parser.add_argument('-A', '--args', help='Specify extra query to supply with each dorks.', dest='ex_query') - -parser.add_argument('-C', '--category', help='Use dorks from this category only.', dest='category') -parser.add_argument('-S', '--severity', help='Specify minimum severity(inclusive) dork file to read, range is [0, 10], defalut: 5.', dest='severity', type=int, choices=range(1, 11)) -parser.add_argument( '--only', help='Use along with severity, to select only a particular value.', dest='s_only', action='store_true') -parser.add_argument( '--upper', help='Use along with severity, to mark provided value as upper limit (exclusive).', dest='s_upper', action='store_true') -parser.add_argument('-a', '--all', help='Use all the dork files to fetch result (overrides --only, --upper flags).', dest='s_all', action='store_true') -parser.add_argument('-Q', '--quality', help='Use only top severity(>=8) dork files (overrides --only, --upper flags). ', dest='s_qual', action='store_true') - -parser.add_argument('-r', '--results', help='Total results to fetch in one request, default is 30.', dest='page_size', type=int) -parser.add_argument('-t', '--total', help='Total results to fetch for each dork, default is 150.', dest='dork_size', type=int) -parser.add_argument('-T', '--max', help='Maximum results to fetch for all the dorks combined.', dest='max_results', type=int) +parser.add_argument('-s', '--site', help='Specify Target site', dest='site') + +parser.add_argument('-q', '--query', help='Dork to use. If specified, \ + other files will not be read.', dest='query') +parser.add_argument('-i', '--inclusive', help='This works with `query` option \ + only, if used, will also read dorks from file. ', dest='inc', + action="store_true") +parser.add_argument('-A', '--args', help='Specify extra query to supply with \ + each dorks.', dest='ex_query') + +parser.add_argument('-C', '--category', help='Use dorks from this category \ + only.', dest='category') +parser.add_argument('-S', '--severity', help='Specify minimum severity\ + (inclusive) dork file to read, range is [0, 10], defalut: 5.', + dest='severity', type=int, choices=range(1, 11)) +parser.add_argument( '--only', help='Use along with severity, to \ + select only a particular value.', dest='s_only', action='store_true') +parser.add_argument( '--upper', help='Use along with severity, to mark \ + provided value as upper limit (exclusive).', dest='s_upper', + action='store_true') +parser.add_argument('-a', '--all', help='Use all the dork files to fetch \ + result (overrides --only, --upper flags).', dest='s_all', + action='store_true') +parser.add_argument('-Q', '--quality', help='Use only top severity(>=8) dork \ + files (overrides --only, --upper flags). ', dest='s_qual', + action='store_true') + +parser.add_argument('-r', '--results', help='Total results to fetch in one \ + request, default is 30.', dest='page_size', type=int) +parser.add_argument('-t', '--total', help='Total results to fetch for each \ + dork, default is 150.', dest='dork_size', type=int) +parser.add_argument('-T', '--max', help='Maximum results to fetch for all the \ + dorks combined.', dest='max_results', type=int) #parser.add_argument('-P', '--proxy', help='proxy', dest='') #parser.add_argument('-f', '--proxy-file', help='list of proxies', dest='') #parser.add_argument('-c', '--conn', help='connections per proxy', dest='') -parser.add_argument('-m', '--mintime', help='Specify minimum sec to wait between requests, If not specified, default 5 sec range is assumed', dest='min', type=int) -parser.add_argument('-M', '--maxtime', help='Specify maximum sec to wait between requests, if not specified, default 5 sec range is assumed.', dest='max', type=int) -parser.add_argument('-d', '--delay', help='Specify fix delay(in sec), if specified, took priority over variable delay.', dest='delay', type=int) -parser.add_argument('-p', '--parallel', help='Specify total no of parallel requests, default is 5.', dest='parallel', type=int) -parser.add_argument('-U', '--user-agent', help='Specify User Agent ', dest='UA') - -parser.add_argument('-o', '--output', help='Specify output directory', dest='output') -parser.add_argument('-j', '--json', help='Save output in JSON format only', dest='json', action="store_true") -parser.add_argument('-R', '--report', help='Create Report along with JSON format ouput, default', dest='report', action="store_true") - -parser.add_argument('--update', help='Update Dorks Repo, and exit', dest='updaterepo', action="store_true") -parser.add_argument('-L', '--list', help='List Repo categories, total dorks and exit', dest='listrepo', action="store_true") -parser.add_argument('-v', '--verbose', help='Be verbose.', dest='verbose', action="store_true") +parser.add_argument('-m', '--mintime', help='Specify minimum sec to wait \ + between requests, If not specified, default 5 sec range is assumed', + dest='min', type=int) +parser.add_argument('-M', '--maxtime', help='Specify maximum sec to wait \ + between requests, if not specified, default 5 sec range is assumed.', + dest='max', type=int) +parser.add_argument('-d', '--delay', help='Specify fix delay(in sec), if \ + specified, took priority over variable delay.', dest='delay', type=int) +parser.add_argument('-p', '--parallel', help='Specify total no of parallel \ + requests, default is 5.', dest='parallel', type=int) +parser.add_argument('-U', '--user-agent', help='Specify User Agent ', + dest='UA') + +parser.add_argument('-o', '--output', help='Specify output directory', + dest='output') +parser.add_argument('-j', '--json', help='Save output in JSON format only', + dest='json', action="store_true") +parser.add_argument('-R', '--report', help='Create Report along with JSON \ + format ouput, default', dest='report', action="store_true") + +parser.add_argument('--update', help='Update Dorks Repo, and exit', + dest='updaterepo', action="store_true") +parser.add_argument('-L', '--list', help='List Repo categories, total \ + dorks and exit', dest='listrepo', action="store_true") +parser.add_argument('-v', '--verbose', help='Be verbose.', dest='verbose', + action="store_true") args = parser.parse_args() banner() +print("\t{} v{}".format(pp.as_bold(pp.red(__NAME__)), pp.blue(__VERSION__))) if not (args.site or \ args.query or \ @@ -84,256 +116,374 @@ def banner(): args.output or \ args.updaterepo or \ args.listrepo): - print ("[ERROR]: no options are specified\n\n") + pp.p_error('No options are specified.') print (parser.format_help()) quit() -#---------------------------------------------------------------------------------- +#------------------------------------------------------------------------------ + +class Fox: + def __init__(self): + self.verbose = None + self.site = None + self.ex_q = None + self.raw_q = None + self.inc = None + self.cat = None + self.sev = None + self.sev_flag = None + self.psize = None + self.dsize = None + self.max_res = None + self.del_range = [None, None] + self.par = None + self.UA = None + self.out_dir = None + self.breport = None + self.mr_achived = 0 + + def set_site(self, site): + self.site = site + + def get_site(self): + return self.site + + def set_ex_query(self, q): + self.ex_q = q + + def get_ex_query(self): + return self.ex_q + + def set_raw_query(self, q): + self.raw_q = q + + def get_raw_query(self): + return self.raw_q + + def set_inc_q(self): + self.inc = True + + def unset_inc_q(self): + self.inc = False -#read config file -def get_value(key): - with open(getFileName(os.path.dirname(os.path.realpath(__file__)), 'f0x.config'), 'r') as config: - for line in config: - if line.startswith(key): - return line.split('=')[1].strip('\n') + def get_inc_q(self): + return self.inc -def getNewDir(o, dn=''): - out_dir = o - if out_dir.endswith('/'): - out_dir += dn - else: - out_dir += '/' + dn + def set_category(self, c): + self.cat = c + + def get_category(self): + return self.cat + + def set_severity(self, s): + self.sev = s + + def get_severity(self): + return self.sev + + def set_severity_flag(self, sf): + self.sev_flag = sf + + def get_severity_flag(self): + return self.sev_flag + + def set_page_size(self, ps): + self.psize = ps + + def get_page_size(self): + return self.psize + + def set_dork_size(self, ds): + self.dsize = ds + + def get_dork_size(self): + return self.dsize + + def set_max_results(self, mr): + self.max_res = mr + + def get_max_results(self): + return self.max_res + + def set_delay_range(self, s, e): + self.del_range = [s, e] + + def get_delay_range(self): + return self.del_range + + def get_delay_start(self): + return self.get_delay_range()[0] + + def get_delay_end(self): + return self.get_delay_range()[1] + + def set_max_parallel(self, p): + self.par = p + + def get_max_parallel(self): + return self.par + + def set_ua(self, ua): + self.UA = ua + + def get_ua(self): + return self.UA + + def has_ua(self): + if self.UA and not (self.UA == ''): + return True + return False + + def set_out_dir(self, d): + self.out_dir = d + + def get_out_dir(self): + return self.out_dir + + def set_build_report(self): + self.breport = True + + def unset_build_report(self): + self.breport = False + + def get_build_report(self): + return self.breport + + def set_verbose(self): + self.verbose = True + + def is_verbose(self): + return self.verbose + + def _load_conf(self): + conf.load('f0x2.config') + + def get_conf(self): + if conf.getConfig is None: + self._load_conf() + return conf.getConfig() + + def get_dork_path(self): + dp = '' + try: + dp = self.get_conf().get('dork_path') + except: + pp.p_error("Dorks path not exists, Check config file.") + return + else: + if dp == '': + pp.p_error("Dorks path not defined, Check config file.") + return + + if dp.startswith('~'): + dp = os.path.expanduser(dp) + elif dp.startswith('/'): + pass + elif dp.startswith('./'): + cwd = os.path.realpath('.') + dp = dutil.join_names(cwd, dp[2:]) + else: + cwd = os.path.realpath('.') + dp = dutil.join_names(cwd, dp) + + return dp + + def list_dorks_stats(self): + dp = self.get_dork_path() + if dp is None or dp == '': + pp.p_error("Dorks path not defined, Check config file.") + return + + dl = dutil.get_dir_list(dp, True) + if len(dl) == 0: + pp.p_log("No Dorks available, Update dork repo.") + + for i in dl: + dc = re.sub('^{}[/]?'.format(dp), '', i) + dc = re.sub('/', '.', dc) + td = len (dutil.get_files_list(i, True)) + pp.p_log("Category: {}".format(dc)) + pp.p_log("Total Dorks: {}\n".format(td), '**') + + def update_dorks_repo(self): + pp.p_log("Building Dork Repo.") + repo_url = self.get_conf().get('repo_url') + pp.p_log("Fetching from '{}'".format(repo_url)) + + tmpdir = dutil.create_temp_dir('f0x', 'repo_') + Repo.clone_from(repo_url, tmpdir) + pp.p_log("Done Fetching.") - if not os.path.exists(out_dir): try: - os.makedirs(out_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - return out_dir - -def getDir(o, dn = ''): - return getNewDir(o, dn) - -def query_encode(query): - return urlparse.quote_plus(query) - -# query, site, extra_query_string -def createURL(q, s, eqs): - u = 'https://www.google.com/search?gbv=1&q=' - if q == '': - print ("Query cannot be empty") - return - u += query_encode(q) - if eqs != '': - u += '+' + query_encode(eqs) - if s != '': - u += '+' + query_encode(s) - u += '&btnG=Google+Search' - return u - -def getSeverities(): - sev = [] - if severity_flag == 0: - sev = list (range (severity, 11)) - elif severity_flag == 1: - sev = [severity] - elif severity_flag == 2: - sev = list (range (1, severity)) #if severity = 1, return empty set - return sev - -def getFiles(f): - l = [] - for j in os.listdir(f): - t = f - if t.endswith('/'): - t += j + g = dutil.get_dir(tmpdir, '.git') + except: + pass else: - t += '/' + j + dutil.rmdir(g) - if os.path.isfile(t): - l += [t] + try: + f = futil.get_file(tmpdir, 'README.md') + except: + pass else: - l += getFiles(t) - return l - -def getDirs(f): - l = [] - for j in os.listdir(f): - t = f - if t.endswith('/'): - t += j + os.remove(f) + + try: + f = futil.get_file(tmpdir, 'LICENSE') + except: + pass else: - t += '/' + j - - if os.path.isdir(t): - l += [t] - l += getDirs(t) - return l - -def getDorks(rq, inc, svr, cat): - dorks = [] - if rq: - if svr == 10: - dorks += [rq] - if not inc: - return dorks + os.remove(f) + + dutil.merge_dirs(tmpdir, self.get_dork_path) + pp.p_log("Dork Repo updated.") + + def get_severity_list(self): + sev = [] + s = self.get_severity() + s_f = self.get_severity_flag() + + if s_f == 0: + sev = list(range(s, 11)) + elif s_f == 1: + sev = [s] + elif s_f == 2: + sev = list(range(1, s)) #if severity = 1, return empty set + + return sev + + def get_dorks(self, svr): + dorks = [] + if self.get_raw_query(): + if svr == 10: + dorks += [self.get_raw_query()] + if not self.get_inc_q(): + return dorks - dpath = get_value('dork_path') - chome = '' + dpath = self.get_dork_path() + chome = '' - if cat != '': - chome = re.sub('\.', '/', cat) + if self.get_category() and self.get_category() != '': + chome = re.sub('\.', '/', self.get_category()) - dpath = getDir(dpath, chome) - - for i in getFiles(dpath): - with open (i, 'r') as dfile: - d = '' - j = '' - for l in dfile: - if l.lstrip().lower().startswith('dork:'): - d = re.sub('^[dD][oO][rR][kK]:', '', l.lstrip()) - d = d.strip() - elif l.lstrip().lower().startswith('severity:'): - j = re.sub('^severity:', '', l.lstrip().lower()) - j = j.strip() + dpath = dutil.get_dir(dpath, chome) + + for i in dutil.get_files_list(dpath, True): + with open (i, 'r') as dfile: + d = '' + j = '' + for l in dfile: + if l.lstrip().lower().startswith('dork:'): + d = re.sub('^[dD][oO][rR][kK]:', '', l.lstrip()) + d = d.strip() + elif l.lstrip().lower().startswith('severity:'): + j = re.sub('^severity:', '', l.lstrip().lower()) + j = j.strip() - if int(j) == svr: - dorks.append(d) - - return dorks - -def getUserAgents(): - uaf = get_value('useragents') - if not uaf.startswith('/'): #relative path - uaf = getFileName(os.path.dirname(os.path.realpath(__file__)), uaf) - - useragents = [] - with open(uaf, 'r') as uas: - useragents = [ua.strip('\n') for ua in uas] - return useragents - - -def wget(u): - hdrs = { - 'Host': 'www.google.com', - 'User-Agent': random.choice(useragents), - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - 'Referer': 'https://www.google.com/', - 'DNT': '1', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - 'Cache-Control': 'max-age=0, no-cache', - 'Pragma': 'no-cache', - 'TE': 'Trailers' - } - req = requests.get(u, headers=hdrs) - return req.text - -def getNewRandomDir(o, dn=''): - return getNewDir(getNewDir(o, dn), str(time.time_ns())) - -def getFileName(o, n): - if o.endswith('/'): - return o + n - else: - return o + '/' + n + if int(j) == svr: + dorks.append(d) -def persist(r, d, s, o, fc): - if fc == 0: - fd = open(getFileName(o, 'dork.info'), 'w') - fd.write("dork: {}\n".format(d)) - fd.write("severity: {}\n".format(s)) + return dorks + + def _get_count(self): + self.mr_achived + + def _set_count(self, c): + self.mr_achived = c + + def _update_results_count(self, c): + self._set_count(self._get_count() + c) + + def _can_fetch_more(self): + return (self.get_max_results() - self._get_count()) > 0 + + def persist(self, r, d, s, o, fc): + if fc == 1: + fd = open(futil.join_names(o, 'dork.info'), 'w') + fd.write("dork: {}\n".format(d)) + fd.write("severity: {}\n".format(s)) + fd.close() + fd = open(futil.join_names(o, 'dork_page_' + str(fc)), 'w') + fd.write(r) fd.close() - fd = open(getFileName(o, 'dork_' + str(fc + 1)), 'w') - fd.write(r) - fd.close() -def getDelay(): - return delay + (random.random() * var_delay) - -def pageHasMoreResults(r): - o = re.search('aria-label="Next page"[^>]*>((Next >)|(]*>>))', r, re.I) - if o: - return True - else: - return False - -mr_achived = 0 -def canFetchMore(): - return (max_results - mr_achived) > 0 - -# TODO: make it synchronised later -def updateResultsCount(c): - global mr_achived - mr_achived += c - -# TODO: make it async later -def extractURLs(o, res): - fd = open(getFileName(o, 'urls.txt'), 'a') - - for pat in re.findall('\s+href="/url\?q=([^"&]*)[^"]*"[^>]*>', res, re.M|re.I): - if not re.search('^http(s)?://(www\.)?[^.]*\.google\.com', pat, re.I): - u = urlparse.unquote(pat) - fd.write("{}\n".format(u)) - - fd.close() - -def processDork(d, s, qe, ps, ds, o, sev): - if not canFetchMore(): - return - - u = createURL(d, s, qe) - i = -1 - rFlag = True - dd = getNewRandomDir(o, 'dorks') - r = 0 - print("[*] Processing dork: {}".format(d)) - while rFlag and canFetchMore() and ((ps * (i + 1)) <= ds): - url = '' - i += 1 - if i == 0: - url = "{}&start=&num={}".format(u, ps) - else: - url = "{}&start={}&num={}".format(u, ps * i, ps) - t = getDelay() - print("[*] Sleeping for {} sec.".format(t)) - time.sleep(t) - print("[*] Processing now.") - print("[*] Next Page Request: {}".format(r + 1)) - response = wget(url) - print("[*] Got Response.") - persist(response, d, sev, dd, r) - r += 1 - updateResultsCount(ps) - rFlag = pageHasMoreResults(response) - extractURLs(dd, response) - -# TODO: implement thread -def dbBuilder(): - for s in getSeverities(): - for i in getDorks(r_query, inclusive, s, category): - processDork(i, site, query_extra, page_size, dork_size, out_dir, s) - print("[*] Finished fetching results.") - -def jsonBuilder(o): - for f in os.listdir(o): - i = getFileName(o, f) - - if os.path.isdir(i): - if os.path.isfile(getFileName(i, 'urls.txt')): + def process_dork(self, d, s): + if not self._can_fetch_more(): + return + + i = 0 + rFlag = True + dd = dutil.create_random_dir(dutil.create_dir(self.get_out_dir(), \ + 'dorks'), 'dork_') + pp.p_log("Processing dork: {}".format(d)) + + while rFlag and self._can_fetch_more() and \ + ((self.get_page_size() * i) <= self.get_dork_size()): + i += 1 + pp.p_log("Page request: {}".format(i)) + url = gsrch.prepare_URL(d, self.get_site(), \ + self.get_ex_query(), i, self.get_page_size()) + t = rand.rand_between(self.get_delay_start(), \ + self.get_delay_end()) + + if self.is_verbose(): + pp.p_debug("Sleeping for {} sec.".format(t)) + + time.sleep(t) + if self.is_verbose(): + pp.p_bebug("Processing now.") + pp.p_debug("#Page to fetch: {}".format(i)) + + if self.has_ua(): + ua = self.get_ua() + else: + ua = UA.get_random_ua() + if self.is_verbose(): + pp.p_debug("Using UA ==> {}".format(ua)) + + response = gsrch.fetch(url, ua) + if self.is_verbose(): + pp.p_debug("Got Response.") + + if self.is_verbose(): + pp.p_debug("Saving Response.") + self.persist(response, d, s, dd, i) + self._update_results_count(self.get_page_size()) + rFlag = gsrch.has_next(response) + if self.is_verbose(): + pp.p_debug("Extracting URLs.") + futil.dump_list(futil.join_names(dd, 'urls.txt'), \ + gsrch.extract_urls(response)) + + def build_db(self): + for s in self.get_severity_list(): + for d in self.get_dorks(s): + self.process_dork(d, s) + + def build_json(self): + dorks = [] + try: + dorks = dutil.get_dir_list(self.get_dir(self.get_out_dir(), \ + "dorks"), False) + except: + pp.p_log("No Dorks Found.") + return + + for i in dorks: + try: + futil.get_file(i, 'urls.txt') + except: + continue + else: l = [] s = '' d = '' - with open(getFileName(i, 'urls.txt'), 'r') as urls: + with open(futil.get_file(i, 'urls.txt'), 'r') as urls: for u in urls: l += [u.strip('\n')] - - with open(getFileName(i, 'dork.info'), 'r') as infos: + + with open(futil.get_file(i, 'dork.info'), 'r') as infos: for line in infos: if line.startswith('dork: '): d = re.sub('dork: ', '', line) @@ -342,47 +492,52 @@ def jsonBuilder(o): s = re.sub('severity: ', '', line) s = s.strip('\n') - fd = open(getFileName(i, 'result.json'), 'w') - - data = { - 'severity' : s, - 'dork' : d, - 'urls' : l - } - - fd.write(json.dumps(data)) - fd.close() - -def buildReportObj(dd): - data = [] - - for i in range(1, 11): - data += [{ - 'severity': i, - 'lists': [] - }] - - for d in os.listdir(dd): - f = getFileName(dd, d) + with open(futil.join_names(i, 'result.json'), 'w') as fd: + data = { + 'severity' : s, + 'dork' : d, + 'urls' : l + } + fd.write(json.dumps(data)) - if os.path.isdir(f): - if os.path.isfile(getFileName(f, 'result.json')): + def get_report_obj(self): + data = [] + for i in range(1, 11): + data += [{ + 'severity': i, + 'lists': [] + }] + + try: + dorks = dutil.get_dir_list(self.get_dir(self.get_out_dir(), \ + "dorks"), False) + except: + pp.p_log("No Dorks Found.") + return + + for f in dutil.get_dir_list(dorks, False): + jf = '' + try: + jf = futil.get_file(f, 'result.json') + except: + continue + else: jd = {} - - with open(getFileName(f, 'result.json'), 'r') as jfile: + with open(jf, 'r') as jfile: jd = json.load(jfile) data[int(jd['severity']) - 1]['lists'] += [{ 'dork': jd['dork'], - 'path': './dorks/' + d + '/result.json', + 'path': re.sub('^{}'.format(self.get_out_dir()), \ + './', jf), 'count': len(jd['urls']) }] - return data + return data -def reportBuilder(o, d): - of = getFileName(o, 'report.html') - fd = open(of, 'w') - banner = '''") fd.write("") #------------------------------------------------------------------------------ -fox = Fox() +fox = None if args.verbose: - fox.set_verbose() + fox = Fox(verbose=True) +else: + fox = Fox(verbose=False) if args.listrepo: fox.list_dorks_stats() diff --git a/lib/google.py b/lib/google.py index 49c47cf..0d30c74 100644 --- a/lib/google.py +++ b/lib/google.py @@ -2,16 +2,24 @@ import urllib.parse as urlparse import requests +import re class GoogleSearch: def __qencode__(q): return urlparse.quote_plus(q) - def prepare_URL(query, site = '', params = '', page_num = 0, page_size = 30): - u = 'https://www.google.com/search?gbv=1&q=' - if query == '': + def prepare_URL(query, site = '', params = '', page_num = 1, \ + page_size = 30): + if not query or query == '': raise Exception("Query cannot be empty") + + if not site: + site = '' + if not params: + params = '' + + u = 'https://www.google.com/search?gbv=1&q=' u += GoogleSearch.__qencode__(query) if params != '': u += '+' + GoogleSearch.__qencode__(params) @@ -29,10 +37,11 @@ def prepare_URL(query, site = '', params = '', page_num = 0, page_size = 30): page_size = page_size * 10 else: page_size = 30 - - fmt = '{}&start={}&num={}' + fmt = '{}&start={}&num={}' + page_size = int(page_size) page_num = int(page_num) + if page_num <= 1: return fmt.format(u, '', page_size) else: @@ -70,7 +79,9 @@ def extract_urls(text): re.M|re.I): if not re.search('^http(s)?://(www\.)?[^.]*\.google\.com', pat, \ re.I): - urls += [urlparse.unquote(pat)] + up = urlparse.unquote(pat) + if not up.startswith('/search?q='): + urls += [up] return urls diff --git a/lib/utils.py b/lib/utils.py index 51d3400..8799452 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -48,21 +48,21 @@ def list_dir(parent): def get_dir_list(parent, recurse = False): l = [] - for i in list_dir(parent): + for i in DirUtil.list_dir(parent): if os.path.isdir(i): l += [i] if recurse: - l += get_dir_list(i, recurse) + l += DirUtil.get_dir_list(i, recurse) return l def get_files_list(parent, recurse = False): l = [] - for i in list_dir(parent): + for i in DirUtil.list_dir(parent): if os.path.isfile(i): l += [i] else: if recurse: - l += get_files_list(i, recurse) + l += DirUtil.get_files_list(i, recurse) return l def merge_dirs(source, dest): @@ -100,9 +100,12 @@ def create_temp_file(dname = '', prefix = ''): return FileUtil.create_random_file(FileUtil.join_names('/tmp/', \ dname), prefix) - def dump_list(ofile, l): + def dump_list(ofile, l, append=True): if l: - with open(ofile, 'w') as f: + m = 'a' + if not append: + m = 'w' + with open(ofile, m) as f: for i in l: f.write("{}\n".format(i)) From dc7f6d75113968f3a19ffa6d3348ad89d0ac7a68 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Fri, 3 Jul 2020 15:51:15 +0530 Subject: [PATCH 10/17] updated --- f0x.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/f0x.py b/f0x.py index 916170d..9d5cf43 100755 --- a/f0x.py +++ b/f0x.py @@ -440,9 +440,6 @@ def process_dork(self, d, s): ua = self.get_ua() else: ua = UA.get_random_ua() - if self.is_verbose(): - pp.p_debug("Using UA ==> {}".format(ua)) - pp.p_debug("Fetching URL ==> {}".format(url)) response = gsrch.fetch(url, ua) if self.is_verbose(): From 353faa9c46394690edda11f5414048360679021d Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Fri, 3 Jul 2020 15:55:44 +0530 Subject: [PATCH 11/17] corrected --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35110c3..6094f68 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ ## example $ python3 f0x.py --update - $ python3 f0x.py --L + $ python3 f0x.py -L $ python3 f0x.py -C 'Files_Containing_Juicy_Info' -o /ghdb/juicyfiles -T 60 -v $ python3 f0x.py -S 9 -o /ghdb/juicyfiles -T 60 -v From 7a1c207e8e37c4d386e3b619e3ab5b4369fb892a Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Tue, 10 Nov 2020 18:27:46 +0530 Subject: [PATCH 12/17] upgraded and added proxy and multithread support --- README.md | 10 +- f0x.config | 2 +- f0x.py | 1292 +++++++++++++++++++++++--------------------- lib/config.py | 6 +- lib/google.py | 79 ++- lib/prettyPrint.py | 95 ++-- lib/useragents.py | 169 +++--- lib/utils.py | 58 +- 8 files changed, 895 insertions(+), 816 deletions(-) diff --git a/README.md b/README.md index 6094f68..8b59d3e 100644 --- a/README.md +++ b/README.md @@ -15,16 +15,16 @@ * Require python3 ## usage - f0x.py [-h] [-s SITE] [-q QUERY] [-i] [-A EX_QUERY] [-C CATEGORY] [-S {1,2,3,4,5,6,7,8,9,10}] - [--only] [--upper] [-a] [-Q] [-r PAGE_SIZE] [-t DORK_SIZE] [-T MAX_RESULTS] [-m MIN] [-M MAX] - [-d DELAY] [-p PARALLEL] [-U UA] [-o OUTPUT] [-j] [-R] [--update] [-L] [-v] + $f0x.py [-h] [-d DOMAIN] [-q QUERY] [-n] [-Q EX_QUERY] [-c CATEGORY] [-cA] [-S SEVERITY] [-SQ] [-SA] [-t THREADS] [-p PROXY] + [-pF PROXY_FILE] [-pO] [-pC PROXY_COUNT] [-C PROXY_CONN] [--no-ssl-check] [--timeout TIME_OUT] [-m DELAY_MIN] + [-M DELAY_MAX] [-w DELAY] [-U UA] [--update] [-v] [-V] [-r PAGE_SIZE] [-R NO_OF_PAGES] [-T MAX_RESULTS] [-l] [-L] + [-o OUT_DIR] [-oJ] [-oL] [-oR] ## example $ python3 f0x.py --update $ python3 f0x.py -L - $ python3 f0x.py -C 'Files_Containing_Juicy_Info' -o /ghdb/juicyfiles -T 60 -v - $ python3 f0x.py -S 9 -o /ghdb/juicyfiles -T 60 -v + $ python3 f0x.py --any --quality -v -p "http://10.10.10.10:4444" --no-ssl-check -t 3 diff --git a/f0x.config b/f0x.config index 730a669..f1799e7 100644 --- a/f0x.config +++ b/f0x.config @@ -1,4 +1,4 @@ [defaults] -repo_url=https://github.com/em-corp/dorks.git +repo_url=https://github.com/dineshsaini/dorks.git dork_path=~/.f0x/dorks diff --git a/f0x.py b/f0x.py index 9d5cf43..10547d4 100755 --- a/f0x.py +++ b/f0x.py @@ -1,22 +1,28 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -__NAME__ = "f0x.py" -__VERSION__ = "1.0" +__NAME__ = "f0x.py" +__VERSION__ = "2.0" import argparse import re import os import time import json +import threading +import asyncio +from proxybroker import Broker +from concurrent.futures import ThreadPoolExecutor from git import Repo + from lib.prettyPrint import Prettify as pp from lib.config import ConfigManager as conf from lib.utils import DirUtil as dutil from lib.utils import FileUtil as futil from lib.google import GoogleSearch as gsrch from lib.utils import Random as rand -from lib.useragents import UA +from lib.useragents import UA + def banner(): print(pp.green(' .o88o. .o o.')) @@ -26,256 +32,380 @@ def banner(): print(pp.green(' 888 88 ' + pp.yellow('Y888') + ' 88')) print(pp.green(' 888 `8. ' + pp.yellow('.o8"\'88b') + ' .8\'')) print(pp.green(' o888o `8. ' + pp.yellow('o88\' 888o') + ' .8\'')) + print() -parser = argparse.ArgumentParser() - -parser.add_argument('-s', '--site', help='Specify Target site', dest='site') -parser.add_argument('-q', '--query', help='Dork to use. If specified, \ - other files will not be read.', dest='query') -parser.add_argument('-i', '--inclusive', help='This works with `query` option \ - only, if used, will also read dorks from file. ', dest='inc', - action="store_true") -parser.add_argument('-A', '--args', help='Specify extra query to supply with \ - each dorks.', dest='ex_query') - -parser.add_argument('-C', '--category', help='Use dorks from this category \ - only.', dest='category') -parser.add_argument('-S', '--severity', help='Specify minimum severity\ - (inclusive) dork file to read, range is [0, 10], defalut: 5.', - dest='severity', type=int, choices=range(1, 11)) -parser.add_argument( '--only', help='Use along with severity, to \ - select only a particular value.', dest='s_only', action='store_true') -parser.add_argument( '--upper', help='Use along with severity, to mark \ - provided value as upper limit (exclusive).', dest='s_upper', - action='store_true') -parser.add_argument('-a', '--all', help='Use all the dork files to fetch \ - result (overrides --only, --upper flags).', dest='s_all', - action='store_true') -parser.add_argument('-Q', '--quality', help='Use only top severity(>=8) dork \ - files (overrides --only, --upper flags). ', dest='s_qual', - action='store_true') - -parser.add_argument('-r', '--results', help='Total results to fetch in one \ - request, default is 30.', dest='page_size', type=int) -parser.add_argument('-t', '--total', help='Total results to fetch for each \ - dork, default is 150.', dest='dork_size', type=int) -parser.add_argument('-T', '--max', help='Maximum results to fetch for all the \ - dorks combined.', dest='max_results', type=int) - -#parser.add_argument('-P', '--proxy', help='proxy', dest='') -#parser.add_argument('-f', '--proxy-file', help='list of proxies', dest='') -#parser.add_argument('-c', '--conn', help='connections per proxy', dest='') - -parser.add_argument('-m', '--mintime', help='Specify minimum sec to wait \ - between requests, If not specified, default 5 sec range is assumed', - dest='min', type=int) -parser.add_argument('-M', '--maxtime', help='Specify maximum sec to wait \ - between requests, if not specified, default 5 sec range is assumed.', - dest='max', type=int) -parser.add_argument('-d', '--delay', help='Specify fix delay(in sec), if \ - specified, took priority over variable delay.', dest='delay', type=int) -parser.add_argument('-p', '--parallel', help='Specify total no of parallel \ - requests, default is 5.', dest='parallel', type=int) -parser.add_argument('-U', '--user-agent', help='Specify User Agent ', - dest='UA') - -parser.add_argument('-o', '--output', help='Specify output directory', - dest='output') -parser.add_argument('-j', '--json', help='Save output in JSON format only', - dest='json', action="store_true") -parser.add_argument('-R', '--report', help='Create Report along with JSON \ - format ouput, default', dest='report', action="store_true") - -parser.add_argument('--update', help='Update Dorks Repo, and exit', - dest='updaterepo', action="store_true") -parser.add_argument('-L', '--list', help='List Repo categories, total \ - dorks and exit', dest='listrepo', action="store_true") -parser.add_argument('-v', '--verbose', help='Be verbose.', dest='verbose', - action="store_true") - -args = parser.parse_args() +parser = argparse.ArgumentParser() -banner() -print("\t{} v{}".format(pp.as_bold(pp.red(__NAME__)), pp.blue(__VERSION__))) - -if not (args.site or \ - args.query or \ - args.category or \ - args.severity or \ - args.s_all or \ - args.s_qual or \ - args.page_size or \ - args.dork_size or \ - args.max_results or \ - args.min or \ - args.max or \ - args.delay or \ - args.parallel or \ - args.UA or \ - args.output or \ - args.updaterepo or \ - args.listrepo): - pp.p_error('No options are specified.') - print (parser.format_help()) - quit() +# input +parser.add_argument('-d', '--domain', help='Specify Target domain.', dest='domain') -#------------------------------------------------------------------------------ +parser.add_argument('-q', '--query', help='Specify query/dork manually, and' + + ' don\'t use more dorks from dorks-db.', dest='query') -class Fox: - def __init__(self, verbose): - self.verbose = verbose - self.site = None - self.ex_q = None - self.raw_q = None - self.inc = None - self.cat = None - self.sev = None - self.sev_flag = None - self.psize = None - self.dsize = None - self.max_res = None - self.del_range = [None, None] - self.par = None - self.UA = None - self.out_dir = None - self.breport = None - self.mr_achived = 0 - self._load_conf() +parser.add_argument('-n', '--no-stop', help='Works with `--query`, If ' + + 'specified dorks from dorks-db will also be used along ' + + 'with manually supplied dork.', dest='query_nostop', + action="store_true") - def set_site(self, site): - self.site = site +parser.add_argument('-Q', '--query-args', help='Specify extra query to ' + + 'supply with each dorks.', dest='ex_query') - def get_site(self): - return self.site +# dork selection +parser.add_argument('-c', '--category', help='Comma (,) separated dorks ' + + 'categories to use.', dest='category') - def set_ex_query(self, q): - self.ex_q = q +parser.add_argument('-cA', '--any', help='Use all available categories.', + dest='cat_any', action="store_true") - def get_ex_query(self): - return self.ex_q +parser.add_argument('-S', '--severity', help='Comma (,) separated severity ' + + 'range(from 1 to 10). eg. consider range expansion as : ' + + '"-4" will become "1,2,3,4", or ' + + '"2-5" will become "2,3,4,5", or ' + + '"7-" will become "7,8,9,10", or ' + + '"1-4,7" will become "1,2,3,4,7".', dest='severity') - def set_raw_query(self, q): - self.raw_q = q +parser.add_argument('-SQ', '--quality', help='Only use quality dorks ' + + 'i.e. with `severity >= 7`.', dest='sev_quality', + action="store_true") - def get_raw_query(self): - return self.raw_q +parser.add_argument('-SA', '--all', help='Use all available severities.', + dest='sev_all', action="store_true") - def set_inc_q(self): - self.inc = True - - def unset_inc_q(self): - self.inc = False +# optimize fox +parser.add_argument('-t', '--threads', help='Max parallel threads ' + + '(Default: 3).', type=int, dest='threads') - def get_inc_q(self): - return self.inc +parser.add_argument('-p', '--proxy', help="Specify proxy to use.", dest='proxy') - def set_category(self, c): - self.cat = c +parser.add_argument('-pF', '--proxy-file', help='Specify file to read proxies ' + +'list (one per line).', dest='proxy_file') - def get_category(self): - return self.cat +parser.add_argument('-pO', '--open-proxies', help='Make use of open public ' + + 'proxies.', dest='proxy_open', action='store_true') - def set_severity(self, s): - self.sev = s +parser.add_argument('-pC', '--open-proxies-count', help='Try collecting ' + + '`n` open proxies. (Default: 20)', dest='proxy_count', + type=int) - def get_severity(self): - return self.sev +parser.add_argument('-C', '--conn', help='Max connections per proxy ' + + '(Default: 2).', dest='proxy_conn', type=int) - def set_severity_flag(self, sf): - self.sev_flag = sf +parser.add_argument('--no-ssl-check', help='Disable certificate check.', + dest='no_ssl_check', action='store_true') - def get_severity_flag(self): - return self.sev_flag +parser.add_argument('--timeout', help='Set request timeout.', dest='time_out', + type=int) - def set_page_size(self, ps): - self.psize = ps +parser.add_argument('-m', '--min-delay', help='Specify minimum delay(sec) ' + + 'between requests. (Default: `max-delay` - 3)s.', + dest='delay_min', type=int) - def get_page_size(self): - return self.psize +parser.add_argument('-M', '--max-delay', help='Specify maximum delay(sec) ' + + 'between requests. (Default: `min-delay` + 3)s.', + dest='delay_max', type=int) - def set_dork_size(self, ds): - self.dsize = ds +parser.add_argument('-w', '--wait', help='Specify fix delay(sec) between ' + + 'requests (Default: 1s).', dest='delay', type=int) - def get_dork_size(self): - return self.dsize - - def set_max_results(self, mr): - self.max_res = mr +parser.add_argument('-U', '--user-agent', help='Specify User Agent.', dest='ua') - def get_max_results(self): - return self.max_res +parser.add_argument('--update', help='Update dorks repo and exit.', + dest='repo_update', action="store_true") - def set_delay_range(self, s, e): - self.del_range = [s, e] +parser.add_argument('-v', '--verbose', help='Be verbose.', dest='verbose', + action="store_true") - def get_delay_range(self): - return self.del_range +parser.add_argument('-V', '--version', help='Display version info and exit.', + dest='version', action="store_true") - def get_delay_start(self): - return self.get_delay_range()[0] +parser.add_argument('-r', '--results', help='Dork results to fetch in one ' + + 'page request (Default: 30).', dest='page_size', type=int) - def get_delay_end(self): - return self.get_delay_range()[1] +parser.add_argument('-R', '--requests', help='Pages to request for each ' + + 'dork (Default: 5).', dest='no_of_pages', type=int) - def set_max_parallel(self, p): - self.par = p +parser.add_argument('-T', '--max-results', help='Maximum results to fetch ' + + 'for all the dorks combined.', dest='max_results', type=int) - def get_max_parallel(self): - return self.par +# output filters +parser.add_argument('-l', '--list-dorks', help='List all dorks to be used ' + + 'and exit. Specify `category` or `severity` to narrow ' + + 'down the list. ', dest='list_dorks', action="store_true") - def set_ua(self, ua): - self.UA = ua +parser.add_argument('-L', '--categories', help='List available categories ' + + 'and exit.', dest='list_cat', action="store_true") - def get_ua(self): - return self.UA +parser.add_argument('-o', '--outdir', help='Specify output directory.', + dest='out_dir') - def has_ua(self): - if self.UA and not (self.UA == ''): - return True - return False +parser.add_argument('-oJ', '--out-json', help='Save output in JSON format.', + dest='out_fmt_json', action="store_true") - def set_out_dir(self, d): - self.out_dir = d +parser.add_argument('-oL', '--out-list', help='Save output in simple list.', + dest='out_fmt_list', action="store_true") - def get_out_dir(self): - return self.out_dir +parser.add_argument('-oR', '--out-report', help='Create html report with ' + + 'JSON format results.', dest='out_report', + action="store_true") - def set_build_report(self): - self.breport = True +args = parser.parse_args() - def unset_build_report(self): - self.breport = False +if args.version: + print("{} v{}".format(pp.as_bold(pp.red(__NAME__)), pp.blue(__VERSION__))) + quit() + +banner() - def get_build_report(self): - return self.breport - def set_verbose(self): - self.verbose = True +class F0x: - def is_verbose(self): - return self.verbose + def __init__(self, verbose=False): + self.verbose = verbose + self.severities = None + self.categories = None + self.domain = None + self.query = None + self.ex_args = None + self.process_dorksdb_flag = True + self.useragent = None + self.threads = 3 + self.delay_min = 1 + self.delay_max = 1 + self.connection_per_proxy = 2 + self.proxies_list = None + self._conn_per_proxy_count = None + self._proxy_ptr = -1 + self.request_timeout = None + self.ssl_check = True + self.open_proxy_count = 20 + self.page_size = gsrch.correct_page_size(30) + self.no_of_pages = 5 + self.max_results = None + self.outdir = None + self.outmode = None + self.results_count = 0 + + self._proxy_lock = threading.Lock() + self._count_lock = threading.Lock() + + self._load_conf() def _load_conf(self): conf.load('./f0x.config') if self.is_verbose(): - pp.p_debug("Loaded Config file, keys: {}".format(self.get_conf()\ - .getKeys())) + pp.p_debug("Loaded Config file, keys: {}" + .format(self.get_conf().getKeys())) def get_conf(self): return conf.getConfig() + def is_verbose(self): + return self.verbose + + def set_categories(self, categories=None): + self.categories = categories + + def get_categories(self): + return self.categories + + def set_severities(self, severities=None): + self.severities = severities + + def get_severities(self): + return self.severities + + def set_domain(self, domain): + self.domain = domain + + def get_domain(self): + return self.domain + + def set_query(self, query): + self.query = query + + def get_query(self): + return self.query + + def set_ex_args(self, ex_args): + self.ex_args = ex_args + + def get_ex_args(self): + return self.ex_args + + def set_useragent(self, useragent): + self.useragent = useragent + + def get_useragent(self): + return self.useragent + + def set_process_dorksdb_flag(self, flag): + self.process_dorksdb_flag = flag + + def get_process_dorksdb_flag(self): + return self.process_dorksdb_flag + + def set_threads(self, threads): + self.threads = threads + + def get_threads(self): + return self.threads + + def set_delay_min(self, delay_min): + self.delay_min = delay_min + + def get_delay_min(self): + return self.delay_min + + def set_delay_max(self, delay_max): + self.delay_max = delay_max + + def get_delay_max(self): + return self.delay_max + + def set_connection_per_proxy(self, conn): + self.connection_per_proxy = conn + + def get_connection_per_proxy(self): + return self.connection_per_proxy + + def add_proxy(self, proxy): + if not self.proxies_list: + self.proxies_list = [] + self._conn_per_proxy_count = [] + + self.proxies_list += [proxy] + self._conn_per_proxy_count += [0] + + def get_proxy_list(self): + return self.proxies_list + + def set_page_size(self, page_size): + self.page_size = page_size + + def get_page_size(self): + return self.page_size + + def set_no_of_pages(self, no_of_pages): + self.no_of_pages = no_of_pages + + def get_no_of_pages(self): + return self.no_of_pages + + def set_max_results(self, max_results): + self.max_results = max_results + + def get_max_results(self): + return self.max_results + + def set_outdir(self, outdir): + self.outdir = outdir + + def get_outdir(self): + return self.outdir + + def set_outmode(self, omode): + self.outmode = omode + + def get_outmode(self): + return self.outmode + + def set_results_count(self, count): + self.results_count = count + + def get_results_count(self): + return self.results_count + + def set_request_timeout(self, timeout): + self.request_timeout = timeout + + def get_request_timeout(self): + return self.request_timeout + + def set_ssl_check(self, ssl_check): + self.ssl_check = ssl_check + + def do_ssl_check(self): + return self.ssl_check + + def set_open_proxy_count(self, count): + self.open_proxy_count = count + + def get_open_proxy_count(self): + return self.open_proxy_count + + async def _record_proxy(self, proxies): + while True: + proxy = await proxies.get() + if proxy is None: + break + + proto = 'https' if 'HTTPS' in proxy.types else 'http' + proxy_url = '%s://%s:%d' % (proto, proxy.host, proxy.port) + self.add_proxy(proxy_url) + + if fox.is_verbose(): + pp.p_log("Found proxy: {}".format(pp.light_green(proxy_url))) + + def collect_open_proxies(self): + proxies = asyncio.Queue() + broker = Broker(proxies) + tasks = asyncio.gather(broker.find(types=['HTTP', 'HTTPS'], limit=self.get_open_proxy_count()), self._record_proxy(proxies)) + + loop = asyncio.get_event_loop() + loop.run_until_complete(tasks) + + def update_dorks_repo(self): + pp.p_log("Building Dork Repo.") + repo_url = self.get_conf().get('repo_url') + pp.p_log("Fetching from '{}'".format(repo_url)) + + tmpdir = dutil.create_temp_dir('f0x', 'repo_') + Repo.clone_from(repo_url, tmpdir) + pp.p_log("Done Fetching.") + + rmdirs = ['.git'] + rmfiles = ['README.md', 'LICENSE'] + + for i in rmdirs: + try: + g = dutil.get_dir(tmpdir, i) + except: + pass + else: + dutil.rmdir(g) + + for i in rmfiles: + try: + f = futil.get_file(tmpdir, i) + except: + pass + else: + os.remove(f) + try: + dutil.merge_dirs(tmpdir, self.get_dork_path()) + except Exception as e: + pp.p_error(e) + quit() + + pp.p_log("Dork Repo updated.") + def get_dork_path(self): dp = '' + flag = False + try: dp = self.get_conf().get('dork_path') except: - pp.p_error("Dorks path not exists, Check config file.") - return + pp.p_error("Dorks path not exists.") + flag = True else: if dp == '': - pp.p_error("Dorks path not defined, Check config file.") - return + pp.p_error("Dorks path not defined.") + flag = True + + if flag: + raise Exception("Error in Config file.") if dp.startswith('~'): dp = os.path.expanduser(dp) @@ -289,495 +419,407 @@ def get_dork_path(self): dp = dutil.join_names(cwd, dp) return dp - - def list_dorks_stats(self): - dp = self.get_dork_path() - if dp is None or dp == '': - pp.p_error("Dorks path not defined, Check config file.") - return + + def list_repo_categories(self): + dp = None + try: + dp = self.get_dork_path() + except Exception as e: + pp.p_error(e) + quit() dl = dutil.get_dir_list(dp, True) if len(dl) == 0: pp.p_log("No Dorks available, Update dork repo.") + return - for i in dl: + for i in dl: dc = re.sub('^{}[/]?'.format(dp), '', i) dc = re.sub('/', '.', dc) - td = len (dutil.get_files_list(i, True)) + td = len(dutil.get_files_list(i, True)) pp.p_log("Category: {}".format(dc)) pp.p_log("Total Dorks: {}\n".format(td), '**') - - def update_dorks_repo(self): - pp.p_log("Building Dork Repo.") - repo_url = self.get_conf().get('repo_url') - pp.p_log("Fetching from '{}'".format(repo_url)) - tmpdir = dutil.create_temp_dir('f0x', 'repo_') - Repo.clone_from(repo_url, tmpdir) - pp.p_log("Done Fetching.") - - try: - g = dutil.get_dir(tmpdir, '.git') - except: - pass + def list_dorks(self): + cat = None + sev = None + + if self.get_categories(): + cat = self.get_categories() else: - dutil.rmdir(g) - - try: - f = futil.get_file(tmpdir, 'README.md') - except: - pass + cat = [""] + + if self.get_severities(): + sev = self.get_severities() else: - os.remove(f) + sev = range(1, 11) - try: - f = futil.get_file(tmpdir, 'LICENSE') - except: - pass - else: - os.remove(f) - - dutil.merge_dirs(tmpdir, self.get_dork_path()) - pp.p_log("Dork Repo updated.") - - def get_severity_list(self): - sev = [] - s = self.get_severity() - s_f = self.get_severity_flag() - - if s_f == 0: - sev = list(range(s, 11)) - elif s_f == 1: - sev = [s] - elif s_f == 2: - sev = list(range(1, s)) #if severity = 1, return empty set - - return sev - - def get_dorks(self, svr): - dorks = [] - if self.get_raw_query(): - if svr == 10: - dorks += [self.get_raw_query()] - if not self.get_inc_q(): - return dorks + for c in cat: + for d in self.get_dorks(c, sev): + pp.p_log(d) - dpath = self.get_dork_path() + def get_dorks(self, category, sev_list): + dorks = [] + dpath = None chome = '' + + if not sev_list or len(sev_list) == 0: + return dorks + + try: + dpath = self.get_dork_path() + except Exception as e: + pp.p_error(e) + return [] + + if category: + chome = re.sub('\.', '/', category) - if self.get_category() and self.get_category() != '': - chome = re.sub('\.', '/', self.get_category()) - dpath = dutil.get_dir(dpath, chome) for i in dutil.get_files_list(dpath, True): with open (i, 'r') as dfile: - d = '' - j = '' + d = None + j = None for l in dfile: - if l.lstrip().lower().startswith('dork:'): - d = re.sub('^[dD][oO][rR][kK]:', '', l.lstrip()) + if l.lstrip().lower().startswith('googledork:'): + d = re.sub('^googledork:', '', l.lstrip().lower()) d = d.strip() elif l.lstrip().lower().startswith('severity:'): j = re.sub('^severity:', '', l.lstrip().lower()) j = j.strip() - - if int(j) == svr: + elif (not d) and l.lstrip().lower().startswith('dork:'): + d = re.sub('^dork:', '', l.lstrip().lower()) + d = d.strip() + + if j and int(j) in sev_list and d: dorks.append(d) return dorks - - def _get_count(self): - return self.mr_achived - - def _set_count(self, c): - self.mr_achived = c - - def _update_results_count(self, c): - self._set_count(self._get_count() + c) - - def _can_fetch_more(self): - return (self.get_max_results() - self._get_count()) > 0 - - def persist(self, r, d, s, o, fc): - if fc == 1: - fd = open(futil.join_names(o, 'dork.info'), 'w') - fd.write("dork: {}\n".format(d)) - fd.write("severity: {}\n".format(s)) - fd.close() - fd = open(futil.join_names(o, 'dork_page_' + str(fc)), 'w') - fd.write(r) - fd.close() - - def process_dork(self, d, s): - if not self._can_fetch_more(): - return + + def fetch_page_response(self, dork, pagenum, proxy): + gurl = gsrch.prepare_URL(dork, self.get_domain(), self.get_query(), + pagenum, self.get_page_size()) + + ua = None + if self.get_useragent(): + ua = self.get_useragent() + else: + ua = UA.get_random_ua() + + return gsrch.fetch(gurl, ua, proxy, self.get_request_timeout(), + self.do_ssl_check()) + + def save_links(self, links_list): # FIXME: output + for l in links_list: + pp.p_log(l) + + def update_results_stats(self, c): + with self._count_lock: + self.set_results_count(self.get_results_count() + c) + + def can_fetch_more(self): + if self.get_max_results(): + return self.get_results_count() < self.get_max_results() - i = 0 - rFlag = True - dd = dutil.create_random_dir(dutil.create_dir(self.get_out_dir(), \ - 'dorks'), 'dork_') - pp.p_log("Processing dork: {}".format(d)) - - while rFlag and self._can_fetch_more() and \ - ((self.get_page_size() * i) <= self.get_dork_size()): - i += 1 - pp.p_log("Page request: {}".format(i)) - url = gsrch.prepare_URL(d, self.get_site(), \ - self.get_ex_query(), i, self.get_page_size()) - t = rand.rand_between(self.get_delay_start(), \ - self.get_delay_end()) - - if self.is_verbose(): - pp.p_debug("Sleeping for {} sec.".format(t)) - - time.sleep(t) - if self.is_verbose(): - pp.p_debug("Processing now.") - pp.p_debug("#Page to fetch: {}".format(i)) - - if self.has_ua(): - ua = self.get_ua() - else: - ua = UA.get_random_ua() - - response = gsrch.fetch(url, ua) - if self.is_verbose(): - pp.p_debug("Got Response.") - - if self.is_verbose(): - pp.p_debug("Saving Response.") - self.persist(response, d, s, dd, i) - self._update_results_count(self.get_page_size()) - rFlag = gsrch.has_next(response) - if self.is_verbose(): - pp.p_debug("Extracting URLs.") - futil.dump_list(futil.join_names(dd, 'urls.txt'), \ - gsrch.extract_urls(response)) - - def build_db(self): - for s in self.get_severity_list(): - for d in self.get_dorks(s): - self.process_dork(d, s) - - def build_json(self): - dorks = [] + return True + + def get_proxy_object(self): + proxy = {'proxy': None, 'loc': None} + p = None + l = None + + if self.get_proxy_list(): + pl = len(self.get_proxy_list()) + c = 0 + + with self._proxy_lock: + while True: + c += 1 + self._proxy_ptr += 1 + if self._proxy_ptr == pl: + self._proxy_ptr = 0 + + if self._conn_per_proxy_count[self._proxy_ptr] < self.get_connection_per_proxy(): + p = self.get_proxy_list()[self._proxy_ptr] + self._conn_per_proxy_count[self._proxy_ptr] += 1 + l = self._proxy_ptr + break + + if c >= pl: + c = 0 + time.sleep((self.get_delay_min() * self.get_no_of_pages()) / 4) + + if p: + proxy = {'proxy': {'http': p, 'https': p}, 'loc': l} + return proxy + + def release_proxy(self, proxyobj): + l = proxyobj['loc'] try: - dorks = dutil.get_dir_list(dutil.get_dir(self.get_out_dir(), \ - "dorks"), False) + if (l or int(l) == 0) and self._conn_per_proxy_count[l] != 0: + self._conn_per_proxy_count[l] -= 1 except: - pp.p_log("No Dorks Found.") - return + pass + + def process_dork(self, dork): + if self.is_verbose(): + pp.p_debug("Processing dork: {}".format(dork)) + + proxy = self.get_proxy_object() - for i in dorks: + for p in range(1, self.get_no_of_pages() + 1): + if not self.can_fetch_more(): + break + + time.sleep(rand.rand_between(self.get_delay_min(), + self.get_delay_max())) + response = None try: - futil.get_file(i, 'urls.txt') - except: - continue - else: - l = [] - s = '' - d = '' - - with open(futil.get_file(i, 'urls.txt'), 'r') as urls: - for u in urls: - l += [u.strip('\n')] - - with open(futil.get_file(i, 'dork.info'), 'r') as infos: - for line in infos: - if line.startswith('dork: '): - d = re.sub('dork: ', '', line) - d = d.strip('\n') - elif line.startswith('severity: '): - s = re.sub('severity: ', '', line) - s = s.strip('\n') - - with open(futil.join_names(i, 'result.json'), 'w') as fd: - data = { - 'severity' : s, - 'dork' : d, - 'urls' : l - } - fd.write(json.dumps(data)) - - def get_report_obj(self): - data = [] - for i in range(1, 11): - data += [{ - 'severity': i, - 'lists': [] - }] + response = self.fetch_page_response(dork, p, proxy['proxy']) + except Exception as e: + gsrch.session_cleanup() + pp.p_error(e) + return - try: - dorks = dutil.get_dir_list(dutil.get_dir(self.get_out_dir(), \ - "dorks"), False) - except: - pp.p_log("No Dorks Found.") - return + if self.is_verbose(): + pp.p_debug("Fetched page : {}".format(p)) + + links = gsrch.extract_urls(response) + if self.is_verbose(): + pp.p_debug("Found {} url(s).".format(len(links))) + + self.save_links(links) + + self.update_results_stats(len(links)) + + if not gsrch.has_next(response): + break + + self.release_proxy(proxy) + gsrch.session_cleanup() + + def execute(self): + dorks = [] + if self.get_query(): + dorks += [self.get_query()] + + if self.get_process_dorksdb_flag(): + cat = [] + if self.get_categories(): + cat = self.get_categories() + + for c in cat: + dorks += self.get_dorks(c, self.get_severities()) + + if self.is_verbose(): + pp.p_debug("{} dorks to fetch.".format(len(dorks))) + + with ThreadPoolExecutor(max_workers=self.get_threads()) as exec: + exec.map(self.process_dork, dorks) - for f in dorks: - jf = '' - try: - jf = futil.get_file(f, 'result.json') - except: - continue - else: - jd = {} - with open(jf, 'r') as jfile: - jd = json.load(jfile) - - data[int(jd['severity']) - 1]['lists'] += [{ - 'dork': jd['dork'], - 'path': re.sub('^{}(/)?'.format(self.get_out_dir()), \ - './', jf), - 'count': len(jd['urls']) - }] - return data - - def build_report(self): - with open(futil.join_names(fox.get_out_dir(), 'report.html'), 'w') \ - as fd: - banner = ''' - -''' - do = self.get_report_obj() - css = ''' - .banner{ - font-weight: 600; - } - - .banner-footer { - font-style: italic; - } - - .severity { - font-size: 2.4em; - } - - .label { - font-size: 1.4em; - } - - .label-value { - font-style: italic; - font-weight: 600; - } - ''' - - fd.write("OSINT Report - [GHDB]" + \ - "".format(css)) - fd.write(banner) - - for i in do: - fd.write('

' + \ - 'Severity {}

'.format(i['severity'], \ - i['severity'])) - for j in i['lists']: - fd.write('

' + \ - 'Dork Used: {}

'.format(\ - j['dork'])) - fd.write('

' + \ - 'URLs Retrived: {}

'.\ - format(j['count'])) - fd.write(('

' + \ - 'JSON File: {}' + \ - '

').format(j['path'], j['path'])) - fd.write('
') - fd.write("
") - fd.write("") - -#------------------------------------------------------------------------------ -fox = None - -if args.verbose: - fox = Fox(verbose=True) -else: - fox = Fox(verbose=False) - -if args.listrepo: - fox.list_dorks_stats() - quit() + +fox = F0x(verbose=args.verbose) -if args.updaterepo: +if args.repo_update: fox.update_dorks_repo() quit() -if args.site: - site = '' - site = args.site.strip() - site = re.sub(r'^http(s)?://(www\.)?', '', site) - site = re.sub('/.*(/)?', '', site) +if args.list_cat: + fox.list_repo_categories() + quit() + +flag_dork_selector = False + +if args.severity: + flag_dork_selector = True + s = [] + l = "1" + m = "10" + + if re.search("[^0-9,-]", args.severity): + pp.p_error("Severity value can only contains numbers or numeral " + + "range, separated by comma (,).") + quit() + + for i in args.severity.split(','): + j = i + if i.startswith('-'): + j = l + i + elif i.endswith('-'): + j = i + m + + k = j.split('-') + for x in range(int(k[0]), int(k[-1]) + 1): + s += [x] + + s = list(set(s)) + fox.set_severities(s) + +if args.sev_all: + flag_dork_selector = True + + s = [] + for x in range(1, 11): + s += [x] + + fox.set_severities(s) - fox.set_site(site) +if args.severity and args.sev_all and fox.is_verbose(): + pp.p_debug("Provided severity range is overridden by `all` switch.") - if fox.is_verbose(): - pp.p_debug("Using target ==> {}".format(fox.get_site())) +if args.sev_quality: + flag_dork_selector = True + + s = [] + for x in range(7, 11): + s += [x] + fox.set_severities(s) + +if (args.severity or args.sev_all) and args.sev_quality and fox.is_verbose(): + pp.p_debug("Provided severity (range | `all` switch) is overridden by " + + "`quality` switch.") + +if args.category: + flag_dork_selector = True + fox.set_categories(args.category.split(',')) -if args.ex_query: - fox.set_ex_query(args.ex_query.strip()) +if args.cat_any: + flag_dork_selector = True + fox.set_categories(["."]) - if fox.is_verbose(): - pp.p_debug("Using extra query parameters ==> {}".format(fox.\ - get_ex_query())) +if args.category and args.cat_any and fox.is_verbose(): + pp.p_debug("Provided categories value is overridden by `any` switch.") +if args.list_dorks: + fox.list_dorks() + quit() + if args.query: - fox.set_raw_query(args.query.strip()) + flag_dork_selector = True + fox.set_query(args.query.strip()) + fox.set_process_dorksdb_flag(args.query_nostop) - if fox.is_verbose(): - pp.p_debug("Using query ==> {}".format(fox.get_raw_query())) +if not flag_dork_selector: + pp.p_error('Please provide atleast one dork selector from ' + + '`category`, `severity` or `query`.') + quit() -if args.inc: - if not args.query: - pp.p_error("Query not found, but inclusive switch is on") - quit() +if args.domain: + domain = args.domain.strip() + domain = re.sub(r'^http(s)?://(www\.)?', '', domain) + domain = re.sub('/.*(/)?', '', domain) - fox.set_inc_q() - if fox.is_verbose(): - pp.p_debug("Including dorks results along with query results") + fox.set_domain(domain) -if args.category: - fox.set_category(args.category.strip()) +if args.ex_query: + fox.set_ex_args(args.ex_query.strip()) - if fox.is_verbose(): - pp.p_debug("Using category ==> {}".format(fox.get_category())) +if args.ua: + fox.set_useragent(args.ua.strip()) -fox.set_severity(5) -fox.set_severity_flag(0) +if args.threads: + if args.threads > 0: + fox.set_threads(args.threads) + else: + pp.p_error("Please provide some +ve value for threads.") + quit() -if args.severity: - fox.set_severity(args.severity) - -# 0 for >= severity -# 1 for = severity -# 2 for < severity -if args.s_only: - fox.set_severity_flag(1) -if args.s_upper: - fox.set_severity_flag(2) - -if args.s_all: - fox.set_severity(0) - fox.set_severity_flag(0) - -if fox.is_verbose() and args.s_all: - if args.severity: - pp.p_debug("Severity is overridden by `--all` switch") - -if args.s_qual: - fox.set_severity(8) - fox.set_severity_flag(0) - -if fox.is_verbose() and args.s_qual: - if args.severity or args.s_all: - pp.p_debug("Severity is overridden by `--quality` switch") - -if fox.is_verbose(): - s_m = '' - if fox.get_severity_flag() == 0: - s_m = 'min' - elif fox.get_severity_flag() == 1: - s_m = 'only' - elif fox.get_severity_flag() == 2: - s_m = 'max' - pp.p_debug("Using severity ==> {}".format(fox.get_severity())) - pp.p_debug("Using Severity as ==> {}".format(s_m)) - -fox.set_page_size(30) -if args.page_size: - fox.set_page_size(args.page_size) +if args.delay: + if args.delay > 0: + fox.set_delay_min(args.delay) + fox.set_delay_max(args.delay) + else: + pp.p_error("Please provide some +ve value for delay.") + quit() -if fox.is_verbose(): - pp.p_debug("Using page size ==> {}".format(fox.get_page_size())) +if args.delay_min: + if args.delay_min > 0: + fox.set_delay_min(args.delay_min) + fox.set_delay_max(args.delay_min + 3) + else: + pp.p_error("Please provide some +ve value for delay_min.") + quit() -fox.set_dork_size(150) -if args.dork_size: - fox.set_dork_size(args.dork_size) +if args.delay_max: + if args.delay_max > 0: + fox.set_delay_max(args.delay_max) + if args.delay_max - 3 > 0: + fox.set_delay_min(args.delay_max - 3) + else: + fox.set_delay_min(0) + else: + pp.p_error("Please provide some +ve value for delay_max.") + quit() + +if args.delay_min and args.delay_max: + fox.set_delay_min(args.delay_min) + fox.set_delay_max(args.delay_max) -if fox.is_verbose(): - pp.p_debug("Max results per dork ==> {}".format(fox.get_dork_size())) +if args.page_size: + if args.page_size <= 0: + pp.p_error("Please provide some +ve value for `dork results`.") + quit() + fox.set_page_size(gsrch.correct_page_size(args.page_size)) -# defaults to 100 dorks -fox.set_max_results(fox.get_dork_size() * 100) -if args.max_results and args.max_results >= 0: +if args.no_of_pages: + if args.no_of_pages <= 0: + pp.p_error("Please provide some +ve value for `pages to request`.") + quit() + fox.set_no_of_pages(int(args.no_of_pages)) + +if args.max_results: + if args.max_results <= 0: + pp.p_error("Please provide some +ve value for `max results`.") + quit() fox.set_max_results(args.max_results) -if fox.is_verbose(): - pp.p_debug("Total results limit to ==> {}".format(fox.get_max_results())) - -s_delay = 2 -e_delay = 7 -if args.delay and args.delay >= 0: - s_delay = e_delay = args.delay -else: - if args.min and args.max: - s_delay = args.min - e_delay = args.max - elif args.min: - s_delay = args.min - e_delay = s_delay + 5 - elif args.max: - e_delay = args.max - s_delay = 0 - if e_delay - 5 > 0: - s_delay = e_delay - 5 - -fox.set_delay_range(s_delay, e_delay) -if fox.is_verbose(): - pp.p_debug("Using delay range ==> [{}, {}] sec".format\ - (fox.get_delay_start(), fox.get_delay_end())) - -fox.set_max_parallel(5) -if args.parallel and args.parallel > 0: - fox.set_max_parallel(args.parallel) - -if fox.is_verbose(): - pp.p_debug("Using parallel requests ==> {}".format(\ - fox.get_max_parallel())) - -if args.UA: - fox.set_ua(args.UA.strip()) - -if fox.is_verbose() and fox.has_ua(): - pp.p_debug("Using User-Agent ==> {}".format(fox.get_ua())) - -if args.output: - fox.set_out_dir(dutil.create_dir(args.output.strip())) -else: - pp.p_error("Output directory is not specified") - quit() +# TODO: FIXME: +# output dir and report logic/code left -fox.set_build_report() -if args.json and not args.report: - fox.unset_build_report() +if args.no_ssl_check: + fox.set_ssl_check(False) -if fox.is_verbose(): - pp.p_debug("Using output directory ==> {}".format(fox.get_out_dir())) - if fox.get_build_report(): - pp.p_debug("Output will be saved in JSON format") +if args.time_out: + if args.time_out <= 0: + pp.p_error("Please provide some +ve value for `request timeout`.") + quit() + fox.set_request_timeout(args.time_out) + +if args.proxy_conn: + if args.proxy_conn > 0: + fox.set_connection_per_proxy(args.proxy_conn) else: - pp.p_debug("Reporting is enabled, along with JSON format") + pp.p_error("Please provide some +ve value for connection per proxy.") + +if args.proxy_open and (args.proxy or args.proxy_file): + pp.p_error("Please use only one option from `open proxies` or " + + "(proxy and proxy_file).") + quit() + +if args.proxy and args.proxy_file: + pp.p_error("Please use only one option from proxy or proxy_file.") + quit() -pp.p_log("Building db.") -fox.build_db() +if args.proxy: + fox.add_proxy(args.proxy.strip()) -pp.p_log("Building JSON.") -fox.build_json() +if args.proxy_file: + for i in futil.get_file_aslist(args.proxy_file): + fox.add_proxy(i) -if fox.get_build_report(): - pp.p_log("Building Report.") - fox.build_report() - pp.p_log("Report saved at '{}'.".format(futil.join_names(\ - fox.get_out_dir(), 'report.html'))) +if args.proxy_count: + if args.proxy_count <= 0: + pp.p_error("Please provide some +ve value for `open proxy count`.") + quit() + + fox.set_open_proxy_count(args.proxy_count) + + if fox.is_verbose() and not args.proxy_open: + pp.p_info('Ignoring `open proxy count` as provided without enabling `open proxies` switch.') + +if args.proxy_open: + fox.collect_open_proxies() -pp.p_log("Results saved at '{}'".format(fox.get_out_dir())) +fox.execute() + \ No newline at end of file diff --git a/lib/config.py b/lib/config.py index 3977c0a..7b4f7de 100644 --- a/lib/config.py +++ b/lib/config.py @@ -2,6 +2,7 @@ import configparser + class ConfigManager: config = None @@ -21,6 +22,7 @@ def load(cfile): return ConfigManager.getConfig() class Configuration(): + def __init__(self): self.properties = {} @@ -28,8 +30,8 @@ def get(self, key): if key in self.properties.keys(): return self.properties[key] else: - raise self.NoSuchKeyException("No such key `{}` found"\ - .format(key)) + raise self.NoSuchKeyException("No such key `{}` found" + .format(key)) def set(self, key, value): self.properties[key] = value diff --git a/lib/google.py b/lib/google.py index 0d30c74..6e424bb 100644 --- a/lib/google.py +++ b/lib/google.py @@ -2,15 +2,52 @@ import urllib.parse as urlparse import requests +import threading import re + class GoogleSearch: + _session_var = threading.local() + + def _init_session(): + if not hasattr(GoogleSearch._session_var, "session"): + GoogleSearch._session_var.session = requests.Session() + + def session_cleanup(): + if hasattr(GoogleSearch._session_var, "session"): + delattr(GoogleSearch._session_var, "session") + + def _get_session(): + GoogleSearch._init_session() + return GoogleSearch._session_var.session def __qencode__(q): return urlparse.quote_plus(q) + + def correct_page_size(page_size): + default = 30 + if not page_size: + return default + + try: + page_size = int(page_size) + except: + return default + + page_size /= 10 + # consider lower bound + if page_size >= 10: + page_size = 100 + elif page_size >= 5: + page_size = 50 + elif page_size > 0: + page_size = page_size * 10 + else: + page_size = default + + return int(page_size) - def prepare_URL(query, site = '', params = '', page_num = 1, \ - page_size = 30): + def prepare_URL(query, site='', params='', page_num=1, page_size=30): if not query or query == '': raise Exception("Query cannot be empty") @@ -27,19 +64,9 @@ def prepare_URL(query, site = '', params = '', page_num = 1, \ u += '+' + GoogleSearch.__qencode__(site) u += '&btnG=Google+Search' - page_size = int(page_size) / 10 - # consider lower bound - if page_size >= 10: - page_size = 100 - elif page_size >= 5: - page_size = 50 - elif page_size > 0: - page_size = page_size * 10 - else: - page_size = 30 + page_size = GoogleSearch.correct_page_size(page_size) fmt = '{}&start={}&num={}' - page_size = int(page_size) page_num = int(page_num) if page_num <= 1: @@ -47,12 +74,13 @@ def prepare_URL(query, site = '', params = '', page_num = 1, \ else: return fmt.format(u, (page_num - 1) * page_size, page_size) - def fetch(url, UA = 'f0x.py/1.0 (Linux; python/requests)'): + def fetch(url, UA='f0x.py (Linux; python/requests)', proxy=None, + time_out=None, sslcheck=True): hdrs = { 'Host': 'www.google.com', 'User-Agent': UA, - 'Accept': 'text/html,application/xhtml+xml,application/xml;\ - q=0.9,*/*;q=0.8', + 'Accept': 'text/html,application/xhtml+xml,application/xml;' + + ' q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate', 'Referer': 'https://www.google.com/', @@ -63,11 +91,14 @@ def fetch(url, UA = 'f0x.py/1.0 (Linux; python/requests)'): 'Pragma': 'no-cache', 'TE': 'Trailers' } - return requests.get(url, headers=hdrs).text + + return GoogleSearch._get_session().get(url, headers=hdrs, + proxies=proxy, timeout=time_out, + verify=sslcheck).text def has_next(text): - o = re.search('aria-label="Next page"[^>]*>((Next >)|\ - (]*>>))', text, re.I) + o = re.search('aria-label="Next page"[^>]*>((Next >)|' + + '(]*>>))', text, re.I) if o: return True else: @@ -75,13 +106,11 @@ def has_next(text): def extract_urls(text): urls = [] - for pat in re.findall('\s+href="/url\?q=([^"&]*)[^"]*"[^>]*>', text, \ - re.M|re.I): - if not re.search('^http(s)?://(www\.)?[^.]*\.google\.com', pat, \ - re.I): + for pat in re.findall('\s+href="/url\?q=([^"&]*)[^"]*"[^>]*>', text, + re.M | re.I): + if not re.search('^http(s)?://(www\.)?[^.]*\.google\.com', pat, + re.I): up = urlparse.unquote(pat) if not up.startswith('/search?q='): urls += [up] return urls - - diff --git a/lib/prettyPrint.py b/lib/prettyPrint.py index bf36e58..2882b52 100644 --- a/lib/prettyPrint.py +++ b/lib/prettyPrint.py @@ -11,7 +11,6 @@ def __cont__(t, r, c): return t.replace(r, r + c) # *** text *** - def red(text): return f.RED + Prettify.__cont__(text, f.RESET, f.RED) + f.RESET @@ -28,8 +27,7 @@ def green(text): return f.GREEN + Prettify.__cont__(text, f.RESET, f.GREEN) + f.RESET def magenta(text): - return f.MAGENTA + Prettify.__cont__(text, f.RESET, f.MAGENTA) + \ - f.RESET + return f.MAGENTA + Prettify.__cont__(text, f.RESET, f.MAGENTA) + f.RESET def white(text): return f.WHITE + Prettify.__cont__(text, f.RESET, f.WHITE) + f.RESET @@ -38,41 +36,39 @@ def yellow(text): return f.YELLOW + Prettify.__cont__(text, f.RESET, f.YELLOW) + f.RESET def light_black(text): - return f.LIGHTBLACK_EX + Prettify.__cont__(text, f.RESET, \ - f.LIGHTBLACK_EX) + f.RESET + return f.LIGHTBLACK_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTBLACK_EX) + f.RESET def light_blue(text): - return f.LIGHTBLUE_EX + Prettify.__cont__(text, f.RESET, \ - f.LIGHTBLUE_EX) + f.RESET + return f.LIGHTBLUE_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTBLUE_EX) + f.RESET def light_cyan(text): - return f.LIGHTCYAN_EX + Prettify.__cont__(text, f.RESET, \ - f.LIGHTCYAN_EX ) + f.RESET + return f.LIGHTCYAN_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTCYAN_EX) + f.RESET def light_green(text): - return f.LIGHTGREEN_EX + Prettify.__cont__(text, f.RESET, \ - f.LIGHTGREEN_EX) + f.RESET + return f.LIGHTGREEN_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTGREEN_EX) + f.RESET def light_magenta(text): - return f.LIGHTMAGENTA_EX + Prettify.__cont__(text, f.RESET, \ - f.LIGHTMAGENTA_EX) + f.RESET + return f.LIGHTMAGENTA_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTMAGENTA_EX) + \ + f.RESET def light_red(text): - return f.LIGHTRED_EX + Prettify.__cont__(text, f.RESET, \ - f.LIGHTRED_EX) + f.RESET + return f.LIGHTRED_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTRED_EX) + f.RESET def light_white(text): - return f.LIGHTWHITE_EX + Prettify.__cont__(text, f.RESET, \ - f.LIGHTWHITE_EX) + f.RESET + return f.LIGHTWHITE_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTWHITE_EX) + f.RESET def light_yellow(text): - return f.LIGHTYELLOW_EX + Prettify.__cont__(text, f.RESET, \ - f.LIGHTYELLOW_EX) + f.RESET - - + return f.LIGHTYELLOW_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTYELLOW_EX) + f.RESET # *** background *** - def on_red(text): return b.RED + Prettify.__cont__(text, b.RESET, b.RED) + b.RESET @@ -89,8 +85,7 @@ def on_green(text): return b.GREEN + Prettify.__cont__(text, b.RESET, b.GREEN) + b.RESET def on_magenta(text): - return b.MAGENTA + Prettify.__cont__(text, b.RESET, b.MAGENTA) + \ - b.RESET + return b.MAGENTA + Prettify.__cont__(text, b.RESET, b.MAGENTA) + b.RESET def on_white(text): return b.WHITE + Prettify.__cont__(text, b.RESET, b.WHITE) + b.RESET @@ -99,54 +94,53 @@ def on_yellow(text): return b.YELLOW + Prettify.__cont__(text, b.RESET, b.YELLOW) + b.RESET def on_light_black(text): - return b.LIGHTBLACK_EX + Prettify.__cont__(text, b.RESET, \ - b.LIGHTBLACK_EX) + b.RESET + return b.LIGHTBLACK_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTBLACK_EX) + b.RESET def on_light_blue(text): - return b.LIGHTBLUE_EX + Prettify.__cont__(text, b.RESET, \ - b.LIGHTBLUE_EX) + b.RESET + return b.LIGHTBLUE_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTBLUE_EX) + b.RESET def on_light_cyan(text): - return b.LIGHTCYAN_EX + Prettify.__cont__(text, b.RESET, \ - b.LIGHTCYAN_EX) + b.RESET + return b.LIGHTCYAN_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTCYAN_EX) + b.RESET def on_light_green(text): - return b.LIGHTGREEN_EX + Prettify.__cont__(text, b.RESET, \ - b.LIGHTGREEN_EX) + b.RESET + return b.LIGHTGREEN_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTGREEN_EX) + b.RESET def on_light_magenta(text): - return b.LIGHTMAGENTA_EX + Prettify.__cont__(text, b.RESET, \ - b.LIGHTMAGENTA_EX) + b.RESET + return b.LIGHTMAGENTA_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTMAGENTA_EX) + \ + b.RESET def on_light_red(text): - return b.LIGHTRED_EX + Prettify.__cont__(text, b.RESET, \ - b.LIGHTRED_EX) + b.RESET + return b.LIGHTRED_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTRED_EX) + b.RESET def on_light_white(text): - return b.LIGHTWHITE_EX + Prettify.__cont__(text, b.RESET, \ - b.LIGHTWHITE_EX) + b.RESET + return b.LIGHTWHITE_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTWHITE_EX) + b.RESET def on_light_yellow(text): - return b.LIGHTYELLOW_EX + Prettify.__cont__(text, b.RESET, \ - b.LIGHTYELLOW_EX) + b.RESET - + return b.LIGHTYELLOW_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTYELLOW_EX) + b.RESET # *** sytle *** def as_bold(text): - return s.BRIGHT + Prettify.__cont__(text, s.RESET_ALL, s.BRIGHT) + \ - s.RESET_ALL + return s.BRIGHT + Prettify.__cont__(text, s.RESET_ALL, + s.BRIGHT) + s.RESET_ALL def as_dim(text): - return s.DIM + Prettify.__cont__(text, s.RESET_ALL, s.BRIGHT) + \ - s.RESET_ALL + return s.DIM + Prettify.__cont__(text, s.RESET_ALL, + s.BRIGHT) + s.RESET_ALL def as_normal(text): - return s.NORMAL + Prettify.__cont__(text, s.RESET_ALL, s.BRIGHT) + \ - s.RESET_ALL + return s.NORMAL + Prettify.__cont__(text, s.RESET_ALL, + s.BRIGHT) + s.RESET_ALL # *** logger *** - def p_error(text): print("[{}]: {}".format(Prettify.as_bold(Prettify.red('ERROR')), text)) @@ -157,12 +151,11 @@ def p_warn(text): print("[{}]: {}".format(Prettify.red('WARN'), text)) def p_debug(text): - print("[{}]: {}".format(Prettify.as_bold(Prettify.yellow('DEBUG')), + print("[{}]: {}".format(Prettify.as_bold(Prettify.yellow('DEBUG')), text)) - def p_log(text, header = '*'): + def p_log(text, header='*'): print("[{}]: {}".format(Prettify.as_bold(Prettify.blue(header)), text)) def p_clog(text, header): print("[{}]: {}".format(header, text)) - diff --git a/lib/useragents.py b/lib/useragents.py index 6fc4ac9..78ae637 100644 --- a/lib/useragents.py +++ b/lib/useragents.py @@ -2,142 +2,143 @@ import random + class UA: __ualist__ = [ - 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) ' + - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.'+ + 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.' + '84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) ' + - 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + + 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + 'Chrome/60.0.3112.107 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 7.0; SM-G930VC Build/NRD90M; wv) ' + - 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Mozilla/5.0 (Linux; Android 7.0; SM-G930VC Build/NRD90M; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + 'Chrome/58.0.3029.83 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G935S Build/MMB29K; wv) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G935S Build/MMB29K; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + 'Chrome/55.0.2883.91 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G920V Build/MMB29K) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.'+ + 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G920V Build/MMB29K) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.' + '98 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.'+ + 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.' + '83 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 6P Build/MMB29P) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.'+ + 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 6P Build/MMB29P) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.' + '83 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 7.1.1; G8231 Build/41.2.A.0.219; wv)'+ - ' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Mozilla/5.0 (Linux; Android 7.1.1; G8231 Build/41.2.A.0.219; wv)' + + ' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + 'Chrome/59.0.3071.125 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 6.0.1; E6653 Build/32.2.A.0.253) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.'+ + 'Mozilla/5.0 (Linux; Android 6.0.1; E6653 Build/32.2.A.0.253) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.' + '98 Mobile Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; '+ + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; ' + 'Trident/5.0)', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 '+ + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 ' + 'Firefox/15.0.1', - 'Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.'+ + 'Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.' + '98 Mobile Safari/537.3', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) '+ - 'AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 '+ + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) ' + + 'AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 ' + 'Mobile/15E148 Safari/604.1', - 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FSL 7.0.7.'+ + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FSL 7.0.7.' + '01001)', 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.02', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) '+ - 'AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/'+ + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) ' + + 'AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/' + '69.0.3497.105 Mobile/15E148 Safari/605.1', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) '+ - 'AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/13.'+ + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) ' + + 'AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/13.' + '2b11866 Mobile/16A366 Safari/605.1.15', - 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; '+ - '.NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET '+ + 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; ' + + '.NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET ' + 'CLR 3.5.30729)', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) '+ - 'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/'+ + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) ' + + 'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/' + '11.0 Mobile/15A372 Safari/604.1', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) '+ - 'AppleWebKit/604.1.34 (KHTML, like Gecko) Version/'+ + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) ' + + 'AppleWebKit/604.1.34 (KHTML, like Gecko) Version/' + '11.0 Mobile/15A5341f Safari/604.1', - 'Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; '+ - 'RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/'+ + 'Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; ' + + 'RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/' + '52.0.2743.116 Mobile Safari/537.36 Edge/15.15254', - 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; '+ - 'RM-1127_16056) AppleWebKit/537.36(KHTML, like Gecko) '+ + 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; ' + + 'RM-1127_16056) AppleWebKit/537.36(KHTML, like Gecko) ' + 'Chrome/42.0.2311.135 Mobile Safari/537.36 Edge/12.10536', - 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like '+ + 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like ' + 'Gecko) Chrome/13.0.782.112 Safari/535.1', - 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; '+ - 'Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) '+ + 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; ' + + 'Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.1058', - 'Mozilla/5.0 (Linux; Android 7.0; SM-T827R4 Build/NRD90M) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.'+ + 'Mozilla/5.0 (Linux; Android 7.0; SM-T827R4 Build/NRD90M) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.' + '3112.116 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T550 Build/LRX22G)'+ - ' AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser'+ + 'Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T550 Build/LRX22G)' + + ' AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser' + '/3.3 Chrome/38.0.2125.102 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 4.4.3; KFTHWI Build/KTU84M) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Silk/47.1.79 like'+ + 'Mozilla/5.0 (Linux; Android 4.4.3; KFTHWI Build/KTU84M) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Silk/47.1.79 like' + ' Chrome/47.0.2526.80 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.0.2; LG-V410/V41020c Build/LRX22G)'+ - ' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Mozilla/5.0 (Linux; Android 5.0.2; LG-V410/V41020c Build/LRX22G)' + + ' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + 'Chrome/34.0.1847.118 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '+ - '(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 '+ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 ' + 'Edge/12.246', - 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 '+ + 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36', - 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET '+ + 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET ' + 'CLR 2.0.50727)', - 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 '+ + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 ' + 'Firefox/12.0', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) '+ - 'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 '+ + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) ' + + 'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 ' + 'Mobile/15A5370a Safari/604.1', - 'Mozilla/5.0 (iPhone9,3; U; CPU iPhone OS 10_0_1 like Mac OS X) '+ - 'AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 '+ + 'Mozilla/5.0 (iPhone9,3; U; CPU iPhone OS 10_0_1 like Mac OS X) ' + + 'AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 ' + 'Mobile/14A403 Safari/602.1', - 'Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/'+ + 'Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/' + '13.0.1', - 'Mozilla/5.0 (iPhone9,4; U; CPU iPhone OS 10_0_1 like Mac OS X) '+ - 'AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 '+ + 'Mozilla/5.0 (iPhone9,4; U; CPU iPhone OS 10_0_1 like Mac OS X) ' + + 'AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 ' + 'Mobile/14A403 Safari/602.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/'+ + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/' + '601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, '+ + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/47.0.2526.111 Safari/537.36', - 'Mozilla/5.0 (Apple-iPhone7C2/1202.466; U; CPU like Mac OS X; en)'+ - ' AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 '+ + 'Mozilla/5.0 (Apple-iPhone7C2/1202.466; U; CPU like Mac OS X; en)' + + ' AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 ' + 'Mobile/1A543 Safari/419.3', - 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 '+ + 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, '+ + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/47.0.2526.111 Safari/537.36', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 '+ + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 ' + 'Firefox/15.0.1', - 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/'+ + 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/' + 'bingbot.htm)', - 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/'+ + 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/' + 'help/us/ysearch/slurp)', - 'Mozilla/5.0 (Linux; Android 6.0.1; SGP771 Build/32.2.A.0.253; '+ - 'wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/'+ + 'Mozilla/5.0 (Linux; Android 6.0.1; SGP771 Build/32.2.A.0.253; ' + + 'wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/' + '52.0.2743.98 Safari/537.36', - 'Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) '+ + 'Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) ' + 'Opera 7.02 Bork-edition [en]', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/'+ + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/' + '601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '+ - '(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 '+ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 ' + 'Edge/12.246', - 'Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) '+ - 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '+ + 'Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + 'Chrome/52.0.2743.98 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 6.0.1; SHIELD Tablet K1 Build/MRA58K'+ - '; wv) AppleWebKit/537.36 (KHTML, like Gecko) '+ + 'Mozilla/5.0 (Linux; Android 6.0.1; SHIELD Tablet K1 Build/MRA58K' + + '; wv) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Version/4.0 Chrome/55.0.2883.91 Safari/537.36', - 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET '+ + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET ' + 'CLR 1.0.3705)', - 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/'+ + 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/' + 'search/spider.html)', 'Opera/9.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.01' ] @@ -147,4 +148,4 @@ def get_ua_list(): def get_random_ua(): return random.choice(UA.__ualist__) - + \ No newline at end of file diff --git a/lib/utils.py b/lib/utils.py index 8799452..84ea661 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -6,7 +6,9 @@ import shutil import random + class DirUtil: + def join_names(parent, child): o = parent if o.endswith('/'): @@ -15,7 +17,7 @@ def join_names(parent, child): o += '/' + child return o - def create_dir(parent, dname = ''): + def create_dir(parent, dname=''): o = DirUtil.join_names(parent, dname) if not os.path.exists(o): try: @@ -25,28 +27,27 @@ def create_dir(parent, dname = ''): raise return o - def get_dir(parent, child = ''): + def get_dir(parent, child=''): d = DirUtil.join_names(parent, child) if not os.path.isdir(d): raise OSError('Directory not exists.') return d - def create_random_dir(parent, prefix = ''): - return DirUtil.create_dir(parent, prefix + str(int(time.time() * \ - 1000))) + def create_random_dir(parent, prefix=''): + return DirUtil.create_dir(parent, prefix + str(int(time.time() * 1000))) - def create_temp_dir(parent = '', prefix = ''): - return DirUtil.create_random_dir(DirUtil.create_dir('/tmp/', \ - parent), prefix) + def create_temp_dir(parent='', prefix=''): + return DirUtil.create_random_dir(DirUtil.create_dir('/tmp/', parent), + prefix) def list_dir(parent): l = [] for j in os.listdir(parent): t = DirUtil.join_names(parent, j) - l += [t] + l += [t] return l - def get_dir_list(parent, recurse = False): + def get_dir_list(parent, recurse=False): l = [] for i in DirUtil.list_dir(parent): if os.path.isdir(i): @@ -55,7 +56,7 @@ def get_dir_list(parent, recurse = False): l += DirUtil.get_dir_list(i, recurse) return l - def get_files_list(parent, recurse = False): + def get_files_list(parent, recurse=False): l = [] for i in DirUtil.list_dir(parent): if os.path.isfile(i): @@ -69,15 +70,18 @@ def merge_dirs(source, dest): DirUtil.create_dir(dest) for i in os.listdir(source): if os.path.isfile(DirUtil.join_names(source, i)): - shutil.move(DirUtil.join_names(source, i), \ - DirUtil.join_names(dest, i)) + shutil.move(DirUtil.join_names(source, i), + DirUtil.join_names(dest, i)) else: - DirUtil.merge_dirs(DirUtil.join_names(source, i), \ - DirUtil.create_dir(dest, i)) + DirUtil.merge_dirs(DirUtil.join_names(source, i), + DirUtil.create_dir(dest, i)) + def rmdir(dname): shutil.rmtree(dname) + class FileUtil: + def join_names(parent, fname): o = parent if o.endswith('/'): @@ -92,13 +96,13 @@ def get_file(parent, fname): raise OSError('File not exists') return f - def create_random_file(parent, prefix = ''): - return FileUtil.join_names(parent, prefix + str(int(time.time() * \ - 1000))) + def create_random_file(parent, prefix=''): + return FileUtil.join_names(parent, prefix + str(int(time.time() * + 1000))) - def create_temp_file(dname = '', prefix = ''): - return FileUtil.create_random_file(FileUtil.join_names('/tmp/', \ - dname), prefix) + def create_temp_file(dname='', prefix=''): + return FileUtil.create_random_file(FileUtil.join_names('/tmp/', dname), + prefix) def dump_list(ofile, l, append=True): if l: @@ -109,7 +113,17 @@ def dump_list(ofile, l, append=True): for i in l: f.write("{}\n".format(i)) + def get_file_aslist(file): + list = [] + + with open(file, 'r') as f: + list = [l.strip() for l in f if l.strip()] + + return list + + class Random: + def rand_between(start, end): if start < 0: raise Exception('Require positive number') @@ -120,5 +134,3 @@ def rand_between(start, end): def rand_no(max_no): return Random.rand_between(0, max_no) - - From fc1c4ad6593a1766ba9ff8986147aa258e84bfcd Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Tue, 10 Nov 2020 18:34:21 +0530 Subject: [PATCH 13/17] formatting fix --- f0x.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/f0x.py b/f0x.py index 10547d4..d94ba1b 100755 --- a/f0x.py +++ b/f0x.py @@ -38,7 +38,8 @@ def banner(): parser = argparse.ArgumentParser() # input -parser.add_argument('-d', '--domain', help='Specify Target domain.', dest='domain') +parser.add_argument('-d', '--domain', help='Specify Target domain.', + dest='domain') parser.add_argument('-q', '--query', help='Specify query/dork manually, and' + ' don\'t use more dorks from dorks-db.', dest='query') @@ -350,7 +351,9 @@ async def _record_proxy(self, proxies): def collect_open_proxies(self): proxies = asyncio.Queue() broker = Broker(proxies) - tasks = asyncio.gather(broker.find(types=['HTTP', 'HTTPS'], limit=self.get_open_proxy_count()), self._record_proxy(proxies)) + tasks = asyncio.gather(broker.find(types=['HTTP', 'HTTPS'], + limit=self.get_open_proxy_count()), + self._record_proxy(proxies)) loop = asyncio.get_event_loop() loop.run_until_complete(tasks) @@ -540,7 +543,8 @@ def get_proxy_object(self): if self._proxy_ptr == pl: self._proxy_ptr = 0 - if self._conn_per_proxy_count[self._proxy_ptr] < self.get_connection_per_proxy(): + if self._conn_per_proxy_count[self._proxy_ptr] < \ + self.get_connection_per_proxy(): p = self.get_proxy_list()[self._proxy_ptr] self._conn_per_proxy_count[self._proxy_ptr] += 1 l = self._proxy_ptr @@ -548,7 +552,8 @@ def get_proxy_object(self): if c >= pl: c = 0 - time.sleep((self.get_delay_min() * self.get_no_of_pages()) / 4) + time.sleep((self.get_delay_min() * + self.get_no_of_pages()) / 4) if p: proxy = {'proxy': {'http': p, 'https': p}, 'loc': l} @@ -816,7 +821,8 @@ def execute(self): fox.set_open_proxy_count(args.proxy_count) if fox.is_verbose() and not args.proxy_open: - pp.p_info('Ignoring `open proxy count` as provided without enabling `open proxies` switch.') + pp.p_info('Ignoring `open proxy count` as provided without ' + + 'enabling `open proxies` switch.') if args.proxy_open: fox.collect_open_proxies() From 10e7c36117dd29515e7672d2cdb8133f3135dc81 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Tue, 10 Nov 2020 18:41:29 +0530 Subject: [PATCH 14/17] proxy conn release count fix --- f0x.py | 1 + 1 file changed, 1 insertion(+) diff --git a/f0x.py b/f0x.py index d94ba1b..bd67870 100755 --- a/f0x.py +++ b/f0x.py @@ -583,6 +583,7 @@ def process_dork(self, dork): try: response = self.fetch_page_response(dork, p, proxy['proxy']) except Exception as e: + self.release_proxy(proxy) gsrch.session_cleanup() pp.p_error(e) return From f4bd5b4b6c2d05f50729f618a575b578939ae979 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Wed, 11 Nov 2020 16:15:05 +0530 Subject: [PATCH 15/17] added output code --- f0x.py | 197 +++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 172 insertions(+), 25 deletions(-) diff --git a/f0x.py b/f0x.py index bd67870..e06601a 100755 --- a/f0x.py +++ b/f0x.py @@ -11,6 +11,7 @@ import json import threading import asyncio +import random from proxybroker import Broker from concurrent.futures import ThreadPoolExecutor from git import Repo @@ -38,7 +39,7 @@ def banner(): parser = argparse.ArgumentParser() # input -parser.add_argument('-d', '--domain', help='Specify Target domain.', +parser.add_argument('-d', '--domain', help='Specify Target domain.', dest='domain') parser.add_argument('-q', '--query', help='Specify query/dork manually, and' + @@ -86,7 +87,7 @@ def banner(): 'proxies.', dest='proxy_open', action='store_true') parser.add_argument('-pC', '--open-proxies-count', help='Try collecting ' + - '`n` open proxies. (Default: 20)', dest='proxy_count', + '`n` open proxies. (Default: 20)', dest='proxy_count', type=int) parser.add_argument('-C', '--conn', help='Max connections per proxy ' + @@ -143,13 +144,17 @@ def banner(): parser.add_argument('-oJ', '--out-json', help='Save output in JSON format.', dest='out_fmt_json', action="store_true") -parser.add_argument('-oL', '--out-list', help='Save output in simple list.', - dest='out_fmt_list', action="store_true") +parser.add_argument('-oX', '--out-xml', help='Save output in XML format.', + dest='out_fmt_xml', action="store_true") parser.add_argument('-oR', '--out-report', help='Create html report with ' + 'JSON format results.', dest='out_report', action="store_true") +parser.add_argument('--silent', help='Do not print fetched links to stdout, ' + + 'just save them in file.', dest='out_silent', + action='store_true') + args = parser.parse_args() if args.version: @@ -162,6 +167,15 @@ def banner(): class F0x: def __init__(self, verbose=False): + self._conn_per_proxy_count = None + self._proxy_ptr = -1 + self._out_fmt_json = 1 << 0 + self._out_fmt_xml = 1 << 1 + self._out_report = 1 << 2 + self._outmode = 0 + self._proxy_lock = threading.Lock() + self._count_lock = threading.Lock() + self.verbose = verbose self.severities = None self.categories = None @@ -175,8 +189,6 @@ def __init__(self, verbose=False): self.delay_max = 1 self.connection_per_proxy = 2 self.proxies_list = None - self._conn_per_proxy_count = None - self._proxy_ptr = -1 self.request_timeout = None self.ssl_check = True self.open_proxy_count = 20 @@ -184,12 +196,9 @@ def __init__(self, verbose=False): self.no_of_pages = 5 self.max_results = None self.outdir = None - self.outmode = None + self.out_silent = False self.results_count = 0 - self._proxy_lock = threading.Lock() - self._count_lock = threading.Lock() - self._load_conf() def _load_conf(self): @@ -305,11 +314,29 @@ def set_outdir(self, outdir): def get_outdir(self): return self.outdir - def set_outmode(self, omode): - self.outmode = omode + def set_outmode_json(self): + self._outmode |= self._out_fmt_json - def get_outmode(self): - return self.outmode + def set_outmode_xml(self): + self._outmode |= self._out_fmt_xml + + def set_outmode_report(self): + self._outmode |= self._out_report + + def get_outmode_json(self): + return (self._outmode & self._out_fmt_json) == self._out_fmt_json + + def get_outmode_xml(self): + return (self._outmode & self._out_fmt_xml) == self._out_fmt_xml + + def get_outmode_report(self): + return (self._outmode & self._out_report) == self._out_report + + def set_output_silent(self, flag): + self.out_silent = flag + + def is_output_silent(self): + return self.out_silent def set_results_count(self, count): self.results_count = count @@ -351,8 +378,8 @@ async def _record_proxy(self, proxies): def collect_open_proxies(self): proxies = asyncio.Queue() broker = Broker(proxies) - tasks = asyncio.gather(broker.find(types=['HTTP', 'HTTPS'], - limit=self.get_open_proxy_count()), + tasks = asyncio.gather(broker.find(types=['HTTP', 'HTTPS'], + limit=self.get_open_proxy_count()), self._record_proxy(proxies)) loop = asyncio.get_event_loop() @@ -510,12 +537,36 @@ def fetch_page_response(self, dork, pagenum, proxy): else: ua = UA.get_random_ua() - return gsrch.fetch(gurl, ua, proxy, self.get_request_timeout(), + return gsrch.fetch(gurl, ua, proxy, self.get_request_timeout(), self.do_ssl_check()) - def save_links(self, links_list): # FIXME: output - for l in links_list: - pp.p_log(l) + def save_links(self, dork, dname, pnum, links_list): + if not self.get_outdir(): + for l in links_list: + pp.p_log(l) + return dname + + append = True + if pnum == 1: + append = False + dname = dutil.create_random_dir(self.get_outdir(), dname) + dname = re.sub('^{}[/]?'.format(self.get_outdir()), '', dname) + + futil.dump_list(futil. + join_names(dutil.get_dir(self.get_outdir(), + dname), + "{}.info".format(dname)), + ["dork: {}".format(dork)], append) + + futil.dump_list(futil. + join_names(dutil.get_dir(self.get_outdir(), dname), + "{}.txt".format(dname)), links_list, append) + + if not self.is_output_silent(): + for l in links_list: + pp.p_log(l) + + return dname def update_results_stats(self, c): with self._count_lock: @@ -570,7 +621,8 @@ def release_proxy(self, proxyobj): def process_dork(self, dork): if self.is_verbose(): pp.p_debug("Processing dork: {}".format(dork)) - + + dname = "dork{}".format(int(random.random() * 1000)) proxy = self.get_proxy_object() for p in range(1, self.get_no_of_pages() + 1): @@ -595,7 +647,7 @@ def process_dork(self, dork): if self.is_verbose(): pp.p_debug("Found {} url(s).".format(len(links))) - self.save_links(links) + dname = self.save_links(dork, dname, p, links) self.update_results_stats(len(links)) @@ -623,6 +675,81 @@ def execute(self): with ThreadPoolExecutor(max_workers=self.get_threads()) as exec: exec.map(self.process_dork, dorks) + + self.make_report() + + def make_report(self): + if not self.get_outdir(): + return + + if not (self.get_outmode_json() or self.get_outmode_report() or + self.get_outmode_xml()): + return + + fdr = None + if self.get_outmode_report(): + fdr = open(futil.join_names(self.get_outdir(), 'index.html') , 'w') + fdr.write(' f0x Report: links' + + '') + + for ddir in dutil.get_dir_list(self.get_outdir()): + dname = re.sub('^{}[/]?'.format(self.get_outdir()), '', ddir) + + links = futil.get_file_aslist( + futil.get_file(ddir, "{}.txt".format(dname))) + + if self.get_outmode_json(): + with open(futil.join_names(ddir, '{}.json'.format(dname)), + 'w') as fd: + fd.write(json.dumps({'urls': links})) + + try: + fdh = None + fdx = None + + if self.get_outmode_report(): + fdh = open(futil.join_names(ddir, '{}.html'.format(dname)), + 'w') + + if self.get_outmode_xml(): + fdx = open(futil.join_names(ddir, '{}.xml'.format(dname)), + 'w') + + except Exception as e: + pp.p_error(e) + else: + if fdh: + fdh.write(' dork urls' + + '') + if fdx: + fdx.write('') + + for link in links: + if fdh: + fdh.write('{}
'.format(link, link)) + + if fdx: + fdx.write('{}'.format(link)) + + if fdx: + fdx.write('') + if fdh: + fdh.write('') + + if fdr: + fdr.write(('dork: {} urls ' + + 'fetched: {}
').format(dname, dname, dname, + len(links))) + + finally: + if fdh: + fdh.close() + if fdx: + fdx.close() + + if fdr: + fdr.write('') + fdr.close() fox = F0x(verbose=args.verbose) @@ -780,9 +907,30 @@ def execute(self): quit() fox.set_max_results(args.max_results) -# TODO: FIXME: -# output dir and report logic/code left +if ((args.out_fmt_json or args.out_fmt_xml or args.out_report) and + not args.out_dir): + pp.p_error("Output format is defined without specifying output directory.") + quit() +if args.out_silent and not args.out_dir: + pp.p_warn("Can't silent links output, as no output directory defined " + + "to save them.") + +if args.out_silent: + fox.set_output_silent(True) + +if args.out_dir: + fox.set_outdir(args.out_dir.strip()) + +if args.out_fmt_json: + fox.set_outmode_json() + +if args.out_fmt_xml: + fox.set_outmode_xml() + +if args.out_report: + fox.set_outmode_report() + if args.no_ssl_check: fox.set_ssl_check(False) @@ -829,4 +977,3 @@ def execute(self): fox.collect_open_proxies() fox.execute() - \ No newline at end of file From cd1a30d00fc3dbd9012ef50bc6b55d9d9cdc4450 Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Wed, 11 Nov 2020 17:10:08 +0530 Subject: [PATCH 16/17] optimize loop --- f0x.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/f0x.py b/f0x.py index e06601a..ddbba99 100755 --- a/f0x.py +++ b/f0x.py @@ -522,6 +522,9 @@ def get_dorks(self, category, sev_list): d = re.sub('^dork:', '', l.lstrip().lower()) d = d.strip() + if d and j: + break + if j and int(j) in sev_list and d: dorks.append(d) From c43ec0846086b78c059242bf88481e0937a9dc8b Mon Sep 17 00:00:00 2001 From: Dinesh Saini Date: Wed, 11 Nov 2020 23:13:39 +0530 Subject: [PATCH 17/17] updated readme --- README.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8b59d3e..09b8e3d 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,27 @@ o888o `8. o88' 888o .8' -* Crawl URL from GHDB. -* Let user defined and saved dorks as per severity/category so dorks can be quickly executed to gather intel -* Generate Report and JSON file of extracted urls -* Require python3 +* Build on python3 +* Fetch exposed URLs using google dorks. +* Support HTML Report to feed results to crawler. +* Support JSON & XML output of extracted urls to feed results to external apis. +* Support grep-able output generation. +* Suport proxies. +* Multithread & optimised for better performance. +* Easy dorks updation. +* Uses huge dork collection. +* Dorks db is updating frequently. +* Easily integrable with other tools. +* Can be easily optimized to generate low noise and fetch quality results. +* Can be used for checking leaked info for tagreted domain. +* Integrate in ecosystem to search for urls for targeted domain and check for any blacklisted/unwanted exposed info. +* Out of the box search queries can be easily extendable to check extra information. ## usage $f0x.py [-h] [-d DOMAIN] [-q QUERY] [-n] [-Q EX_QUERY] [-c CATEGORY] [-cA] [-S SEVERITY] [-SQ] [-SA] [-t THREADS] [-p PROXY] [-pF PROXY_FILE] [-pO] [-pC PROXY_COUNT] [-C PROXY_CONN] [--no-ssl-check] [--timeout TIME_OUT] [-m DELAY_MIN] [-M DELAY_MAX] [-w DELAY] [-U UA] [--update] [-v] [-V] [-r PAGE_SIZE] [-R NO_OF_PAGES] [-T MAX_RESULTS] [-l] [-L] - [-o OUT_DIR] [-oJ] [-oL] [-oR] + [-o OUT_DIR] [-oJ] [-oX] [-oR] [--silent] ## example