VulnSpider_cloud/vulbox.py at master · opencloudshare/VulnSpider_cloud · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
# encoding=utf8
import os
import sys
reload(sys)
sys.setdefaultencoding('utf8')

from selenium import webdriver
from bs4 import BeautifulSoup
import time

browser = webdriver.Chrome()
url = 'https://www.vulbox.com/board/search/q/%E4%BA%91/page/{page}'
dirpath = os.getcwd()
filepath = os.path.join(dirpath,'vulbox_{page}.html')

for i in range(1,19):
    browser.get(url.format(page=i))
    soup = BeautifulSoup(browser.page_source, 'lxml')
    print soup.encode("utf8")

    with open(filepath.format(page=i),'w') as f:
        f.write(browser.page_source.encode("utf8"))
    f.close()
    #browser.delete_all_cookies()
    time.sleep(2)

browser.close()

respath = os.path.join(dirpath,'res_vulbox')
for i in range(1,19):
    soup = BeautifulSoup(open(filepath.format(page=i)),'lxml')

    f = open(respath,'a')

    divlist = soup.find_all('div')
    for i in divlist:
        try:
            if "blackboard" in i.attrs["class"]:
                bb = i
        except:
            pass

    divlist2 = bb.find_all('div')
    alist = bb.find_all('a')

    datelist = []
    complist = []
    titlist = []

    for i in divlist2:
        try:
            if "desc-tit" in i.attrs["class"]:
                titlist.append(i.text.strip())

            if "desc-date" in i.attrs["class"]:
                datelist.append(i.text)
        except:
            pass
    for i in alist:
        try:
            if "desc-comp" in i.attrs["class"]:
                complist.append(i.text)
        except:
            pass

    for i in range(0,len(datelist)):
        print datelist[i],complist[i],titlist[i]
        f.write(datelist[i].encode('utf8')+' '+complist[i].encode('utf8')+' '+titlist[i].encode('utf8'))
        f.write('\n')

    f.close()