从0到1，打造专属自己的fofa爬虫脚本

前言：

---

一直想找一个功能不错的fofa爬虫脚本，奈何github上都似乎都不能满足我的需求，我又穷无法充值高级会员，每天只能使用100次的API，只好自己写个批量查询脚本，顺便锻炼下python脚本编写能力。

header信息：

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 OPR/52.0.2871.40',
    'Cookie': '_fofapro_ars_session=a497d09beffa61681c2909701e1198d0'       # 请输入你的session
}

获得总页数：

def get_page(key):
    key_base64 = base64.b64encode(key.encode('utf-8')).decode()
    key_base64 = urllib.parse.quote(key_base64)
    url = f'https://fofa.so/result?page=1&qbase64={key_base64}'
    r = requests.get(url=url, headers=headers, verify=False)
    html = r.text
    response = HtmlResponse(html, body=html, encoding='utf-8')
    selector = Selector(response=response)
    for i in [7, 6, 5, 4, 3, 2, 1]:
        path_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[11]/div[2]/a[{i}])'
        page = selector.xpath(path_xpath).extract()
        page = " ".join(page)
        if page:
            break
    if not page:
        page = 1
    return page

获取查询url链接：

def get_url(key, count):
    key_base64 = base64.b64encode(key.encode('utf-8')).decode()
    key_base64 = urllib.parse.quote(key_base64)
    scanurl = f'https://fofa.so/result?page={count}&qbase64={key_base64}'
    return scanurl

通过xpath定位获取数据：

def get_data(xpath):
    result = selector.xpath(xpath).extract()
    data = " ".join(result)
    return data

主功能模块，通过正则获取目标信息：

def scan(target):
    global count, page_count, selector
    host = ''
    count += 1
    leave_count = page_count - count - 1
    print(f'这是第{count}页的内容，还有{leave_count}页的内容：')
    result = ''
    r = requests.get(url=target, headers=headers, verify=False)
    if headers['Cookie'] in str(r.cookies):
        result = True
    html = r.text
    if '出错了' in html:
        print(html)
        print('某个地方出现了问题，请查看html代码')
        sys.exit()
    response = HtmlResponse(html, body=html, encoding='utf-8')
    selector = Selector(response=response)
    if result:
        for i in range(1, 11):
            for j in range(1, 3):
                host_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[1]/div[1]/a[{j}])'
                host_result = get_data(host_xpath)
                if host_result:
                    host = host_result
            port_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[2]/div[1]/a)'
            title_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[1]/div[2])'
            header_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[2]/div[2]/div/div[1])'
            certificate_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[2]/div[4])'
            server_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[1]/div[8]/a)'
            isp_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[1]/div[6]/a)'
            port = get_data(port_xpath)
            title = get_data(title_xpath)
            header = get_data(header_xpath)
            certificate = get_data(certificate_xpath)
            server = get_data(server_xpath)
            isp = get_data(isp_xpath)
            if port:
                port = int(port)
            ssl_domain = re.findall(r'(?<=CommonName: ).*(?=Subject Public)', certificate)
            ssl_domain = " ".join(ssl_domain).strip()
            language = re.findall(r'(?<=X-Powered-By: ).*(?=)', header)
            language = " ".join(language).strip()
            if 'PHPSESSID' in header and language == '':
                language = 'php'
            elif 'JSESSIONID' in header and language == '':
                language = 'jsp'
            try:
                ssl_domain = ssl_domain.split(' CommonName: ')[1]
            except:
                ssl_domain = ''
            if not ssl_domain and 'domain=' in header:
                ssl_domain = re.findall(r'(?<=domain=).*(?=;)', header)
                ssl_domain = " ".join(ssl_domain).strip()
                ssl_domain = ssl_domain.split(';')[0]
            try:
                status = int(header.split(' ')[1].strip())
                if status not in [200, 301, 302, 303, 304, 307, 400, 401, 403, 404, 405, 407,
                                  500, 501, 502, 503, 504, 508]:
                    status = ''
            except:
                status = ''
            if port == '' and status == '' and isp == '' and title == '':
                host = ''
            print(f'{host} {port} {status} {ssl_domain} {language} {server} {title}')
            # print(certificate)
            # print(header)
    else:
        print('cookie无效，请重新获取cookie')

主函数模块

def main():
    global count, page_count
    count = 0
    key = 'title="小程序后台管理系统"'
    page_count = get_page(key)
    page_count = int(page_count) + 1
    for page in range(1, page_count):
        url = get_url(key, page)
        print(url)
        scan(url)
        time.sleep(5)

使用条件：拥有fofa会员账号

使用说明：

单线程同步爬取，而且每翻一页需等待5秒，主要是防止被Ban
1.需先自行登录fofa账号，把获取到的cookie值填入脚本中：_fofapro_ars_session
2.需填写查询关键词，脚本里有注释

这只是一个demo，可自行修改，比如生成.csv或者.xls文件，或者写入数据库都行，还可以改成批量查询都没有问题。

我还是习惯用.xls文件查看，效果示图：

代码本身还存在不足的地方，我只是个菜鸟，大佬轻喷~

完整代码链接：https://github.com/book4yi/fofascan