python - bing_api_search

bing api search

#!/usr/bin/python
# -*- coding: utf-8 -*-

# Author: Nixawk
# Name:   bing api search
# Date:   Sat Sep 26 11:39:32 UTC 2015
# Docs:   https://datamarket.azure.com/dataset/bing/search

import requests
import json
import urllib
import logging


logging.basicConfig(level=logging.INFO, format="[+] %(message)s")
logger = logging.getLogger('bing_api_search')


class Bing(object):
    def __init__(self):
        """Get results from bing api
        """
        self.api_url = 'http://api.datamarket.azure.com/Bing/Search/Web'
        self.bing_urls = []
        self.bing_titles = []
        self.bing_results = []

    def json(self, content):
        """Json data format
        """
        logger.debug('translate data as json format')

        try:
            return json.loads(content)
        except ValueError:
            return {}

    def json_parse_Data(self, bing_json):
        """valid json data
        """
        logger.debug('if valid json data or not')
        if (bing_json) and ('d' in bing_json):
            return bing_json['d']

        return {}

    def json_parse_Results(self, bing_json_d):
        """parse results records from bing json data
        """
        logger.debug('parse results records from bing json data')
        if (bing_json_d) and ('results' in bing_json_d):
            return bing_json_d['results']

        return []

    def json_parse_Url(self, bing_json_r):
        """Parse url from (bing json results records)
        """
        logger.debug('parse urls from bing results records')
        return [_['Url'] for _ in bing_json_r if 'Url' in _]

    def json_parse_Title(self, bing_json_r):
        """Parse title from (bing json results records)
        """
        logger.debug('parse titles from bing results records')
        return [_['Title'] for _ in bing_json_r if 'Title' in _]

    def uniq(self, totalrecords=[], newrecords=[]):
        """uniq records
        """
        for i in newrecords:
            if i not in totalrecords:
                totalrecords.append(i)

        return totalrecords

    def api_request(self, url, api_key):
        """send api http request
        """
        sess = requests.Session()
        return sess.get(url, auth=("", api_key))

    def api_search(self, api_key, query, top=10, bing_limit=0):
        """bing search api
        """
        logger.debug('use bing search api for an new search')
        num = 1
        payload = {
            'Query': "'%s'" % query,
            '$format': 'json',
            '$top': top
        }

        while True:
            try:
                bing_json_d = None  # {'d':{}}
                bing_json_r = None  # {'d':{'results'}}

                url = "%s?%s" % (self.api_url, urllib.urlencode(payload))
                logger.info("%08d : %s" % (num, url))

                bing_resp = self.api_request(url, api_key)
                logger.info("%d - %s" % (bing_resp.status_code, url))

                # parse bing json data
                bing_json = self.json(bing_resp.text)
                bing_json_d = self.json_parse_Data(bing_json)

                # check if exists valid 'd' record
                if bing_json_d:
                    bing_json_r = self.json_parse_Results(bing_json_d)
                else:
                    break

                # check if exists valid ['d']['Results'] record
                if bing_json_r:
                    # parse results from json
                    self.uniq(self.bing_results, bing_json_r)

                    # parse urls from json
                    self.uniq(
                        self.bing_urls,
                        self.json_parse_Url(bing_json_r)
                    )

                    # parse titles from json
                    self.uniq(
                        self.bing_titles,
                        self.json_parse_Title(bing_json_r)
                    )
                else:
                    break

                # limit search number
                if (bing_limit != 0) and (bing_limit <= num):
                    break
                else:
                    num = num + (top * 1)

                if '__next' not in bing_json_d:
                    break
                else:
                    payload['$skip'] = num
            except KeyboardInterrupt:
                break

        return self.bing_results, self.bing_urls, self.bing_titles


if __name__ == "__main__":
    api_key = ''  # put your key here.
    query = 'site:exploit-db.com'

    bing = Bing()
    bing.api_search(api_key, query)

    # bing.api_search(api_key, query, top=5, bing_limit=100)
    #   api_key    :  bing search api key
    #   query      :  bing search syntax
    #   top        :  number of every query records
    #   bing_limit :  limit search records number

    for url in bing.bing_urls:
        print(url)

┌─[lab@core]─[~]
└──╼  python2 ~/bing_api.py 
[+] 00000001 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24top=5
[+] 00000006 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=6&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=6&%24top=5
[+] 00000011 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=11&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=11&%24top=5
[+] 00000016 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=16&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=16&%24top=5
[+] 00000021 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=21&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=21&%24top=5
[+] 00000026 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=26&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=26&%24top=5
[+] 00000031 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=31&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=31&%24top=5
[+] 00000036 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=36&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=36&%24top=5
[+] 00000041 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=41&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=41&%24top=5
[+] 00000046 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=46&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=46&%24top=5
[+] 00000051 : http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=51&%24top=5
[+] Starting new HTTP connection (1): api.datamarket.azure.com
[+] Starting new HTTPS connection (1): api.datamarket.azure.com
[+] 200 - http://api.datamarket.azure.com/Bing/Search/Web?Query=%27site%3Aexploit-db.com%27&%24format=json&%24skip=51&%24top=5
http://www.exploit-db.com/exploits/16314/
https://www.exploit-db.com/
https://www.exploit-db.com/google-hacking-database/
https://www.exploit-db.com/shellcode/
https://www.exploit-db.com/webapps/
https://www.exploit-db.com/remote/
https://www.exploit-db.com/dos/
https://www.exploit-db.com/platform/?order_by=date&order=asc&pg=1&p=linux
https://www.exploit-db.com/vbseo-from-xss-to-reverse-php-shell/
https://www.exploit-db.com/local/
https://www.exploit-db.com/exploits/38146/
https://www.exploit-db.com/exploits/38126/
https://www.exploit-db.com/exploits/38151/
https://www.exploit-db.com/exploits/38087/
https://www.exploit-db.com/exploits/38176
https://www.exploit-db.com/exploits/38089/
https://www.exploit-db.com/exploits/38085/
https://www.exploit-db.com/exploits/37976/
https://www.exploit-db.com/exploits/38202
https://www.exploit-db.com/exploits/37822/
https://www.exploit-db.com/exploits/38190
https://www.exploit-db.com/exploits/37932/
https://www.exploit-db.com/exploits/37778/
https://www.exploit-db.com/exploits/37890/
https://www.exploit-db.com/exploits/37927/
https://www.exploit-db.com/exploits/38073/
https://www.exploit-db.com/exploits/37818/
https://www.exploit-db.com/exploits/37774/
https://www.exploit-db.com/exploits/38091/
https://www.exploit-db.com/exploits/38000/
https://www.exploit-db.com/exploits/37929/
https://www.exploit-db.com/exploits/37920/
https://www.exploit-db.com/exploits/37931/
https://www.exploit-db.com/exploits/37773/
https://www.exploit-db.com/exploits/37817/
https://www.exploit-db.com/exploits/37923/
https://www.exploit-db.com/exploits/37814/
https://www.exploit-db.com/exploits/37977/
https://www.exploit-db.com/exploits/37966/
https://www.exploit-db.com/exploits/37812/
https://www.exploit-db.com/exploits/38051/
https://www.exploit-db.com/exploits/38072/
https://www.exploit-db.com/exploits/37988/
https://www.exploit-db.com/exploits/37771/
https://www.exploit-db.com/exploits/37852/
https://www.exploit-db.com/exploits/37926/
https://www.exploit-db.com/exploits/37843/
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值