世纪佳缘

最新推荐文章于 2022-04-07 20:31:55 发布

老猫4825445

最新推荐文章于 2022-04-07 20:31:55 发布

阅读量491

点赞数

本文链接：https://blog.csdn.net/m0_45118927/article/details/102517897

版权

#世纪佳缘（但数据少，采集男的好像仅有3000个，其中好像还有重复的）

import requests
import time
import socket
import json
from pymongo import MongoClient
MC = MongoClient("192.168.210.45")
table = MC.sjjy.cs09
def fetchURL(url):
    headers = {
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
        'Cookie': 'guider_quick_search=on; SESSION_HASH=f09e081981a0b33c26d705c2f3f82e8f495a7b56; PHPSESSID=e29e59d3eacad9c6d809b9536181b5b4; is_searchv2=1; save_jy_login_name=18511431317; _gscu_1380850711=416803627ubhq917; stadate1=183524746; myloc=11%7C1101; myage=23; mysex=m; myuid=183524746; myincome=30; COMMON_HASH=4eb61151a289c408a92ea8f4c6fabea6; sl_jumper=%26cou%3D17%26omsg%3D0%26dia%3D0%26lst%3D2018-11-07; last_login_time=1541680402; upt=4mGnV9e6yqDoj%2AYFb0HCpSHd%2AYI3QGoganAnz59E44s4XkzQZ%2AWDMsf5rroYqRjaqWTemZZim0CfY82DFak-; user_attr=000000; main_search:184524746=%7C%7C%7C00; user_access=1; PROFILE=184524746%3ASmartHe%3Am%3Aimages1.jyimg.com%2Fw4%2Fglobal%2Fi%3A0%3A%3A1%3Azwzp_m.jpg%3A1%3A1%3A50%3A10; pop_avatar=1; RAW_HASH=n%2AazUTWUS0GYo8ZctR5CKRgVKDnhyNymEBbT2OXyl07tRdZ9PAsEOtWx3s8I5YIF5MWb0z30oe-qBeUo6svsjhlzdf-n8coBNKnSzhxLugttBIs.; pop_time=1541680493356'
    }

    try:
        r = requests.get(url, headers=headers)
        if r.text:
            r=r
        else:
            r.raise_for_status()
            r.encoding = 'unicode_escape'
            # print(r.url)
            # print(r.text)
        table.insert_one({"flag":page,"wb":r.text})
        time.sleep(10)
        # return r.text
    except requests.HTTPError as e:
        print(e)
        print("HTTPError")
    except requests.RequestException as e:
        print(e)
    # except:
    #     print("Unknown Error !")
def parserHtml(html):
    if html.find("'false'"):
        m=html.replace("'false'","false")
    else:
        m=html
    if m.find("false"):
        mm=m.replace("false","'false'")
    else:
        mm=m
    s1=mm
    if s1.find("'null'"):
        s2=s1.replace("'null'","null")
    else:
        s2=s1
    if s2.find("null"):
        s3=s2.replace("null","'null'")
    else:
        s3=s2
    s=eval(s3)
    # print("*")
    # print(s)
    for f in s['userInfo']:
        blist = {}
        blist["realUid"] = f["realUid"]
        print(blist)
        table.insert_one(blist)
if __name__ == '__main__':
    for page in range(1, 3425):
        url = 'http://search.jiayuan.com/v2/search_v2.php?key=&sex=f&stc=1:50&sn=default&sv=1&p={}&f=select'.format(page)
        html1 = fetchURL(url)
        print("{}".format(page))
        time.sleep(2)
        # parserHtml(html1)
        # 为了降低被封ip的风险，每爬100页便歇5秒。
        if page % 100 == 99:
            time.sleep(5)

"""
MongoDB处理
"""
from pymongo import MongoClient
MC = MongoClient("192.168.210.45")
table = MC.sjjy.cs09
data = table.find({})#MongoDB有永不超时
table1 = MC.sjjy.chul0925
m=0
for page in data:
    m+=1
    wb=page['wb']
    ab=wb.find('[{"uid"')
    ae=wb.rfind('}],\"condi\"')
    d=wb[ab+1:ae+1]
    print(m)
    n=0
    for i in range(50):
        b=d
        if b.find(',{"uid":'):
            bb=b.find('{"uid"')
            be=b.find(',{"uid":')
            a={}
            c=b[bb:be]
            cb=c.find('"realUid"')
            e=c[cb+10:]
            ee=e.find(',')
            a['wb']=e[:ee]
            table1.insert_one(a)
            d=b[be+1:]
            n+=1
        else:
            f=d.find('"realUid"')
            f1=d[f+10:]
            f2=f1.find(',')
            table1.insert_one({'wb':f1[:f2]})
MC.close()