python爬取天天基金_「天天基金网每日净值」python爬虫简单爬取天天基金的基金信息和每日净值 - 金橙教程网...

天天基金网每日净值

fROM bs4 import BeautifulSoup

import Requests

import JSON

import js2py

import time

from elasticsearch import Elasticsearch

from elasticsearch import helpers

import threadpool

code_List = []

actions = []

time_ms = int(round(time.time() * 1000))

# 获取基金代码和名称

def getJJCode_Name(page=1):

URL = "http://fund.eaSTMoney.com/DATa/Fund_JJJZ_Data.aspx?t=1&lx=1&letter=&gsid=&text=&sort=zdf,desc&page=" \

+ str(page) + ",200&dt=1562661541203&atfc=&onlySale=0"

js_var = requests.get(url).text

db = js2py.eval_js(js_var)

pages = db['pages']

curpage = db['curpage']

datas = db['datas']

for data in datas:

print(data[0] + ' ' + data[1])

code_list.append(data[0])

action =fund_breif(data[0])

actions.append(action)

helpers.bulk(es, actions)

actions.clear()

print('current page --->' + curpage)

if int(pages) > int(curpage):

getJJCode_Name(int(curpage) + 1)

# 获取基金概况

def fund_breif(code):

url = "http://fundf10.eastmoney.com/jbgk_" + code + ".html"

html = requests.get(url)

html.encoding = 'utf-8'

currentPage = BeautifulSoup(html.text, 'lxml')

jj_table = currentPage.find('table', {'class': 'info'})

row_1 = jj_table.find('tr')

row_1_td_1 = row_1.find('td')

row_1_td_2 = row_1_td_1.find_next('td')

print(row_1_td_1.get_text() + " " + row_1_td_2.get_text())

row_2 = row_1.next_sibling

row_2_td_1 = row_2.find('td')

row_2_td_2 = row_2_td_1.find_next('td')

print(row_2_td_1.get_text() + " " + row_2_td_2.get_text())

row_3 = row_2.next_sibling

row_3_td_1 = row_3.find('td')

row_3_td_2 = row_3_td_1.find_next('td')

print(row_3_td_1.get_text() + " " + row_3_td_2.get_text())

row_4 = row_3.next_sibling

row_4_td_1 = row_4.find('td')

row_4_td_2 = row_4_td_1.find_next('td')

print(row_4_td_1.get_text() + " " + row_4_td_2.get_text())

row_5 = row_4.next_sibling

row_5_td_1 = row_5.find('td')

row_5_td_2 = row_5_td_1.find_next('td')

print(row_5_td_1.get_text() + " " + row_5_td_2.get_text())

action = {

"_index": "fund_breif",

"_type": "_doc",

"_source": {

"fund_full_name": row_1_td_1.get_text(),

'fund_name': row_1_td_2.get_text(),

'fund_code': code,

"fund_type": row_2_td_2.get_text(),

"issue_date": row_3_td_1.get_text(),

"establish_date": row_3_td_2.get_text(),

"aum": row_4_td_1.get_text(),

"share_size": row_4_td_2.get_text(),

"company": row_5_td_1.get_text()

}

}

return action

# 获取净值

def net_value(code, curr_page=1):

url = "http://api.fund.eastmoney.com/f10/lsjz?callback=fun&fundCode=000209&pageIndex="\

+ str(curr_page) + "&pageSize=200&startDate=&endDate=&_="+ str(time_ms)

headers = {

'User-Agent': 'Mozilla/5.0 (windows NT 6.1; WOW64) AppleWEBKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE',

'Cookie': 'st_pvi=01936466598903; st_sp=2019-07-02%2020%3A04%3A13; st_inirUrl=Https%3A%2F%2Fwww.baidu.com%2Flink',

'Referer': 'http://fundf10.eastmoney.com/jjjz_' + code + '.html'

}

v_actions = []

js_var = requests.get(url, headers=headers).text

mes = json.loads(js_var[4:-1])

for d in mes['Data']['LSJZList']:

danweijingzhi = 0

lishijingzhi = 0

zengzhanglv = 0

if d['DWJZ'].isdecimal() :

danweijingzhi = float(d['DWJZ'])

if d['LJJZ'].isdecimal() :

lishijingzhi = float(d['LJJZ'])

if d['JZZZL'].isdecimal():

print("d = " + d['JZZZL'])

print(d['JZZZL'].isspace())

zengzhanglv = float(d['JZZZL'])

action = {

"_index": "fund_netvalue",

"_type": "_doc",

"_source": {

"code" : code,

"date" : d['FSRQ'],

"danweijingzhi": danweijingzhi,

"lishijingzhi": lishijingzhi,

"zengzhanglv": zengzhanglv

}

}

v_actions.append(action)

helpers.bulk(es, v_actions)

print('current page --->' + str(mes['PageIndex']))

if int(mes['TotalCount']) > int(mes['PageSize'])* int(mes['PageIndex']) :

c_page = int(mes['PageIndex']) + 1

net_value(code, c_page)

def netvalue_threadpool():

pool = threadpool.ThreadPool(10)

t_reqs = threadpool.makeRequests(net_value, code_list)

[pool.putRequest(req) for req in t_reqs]

pool.wait()

if __name__ == '__main__':

es = Elasticsearch([{"host": "192.168.31.213", "port": 9200}])

getJJCode_Name()

netvalue_threadpool()

相关阅读

不少朋友觉得每次登录win10系统的时候都需要输入登录密码很麻烦，有没有win10取消登陆密码的方法呢？别着急，方法肯定是有的，下面小编就

一、变量的概念

变量名只有在第一次出现的时候，才是定义变量。当再次出现时，不是定义变量，而是直接使用之前定义的变量。

1.变量命

1、安装Pecan

更改pip镜像源mkdir ~/.pip

echo """

[global]

trusted-host=mirrors.aliyun.com

index-url=http://mirrors.aliyun

bisect是python内置模块，用于有序序列的插入和查找。

查找： bisect(array, item)

插入： insort(array,item)

查找

import bisect

DEM除了包括地面高程信息外，还可以派生地貌特性，包括坡度、坡向等，还可以计算地形特征参数，包括山峰、山脊、平原、位面、河道和沟谷