python 爬取網頁天天基金
阿新 • • 發佈：2020-08-17
# encoding=utf-8
import pandas as pd
import requests
from lxml import etree
import re
import collections
def fund_code_name():
""" 篩選天天基金，6千多基金機構的，最近一週收益率排在前50強基金"""
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
'Referer': 'http://fund.eastmoney.com/data/fundranking.html',
'Cookie': 'st_si=51694067779834; st_asi=delete; ASP.NET_SessionId=e1pno0koqkcp5es3xyzyrg1n; EMFUND1=null; EMFUND2=null; EMFUND3=null; EMFUND4=null; EMFUND5=null; EMFUND6=null; EMFUND7=null; EMFUND8=null; EMFUND0=null; _adsame_fullscreen_18503=1; EMFUND9=08-16 01:16:38@#$%u4E07%u5BB6%u65B0%u5229%u7075%u6D3B%u914D%u7F6E%u6DF7%u5408@%23%24519191; st_pvi=87492384111747; st_sp=2020-08-16%2000%3A05%3A17; st_inirUrl=http%3A%2F%2Ffund.eastmoney.com%2Fdata%2Ffundranking.html; st_sn=15; st_psi=20200816011636912-0-9218336114'
}
response = requests.get(
url='http://fund.eastmoney.com/data/rankhandler.aspx?op=ph&dt=kf&ft=all&rs=&gs=0&sc=zzf&st=desc&sd=2018-11-26&ed=2019-11-26&qdii'
'=&tabSubtype=,,,,,&pi=1&pn=6450&dx=1&v=0.6516597604405057', headers=header)
text = response.text
data = text.split('=')[1]
# print(data)
compile_data = re.findall("{datas:\\[(.*)\\],allRecords", str(data))[0]
strip_data = str(compile_data).strip('[').strip(']')
replace_quta = strip_data.replace('"', "")
quota_arrays = replace_quta.split(",")
intervals = [[i * 25, (i + 1) * 25] for i in range(258)]
narrays = []
for k in intervals:
start, end = k[0], k[1]
line = quota_arrays[start:end]
narrays.append(line)
header = ["基金程式碼", "基金簡稱", "基金條碼", "日期",
"單位淨值", "累計淨值", "日增長率", "近1周增長率", "近1月增長率", "近3月", "近半年", "近1年", "近2年", "近3年",
"今年來", "成立來", "其他1", "其他2", "其他3", "其他4", "其他5", "其他6", "其他7", "其他8", "其他9"]
df = pd.DataFrame(narrays, columns=header)
df_part = df[["基金程式碼", "基金簡稱", "日期",
"單位淨值", "累計淨值", "日增長率", "近1周增長率", "近1月增長率", "近3月", "近半年"]]
df_tmp = df_part.sort_values(by=["近1周增長率"], ascending=False, axis=0)
rank_fund_code = df_tmp.head(50)["基金程式碼"]
fund_codes_list = rank_fund_code.values.tolist()
print("前50強基金：", fund_codes_list)
df_tmp.head(50).to_csv("./本季度前50強基金收益.csv", encoding="utf_8_sig")
return fund_codes_list
def get_one_fund_stocks(fund_code):
"""根據基金碼,獲取每一支基金的最新一季度所有持倉股票池前10支股票"""
url = "http://fundf10.eastmoney.com/FundArchivesDatas.aspx?type=jjcc&code={}&topline=10&year=&month=&rt=0.5032668912422176".format(
fund_code)
head = {
"Cookie": "EMFUND1=null; EMFUND2=null; EMFUND3=null; EMFUND4=null; EMFUND5=null; EMFUND6=null; EMFUND7=null; EMFUND8=null; EMFUND0=null; st_si=44023331838789; st_asi=delete; EMFUND9=08-16 22:04:25@#$%u4E07%u5BB6%u65B0%u5229%u7075%u6D3B%u914D%u7F6E%u6DF7%u5408@%23%24519191; ASP.NET_SessionId=45qdofapdlm1hlgxapxuxhe1; st_pvi=87492384111747; st_sp=2020-08-16%2000%3A05%3A17; st_inirUrl=http%3A%2F%2Ffund.eastmoney.com%2Fdata%2Ffundranking.html; st_sn=12; st_psi=2020081622103685-0-6169905557"
,
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36"}
response = requests.get(url, headers=head)
text = response.text # html subsitue text
div = re.findall('content:\\"(.*)\\",arryear', text)[0]
html_body = '<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><title>test</title></head><body>%s</body></html>' % (
div)
html = etree.HTML(html_body)
stock_info = html.xpath('//div[1]/div/table/tbody/tr/td/a')
stock_money = html.xpath('//div[1]/div/table/tbody/tr/td')
stock_one_fund = []
for stock in stock_info:
if stock.text and stock.text.isdigit():
stock_one_fund.append(stock.text)
if len(stock_one_fund)>1:
print("基金程式碼：{}".format(fund_code), "基金持有前10股票池", stock_one_fund)
return stock_one_fund # can return empty list
def static_best_stock(rank=20):
""" 統計收益最佳前50機構共同持有股票程式碼情況,修改rank數量可調整展示股票排名數目"""
rank_codes = fund_code_name()
stocks_array = []
for index, code in enumerate(rank_codes):
if index < 1:
print("<" * 30 + "FBI WARNING近1周收益最高基金的排名高到低排序以及股票池情況" + ">" * 30)
stocks = get_one_fund_stocks(code)
if len(stocks) > 1 and stocks:
stocks_array.extend(stocks)
count_each_stock = collections.Counter(stocks_array)
print("<" * 30 + "FBI WARNING,{}".format(static_best_stock.__doc__) + ">" * 30)
print("#" * 30 + "本季度基金機構共同持有股票數目排行前{}股票程式碼情況".format(rank) + "#" * 30)
df=pd.DataFrame.from_dict(count_each_stock,orient='index',columns=["持有該股機構數目"])
df=df.reset_index().rename(columns={"index":"股票程式碼"})
# for k, v in count_each_stock.items():
# print("股票程式碼: ", k, "持有該股票機構數量: ", v)
df=df.sort_values(by="持有該股機構數目",ascending=False)
print(df.head(rank))
if __name__ == '__main__':
static_best_stock()
備註：本文只為個人練習學習，如果用於違法行為概不負責