數據化結構與保存
阿新 • • 發佈:2018-04-12
點擊 AR title ews page fin lis ttr brush
import requests from bs4 import BeautifulSoup from datetime import datetime import re import pandas #獲取點擊次數 def getClickCount(newsUrl): newsId = re.findall(‘\_(.*).html‘, newsUrl)[0].split(‘/‘)[1] clickUrl = ‘http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80‘.format(newsId) clickStr = requests.get(clickUrl).text count = int(re.search("hits‘\).html\(‘(.*)‘\);",clickStr).group(1)) return count # def writeNewsContent(content): # f = open(‘getNewsDetail.txt‘,‘a‘,encoding=‘utf-8‘) # f.write(content) # f.close() # 獲取新聞詳情 def getNewDetail(url): resd = requests.get(url) resd.encoding = ‘utf-8‘ soupd = BeautifulSoup(resd.text, ‘html.parser‘) news = {} news[‘title‘] = soupd.select(‘.show-title‘)[0].text info = soupd.select(‘.show-info‘)[0].text news[‘dt‘] = datetime.strptime(info.lstrip(‘發布時間:‘)[0:19], ‘%Y-%m-%d %H:%M:%S‘) if info.find(‘來源:‘) > 0: news[‘source‘] = info[info.find(‘來源:‘):].split()[0].lstrip(‘來源:‘) else: news[‘source‘] = ‘none‘ # if info.find(‘作者:‘) > 0: # author = info[info.find(‘作者:‘):].split()[0].lstrip(‘作者:‘) # else: # author = ‘none‘ news[‘clickCount‘] = getClickCount(url) return news def getListPage(listPageUrl): res = requests.get(listPageUrl) res.encoding = ‘utf-8‘ soup = BeautifulSoup(res.text, ‘html.parser‘) newsList = [] for news in soup.select(‘li‘): if len(news.select(‘.news-list-title‘)) > 0: # 獲取新聞模塊鏈接 a = news.a.attrs[‘href‘] # 調用函數獲取新聞正文 newsList.append(getNewDetail(a)) return newsList # 首頁列表新聞 newsTotal = [] firstPageUrl = ‘http://news.gzcc.cn/html/xiaoyuanxinwen/‘ newsTotal.extend(getListPage(firstPageUrl)) #計算總頁數 def getPageNum(): resn = requests.get(‘http://news.gzcc.cn/html/xiaoyuanxinwen/‘) resn.encoding = ‘utf-8‘ soupn = BeautifulSoup(resn.text, ‘html.parser‘) n = int(soupn.select(‘.a1‘)[0].text.rstrip(‘條‘)) return (n // 10 + 1) n = getPageNum() for i in range(n,n+1): pageUrl = ‘http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html‘.format(i) newsTotal.extend(getListPage(pageUrl)) # print(newsTotal) df = pandas.DataFrame(newsTotal) df.to_excel(‘gzccnews.xlsx‘) #提取包含點擊次數、標題、來源的前6行數據 print(df.head(6)) #提取‘學校綜合辦’發布的,‘點擊次數’超過3000的新聞。 print(df[(df[‘clickCount‘]>3000)&(df[‘source‘]==‘學校綜合辦‘)]) #提取‘國際學院‘和‘學生工作處‘發布的全部新聞。 sour = [‘國際學院‘,‘學生工作處‘] print(df[df[‘source‘].isin(sour)])
數據化結構與保存