#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time  : 2020/02/11 21:44
# @Author : dangxusheng
# @Email  : [email protected]
# @File  : download_by_href.py
'''
自動從arxiv.org 下載文獻
'''

import os
import os.path as osp
import requests
from lxml import etree
from pprint import pprint
import re
import time
import glob

headers = {
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/80.0.3987.87 Safari/537.36","Host": 'arxiv.org'
}

HREF_CN = 'http://cn.arxiv.org/pdf/'
HREF_SRC = 'http://cn.arxiv.org/pdf/'
SAVE_PATH = '/media/dangxs/E/Paper/download_at_20200730'
os.makedirs(SAVE_PATH,exist_ok=True)

FAIL_URLS = []
FAIL_URLS_TXT = f'{SAVE_PATH}/fail_urls.txt'


def download(url,title):
  pattern = r'[\\/:*?"\'<>|\r\n]+'
  new_title = re.sub(pattern," ",title)
  print(f'new title: {new_title}')
  save_filepath = '%s/%s.pdf' % (SAVE_PATH,new_title)
  if osp.exists(save_filepath) and osp.getsize(save_filepath) > 50 * 1024:
    print(f'this pdf is be existed.')
    return True
  try:
    with open(save_filepath,'wb') as file:
      # 分位元組下載
      r = requests.get(url,stream=True,timeout=None)
      for i in r.iter_content(2048):
        file.write(i)
    if osp.getsize(save_filepath) >= 10 * 1024:
      print('%s 下載成功.' % title)
      return True
  except Exception as e:
    print(e)
  return False


# 從arxiv.org 去下載
def search(start_size=0,title_keywords='Facial Expression'):
  # 訪問地址: https://arxiv.org/find/grp_eess,grp_stat,grp_cs,grp_econ,grp_math/1/ti:+Face/0/1/0/past,2018,2019/0/1?skip=200&query_id=1c582e6c8afc6146&client_host=cn.arxiv.org
  req_url = 'https://arxiv.org/search/advanced'
  req_data = {
    'advanced': 1,'terms-0-operator': 'AND','terms-0-term': title_keywords,'terms-0-field': 'title','classification-computer_science': 'y','classification-physics_archives': 'all','classification-include_cross_list': 'include','date-filter_by': 'date_range',# date_range | specific_year
    # 'date-year': DOWN_YEAR,'date-year': '','date-from_date': '2015','date-to_date': '2020','date-date_type': 'announced_date_first',# submitted_date | submitted_date_first | announced_date_first
    'abstracts': 'show','size': 50,'order': '-announced_date_first','start': start_size,}
  res = requests.get(req_url,params=req_data,headers=headers)
  html = res.content.decode()
  html = etree.HTML(html)

  total_text = html.xpath('//h1[@class="title is-clearfix"]/text()')
  total_text = ''.join(total_text).replace('\n','').lstrip(' ').strip(' ')
  # i.e. : Showing 1–50 of 355 results
  num = re.findall('\d+',total_text)
  # Sorry,your query returned no results
  if len(num) == 0: return [],0

  total = int(num[-1]) # 查詢總條數
  paper_list = html.xpath('//ol[@class="breathe-horizontal"]/li')
  info_list = []
  for p in paper_list:
    title = p.xpath('./p[@class="title is-5 mathjax"]//text()')
    title = ''.join(title).replace('\n','').lstrip(' ').strip(' ')
    href = p.xpath('./div/p/a/@href')[0]
    info_list.append({'title': title,'href': href})

  return info_list,total


# 去指定頁面下載
def search_special():
  res = requests.get('https://gitee.com/weberyoung/the-gan-zoo?_from=gitee_search')
  html = res.content.decode()
  html = etree.HTML(html)

  paper_list = html.xpath('//div[@class="file_content markdown-body"]//li')
  info_list = []
  for p in paper_list:
    title = p.xpath('.//text()')
    title = ''.join(title).replace('\n','').lstrip(' ').strip(' ')
    href = p.xpath('./a/@href')[0]
    info_list.append({'title': title,'href': href})

  pprint(info_list)
  return info_list


if __name__ == '__main__':
  page_idx = 0
  total = 1000
  keywords = 'Facial Action Unit'
  while page_idx <= total // 50:
    paper_list,total = search(page_idx * 50,keywords)
    print(f'total: {total}')
    if total == 0:
      print('no found .')
      exit(0)

    for p in paper_list:
      title = p['title']
      href = HREF_CN + p['href'].split('/')[-1] + '.pdf'
      print(href)
      if not download(href,title):
        print('從國內映象下載失敗，從源地址開始下載 >>>>')
        # 使用國際URL再下載一次
        href = HREF_SRC + p['href'].split('/')[-1] + '.pdf'
        if not download(href,title):
          FAIL_URLS.append(p)
    page_idx += 1

  # 下載最後的部分
  last_1 = total - page_idx * 50
  paper_list,total = search(last_1,keywords)
  for p in paper_list:
    title = p['title']
    href = HREF_CN + p['href'].split('/')[-1] + '.pdf'
    if not download(href,title):
      FAIL_URLS.append(p)
    time.sleep(1)

  pprint(FAIL_URLS)
  with open(FAIL_URLS_TXT,'a+') as f:
    for item in FAIL_URLS:
      href = item['href']
      title = item['title']
      f.write(href + '\n')

  print('done.')

以上就是python自動從arxiv下載paper的示例程式碼的詳細內容，更多關於python 從arxiv下載paper的資料請關注我們其它相關文章！