Download arxiv paper

1. Code

#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
###########
Usage:
python download.py site.txt(containing https://...)

'''

from selenium import webdriver
import time
from pymouse import PyMouse

m = PyMouse()

def pause(length=1):
    time.sleep(length)

def download(url):
    b = webdriver.Firefox()
    #b.set_page_load_timeout(60) # useless
    b.maximize_window()
    pause(1)

    b.get(url)
    pause(2)

    loading_time = 60

    dt = b.find_elements_by_tag_name('dt')
    dd = b.find_elements_by_tag_name('dd')
    assert(len(dt) == len(dd))
    dst_type = "Computer Vision"

    print b.get_window_size()
    bias = [254, 171]
    screenIsVertical = False
    if screenIsVertical:
        print "No implement when screen is vertical"
        return
    else:
        pos = [b.get_window_size()['width']/2 + bias[0], b.get_window_size()['height']/2 + bias[1]]

    for i in xrange(4, len(dt)):

        # no Computer Vision paper
        if dst_type not in dd[i].find_element_by_class_name('primary-subject').text:
            continue

        # no 'pdf' button
        try:
            dt[i].find_element_by_link_text('pdf').click()
        except Exception, e:
            continue

        pause(loading_time)

        b.find_element_by_id('download').click()
        pause(2)

        m.click(pos[0], pos[1], 1, 1)
        time.sleep(1)

        b.back()
        time.sleep(1)
        dt = b.find_elements_by_tag_name('dt')
        dd = b.find_elements_by_tag_name('dd')

    b.close()

def main():
    import sys
    if len(sys.argv) != 2:
        print(__doc__)
        return

    with open(sys.argv[1], 'r') as fid:
        urls = [x.split('\n')[0] for x in fid.readlines()]

    for url in urls:
        if url.startswith('#'):
            continue
        else:
            download(url)


if __name__ == "__main__":
    main()

2. Usage

python download.py site.txt

site.txt (example)

https://arxiv.org/find/all/1/ti:+AND+object+detection/0/1/0/all/0/1
https://arxiv.org/find/all/1/ti:+AND+object+detection/0/1/0/all/0/1?skip=25&query_id=a6b6ed358647ff57
#https://arxiv.org/find/all/1/ti:+AND+object+detection/0/1/0/all/0/1?skip=50&query_id=a6b6ed358647ff57
https://arxiv.org/find/all/1/ti:+AND+object+detection/0/1/0/all/0/1?skip=75&query_id=a6b6ed358647ff57

You can use # to ignore specific url.

Refer this post for installing requirement.