1
import fire
import requests
from crawl import exe_sql
class Jk:
def search(self, name):
res = exe_sql(f'select * from jenkins_plugins where name like "{name}%"')
print(res)
def download(self, name):
url = exe_sql(f'select url from jenkins_plugins where name="{name}"')[0][0]
with open(f'{name}.hpi', 'wb') as fwb:
fwb.write(requests.get(url).content)
if __name__ == '__main__':
fire.Fire(Jk)
# Jk().download('AdaptivePlugin')
2
import requests
from bs4 import BeautifulSoup
import sqlite3
def get_from_remote():
plugins_url = 'https://mirrors.huaweicloud.com/jenkins/plugins/'
soup = BeautifulSoup(requests.get(plugins_url).text, 'lxml')
name_url = []
for ll in soup.select('a[title]'):
name = ll['title']
url = plugins_url + ll['href'] + 'latest/' + '{}.hpi'.format(name)
name_url.append({name: url})
return name_url
def exe_sql(sql_str):
conn = sqlite3.connect('name_url.db')
c = conn.cursor()
res = c.execute(sql_str).fetchall()
conn.commit()
conn.close()
return res
def save():
for item in get_from_remote():
for k, v in item.items():
sql_str = f'insert into jenkins_plugins values("{k}","{v}");'
exe_sql(sql_str)
if __name__ == '__main__':
# sql('''create table jenkins_plugins
# (name char(50) primary key not null,
# url text not null
# );''')
save()
# pass