暑期 ---------0714爬虫

#0716------------
'''
import csv
csvFile = open("csv test","w+")
try:
    writer = csv.writer(csvFile)
    writer.writerow(('number','number plus 2','number times 3'))
    for i in range(10):
        writer.writerow((i,i+2,i*2))
finally:
    csvFile.close()
'''

'''
import csv
from urllib.request import urlopen
from bs4 import BeautifulSoup

html = urlopen("https://www.runoob.com/html/html-tables.html")
bs = BeautifulSoup(html,"html.parser")

table = bs.findAll('table',{'class':'reference'})[0]
rows = table.findAll('tr')

csvFile = open("csv_test","wt+")
writer = csv.writer(csvFile)
try:
    for row in rows:
        csvRow = []
        for cell in row.findAll(['td','th']):
            csvRow.append(cell.get_text())
        writer.writerow(csvRow)
finally:
    csvFile.close()
'''

#  use scraping  ----创建事务  sql语句
'''
import pymysql,random,datetime,re
from urllib.request import urlopen
from bs4 import BeautifulSoup

#创建链接
conn = pymysql.connect(
    host = "127.0.0.1",
    unix_socket = "/tmp/mysql.sock",
    user = "root",
    password = "root",
    db = "gly",
    charset = "utf-8"
)
cur = conn.curser()#创建游标
cur.execute("use scraping")#执行
random.seed(datetime.datetime.now())

#存储
def store(title,content):
    cur.execute("insert into pages(title,content) values ('%s','%s'),(title,content)")
    cur.connection.commit()
#爬数据
def getLinks(articleUrl):
    html = urlopen("http://en.wikipedia.org"+articleUrl)
    bs0bj = BeautifulSoup(html,"html.parser")
    title = bs0bj.find("h1").get_text()
    content = bs0bj.find("div",{"id":"mw-content-text"}).find("p").get_text()
    store(title,content)
    return bs0bj.find("div",{"id":"babyContent"}).findAll("a",href=re.compile("^(/wiki/)((?!:).)*$"))
links = getLinks("/wiki/kevin_Bacon")
try:
    while len(links) > 0:
        newArticle = links[random.randint(0,len(links)-1)].attrs["href"]
        print(newArticle)
        links = getLinks(newArticle)
finally:
    cur.close()
    conn.close()
'''

'''
import smtplib
from email.mime.text import MIMEText

def send_mail(username,passwd,recv,content):
    mailserver = "smtp.qq.com"
    username_send = username
    password = passwd
    username_recv = recv
    mail = MIMEText(content,_subtype = 'html',_charset = 'utf-8')

    mail['Subject'] = '工资条'
    mail['From'] = username_send
    mail['To'] = username_recv
    smtp = smtplib.SMTP(mailserver,port=587)
    smtp.login(username_send,password)
    smtp.sendmail(username_send,username_recv,mail.as_string())
    smtp.quit()
    print("successful")

send_mail("756347780@qq.com","15045491532q","756347780@qq.com","你好")
'''

'''
import smtplib
from email.mime.text import MIMEText
def send_mail(username, passwd, recv, content):
    mailserver = "smtp.163.com"  # 邮箱服务器地址
    username_send = username  # 邮箱用户名
    password = passwd  # 邮箱密码：需要使用授权码
    username_recv = recv  # 收件人，多个收件人用逗号隔开
    mail = MIMEText(content, _subtype='html', _charset='utf-8')

    mail['Subject'] = '工资条'
    mail['From'] = username_send  # 发件人
    mail['To'] = username_recv  # 收件人；[]里的三个是固定写法，别问为什么，我只是代码的搬运工
    smtp = smtplib.SMTP(mailserver, port=25)  # 连接邮箱服务器，smtp的端口号是25
    smtp.login(username_send, password)  # 登录邮箱
    smtp.sendmail(username_send, username_recv, mail.as_string())  # 参数分别是发送者，接收者，第三个是把上面的发送邮件的内容变成字符串
    smtp.quit()  # 发送完毕后退出smtp
    print('发送成功')
send_mail("15045491532@163.com","333355wo","15045491532@163.com","你好")
'''

#读取纯文本
'''
from urllib.request import urlopen

textPage = urlopen("http://www.pythonscraping.com/pages/warandpeace/chapter1-ru.txt")
print(str(textPage.read(),'utf-8'))
'''

#读取csv
'''
from urllib.request import urlopen
from io import StringIO
import csv

data = urlopen("http://www.pythonscraping.com/files/MontyPythonAlbums.csv")\
    .read().decode("ascii","ignore")
dataFile = StringIO(data)
csvReader = csv.reader(dataFile)
for row in csvReader:
    print(row)
'''

#q去除头信息
'''
from urllib.request import urlopen
from io import StringIO
import csv

data = urlopen("http://www.pythonscraping.com/files/MontyPythonAlbums.csv")\
    .read().decode("ascii","ignore")
dataFile = StringIO(data)
csvReader = csv.DictReader(dataFile)
print(csvReader.fieldnames)
print("---------------")
for row in csvReader:
    print(row)
'''

#读取pdf----pdfminer(pdfminer3k)
'''
from urllib.request import urlopen
from pdfminer.pdfinterp import PDFResourceManager,process_pdf
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from io import StringIO
from io import open

def readPDF(pdfFile):
    rsrcmgr = PDFResourceManager()
    retstr = StringIO()
    laparams = LAParams()
    device = TextConverter(rsrcmgr,retstr,laparams=laparams)

    process_pdf(rsrcmgr,device,pdfFile)
    device.close()

    content = retstr.getvalue()
    retstr.close()
    return content
pdfFile = urlopen("http://www.pythonscraping.com/pages/warandpeace/chapter1.pdf")
outputString = readPDF(pdfFile)
print(outputString)
pdfFile.close()
'''

#读取word .docx
'''
from zipfile import ZipFile
from urllib.request import urlopen
from io import BytesIO

#读取数据
wordFile = urlopen("http://www.pythonscraping.com/pages/AWordDocument.docx").read()
wordFile = BytesIO(wordFile)#转字节
document = ZipFile(wordFile)#解压
xml_content = document.read('word/document.xml')#转xml格式
print(xml_content.decode("utf-8"))
'''

#解析BeautifulSoup
#w:t==========word标签
'''
from zipfile import ZipFile
from urllib.request import urlopen
from io import BytesIO
from bs4 import BeautifulSoup

wordFile = urlopen("http://www.pythonscraping.com/pages/AWordDocument.docx").read()
wordFile = BytesIO(wordFile)
document = ZipFile(wordFile)
xml_content = document.read('word/document.xml')

print(xml_content)
#wordobj = BeautifulSoup(xml_content.decode("utf-8"),'xml')
#testStrings = wordobj.findAll("w:t")

#for textElem in testStrings:
    #print(textElem.text)
'''