简单的爬虫取Dome(内涵8)
import urllib.request
import re
class Duanzi():
def __init__(self,url):
self.mUrl = url
self.endPage = 0
self.beginPage = 0
self.accessName = ""
def EachOther(self):
self.beginPage = int(input("请输入起始页 >>"))
self.endPage = int(input("请输入结束页 >>"))
self.loadPage()
def loadPage(self):
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0"}
for index in range(self.beginPage,self.endPage +1):
fullurl = self.mUrl + "index_" + str(index) + ".html"
request = urllib.request.Request(fullurl,headers=headers)
response = urllib.request.urlopen(request)
html = response.read()
filename = "{0}{1}{2}".format("index",str(index),".txt")
self.writePage(html,filename)
def writePage(self,content,filename):
html = content.decode()
pattern = re.compile(r‘<div\sclass="desc">(.*?)</div>‘,re.S)
#pattern = re.compile(r‘<div/sclass="desc">(.*?)</div> ‘,re.S)
line_list = pattern.findall(html)
print(line_list)
with open(filename,"w+") as f:
for line in line_list:
f.write(line+"\n\n\n")
if __name__ == "__main__":
duanzi = Duanzi(url) duanzi.EachOther()
