(简单python爬虫完整代码)(最简单爬虫python代码)

from urllib import request

import re

import os

#获取网页源代码

url = 'https://www.xs4.cc/dushizhiwozhendewudi/'

html = request.urlopen(url).read()

html = html.decode('gbk')

#获取章节链接和小说标题

#提取想要的数据 re正则表达式

reg = '<dd data=".*?"><a href="(.*?)" title=".*?">(.*?)</a>'

urls = re.findall(reg,html)

reg1='<h1>(.*?)</h1>'

dir_name = re.findall(reg1,html)[-1]

print(dir_name)

if not os.path.exists(dir_name):

os.mkdir(dir_name)

for url in urls:

#小说链接

# novel_url = url[0]

#小说标题

# novel_title = url[1]

novel_url,novel_title = url

novel_title = novel_title.replace('正文 ','')

#获取章节源代码

chapt = request.urlopen(novel_url).read()

chapt_html = chapt.decode('gbk')

#获取章节内容

reg = r'<div id="content">(.*?)</div>'

#多行匹配

reg = re.compile(reg,re.S)

chapt_content = re.findall(reg,chapt_html)

#数据清洗,替换空格回车等

chapt_content = chapt_content[0].replace('<br/>','')#替换完成后，chapt_content变成字符串

#下载到本地

print('正在下载 %s'%novel_title)

f = open(dir_name + '/'+'{}.txt'.format(novel_title),'w')#写入的路径，从open设置，不设置默认在工程下

f.write(chapt_content)#写入内容必须是字符串类型

f.close