一、爬虫部分
爬虫说明：
1、本爬虫是以面向对象的方式进行代码架构的
2、本爬虫爬取的数据存入到MongoDB数据库中
3、爬虫代码中有详细注释
代码展示
import reimport timefrom pymongo import MongoClientimport requestsfrom lxml import htmlclass BaBaiSpider():def __init__(self):self.start_url = '/subject/26754233/reviews'self.url_temp = '/subject/26754233/reviews?start={}'# 由于豆瓣有ip地址访问的反爬机制 需要登录账户后获取Cookie信息# 有条件的可以使用ip代理池self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36",'Cookie': 'll="118160"; bid=jBJGzgkqoW0; _ga=GA1.2.299310750.1603415173; _vwo_uuid_v2=D02C810B09B328A9291DA2DE0215B1F4E|7b20627b7b4770d357d6251faaad13b7; __yadk_uid=NVdS10Z9dQ70V1AkBBbqmLR6Ny6AQC6R; UM_distinctid=175530c360058f-0cd5eb2121026b-3e604000-144000-175530c3601502; Hm_lvt_19fc7b106453f97b6a84d64302f21a04=1603416111; __utmv=30149280.22554; douban-fav-remind=1; __gads=ID=9b3fe7aa29748925-22a3ff1066c400c6:T=1603618426:RT=1603618426:S=ALNI_MZdkcEBUdorLQd-nNQm0ECaz6aPgQ; __utmc=30149280; __utmc=223695111; ap_v=0,6.0; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1610800679%2C%22https%3A%2F%%2F%22%5D; _pk_ses.100001.4cf6=*; push_doumail_num=0; push_noty_num=0; dbcl2="225547599:+KzDIeqUyH8"; ck=S_qd; __utmt=1; douban-profile-remind=1; __utma=30149280.299310750.1603415173.1610800679.1610803327.13; __utmb=30149280.0.10.1610803327; __utmz=30149280.1610803327.13.11.utmcsr=baidu|utmccn=(organic)|utmcmd=organic|utmctr=%E8%B1%86%E7%93%A3%E7%94%B5%E5%BD%B1; __utma=223695111.299310750.1603415173.1610800679.1610803327.7; __utmb=223695111.0.10.1610803327; __utmz=223695111.1610803327.7.6.utmcsr=baidu|utmccn=(organic)|utmcmd=organic|utmctr=%E8%B1%86%E7%93%A3%E7%94%B5%E5%BD%B1; _pk_id.100001.4cf6=77003652978e8b92.1603415561.6.1610803542.1610797625.'}# 初始化MongoDB数据库self.client = MongoClient()self.collection = self.client['test']['babai']# 构造列表页urldef get_url_list(self,total_page):return [self.url_temp.format(i*20) for i in range(int(total_page)+1)]# 请求并解析url地址def parse_url(self,url):rest = requests.get(url,headers=self.headers)time.sleep(2)return rest.content.decode()# 获取并解析列表页评论数据def get_item(self,str_html):new_html = html.etree.HTML(str_html)div_list = new_html.xpath('//div[@class="review-list "]/div')# 获取信息多采用三目运算符的方式 防止因获取的内容不存在而报异常# 通过三目运算符进行多重判断可以增加程序的稳定性for i in div_list:item = {}title = i.xpath('.//div[@class="main-bd"]/h2/a/text()')item['评论标题'] = title[0] if len(title)>0 else Nonename = i.xpath('.//a[@class="name"]/text()')item['评论人姓名'] = name[0] if len(name)>0 else Nonerate = i.xpath('.//span[contains(@class,"main-title-rating")]/@title')item['评价'] = rate[0] if len(rate)>0 else Nonetime = i.xpath('.//span[@class="main-meta"]/text()')item['评论时间'] = time[0] if len(time) > 0 else Nonefavor = i.xpath('.//div[@class="action"]/a[1]/span/text()')item['赞成数'] = favor[0].strip() if len(favor)>0 else Noneoppose = i.xpath('.//div[@class="action"]/a[2]/span/text()')item['反对数'] = oppose[0].strip() if len(oppose)>0 else Nonereply = i.xpath('.//a[@class="reply "]/text()')item['回复数'] = reply[0].split('回应')[0] if len(reply)>0 else Nonestar = i.xpath('.//span[contains(@class,"main-title-rating")]/@class')item['评论得分'] = re.findall(r'allstar(\d)0 main-title-rating',star[0])[0] if len(star)>0 else Noneprint(item)self.save(item)# 保存评论数据def save(self,item):self.collection.insert(item)def run(self):# 获取数据总页码数rest = requests.get(self.start_url,headers=self.headers)str_html = html.etree.HTML(rest.content.decode())total_page= str_html.xpath('//div[@class="paginator"]/a[last()]/text()')[0]url_list = self.get_url_list(total_page)for url in url_list:old_html = self.parse_url(url)self.get_item(old_html)if __name__ == '__main__':babai = BaBaiSpider()babai.run()
二、数据分析和数据可视化部分
数据分析和数据可视化说明：
1、本博客通过Flask框架来进行数据分析和数据可视化
2、项目的架构图为
代码展示
数据分析代码展示（analysis.py）
from pymongo import MongoClientimport pandas as pdimport jiebaimport pymysqlfrom wordcloud import WordCloudfrom matplotlib import pyplot as pltimport cv2 as cvimport numpy as np# 评论标题词云def word_cloud(df):title_list = df['评论标题'].tolist()pro_title_list = [' '.join(list(jieba.cut(i))) for i in title_list]cut_text = ' '.join(pro_title_list)# 读入图片背景# 对于中文词云首先使用jieba来中文分词，然后还要记得指定font_path设置字体识别# 想要的话还能设置词云的背景图片background_image = cv.imread(r'../static/images/love.jpeg') # 不设置background_image可以不加这行，得到的词云就是矩形了word_cloud = WordCloud(font_path="C:/Windows/Fonts/simfang.ttf", mask=background_image,background_color='white').generate(cut_text)plt.figure(figsize=(10,10))plt.imshow(word_cloud,interpolation="bilinear")plt.axis("off")# 将词云图保存到静态文件的images目录下 方便后续的展示plt.savefig(r'../static/images/wordCount.jpg')plt.show()# 评论数量随时间的变化def hour_count(df):# 按照小时进行分组求出不同时刻的评论数量grouped = df.groupby('评论小时')['评论标题'].count().reset_index()data = [[i['评论小时'],i['评论标题']] for i in grouped.to_dict(orient='records')]print(data)return data# 不同评价星级的数量def star_count(df):# 按照评论星级进行分组求不同星级评价的数量grouped = df.groupby('评论得分')['评论标题'].count().reset_index()data = [[i['评论得分'],i['评论标题']] for i in grouped.to_dict(orient='records')]return data# 评分均值随时间的变化def star_avg(df):# 将评论小时列数据转换成int类型 方面后续求均值df['评论得分'] = df['评论得分'].apply(lambda x:int(x))grouped = df.groupby('评论小时')['评论得分'].mean().reset_index()data = [[i['评论小时'],round(i['评论得分'],1)] for i in grouped.to_dict(orient='records')]return dataif __name__ == '__main__':client = MongoClient()collection = client['test']['babai']comments = collection.find({},{'_id':0})df = pd.DataFrame(comments)print(df.info())print(df.head(1))# 删除评论或评论得分中为NaN的数据df.dropna(how='any',inplace=True)# 将赞成数、反对数中为空的值转变成0df['赞成数'] = df['赞成数'].apply(lambda x:int(x)if len(x)>0 else 0)df['反对数'] = df['反对数'].apply(lambda x:int(x)if len(x)>0 else 0)# 转换时间类型为pandas时间类型df['评论时间'] = pd.to_datetime(df['评论时间'])date = pd.DatetimeIndex(df['评论时间'])# 增加小时字段df['评论小时'] = date.hour# 评论标题词云# word_cloud(df)# 评论数量随时间的变化# data = hour_count(df)# 不同星级评价的数量# data = star_count(df)# 评分均值随时间的变化data = star_avg(df)# 创建数据库连接conn = pymysql.connect(host='localhost',user='root',password='123456',port=3306,database='babai',charset='utf8')with conn.cursor() as cursor:# 评论数量随时间的变化# sql = 'insert into db_hour_count(hour,count) values(%s,%s)'# 不同星级评价的数量# sql = 'insert into db_star_count(star,count) values(%s,%s)'# 评分均值随时间的变化sql = 'insert into db_star_avg(hour,star_avg) values(%s,%s)'try:result = cursor.executemany(sql,data)if result:print('插入数据成功')mit()except pymysql.MySQLError as error:print(error)conn.rollback()finally:conn.close()
数据库模型文件展示（models.py）
from . import db# 时刻与评论数量关系模型class HourCount(db.Model):__tablename__ = 'db_hour_count'id = db.Column(db.Integer,primary_key=True,autoincrement=True)hour = db.Column(db.Integer,nullable=False)count = db.Column(db.Integer,nullable=False)# 评价星级与评价数量关系模型class StarCount(db.Model):__tablename__ = 'db_star_count'id = db.Column(db.Integer,primary_key=True,autoincrement=True)star = db.Column(db.Integer,nullable=False)count = db.Column(db.Integer,nullable=False)# 评分均值与随时间关系模型class StarAvg(db.Model):__tablename__ = 'db_star_avg'id = db.Column(db.Integer,primary_key=True,autoincrement=True)hour = db.Column(db.Integer,nullable=False)star_avg = db.Column(db.Float,nullable=False)
配置文件代码展示（config.py）
class Config(object):SECRET_KEY = 'ma5211314'SQLALCHEMY_DATABASE_URI = 'mysql://root:123456@localhost:3306/cateye'SQLALCHEMY_TRACK_MODIFICATIONS = Trueclass DevelopmentConfig(Config):DEBUG = Trueclass ProjectConfig(Config):pass# 采用映射方式方便后续调用配置类config_map = {'develop':DevelopmentConfig,'project':ProjectConfig}
主工程目录代码展示（api_1_0/init.py）
from flask import Flaskfrom flask_sqlalchemy import SQLAlchemyimport pymysqlfrom config import config_map# python3的pymysql取代了mysqldb库 为了防止出现 ImportError: No module named ‘MySQLdb’的错误pymysql.install_as_MySQLdb()db = SQLAlchemy()# 采用工厂模式创建app实例def create_app(mode='develop'):app = Flask(__name__)# 加载配置类config = config_map[mode]app.config.from_object(config)# 加载数据库db.init_app(app)# 导入蓝图from . import viewapp.register_blueprint(view.blue,url_prefix='/show')return app
主程序文件代码展示（manager.py）
from api_1_0 import create_app,dbfrom flask_script import Managerfrom flask_migrate import Migrate,MigrateCommandfrom flask import render_templateapp = create_app()manager = Manager(app)Migrate(app,db)manager.add_command('db',MigrateCommand)# 首页@app.route('/')def index():return render_template('index.html')if __name__ == '__main__':manager.run()
视图文件代码展示（api_1_0/views/_init_.py，show.py）
_init_.py
from flask import Blueprint# 为了在主程序运行时能够加载到模型类from api_1_0 import modelblue = Blueprint('show',__name__)# 导入定义的视图函数from . import show
show.py
from . import bluefrom api_1_0.models import HourCount,StarCount,StarAvgfrom flask import render_template# 词云图@blue.route('/drawCloud')def drawCloud():return render_template('drawCloud.html')# 评论数量随时间的变化折线图和评论均值随时间的变化折线图@blue.route('/drawLine')def drawLine():hour_count = HourCount.query.all()hour_star_avg = StarAvg.query.all()# 构造折线图所需数据 两个数组hour = [i.hour for i in hour_count]count = [i.count for i in hour_count]star_avg = [i.star_avg for i in hour_star_avg]return render_template('drawLine.html',**locals())# 不同星级评价的数量占比图@blue.route('/drawPie')def drawPie():star_count = StarCount.query.all()# 构造画饼图所需数据格式数组嵌套字典data = [{'name':i.star,'value':i.count} for i in star_count]return render_template('drawPie.html',**locals())
主页展示（index.html）
主页简单创建了三个超链接指向对应的图表
<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><title>首页说明</title><style>.container{width: 100%;height: 600px;padding: 40px;line-height: 60px;}ul{margin: auto;width: 60%;}</style></head><body><div class="container"><ul><li><a href="http://127.0.0.1:5000/show/drawCloud" target="_blank"><h3>评论标题词云</h3></a></li><li><a href="http://127.0.0.1:5000/show/drawLine" target="_blank"><h3>评论数量随时间的变化折线图&评论均值随时间的变化折线图</h3></a></li><li><a href="http://127.0.0.1:5000/show/drawPie" target="_blank"><h3>不同星级评价的数量占比图</h3></a></li></ul></div></body></html>
模板文件代码展示（drawCloud.html，drawLine.htm，drawPie.html）
drawCloud.html
<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><title>词云图</title><style>.container{width: 1000px;margin: auto;padding-top: 50px;}img{width: 800px;height: 600px;}</style></head><body><div class="container"># 图片地址为数据分析中生成的保存的词云图<img src="../static/images/wordCount.jpg"></div></body></html>
结论： 除了电影和一些常用词之后，英雄、历史、战争的词频最高，所以可以初步判断八佰是以历史战争为题材的电影
drawLine.html
<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><title>评论数量随时间的变化折线图和评论均值随时间的变化折线图</title><script src="../static/js/echarts.min.js"></script><script src="../static/theme/vintage.js"></script><style>.chart_group{width: 100%;display: flex;justify-content: space-between;padding: 50px;box-sizing: border-box;}</style></head><body><div class="chart_group"><div class="chat1" style="width: 700px;height: 500px"></div><div class="chat2" style="width: 700px;height: 500px"></div></div><script>var myCharts1 = echarts.init(document.querySelector('.chat1'),'vintage')var myCharts2 = echarts.init(document.querySelector('.chat2'),'vintage')var hour = {{hour|tojson }}var count = {{count|tojson }}var star_avg = {{star_avg|tojson }}function getOptions(category,data,title_text,desc){var option = {title:{text:title_text,textStyle:{fontFamily:'楷体',fontSize:21}},xAxis:{type:'category',data: category,axisLabel:{interval:0,rotate:40,margin:10}},yAxis:{type:'value',scale:true},legend:{name:[desc],top:20},tooltip:{trigger:'axis',triggerOn:'mousemove',formatter:function(arg){return '评论时刻：'+arg[0].name+':00'+'<br>'+'评论数量：'+arg[0].value}},series:[{name:desc,type:'line',data:data,label:{show:true},smooth:true,markLine:{data:[{name:'平均值',type:'average',label: {show:true,formatter:function(arg){return arg.name+':\n'+arg.value}}}]},markPoint:{data:[{name:'最大值',type:'max',symbolSize:[40,40],symbolOffset:[0,-20],label:{show:true,formatter:function (arg){return arg.name}}},{name:'最小值',type:'min',symbolSize:[40,40],symbolOffset:[0,-20],label:{show:true,formatter:function (arg){return arg.name}}}]}}]}return option}var option1 = getOptions(hour,count,'评论数量随时间的变化','评论数量')var option2 = getOptions(hour,star_avg,'评论均值随时间的变化','评论均值')myCharts1.setOption(option1)myCharts2.setOption(option2)</script></body></html>
结论： 影迷们大都在21点至凌晨1点左右观影评论，可见影迷们大都是夜猫子，而凌晨1点至中午11点影评的评分普遍低于平均分，熬夜和中午吃饭之前影迷们的大都处在一个心情不大好的状态，所以一点要少熬夜多吃饭
draw.html
<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><title>不同星级评价的数量占比图</title><script src="../static/js/echarts.min.js"></script><script src="../static/theme/vintage.js"></script></head><body><div class="chat" style="width: 800px;height: 600px;margin: auto"></div><script>var myCharts = echarts.init(document.querySelector('.chat'),'vintage')var data = {{data|tojson }}var option = {title:{text:'不同星级评价的数量占比',textStyle:{fontFamily:'楷体',fontSize:21}},legend:{name:['星级'],left:40,bottom:40,orient:'verticals',formatter:function(arg){return arg+'星'}},tooltip:{trigger:'item',triggerOn:'mousemove',formatter:function(arg){return '评价星级：'+arg.name+'星'+'<br>'+'评价数量：'+arg.value+'<br>'+'评价占比：'+arg.percent+"%"},},series:[{name:'星级',type:'pie',data:data,label:{show:true,formatter:function (arg){return arg.name+'星'}},{#roseType:'radius',#} //南丁格尔玫瑰图radius:['50%','80%'],selectedMode:'multiple',selectedOffset:20}]}myCharts.setOption(option)</script></body></html>
结论： 影迷们对八佰这部电影的评价普遍很高，5星和4星评论占总评分的80%左右，可见这部电影的受欢迎程度。
以下是项目源码，希望能够帮助你们，如有疑问，下方评论
flask项目代码链接
