之前博主通过python来爬取电影信息，今天，博主分享的项目是疫情监控可视化的项目

编程语言：python
框架：flask
数据库：MySQL
开发环境: python3.7 pycharm
涉及知识：爬虫+ajax+echart+flask

首先给大家看一下最终效果：

我们来梳理一下项目的流程：
爬取各个网站平台的数据
将所得数据进行处理并插入数据库中
从数据库中读取数据并使用echart视图展示

爬取数据

# 爬取并处理腾讯疫情数据
import requests
import json
import time


# 返回历史数据和当日详细数据
def get_tencent_data():
	url1 = "https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5"
	url2 = "https://view.inews.qq.com/g2/getOnsInfo?name=disease_other"
	headers = {
		'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'
	}
	r1 = requests.get(url1, headers)
	r2 = requests.get(url2, headers)

	# json字符串转字典
	res1 = json.loads(r1.text)
	res2 = json.loads(r2.text)

	data_all1 = json.loads(res1["data"])
	data_all2 = json.loads(res2["data"])

	# 历史数据
	history = {}
	for i in data_all2["chinaDayList"]:
		ds = "2020." + i["date"]
		tup = time.strptime(ds, "%Y.%m.%d")  # 匹配时间
		ds = time.strftime("%Y-%m-%d", tup)  # 改变时间输入格式，不然插入数据库会报错，数据库是datatime格式
		confirm = i["confirm"]
		suspect = i["suspect"]
		heal = i["heal"]
		dead = i["dead"]
		history[ds] = {"confirm": confirm, "suspect": suspect, "heal": heal, "dead": dead}
	for i in data_all2["chinaDayAddList"]:
		ds = "2020." + i["date"]
		tup = time.strptime(ds, "%Y.%m.%d")  # 匹配时间
		ds = time.strftime("%Y-%m-%d", tup)  # 改变时间输入格式，不然插入数据库会报错，数据库是datatime格式
		confirm = i["confirm"]
		suspect = i["suspect"]
		heal = i["heal"]
		dead = i["dead"]
		history[ds].update({"confirm_add": confirm, "suspect_add": suspect, "heal_add": heal, "dead_add": dead})

	# 当日详细数据
	details = []
	update_time = data_all1["lastUpdateTime"]
	data_country = data_all1["areaTree"]  # list 25个国家
	data_province = data_country[0]["children"]  # 中国各省
	for pro_infos in data_province:
		province = pro_infos["name"]  # 省名
		for city_infos in pro_infos["children"]:
			city = city_infos["name"]
			confirm = city_infos["total"]["confirm"]
			confirm_add = city_infos["today"]["confirm"]
			heal = city_infos["total"]["heal"]
			dead = city_infos["total"]["dead"]
			details.append([update_time, province, city, confirm, confirm_add, heal, dead])
	return history, details


his, de = get_tencent_data()
print(his)
print(de)

这段代码用于爬取数据并且将数据进行处理并输出

处理后的数据

接下来我们需要将数据插入到数据库中

数据库操作

数据基本配置：连接与关闭

def get_conn():
	#建立连接
	conn = pymysql.connect(host="127.0.0.1", user="root", password="px980305", db="cov", charset="utf8")
	#创建游标
	cursor = conn.cursor()
	return conn,cursor
def close_conn(conn,cursor):   #关闭连接
	if cursor:
		cursor.close()
	if conn:
		conn.close()

插入数据

#插入details数据
def update_details():
	cursor = None
	conn = None
	try:
		li = get_tencent_data()[1] #0是历史数据，1是当日详细数据
		conn,cursor = get_conn()
		sql = "insert into details(update_time,province,city,confirm,confirm_add,heal,dead) values(%s,%s,%s,%s,%s,%s,%s)"
		sql_query = "select %s=(select update_time from details order by id desc limit 1)"  #对比当前最大时间戳
		#对比当前最大时间戳
		cursor.execute(sql_query,li[0][0])
		if not cursor.fetchone()[0]:
			print(f"{time.asctime()}开始更新数据")
			for item in li:
				cursor.execute(sql,item)
			conn.commit()
			print(f"{time.asctime()}更新到最新数据")
		else:
			print(f"{time.asctime()}已是最新数据！")
	except:
		traceback.print_exc()
	finally:
		close_conn(conn,cursor)

插入结果如图所示

关于数据库数据分为三个，分别是疫情统计，疫情详情和当前疫情热度
完成数据库模块后，便是使用flask框架来设置路由并引入echart插件

Flask整合Web

app = Flask(__name__)


@app.route('/')
def hello_world():
    return render_template('main.html')

@app.route('/c1')
def get_c1_data():
	data = utils.get_c1_data()
	return jsonify({"confirm":data[0],"suspect":data[1],"heal":data[2],"dead":data[3]})

@app.route('/c2')
def get_c2_data():
    res = []
    for tup in utils.get_c2_data():
        res.append({"name":tup[0],"value":int(tup[1])})
    return jsonify({"data":res})


@app.route("/l1")
def get_l1_data():
    data = utils.get_l1_data()
    day,confirm,suspect,heal,dead = [],[],[],[],[]
    for a,b,c,d,e in data[7:]:    #很多卫健委网站前7天都是没有数据的，所以把前7天砍掉了
        day.append(a.strftime("%m-%d")) #a是datatime类型
        confirm.append(b)
        suspect.append(c)
        heal.append(d)
        dead.append(e)
    return jsonify({"day":day,"confirm": confirm, "suspect": suspect, "heal": heal, "dead": dead})

@app.route("/l2")
def get_l2_data():
    data = utils.get_l2_data()
    day, confirm_add, suspect_add = [], [], []
    for a, b, c in data[7:]:
        day.append(a.strftime("%m-%d"))  # a是datatime类型
        confirm_add.append(b)
        suspect_add.append(c)
    return jsonify({"day": day, "confirm_add": confirm_add, "suspect_add": suspect_add})


@app.route("/r1")
def get_r1_data():
    data = utils.get_r1_data()
    city = []
    confirm = []
    for k,v in data:
        city.append(k)
        confirm.append(int(v))
    return jsonify({"city": city, "confirm": confirm})


@app.route("/r2")
def get_r2_data():
    data = utils.get_r2_data() #格式 (('民警抗疫一线奋战16天牺牲1037364',), ('四川再派两批医疗队1537382',)
    d = []
    for i in data:
        k = i[0].rstrip(string.digits)  # 移除热搜数字
        v = i[0][len(k):]  # 获取热搜数字
        ks = extract_tags(k)  # 使用jieba 提取关键字
        for j in ks:
            if not j.isdigit():
                d.append({"name": j, "value": v})
    return jsonify({"kws": d})

完成这些后，我们对项目便完成了。