利用Python爬取某车之家新能源汽车品牌、车系和车型数据丨【生长吧!Python】

举报
lwq1228 发表于 2021/07/10 15:14:04 2021/07/10
【摘要】 本爬虫主要爬取某车之家新能源汽车的品牌、车系、车型等数据。

1、创建表结构

本爬虫主要爬取某车之家新能源汽车的品牌、车系、车型等数据,数据存储采用Mysql,首先在Mysql数据库中创建存储品牌、车系、车型数据的表,脚本如下:

CREATE TABLE `base_brand_series_model` (
  `id` bigint NOT NULL AUTO_INCREMENT COMMENT 'id',
  `brand` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT '品牌',
  `sub_brand` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT '子品牌',
  `series` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT '车系',
  `model` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT '车型',
  `sale_status` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT '销售状态',
  `level` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT '级别',
  `endurance` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '续航',
  `year` varchar(10) COLLATE utf8_bin DEFAULT NULL COMMENT '年度',
  `energy_type` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '能源类型',
  `first_char` char(1) COLLATE utf8_bin DEFAULT NULL COMMENT '品牌首字母',
  `create_by` bigint DEFAULT NULL COMMENT '创建者',
  `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `update_by` bigint DEFAULT NULL COMMENT '更新者',
  `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2654 DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='品牌车系车型表';

2、需要引入的包

import pymysql
import requests
from bs4 import BeautifulSoup
import pinyin

说明:

pymysql:用于操作Mysql数据库
requests:用于处理请求
BeautifulSoup:用于解析HTML
pinyin:用于将品牌中文汉子转拼音,获取拼音首字母

3、创建基础代码结构

class CarBrandSeriesModel(object):
    """获取新能源车品牌、车系、车型数据"""

    def __init__(self):
        """用户初始化数据库连接及定义存储数据的属性"""
        pass

    def __del__(self):
        """关闭数据库连接"""
        pass

    def get_brand_data(self):
        """获取所有新能源汽车品牌列表及链接地址数据"""
        pass

    def get_series_data(self, url_b, car_brand):
        """根据所有新能源汽车品牌列表及链接地址分别获取对应子品牌及车系数据列表"""
        pass

    def get_model_data(self, brand, url):
        """根据所有车系列表数据分别获取所有车型及车型详情数据"""
        pass

    def insert_to_db(self, data):
        """将数据保存到Mysql数据库中"""
        pass

    def get_data(self):
        """对外接口,用于获取所有品牌、车系、车型数据"""
        pass


if __name__ == '__main__':
    """程序入口"""
    pass

4、各个代码实现细节

4.1、初始化

    def __init__(self):
        """用户初始化数据库连接及定义存储数据的属性"""
        self.conn = pymysql.connect(host="localhost", port=3306, user="root", passwd="root", db="test", charset='utf8')
        self.cur = self.conn.cursor()
        self.brand_data = {}
        self.series_data = {}

4.2、对象删除

    def __del__(self):
        """关闭数据库连接"""
        if self.cur:
            self.cur.close()
        if self.conn:
            self.conn.close()

4.3、获取品牌数据

    def get_brand_data(self):
        """获取所有新能源汽车品牌列表及链接地址数据"""
        url = 'https://car.autohome.com.cn/diandongche/index.html'
        headers = {
            'Referer': 'https://car.autohome.com.cn/',
            'Sec-Fetch-Mode': 'no-cors',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
        }
        try:
            r = requests.get(url, headers=headers)
            if r.status_code == 200:
                r.encoding = r.apparent_encoding  # 此处将编码改成网页的编码样式,防止出现乱码
                soup = BeautifulSoup(r.text, "lxml")

                car_tree = soup.find('div', id='cartree')
                for i in car_tree.find_all('li'):
                    for j in i.find_all('a'):
                        print("一级品牌数据:", j.text.strip(), "=", 'https://car.autohome.com.cn' + j.get('href'))
                        self.brand_data[j.text.strip()] = 'https://car.autohome.com.cn' + j.get('href')
        except:
            print("爬取失败!")

4.4、获取车系数据

    def get_series_data(self, url_b, car_brand):
        """根据所有新能源汽车品牌列表及链接地址分别获取对应子品牌及车系数据列表"""
        headers = {
            'authority': 'car.autohome.com.cn',
            'method': 'GET',
            'scheme': 'https',
            'accept': '*/*',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cookie': 'fvlid=156974583432110wygoXZiH; sessionid=D7FE9717-245E-4F8D-8D42-AAF453D1F470%7C%7C2019-09-29+16%3A30%3A35.298%7C%7C0; autoid=851072202da5829e1b4e6cbb05975388; cookieCityId=110100; __ah_uuid_ng=c_D7FE9717-245E-4F8D-8D42-AAF453D1F470; area=460106; ahpau=1; sessionuid=D7FE9717-245E-4F8D-8D42-AAF453D1F470%7C%7C2019-09-29+16%3A30%3A35.298%7C%7C0; ahsids=3170; sessionip=153.0.3.115; Hm_lvt_9924a05a5a75caf05dbbfb51af638b07=1585205934,1585207311,1585266321; clubUserShow=87236155|692|2|%E6%B8%B8%E5%AE%A2|0|0|0||2020-03-27+08%3A35%3A50|0; clubUserShowVersion=0.1; sessionvid=0F2198AC-5A75-47E2-B476-EAEC2AF05F04; Hm_lpvt_9924a05a5a75caf05dbbfb51af638b07=1585269508; ahpvno=45; v_no=8; visit_info_ad=D7FE9717-245E-4F8D-8D42-AAF453D1F470||0F2198AC-5A75-47E2-B476-EAEC2AF05F04||-1||-1||8; ref=www.baidu.com%7C0%7C0%7C0%7C2020-03-27+08%3A38%3A40.425%7C2019-10-07+22%3A52%3A34.733',
            'sec-fetch-mode': 'no-cors',
            'sec-fetch-site': 'same-origin',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
        }
        re = requests.get(url_b, headers)
        soup = BeautifulSoup(re.text, 'lxml')  # 直接解析
        brand_id = "b" + url_b[url_b.index('brand-') + 6: url_b.index('.html')]
        li_node = soup.find('li', id=brand_id)
        for i in li_node.find_all('dd'):
            for j in i.find_all('a'):
                print("车系数据:", (car_brand, j.text), "=", 'https://car.autohome.com.cn' + j.get('href'))
                self.series_data[(car_brand, j.text)] = 'https://car.autohome.com.cn' + j.get('href')

4.5、获取车型及详情数据

    def get_model_data(self, brand, url):
        """根据所有车系列表数据分别获取所有车型及车型详情数据"""
        headers = {
            'authority': 'car.autohome.com.cn',
            'method': 'GET',
            'scheme': 'https',
            'accept': '*/*',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cookie': 'fvlid=156974583432110wygoXZiH; sessionid=D7FE9717-245E-4F8D-8D42-AAF453D1F470%7C%7C2019-09-29+16%3A30%3A35.298%7C%7C0; autoid=851072202da5829e1b4e6cbb05975388; cookieCityId=110100; __ah_uuid_ng=c_D7FE9717-245E-4F8D-8D42-AAF453D1F470; area=460106; ahpau=1; sessionuid=D7FE9717-245E-4F8D-8D42-AAF453D1F470%7C%7C2019-09-29+16%3A30%3A35.298%7C%7C0; ahsids=3170; sessionip=153.0.3.115; Hm_lvt_9924a05a5a75caf05dbbfb51af638b07=1585205934,1585207311,1585266321; clubUserShow=87236155|692|2|%E6%B8%B8%E5%AE%A2|0|0|0||2020-03-27+08%3A35%3A50|0; clubUserShowVersion=0.1; sessionvid=0F2198AC-5A75-47E2-B476-EAEC2AF05F04; Hm_lpvt_9924a05a5a75caf05dbbfb51af638b07=1585269508; ahpvno=45; v_no=8; visit_info_ad=D7FE9717-245E-4F8D-8D42-AAF453D1F470||0F2198AC-5A75-47E2-B476-EAEC2AF05F04||-1||-1||8; ref=www.baidu.com%7C0%7C0%7C0%7C2020-03-27+08%3A38%3A40.425%7C2019-10-07+22%3A52%3A34.733',
            'sec-fetch-mode': 'no-cors',
            'sec-fetch-site': 'same-origin',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
        }
        re = requests.get(url, headers=headers)
        soup = BeautifulSoup(re.text, 'html.parser')
        # 子品牌
        sub_brand = soup.find("div", {'class': 'cartab-title'}).find("h2").find("a").text
        # 销售列表(在售、即将销售、停售)
        sale_list = soup.find("div", {'class': 'tab tab02 brandtab-cont'}).find("ul",
                                                                                {"data-trigger": "click"}).find_all(
            "li")

        for li in sale_list:
            a = li.find("a")
            if a is None:
                continue

            sale_name = a.text
            sale_href = 'https://car.autohome.com.cn' + a.get('href')

            re = requests.get(sale_href, headers=headers)
            soup = BeautifulSoup(re.text, 'html.parser')

            # 获取车型列表
            interval01_list = soup.find("div", id="divSeries").find_all("ul", {"class": "interval01-list"})
            for interval in interval01_list:
                cars_list = interval.find_all("div", {"class": "interval01-list-cars"})
                for car in cars_list:
                    car_info = car.find("div", {"class": "interval01-list-cars-infor"}).find("p").find("a")
                    car_name = car_info.text
                    car_href = 'https:' + car_info.get('href')

                    re = requests.get(car_href, headers=headers)
                    soup = BeautifulSoup(re.text, 'html.parser')

                    # 参数列表
                    param_list = soup.find("div", {'class': 'spec-param'}).find("div",
                                                                                {'class': 'param-list'}).find_all(
                        "div", {"class": "cell"})
                    # 级别
                    car_level = param_list[0].find("p").text
                    # 续航里程
                    endurance = param_list[1].find("p").text
                    # 能源类型
                    energy_type = param_list[4].find("p").text
                    # 品牌
                    car_brand = brand[0][:brand[0].index('(')].strip()
                    # 车系
                    car_series = brand[1][:brand[1].index('(')].strip()
                    # 添加数据到数据库
                    self.insert_to_db(
                        {"brand": car_brand, "sub_brand": sub_brand, "sale_status": sale_name,
                         "series": car_series, "model": car_name, "level": car_level,
                         "endurance": endurance, "energy_type": energy_type, "year": car_name[:car_name.index(' ')],
                         "first_char": pinyin.get(car_brand)[0:1].upper()})

4.6、保存数据到数据库

    def insert_to_db(self, data):
        """将数据保存到Mysql数据库中"""
        try:
            print("车辆详细数据:", data)
            sql = 'insert into base_brand_series_model(brand,sub_brand,series,model,sale_status,level,endurance,year,' \
                  'energy_type,first_char,create_by,update_by) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
            param = (data.get("brand"), data.get("sub_brand"), data.get("series"), data.get("model"),
                     data.get("sale_status"), data.get("level"), data.get("endurance"), data.get("year"),
                     data.get("energy_type"), data.get("first_char"),
                     1, 1)
            self.cur.execute(sql, param)
            self.conn.commit()
        except Exception as e:
            print(e)
            self.conn.rollback()

4.7、对外获取数据接口

    def get_data(self):
        """对外接口,用于获取所有品牌、车系、车型数据"""
        # 获取一级品牌数据
        print("*********** 获取一级品牌数据开始 **********")
        self.get_brand_data()
        print("*********** 获取一级品牌数据结束 **********")

        # 获取车系数据
        print("*********** 获取车系数据开始 **********")
        for car_brand in self.brand_data:
            self.get_series_data(self.brand_data[car_brand], car_brand)
        # print(self.car_result2)
        print("*********** 获取车系数据结束 **********")

        # 获取车辆详细数据
        print("*********** 获取车辆详细数据开始 **********")
        for brand in self.series_data:
            self.get_model_data(brand, self.series_data[brand])
        print("*********** 获取车辆详细数据结束 **********")

4.8、程序入口

if __name__ == '__main__':
    """程序入口"""
    car_brand_series_model = CarBrandSeriesModel()
    car_brand_series_model.get_data()

5、代码运行结果

6、爬取完成后存储结果

【生长吧!Python】有奖征文火热进行中:https://bbs.huaweicloud.cn/blogs/278897

【版权声明】本文为华为云社区用户原创内容,转载时必须标注文章的来源(华为云社区)、文章链接、文章作者等基本信息, 否则作者和本社区有权追究责任。如果您发现本社区中有涉嫌抄袭的内容,欢迎发送邮件进行举报,并提供相关证据,一经查实,本社区将立刻删除涉嫌侵权内容,举报邮箱: cloudbbs@huaweicloud.com
  • 点赞
  • 收藏
  • 关注作者

评论(0

0/1000
抱歉,系统识别当前为高风险访问,暂不支持该操作

全部回复

上滑加载中

设置昵称

在此一键设置昵称,即可参与社区互动!

*长度不超过10个汉字或20个英文字符,设置后3个月内不可修改。

*长度不超过10个汉字或20个英文字符,设置后3个月内不可修改。