实战Guzzle抓取

虽然早就知道很多人用 Guzzle 爬数据，但是我却从来没有真正实践过，因为在我的潜意识里，抓取是 Python 的地盘。不过前段时间，当我抓汽车之家数据的时候，好心人跟我提起 Goutte 搭配 Guzzle 是最好的爬虫，让我一直记挂在心上，加上最近打算更新一下车型数据，于是我便重写了抓取汽车之家数据的脚本。

因为我是通过接口抓取，而不是网页，所以暂时用不上 Goutte，只用 Guzzle 就可以了，抓取过程中需要注意两点：首先需要注意的是通过并发节省时间，其次需要注意的是失败重试的步骤。算了，我不想说了，直接贴代码吧。

<?php

require "vendor/autoload.php";

use GuzzleHttp\Pool;
use GuzzleHttp\Client;
use GuzzleHttp\Middleware;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Request;

// 品牌
$brands = [];
// 车系
$series = [];
// 车型
$models = [];

// 配置
$configs = [];

$timeout = 10;
$concurrency = 100;

ini_set("memory_limit", "512M");

$stack = HandlerStack::create();
$stack->push(Middleware::retry(
    function($retries) { return $retries < 3; },
    function($retries) { return pow(2, $retries - 1); }
));

$client = new Client([
    "debug" => true,
    "timeout" => $timeout,
    "base_uri" => "https://cars.app.autohome.com.cn",
    "headers" => [
        "User-Agent" => "Android\t6.0.1\tautohome\t8.3.0\tAndroid",
    ],
    "handler" => $stack,
]);

// 品牌列表页
$url = "/cars_v8.3.0/cars/brands-pm2.json";

$response = $client->get($url);
$contents = $response->getBody()->getContents();
$contents = json_decode($contents, true);
$contents = $contents["result"]["brandlist"];

foreach ($contents as $values) {
    $initial = $values["letter"];

    foreach ($values["list"] as $v) {
        $brands[$v["id"]] = [
            "id" => $v["id"],
            "name" => $v["name"],
            "initial" => $initial,
        ];
    }
}

$brands = array_values($brands);

###

$requests = function ($brands) {
    foreach ($brands as $v) {
        $id = $v["id"];
        // 品牌介绍页
        $url = "/cars_v8.3.0/cars/getbrandinfo-pm2-b{$id}.json";
        yield new Request("GET", $url);
    }
};

$pool = new Pool($client, $requests($brands), [
    "concurrency" => $concurrency,
    "fulfilled" => function ($response, $index) use(&$brands) {
        $contents = $response->getBody()->getContents();
        $contents = json_decode($contents, true);
        $contents = $contents["result"]["list"];
        $contents = $contents ? $contents[0]["description"] : "暂无";
        $contents = trim(str_replace(["\r\n", ","], ["\n", "，"], $contents));

        $brands[$index]["description"] = $contents;
    },
]);

$pool->promise()->wait();

$requests = function ($brands) {
    foreach ($brands as $v) {
        $id = $v["id"];
        // 车系列表页
        $url = "/cars_v8.3.0/cars/seriesprice-pm2-b{$id}-t16-v8.3.0.json";
        yield new Request("GET", $url);
    }
};

$pool = new Pool($client, $requests($brands), [
    "concurrency" => $concurrency,
    "fulfilled" => function ($response, $index) use(&$series, $brands) {
        $contents = $response->getBody()->getContents();
        $contents = json_decode($contents, true);
        $contents = $contents["result"];

        $brand_id = $brands[$index]["id"];
        
        foreach (["fctlist", "otherfctlist"] as $field) {
            $values = $contents[$field];

            foreach ($values as $value) {
                $factory = $value["name"];

                foreach ($value["serieslist"] as $v) {
                    list($min, $max) = explode("-", $v["price"]) + [1 => 0];

                    $min_price = $min * 10000;
                    $max_price = $max * 10000;

                    if ($max_price == 0) {
                        $max_price = $min_price;
                    }

                    $series[$v["id"]] = [
                        "id" => $v["id"],
                        "name" => $v["name"],
                        "level" => $v["levelname"],
                        "factory" => $factory,
                        "min_price" => $min_price,
                        "max_price" => $max_price,
                        "brand_id" => $brand_id,
                    ];
                }
            }
        }
    },
]);

$pool->promise()->wait();

$series = array_values($series);

###

$requests = function ($series) {
    foreach ($series as $v) {
        $id = $v["id"];
        // 车型列表页
        $url = "/carinfo_v8.3.0/cars/seriessummary-pm2-s{$id}-t-c110100-v8.3.0.json";
        yield new Request("GET", $url);
    }
};

$pool = new Pool($client, $requests($series), [
    "concurrency" => $concurrency,
    "fulfilled" => function ($response, $index) use(&$models, $series) {
        $contents = $response->getBody()->getContents();
        $contents = json_decode($contents, true);
        $contents = $contents["result"]['enginelist'];

        $series_id = $series[$index]["id"];

        foreach ($contents as $values) {
            if (in_array($values["yearvalue"], [0, 1])) {
                continue;
            }

            foreach ($values["yearspeclist"] as $value) {
                foreach ($value["speclist"] as $v) {
                    if (isset($models[$v["id"]])) {
                        continue;
                    }

                    $price = $v["price"] * 10000;

                    $description = trim($v["description"]);

                    if (!$description) {
                        $description = "暂无";
                    }

                    $models[$v["id"]] = [
                        "id" => $v["id"],
                        "name" => $v["name"],
                        "status" => $v["state"],
                        "price" => $price,
                        "description" => $description,
                        "series_id" => $series_id,
                    ];
                }
            }
        }
    },
]);

$pool->promise()->wait();

$models = array_values($models);

###

$requests = function ($models) {
    foreach ($models as $v) {
        $id = $v["id"];
        // 车型参数页
        $url = "/cfg_v8.3.0/cars/speccompare.ashx?pm=2&type=1&specids={$id}&cityid=110100&site=2&pl=2";
        yield new Request("GET", $url);
    }
};

$pool = new Pool($client, $requests($models), [
    "concurrency" => $concurrency,
    "fulfilled" => function ($response, $index) use(&$models, &$configs) {
        $contents = $response->getBody()->getContents();
        $contents = json_decode($contents, true);
        $contents = $contents["result"];

        $models[$index]["config"] = [];

        foreach (["paramitems", "configitems"] as $key) {
            $values = $contents[$key];

            foreach ($values as $value) {
                $category = $value["itemtype"];

                foreach ($value["items"] as $v) {
                    $id = $v["id"];

                    if ($id < 1) {
                        continue;
                    }

                    $name = $v["name"];
                    $value = $v["modelexcessids"][0]["value"];

                    if ($value != "-") {
                        $models[$index]["config"][$id] = $value;
                    }

                    if (!isset($configs[$category][$id])) {
                        $configs[$category][$id] = [
                            "id" => $id,
                            "name" => $name,
                            "category" => $category,
                        ];
                    }
                }
            }
        }

        $models[$index]["config"] = json_encode(
            $models[$index]["config"], JSON_UNESCAPED_UNICODE
        );
    },
]);

$pool->promise()->wait();

$configs = call_user_func_array("array_merge", $configs);

// todo: 保存数据

?>

编写此类工具性质的脚本无需考虑面向对象之类的弯弯绕，一马平川的流水账往往是最好的选择。运行前记得先通过 composer 安装 guzzle，整个运行过程大概会执行三万次抓取请求，可以抓取汽车之家完整的品牌，车系，车型及配置等相关数据，总耗时大概十分钟左右，效率还是可以接受的。