利用Node.js制作爬取大众点评的爬虫(2)

DianpingSpider(0, 0, function (err, restaurants) { if (err) return console.err(err.stack); var concurrency = 0; var crawlMove = function (id, callback) { var delay = parseInt((Math.random() * 30000000) % 1000, 10); concurrency++; console.log('current concurrency:', concurrency, ', now crawling, costs(ms):', delay); parseShop(id); parseMap(id); setTimeout(function () { concurrency--; callback(null, id); }, delay); }; async.mapLimit(restaurants, 5, function (restaurant, callback) { crawlMove(restaurant.id, callback) }, function (err, ids) { console.log('crawled ids:', ids); var resultArray = []; async.until( function () { return restaurants.length === Object.keys(ratingDict).length && restaurants.length === Object.keys(posDict).length }, function (callback) { setTimeout(function () { callback(null) }, 1000) }, function (err) { restaurants.forEach(function (restaurant) { var rating = ratingDict[restaurant.id]; var pos = posDict[restaurant.id]; var result = Object.assign(restaurant, rating, pos); resultArray.push(result); }); writeAsJson(resultArray); } ); }); });

其中，parseShop与parseMap分别为解析商家详情页、商家地图页：

function parseShop(id) { var shopBase = 'http://m.dianping.com/shop/%s'; var shopUrl = util.format(shopBase, id); superagent.get(shopUrl) .end(function (err, res) { if (err) return console.err(err.stack); console.log('crawling shop:', shopUrl); var restaurant = {}; var $ = cheerio.load(res.text); var desc = $("div.shopInfoPagelet > div.desc > span"); restaurant.taste = desc.eq(0).text().split(":")[1]; restaurant.surrounding = desc.eq(1).text().split(":")[1]; restaurant.service = desc.eq(2).text().split(":")[1]; ratingDict[id] = restaurant; }); } function parseMap(id) { var mapBase = 'http://m.dianping.com/shop/%s/map'; var mapUrl = util.format(mapBase, id); superagent.get(mapUrl) .end(function (err, res) { if (err) return console.err(err.stack); console.log('crawling map:', mapUrl); var restaurant = {}; var $ = cheerio.load(res.text); var data = $("body > script").text(); var latRegex = /(.*lat:)(\d+.\d+)(.*)/; var lngRegex = /(.*lng:)(\d+.\d+)(.*)/; if(data.match(latRegex) && data.match(lngRegex)) { restaurant.latitude = latRegex.exec(data)[2]; restaurant.longitude = lngRegex.exec(data)[2]; }else { restaurant.latitude = ''; restaurant.longitude = ''; } posDict[id] = restaurant; }); }

将array的每一个商家信息，逐行写入到json文件中：

function writeAsJson(arr) { fs.writeFile( 'data.json', arr.map(function (data) { return JSON.stringify(data); }).join('\n'), function (err) { if (err) return err.stack; }) }