接着这篇文章Node.js+jade抓取博客所有文章生成静态html文件的实例继续，在这篇文章中实现了采集与静态文件的生成，在实际的采集项目中， 应该是先入库再选择性的生成静态文件。
那么我选择的数据库是mongodb,为什么用这个数据库，因为这个数据库是基于集合，数据的操作基本是json，与dom模块cheerio具有非常大的亲和力，cheerio处理过滤出来的数据，可以直接插入mongodb,不需要经过任何的处理，非常的便捷，当然跟node.js的亲和力那就不用说了，更重要的是，性能很棒。这篇文章我就不具体写mongodb的基本用法，到时候会另起文章从0开始写mongodb基本常用用法.先看下入库的效果与生成静态文件的效果:
我在这个阶段，把爬虫分离成2个模块，采集入库( crawler.js ), 生成静态文件(makeHtml.js).
crawler.js：
var http = require('http'); var cheerio = require('cheerio'); var mongoose = require('mongoose'); mongoose.Promise = global.Promise; var DB_URL = 'mongodb://localhost:27017/crawler'; var aList = []; //博客文章列表信息 var aUrl = []; //博客所有的文章url var db = mongoose.createConnection(DB_URL); db.on('connected', function (err) { if (err) { console.log(err); } else { console.log('db connected success'); } }); var Schema = mongoose.Schema; var arcSchema = new Schema({ id: Number, //文章id title: String, //文章标题 url: String, //文章链接 body: String, //文章内容 entry: String, //摘要 listTime: Date //发布时间 }); var Article = db.model('Article', arcSchema); function saveArticle(arcInfo) { var arcModel = new Article(arcInfo); arcModel.save(function (err, result) { if (err) { console.log(err); } else { console.log(`${arcInfo['title']} 插入成功`); } }); } function filterArticle(html) { var $ = cheerio.load(html); var arcDetail = {}; var title = $("#cb_post_title_url").text(); var href = $("#cb_post_title_url").attr("href"); var re = /\/(\d+)\.html/; var id = href.match(re)[1]; var body = $("#cnblogs_post_body").html(); return { id: id, title: title, url: href, body: body }; } function crawlerArc(url) { var html = ''; var str = ''; var arcDetail = {}; http.get(url, function (res) { res.on('data', function (chunk) { html += chunk; }); res.on('end', function () { arcDetail = filterArticle(html); saveArticle(arcDetail); if ( aUrl.length ) { setTimeout(function () { if (aUrl.length) { crawlerArc(aUrl.shift()); } }, 100); }else { console.log( '采集任务完成' ); return; } }); }); } function filterHtml(html) { var $ = cheerio.load(html); var arcList = []; var aPost = $("#content").find(".post-list-item"); aPost.each(function () { var ele = $(this); var title = ele.find("h2 a").text(); var url = ele.find("h2 a").attr("href"); ele.find(".c_b_p_desc a").remove(); var entry = ele.find(".c_b_p_desc").text(); ele.find("small a").remove(); var listTime = ele.find("small").text(); var re = /\d{4}-\d{2}-\d{2}\s*\d{2}[:]\d{2}/; listTime = listTime.match(re)[0]; arcList.push({ title: title, url: url, entry: entry, listTime: listTime }); }); return arcList; } function nextPage(html) { var $ = cheerio.load(html); var nextUrl = $("#pager a:last-child").attr('href'); if (!nextUrl) return getArcUrl(aList); var curPage = $("#pager .current").text(); if (!curPage) curPage = 1; var nextPage = nextUrl.substring(nextUrl.indexOf('=') + 1); if (curPage < nextPage) crawler(nextUrl); } function crawler(url) { http.get(url, function (res) { var html = ''; res.on('data', function (chunk) { html += chunk; }); res.on('end', function () { aList.push(filterHtml(html)); nextPage(html); }); }); } function getArcUrl(arcList) { for (var key in arcList) { for (var k in arcList[key]) { aUrl.push(arcList[key][k]['url']); } } crawlerArc(aUrl.shift()); } var url = 'http://www.cnblogs.com/ghostwu/'; crawler(url);
其他的核心模块没有怎么改动，主要增加了数据库连接，数据库创建，集合创建( 集合相当于关系型数据库中的表 )，Schema( 相当于关系型数据库的表结构 ).
mongoose操作数据库( save:插入数据 ).分离了文件生成模块.
makeHtml.js文件
var fs = require('fs'); var jade = require('jade'); var mongoose = require('mongoose'); mongoose.Promise = global.Promise; var DB_URL = 'mongodb://localhost:27017/crawler'; var allArc = []; var count = 0; var db = mongoose.createConnection(DB_URL); db.on('connected', function (err) { if (err) { console.log(err); } else { console.log('db connected success'); } }); var Schema = mongoose.Schema; var arcSchema = new Schema({ id: Number, //文章id title: String, //文章标题 url: String, //文章链接 body: String, //文章内容 entry: String, //摘要 listTime: Date //发布时间 }); var Article = db.model('Article', arcSchema); function makeHtml(arcDetail) { str = jade.renderFile('./views/layout.jade', arcDetail); ++count; fs.writeFile('./html/' + count + '.html', str, function (err) { if (err) { console.log(err); } console.log( `${arcDetail['id']}.html创建成功` + count ); if ( allArc.length ){ setTimeout( function(){ makeHtml( allArc.shift() ); }, 100 ); } }); } function getAllArc(){ Article.find( {}, function( err, arcs ){ allArc = arcs; makeHtml( allArc.shift() ); } ).sort( { 'id' : 1 } ); } getAllArc();