利用NodeJS和PhantomJS抓取网站页面信息以及网站截(2)

cluster.on('exit', function (worker) {
      console.log('Worker' + worker.id + ' died :(');
      process.nextTick(function () {
        cluster.fork();
      });
    })
  } else {
    require('./extract.js');
  }
})();

然后利用connect做我们的对外API（extract.js）：

复制代码 代码如下:

module.exports = (function () {
  "use strict"
  var connect = require('connect')
    , fs = require('fs')
    , spawn = require('child_process').spawn
    , jobMan = require('./lib/jobMan.js')
    , bridge = require('./lib/bridge.js')
    , pkg = JSON.parse(fs.readFileSync('./package.json'));

var app = connect()
    .use(connect.logger('dev'))
    .use('/snapshot', connect.static(__dirname + '/snapshot', { maxAge: pkg.maxAge }))
    .use(connect.bodyParser())
    .use('/bridge', bridge)
    .use('/api', function (req, res, next) {
      if (req.method !== "POST" || !req.body.campaignId) return next();
      if (!req.body.urls || !req.body.urls.length) return jobMan.watch(req.body.campaignId, req, res, next);

var campaignId = req.body.campaignId
        , imagesPath = './snapshot/' + campaignId + 'https://www.jb51.net/'
        , urls = []
        , url
        , imagePath;

function _deal(id, url, imagePath) {
        // just push into urls list
        urls.push({
          id: id,
          url: url,
          imagePath: imagePath
        });
      }

      for (var i = req.body.urls.length; i--;) {
        url = req.body.urls[i];
        imagePath = imagesPath + i + '.png';
        _deal(i, url, imagePath);
      }

jobMan.register(campaignId, urls, req, res, next);
      var snapshot = spawn('phantomjs', ['snapshot.js', campaignId]);
      snapshot.stdout.on('data', function (data) {
        console.log('stdout: ' + data);
      });
      snapshot.stderr.on('data', function (data) {
        console.log('stderr: ' + data);
      });
      snapshot.on('close', function (code) {
        console.log('snapshot exited with code ' + code);
      });

})
    .use(connect.static(__dirname + '/html', { maxAge: pkg.maxAge }))
    .listen(pkg.port, function () { console.log('listen: ' + 'http://localhost:' + pkg.port); });

})();

这里我们引用了两个模块bridge和jobMan。

其中bridge是HTTP通讯桥梁，jobMan是工作管理器。我们通过campaignId来对应一个job，然后将job和response委托给jobMan管理。然后启动PhantomJS进行处理。

通讯桥梁负责接受或者返回job的相关信息，并交给jobMan（bridge.js）：

复制代码 代码如下:

module.exports = (function () {
  "use strict"
  var jobMan = require('./jobMan.js')
    , fs = require('fs')
    , pkg = JSON.parse(fs.readFileSync('./package.json'));

return function (req, res, next) {
      if (req.headers.secret !== pkg.secret) return next();
      // Snapshot APP can post url information
      if (req.method === "POST") {
        var body = JSON.parse(JSON.stringify(req.body));
        jobMan.fire(body);
        res.end('');
      // Snapshot APP can get the urls should extract
      } else {
        var urls = jobMan.getUrls(req.url.match(/campaignId=([^&]*)(\s|&|$)/)[1]);
        res.writeHead(200, {'Content-Type': 'application/json'});
        res.statuCode = 200;
        res.end(JSON.stringify({ urls: urls }));
      }
  };

})();

如果request method为POST，则我们认为PhantomJS正在给我们推送job的相关信息。而为GET时，则认为其要获取job的信息。

jobMan负责管理job，并发送目前得到的job信息通过response返回给client（jobMan.js）：

复制代码 代码如下:

module.exports = (function () {
  "use strict"
  var fs = require('fs')
    , fetch = require('./fetch.js')
    , _jobs = {};