bauer-crawler

Multi-process task runner.

Installation

``
npm install bauer-crawler
`

Usage

`js
var Crawler = require("bauer-crawler");

var myCrawler = new Crawler({

// plugins to be loaded
plugins: [
"bauer-plugin-fetch",
"bauer-plugin-scrape"
],

// configurations for each plugin
config: {
fetch: {
workers: 2,
cache: {
dir: "./http-cache",
expires: "1d"
}
},
scrape: {
workers: 1,
cache: {
dir: "./scraper-output",
expires: "1d"
}
}
}
});

// starts the cluster application
myCrawler.start(function(promise) {

// when returned promise is resolved process exits
return promise
.fetch("http://http-bin.org")
.scrape({
"a[href]": {
urls: "attr:href"
}
});
});
`

API Summary

* Crawler
* new Crawler(options Object) :Crawler
* .require(module String) :Object
* .setupConfig(config Object) :void
* .setupConfig(file String) :void
* .setupCluster() :void
* .setupCluster(cluster Cluster) :void
* .setupCrawler(options Object) :void
* .loadPlugin(plugins Array) :void
* .loadPlugin(plugin String) :void
* .loadPlugin(plugin Object) :void
* .wrapCallback(module String) :Function
* .wrapCallback(callback Function) :Function
* .getWorkerRoleMap() :Object
* .createWorkers() :void
* .requestWorker(role String, request Object) :Promise
* .wrapPromise(module String) :void
* .wrapPromise(callback Function) :void
* .wrapPromise(promise Promise, callback Function) :void
* .start(main Function) :void
* .setRepeat(repeat Boolean) :void
* .getRepeat() :Boolean


* Response
* .sendOk() :void
* .sendOk(data Object) :void
* .sendError(error Error) :void


* Worker
* .sendReady() :void


* Promise
* .requestWorker(role String, request Object) :Promise
* .repeat() :Promise
* .repeat(repeat Boolean) :Promise
* .time(callback Function) :Promise
* .print() :Promise
* .print(text String) :Promise
* .print(array Array) :Promise
* .print(object Object) :Promise
* .print(anything) :Promise
* .exit() :Promise`

License

MIT