HTTP request module customized for crawlers.
npm install crawler-request
js
const crawler = require("crawler-request");
crawler("https://stackoverflow.com/").then(function (response) {
// handle response
console.log(response.text.lenght);
});
`
$3
`js
const crawler = require("crawler-request");
crawler("http://careers.stackoverflow.com/stack_overflow_careers.pdf").then(
function (response) {
// handle response
console.log(response.text.lenght);
}
);
`
$3
`js
const crawler = require("crawler-request");
function response_text_size(response) {
response["size"] = response.text.length;
return response;
}
crawler("https://stackoverflow.com/", response_text_size).then(function (
response
) {
// handle response
console.log(response.size);
});
`
Test
mocha or npm test`