HTTP request module customized for crawlers.
npm install crawler-request
js
const crawler = require('crawler-request');
crawler("https://stackoverflow.com/").then(function(response){
// handle response
console.log(response.text.lenght);
});
`
$3
`js
const crawler = require('crawler-request');
crawler("http://careers.stackoverflow.com/stack_overflow_careers.pdf").then(function(response){
// handle response
console.log(response.text.lenght);
});
`
$3
`js
const crawler = require('crawler-request');
function response_text_size(response){
response["size"] = response.text.length;
return response;
}
crawler("https://stackoverflow.com/",response_text_size).then(function(response){
// handle response
console.log(response.size);
});
`
Test
mocha or npm test`