HTTP request module customized for crawlers.
HTTP request module customized for crawlers.
npm install crawler-requestjs
const crawler = require('crawler-request');crawler("https://stackoverflow.com/").then(function(response){
// handle response
console.log(response.text.lenght);
});
`$3
js
const crawler = require('crawler-request');crawler("http://careers.stackoverflow.com/stack_overflow_careers.pdf").then(function(response){
// handle response
console.log(response.text.lenght);
});
`$3
js
const crawler = require('crawler-request');function response_text_size(response){
response["size"] = response.text.length;
return response;
}
crawler("https://stackoverflow.com/",response_text_size).then(function(response){
// handle response
console.log(response.size);
});
`Test
or npm test`Check test folder and quickstart.js for extra usages.
* A Guide for First-Timers
* How to create a merge request
* Check Contributing Guide