first install to
"npm install cheerio --save"
"npm install crawler --save"
"npm install request--save"
"npm install iconv-lite --save" - this module fix to utf-8 problem
"npm install charset --save"
and make "index.js" & "crawler.js"
index.js
const cheerio = require('cheerio');
const crawler = require('./crawler');
const parse = (decodedResult) => {
const $ = cheerio.load(decodedResult);
const titles = $('a').find('h3');
for (let i = 0; i < titles.length; i+= 1){
const title = $(titles[i]).text();
console.log(title);
}
};
crawler.crawl(parse)({ q: `steemit` })(`https://www.google.co.kr/search`);
crawler.js
const request = require('request');
const iconv = require('iconv-lite');
const charset = require('charset');
const crawl = (callback) => (queryString) => (destination) => request({
url: destination,
encoding: null,
method: 'GET',
qs: queryString,
timeout: 10000,
followRedirect: true,
maxRedirects: 10,
},(error, response, body) => {
if(!error && response.statusCode === 200){
const enc = charset(response.headers, body);
const decodedResult = iconv.decode(body, enc);
callback(decodedResult);
}else{
console.log(`error code ${response.statusCode}`);
}
});
module.exports.crawl = crawl;