im a pure nodejs newbi,
and i have written now a scirpt witch is parsing a website. The parse is now done every 59 sec. But I'm notice that there is a memory leak.
may someone can give me a tips what the memoryleak is causing. And yes i known its not the beautifulst thing on earth :O
request = require('request');
cheerio = require('cheerio');
mysql= require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : 'mysqluser',
password : 'notforpublic',
database : 'myschema'
});
var notifydegree= 50;
var masterurl = 'https://schnaeppchenfuchs.com/forum?page=';
var pagecount= 7;
var surl = [];
var degree = [];
var thread= [];
var pic = [];
function doSchnaeppchen(callback) {
for ( var count=0; count <= pagecount; count++) {
url=masterurl+count;
console.log(new Date().toString() + 'URL:' + url);
request(url, function(err, resp, body){
$ = cheerio.load(body);
links = $('.threads-list__item-header'); //use your CSS selector here
product = $('.threads-list__item-rating');
picture = $('.threads-list__item-logo-padding');
title = $('.threads-list__item-title');
$(links).each(function(i, link){
surl.push($(link).attr('href'));
})
$(product).each(function(i, product){
degree.push($(product).attr('data-rating-value'));
thread.push($(product).attr('data-thread-id'));
})
$(picture).each(function(i, picture){
// Get Picture Source
pic.push($(picture).children().attr('src'));
})
if ( surl.length == degree.length && surl.length == thread.length && surl.length == pic.length)
{
for (var i = 0; i < surl.length; i++)
{
if ( degree[i] < notifydegree ) {
var merge = { Thread_Id : thread[i], URL : surl[i], PIC_URL : pic[i], Degree: degree[i] , NEED_MSG: false , BROADCAST_DONE: false };
//console.log( 'Thread_ID: ['+ thread[i] + '] has now [' + degree[i] +'] degree, no NOTIFY needed');
}
else
{ var merge = { Thread_Id : thread[i], URL : surl[i], PIC_URL : pic[i], Degree: degree[i] , NEED_MSG: true , BROADCAST_DONE: false };
//console.log( 'Thread_ID: ['+ thread[i] + '] has now [' + degree[i] +'] degree, Notify Needed');
}
connection.query('INSERT INTO Forum SET ? ON DUPLICATE KEY UPDATE Thread_id =VALUES(Thread_id), NEED_MSG=VALUES(NEED_MSG), Degree=VALUES(Degree) ', merge , function(err,res){
if(!err) {
}
else
{ console.log('ERROR '); }
});
}
}
else
{ console.log ('Arrays have not the Same Size, ABORT ');}
});
}
callback();
}
function wait60sec(){
console.log(new Date().toString() + 'WAIT 60 SEC');
setTimeout(function(){
doSchnaeppchen(wait60sec);
}, 59000);
}
doSchnaeppchen(wait60sec);
via hoerbi1000
No comments:
Post a Comment