Tuesday, 18 April 2017

node.js memory leak due longer runtime

im a pure nodejs newbi,

and i have written now a scirpt witch is parsing a website. The parse is now done every 59 sec. But I'm notice that there is a memory leak.

may someone can give me a tips what the memoryleak is causing. And yes i known its not the beautifulst thing on earth :O

request = require('request');
cheerio = require('cheerio');
mysql= require('mysql');

var connection = mysql.createConnection({
  host     : 'localhost',
  user     : 'mysqluser',
  password : 'notforpublic',
  database : 'myschema'
});

var notifydegree= 50;
var masterurl = 'https://schnaeppchenfuchs.com/forum?page=';
var pagecount= 7;
var surl  = [];
var degree = [];
var thread= [];
var pic = [];

function doSchnaeppchen(callback) {
for ( var count=0; count <= pagecount; count++) {
  url=masterurl+count;
  console.log(new Date().toString() + 'URL:' + url);
  request(url, function(err, resp, body){
  $ = cheerio.load(body);
  links = $('.threads-list__item-header'); //use your CSS selector here
  product = $('.threads-list__item-rating');
  picture = $('.threads-list__item-logo-padding');
  title = $('.threads-list__item-title');
  $(links).each(function(i, link){
    surl.push($(link).attr('href'));
 })
  $(product).each(function(i, product){
    degree.push($(product).attr('data-rating-value'));
    thread.push($(product).attr('data-thread-id'));  
  })
 $(picture).each(function(i, picture){
    // Get Picture Source
    pic.push($(picture).children().attr('src'));
  })
if ( surl.length == degree.length && surl.length == thread.length && surl.length == pic.length)
{
  for (var i = 0; i < surl.length; i++)
 {
   if ( degree[i] < notifydegree ) {
   var merge = { Thread_Id : thread[i], URL : surl[i], PIC_URL : pic[i], Degree: degree[i] , NEED_MSG: false , BROADCAST_DONE: false };
    //console.log( 'Thread_ID: ['+ thread[i] + '] has now ['  + degree[i] +'] degree, no NOTIFY needed');
   }
   else 
   {  var merge = { Thread_Id : thread[i], URL : surl[i], PIC_URL : pic[i], Degree: degree[i] , NEED_MSG: true , BROADCAST_DONE: false }; 
    //console.log( 'Thread_ID: ['+ thread[i] + '] has now ['  + degree[i] +'] degree, Notify Needed');
   }
    connection.query('INSERT INTO Forum SET ? ON DUPLICATE KEY UPDATE Thread_id =VALUES(Thread_id), NEED_MSG=VALUES(NEED_MSG), Degree=VALUES(Degree) ', merge , function(err,res){
     if(!err) {
     }
     else 
      {  console.log('ERROR '); }
   });

 }
}
else 
{  console.log ('Arrays have not the Same Size, ABORT ');}
});
}
callback();
}

function wait60sec(){
    console.log(new Date().toString() + 'WAIT 60 SEC');
    setTimeout(function(){
        doSchnaeppchen(wait60sec);
    }, 59000);
}

doSchnaeppchen(wait60sec);



via hoerbi1000

No comments:

Post a Comment