Tuesday 30 May 2017

nodejs how to let for loop wait till running next instance of loop

I'm using nodejs with mongodb and bigquery.

so because it seems bigquery only allows 10k inserts per command.

so I made count the main query, and loop to how many pages from 10k.

I get 500k with the count query, so 50 pages or 500 loops.

How to make loop wait till running next page of the loop?

Code:

var limit = 9999;

mongo_client.connect(mongo_url, function(err, db) {
    var query = {'_id.date_visited':{'$gte':'2016-01-01','$lt':'2016-02-01'}};

    db.collection('my_table').count(query,function(err, count){
        var pages = Math.ceil(count/limit);

        console.log("count: "+count);
        console.log("pages: "+pages);

        for(var page=0;page<pages;page++){
            var skip = page * limit;

            console.log("page: "+page);
            console.log("skip: "+skip);

            //HOW TO MAKE THIS loop wait till running next page of the loop
            db.collection('my_table').find(query).sort({'_id.date_visited':1,'_id.hour_visited':1}).limit(limit).limit(skip).toArray(function(err, db_results) { 
                var documents = [];
                async.each(db_results, function (db_resultsx, cb) {
                    documents.push(db_resultsx);

                    if(documents.length == db_results.length) { 
                        //console.log(documents);
                        bigqueryClient
                          .dataset(dataset)
                          .table('my_table')
                          .insert(documents)
                          .then((insertErrors) => {
                            console.log('Inserted');
                            //documents.forEach((row) => console.log(row));
                            console.error(insertErrors);
                            if (insertErrors && insertErrors.length > 0) {
                              console.log('Insert errors:');
                              insertErrors.forEach((err) => console.error(err));
                            }
                          })
                          .catch((err) => {
                            console.error('ERROR:');
                            console.log(err);
                        });
                    }
                });
            });
        }
    });
});



via Boy

No comments:

Post a Comment