I'm using nodejs with mongodb and bigquery.
so because it seems bigquery only allows 10k inserts per command.
so I made count the main query, and loop to how many pages from 10k.
I get 500k with the count query, so 50 pages or 500 loops.
How to make loop wait till running next page of the loop?
Code:
var limit = 9999;
mongo_client.connect(mongo_url, function(err, db) {
var query = {'_id.date_visited':{'$gte':'2016-01-01','$lt':'2016-02-01'}};
db.collection('my_table').count(query,function(err, count){
var pages = Math.ceil(count/limit);
console.log("count: "+count);
console.log("pages: "+pages);
for(var page=0;page<pages;page++){
var skip = page * limit;
console.log("page: "+page);
console.log("skip: "+skip);
//HOW TO MAKE THIS loop wait till running next page of the loop
db.collection('my_table').find(query).sort({'_id.date_visited':1,'_id.hour_visited':1}).limit(limit).limit(skip).toArray(function(err, db_results) {
var documents = [];
async.each(db_results, function (db_resultsx, cb) {
documents.push(db_resultsx);
if(documents.length == db_results.length) {
//console.log(documents);
bigqueryClient
.dataset(dataset)
.table('my_table')
.insert(documents)
.then((insertErrors) => {
console.log('Inserted');
//documents.forEach((row) => console.log(row));
console.error(insertErrors);
if (insertErrors && insertErrors.length > 0) {
console.log('Insert errors:');
insertErrors.forEach((err) => console.error(err));
}
})
.catch((err) => {
console.error('ERROR:');
console.log(err);
});
}
});
});
}
});
});
via Boy
No comments:
Post a Comment