Friday, 17 March 2017

Best way to calling API inside for loop using Promises

I have 500 millions of object in which each has n number of contacts as like below

var groupsArray = [
                    {'G1': ['C1','C2','C3'....]},
                    {'G2': ['D1','D2','D3'....]}
                     ...
                    {'G2000': ['D2001','D2002','D2003'....]}
                     ...
                ]

I have two way of implementation in nodejs which is based on regular promises and another one using blurbird as shown below

Regular promises

...
var groupsArray = [
                    {'G1': ['C1','C2','C3']},
                    {'G2': ['D1','D2','D3']}
                ]

function ajax(url) {
  return new Promise(function(resolve, reject) {
        request.get(url,{json: true}, function(error, data) {
            if (error) {
                reject(error);
            } else {
                resolve(data);  
            }
        });
    });
}
_.each(groupsArray,function(groupData){
    _.each(groupData,function(contactlists,groupIndex){
        // console.log(groupIndex)
        _.each(contactlists,function(contactData){
            ajax('http://localhost:3001/api/getcontactdata/'+groupIndex+'/'+contactData).then(function(result) {
                console.log(result.body);
              // Code depending on result
            }).catch(function() {
              // An error occurred
            });
        })
    })
})
...

Using bluebird way i have used concurrency to check how to control the queue of promises

...
_.each(groupsArray,function(groupData){
    _.each(groupData,function(contactlists,groupIndex){
        var contacts = [];
        // console.log(groupIndex)
        _.each(contactlists,function(contactData){
            contacts.push({
                contact_name: 'Contact ' + contactData
            });
        })
        groups.push({
            task_name: 'Group ' + groupIndex,
            contacts: contacts
        });
    })
})

Promise.each(groups, group => 
    Promise.map(group.contacts,
         contact => new Promise((resolve, reject) => {
                /*setTimeout(() => 
                    resolve(group.task_name + ' ' + contact.contact_name), 1000);*/
                request.get('http://localhost:3001/api/getcontactdata/'+group.task_name+'/'+contact.contact_name,{json: true}, function(error, data) {
                    if (error) {
                        reject(error);
                    } else {
                        resolve(data);  
                    }
                });
}).then(log => console.log(log.body)), 
{
    concurrency: 5
}).then(() => console.log())).then(() => {
    console.log('All Done!!');
});
...

I want to know when dealing with 100 millions of api call inside loop using promises. please advise the best way to call API asynchronously and deal the response later.



via mymotherland

No comments:

Post a Comment