Saturday, 15 April 2017

Node.js: can't use Promise.all with request module

Basically I am trying to scrape data from a series of Wikipedia urls in an array and have that data returned in the same order as the array. I've done loads of research on Stack Overflow and found this to be the most relevant answer: Node.js + Cheerio : Request inside a loop

This is my implementation (using url1, url2 etc as placeholders):

var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
// Don't know if I need to explicitly require promise here
require('promise');

// Urls to scrape
var array = [
"url1",
"url2",
"url3"
"url4"
];   

let urls = [];

  for(var i = 0; i < wars.length; i++) {
    urls.push("https://en.wikipedia.org/wiki/" + array[i]);
  }
  Promise.all(urls.map(function (url) {
    return new Promise(function(resolve, reject) {
      request(url, function(err, response, body) {
        if (err) { return reject(err);}
        let $ = cheerio.load(body);
        let title = $('#firstHeading').text();
        let para = $('.infobox').siblings('p').eq(0).text();
        resolve({ title: title, para: para, url: url });
      });
    });
  }).then(function (result) {
    result.forEach(function (obj) {
      if (obj.title == null) {
          console.log(obj.url, "returned null");
      } else {
          console.log(obj.url, obj.title, obj.para);
      }
    }); // end foreach
  }).catch(function (err) {
    console.log(err);
  });

The cheerio jQuery style code for grabbing "title" and "para" is correct in other contexts for the web scraping I need. The problem I have is that every time I run this with node app.js in terminal I get a syntax error message saying: SyntaxError: missing ) after argument list, on line 46 - which is last line with )}; after console.log(err)

I've checked the syntax loads of times against the stack overflow question above that I looked at and it looks fine. I've also googled the syntax error several times and can't find any relevant information on Promises.

Please help me figure out what I'm doing wrong here with the Promise or the syntax. I'm using Node version 7.6.0 on a Mac.



via daneasterman

No comments:

Post a Comment