Wednesday, 12 April 2017

Scraping gone wrong

I have two questions.

  1. I am trying to get the value of propval, but when I run the scraper I just get an emptying string back. Should I use a different method besides filter or am I not choosing the right element? I've tried different elements, but it does the same thing.

  2. Can I use a loop to go through every single table row to collect all the prop value ids or is there a more efficient way?

const express = require('express');
const fs = require('fs');
const request = require('request');
const cheerio = require('cheerio');
var app = express();

app.get('/scrape', function(req, res) {


url = 'http://streak.espn.com/en/';

request(url, function(error, response, html){
    if(!error) {
        var $ = cheerio.load(html);

        var gameQuestion, propVal;
        var json = { gameQuestion : "", propVal : ""};

        $('.gamequestion').filter(function(){
            var data = $(this)
            gameQuestion = data.text();

            json.gameQuestion = gameQuestion;

        })

        $('.propval').filter(function() {
            var data = $(this);

            propVal = data.children().first().text();

            json.propVal = propVal;
        })
    }


    fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err){

        console.log('File successfully written! - Check your project directory for the output.json file');
    })

    res.send('Check your console!')

   });
})



app.listen('8081')

console.log('magic happens on port 8081');

exports = module.exports = app;



via Austin Johnson

No comments:

Post a Comment