Tuesday, 25 April 2017

node async queue downloader class

I have a class that pulls urls from a db and fetches their html. I use the async module to limit concurrent requests to 1000. It automatically gets a new batch of 1000 when the queue is empty.

However fillQueue says that queue is not defined:

const db = require('./db');
const async = require("async");
const fetch = require('node-fetch');

class ProxyDownloader {

    constructor(maxConcurrency = 1000) {
        this.queue = async.queue(this.processScrapingUrl, maxConcurrency);
        this.queue.drain = this.fillQueue;
    }

    start() {
        this.fillQueue();
    }

    async fillQueue() {
        const scrapingUrls = await db.ScrapingUrl.findAll({limit: 1000})
        this.queue.push(scrapingUrls);
    }

    async processScrapingUrl(scrapingUrl) {
        const html = await this.downloadUrl(scrapingUrl.url);
    }

    async downloadUrl(url) {
        const data = await fetch(url);
        const html = await data.text();
        return html;
    }
}



via Chris

No comments:

Post a Comment