I have a class that pulls urls from a db and fetches their html. I use the async module to limit concurrent requests to 1000. It automatically gets a new batch of 1000 when the queue is empty.
However fillQueue says that queue is not defined:
const db = require('./db');
const async = require("async");
const fetch = require('node-fetch');
class ProxyDownloader {
constructor(maxConcurrency = 1000) {
this.queue = async.queue(this.processScrapingUrl, maxConcurrency);
this.queue.drain = this.fillQueue;
}
start() {
this.fillQueue();
}
async fillQueue() {
const scrapingUrls = await db.ScrapingUrl.findAll({limit: 1000})
this.queue.push(scrapingUrls);
}
async processScrapingUrl(scrapingUrl) {
const html = await this.downloadUrl(scrapingUrl.url);
}
async downloadUrl(url) {
const data = await fetch(url);
const html = await data.text();
return html;
}
}
via Chris
No comments:
Post a Comment