Wednesday, 19 April 2017

Node kafka consumer to process messages in sequence

I have a kafka topic that I want to consume with a node app. The node app must process the messages from the topic in sequence, one by one, not many at the same time.

I tried this kind of code but this is not doing what I want. When there is messages in the topic waiting for processing and this code is started the on 'message' event gets triggered immediately for all the the messages. The first message gets mutex lock first but the rest of the messages are processed in random order.

var mutex = require( 'node-mutex' )();
var crypto = require('crypto');
var mutexToken = crypto.randomBytes(64).toString('hex');
var kafka = require('kafka-node');
var Consumer = kafka.Consumer;
var client = new kafka.Client('localhost:2181');
var consumer = new Consumer(
        client,
        [
                { topic: 'my_topic' }
        ]
);
consumer.on('message', function(message) {
        console.log("new message")
        mutex
                .lock( mutexToken )
                .then( function( unlock ) {
                        console.log(message);
                        unlock();
                } );
});

Is it possible to consume the messages one by one, synchronously? Maybe with some other library?



via Mika

No comments:

Post a Comment