Tuesday 14 March 2017

how to use kibana and logstach in order to create a visualization of how many time someone access certain rest service

I have to provide a solution in order to record every request to our server. I am trying to implemt based on ELK (ElasticSearch, LogStash and Kibana). I mean, save the request detais in ElasticSearch via LogStash and then provide a visualization via Kibana.

I am trying to save in ElasticSearch with one of these two approaches. The one that seems to me a bit more elaborated depends from me to buid a filter in logstash.conf but I am very confused how to.

Imagine there will always be certain parameter in body labeled cpf (client id). How can I filter that?

In order to not stuck at all, I tried a bit more simple idea without filter but I am facing another issue: although I see the value in ElasticSearch (index myindex, type mytype and number of cpf accordingly to how many time I call the rest service), I got stuck in another issue: how to create a visualization in Kibana that allows me to query how many cpf are found in for certain period?

It is my first time working with ElasticSearch, LogStash and Kibana and I do have a considerable path to learn NodeJs Best Practices. Any suggestion will be highly appreciatted.

Relevant part of server.js

var express = require('express');
var bodyParser = require('body-parser');
var Client = require('node-rest-client').Client;
var expressWinston = require('express-winston');
var winston = require('winston');
require('winston-logstash');

// Aproach 1 - without filter in logstash.conf
// it seems to me a bit more simple
var Logstash = require('logstash-client');
var logstash = new Logstash({
  type: 'tcp', 
  host: '127.0.0.1',
  port: 5000
});


var client = new Client();
var app = express();

// Aproach 2 - depends on certain filter in logstash.conf
// it seems to me a bit more advanced idea 
expressWinston.requestWhitelist.push('body');
expressWinston.responseWhitelist.push('body')


app.use(expressWinston.logger({
  transports: [
     new winston.transports.Logstash({
       type: 'tcp', // udp, tcp, memory 
       host: '127.0.0.1',
       port: 5000
    })
  ],
  meta: true, 
  msg: "HTTP  ", 
  expressFormat: true, 
  colorStatus: true, 
  ignoreRoute: function (req, res) { return false; } 
}));

app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
  extended: true
}));

var port = process.env.PORT || 3000;
var router = express.Router();
var tokenRoute = router.route('/myapi');


tokenRoute.post(function (req, res) {

  // Part of the Aproach 1
  logstash.send(req.body.username);

...

logstash.conf

input {
  tcp {
    port => 5000
    type => mytype
  }
}

output {
  elasticsearch { hosts => ["localhost:9200"] index => "myindex"}
}

Request sample

node --debug-brk=22462 --nolazy server.js 
Debugger listening on port 22462
{
  "res": {
    "statusCode": 200,
    "body": {
      "tokenBackEnd": {
        "OAuth2AccessToken": {
          "access_token": "397b606e-33bf-4d72-a078-c8c01a6d6106",
          "token_type": "bearer",
          "refresh_token": "263a9253-1b87-412f-a164-fea1f5ff1fee",
          "expires_in": "119",
          "scope": "read write trust"
        }
      }
    }
  },
  "req": {
    "url": "/myapi",
    "headers": {
      "host": "localhost:3000",
      "authorization": "Basic Z3JlZW5jYXJkLXRydXN0ZWQtY2xpZW50OmdyZWVuY2FyZC1zZWNyZXQ=",
      "user-agent": "curl/7.47.0",
      "accept": "*/*",
      "content-length": "41",
      "content-type": "application/x-www-form-urlencoded"
    },
    "method": "POST",
    "httpVersion": "1.1",
    "originalUrl": "/myapi",
    "query": {},
    "body": {
      "grant_type": "password",
      "cpf": "123456",
      "password": "a"
    }
  },
  "responseTime": 454,
  "level": "info",
  "message": "POST /token 200 454ms"
}

The above request is the consequence of

curl -u my-trusted-client:mysecret -k -d "grant_type=password&cpf=123456&password=a" http://localhost:3000/myapi



via DemeCarvO

No comments:

Post a Comment