I have to provide a solution in order to record every request to our server. I am trying to implemt based on ELK (ElasticSearch, LogStash and Kibana). I mean, save the request detais in ElasticSearch via LogStash and then provide a visualization via Kibana.
I am trying to save in ElasticSearch with one of these two approaches. The one that seems to me a bit more elaborated depends from me to buid a filter in logstash.conf but I am very confused how to.
Imagine there will always be certain parameter in body labeled cpf (client id). How can I filter that?
In order to not stuck at all, I tried a bit more simple idea without filter but I am facing another issue: although I see the value in ElasticSearch (index myindex, type mytype and number of cpf accordingly to how many time I call the rest service), I got stuck in another issue: how to create a visualization in Kibana that allows me to query how many cpf are found in for certain period?
It is my first time working with ElasticSearch, LogStash and Kibana and I do have a considerable path to learn NodeJs Best Practices. Any suggestion will be highly appreciatted.
Relevant part of server.js
var express = require('express');
var bodyParser = require('body-parser');
var Client = require('node-rest-client').Client;
var expressWinston = require('express-winston');
var winston = require('winston');
require('winston-logstash');
// Aproach 1 - without filter in logstash.conf
// it seems to me a bit more simple
var Logstash = require('logstash-client');
var logstash = new Logstash({
type: 'tcp',
host: '127.0.0.1',
port: 5000
});
var client = new Client();
var app = express();
// Aproach 2 - depends on certain filter in logstash.conf
// it seems to me a bit more advanced idea
expressWinston.requestWhitelist.push('body');
expressWinston.responseWhitelist.push('body')
app.use(expressWinston.logger({
transports: [
new winston.transports.Logstash({
type: 'tcp', // udp, tcp, memory
host: '127.0.0.1',
port: 5000
})
],
meta: true,
msg: "HTTP ",
expressFormat: true,
colorStatus: true,
ignoreRoute: function (req, res) { return false; }
}));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
var port = process.env.PORT || 3000;
var router = express.Router();
var tokenRoute = router.route('/myapi');
tokenRoute.post(function (req, res) {
// Part of the Aproach 1
logstash.send(req.body.username);
...
logstash.conf
input {
tcp {
port => 5000
type => mytype
}
}
output {
elasticsearch { hosts => ["localhost:9200"] index => "myindex"}
}
Request sample
node --debug-brk=22462 --nolazy server.js
Debugger listening on port 22462
{
"res": {
"statusCode": 200,
"body": {
"tokenBackEnd": {
"OAuth2AccessToken": {
"access_token": "397b606e-33bf-4d72-a078-c8c01a6d6106",
"token_type": "bearer",
"refresh_token": "263a9253-1b87-412f-a164-fea1f5ff1fee",
"expires_in": "119",
"scope": "read write trust"
}
}
}
},
"req": {
"url": "/myapi",
"headers": {
"host": "localhost:3000",
"authorization": "Basic Z3JlZW5jYXJkLXRydXN0ZWQtY2xpZW50OmdyZWVuY2FyZC1zZWNyZXQ=",
"user-agent": "curl/7.47.0",
"accept": "*/*",
"content-length": "41",
"content-type": "application/x-www-form-urlencoded"
},
"method": "POST",
"httpVersion": "1.1",
"originalUrl": "/myapi",
"query": {},
"body": {
"grant_type": "password",
"cpf": "123456",
"password": "a"
}
},
"responseTime": 454,
"level": "info",
"message": "POST /token 200 454ms"
}
The above request is the consequence of
curl -u my-trusted-client:mysecret -k -d "grant_type=password&cpf=123456&password=a" http://localhost:3000/myapi
via DemeCarvO
No comments:
Post a Comment