Saturday 27 May 2017

NodeJS and robots.txt that returns ContentType -> text/html

I try to set robots.txt file in ExpressJS. I found that to make all fine needed to add middleware function into app.js . But all time when i test my robots.txt i get ContentType = text/html . What i do wrong?

// app.js
var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');

var index = require('./routes/index');
var users = require('./routes/users');
var test = require('./routes/test');
var test1 = require('./routes/test1');
var test2 = require('./routes/test2');

var app = express();

// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');

// uncomment after placing your favicon in /public
//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));

app.get('/robots.txt', function (req, res) {
    res.type('text/plain');
    res.send("User-agent: *\nDisallow: /engine/go.php\nDisallow: /engine/download.php\nDisallow: /user/\nDisallow: /newposts/\nDisallow: /statistics.html\nDisallow: /*subaction=userinfo\nDisallow: /*subaction=newposts\nDisallow: /*do=lastcomments #comments into text\nDisallow: /*do=feedback\nDisallow: /*do=register\nDisallow: /*do=lostpassword\nDisallow: /*do=addnews\nDisallow: /*do=stats\nDisallow: /*do=pm\nDisallow: /*do=search\n\n# comment in text\n\nUser-agent: Yandex\nDisallow: /*sid= # comment\nDisallow: /*do=addnews\nDisallow: /*do=stats\nDisallow: /*do=pm\nDisallow: /*do=search\n\nUser-agent: Googlebot\nDisallow: /newposts/\nDisallow: /statistics.html\nDisallow: /*subaction=userinfo\n\n# some more comments\n# and one more comment");
});

app.use('/', index);
app.use('/users', users);
app.use('/test', test);
app.use('/test1', test1);
app.use('/test2', test2);

// catch 404 and forward to error handler
app.use(function(req, res, next) {
  var err = new Error('Not Found');
  err.status = 404;
  next(err);
});

// error handler
app.use(function(err, req, res, next) {
  // set locals, only providing error in development
  res.locals.message = err.message;
  res.locals.error = req.app.get('env') === 'development' ? err : {};

  // render the error page
  res.status(err.status || 500);
  res.render('error');
});

module.exports = app;



via Nikita M

No comments:

Post a Comment