Wednesday, 12 April 2017

Why closing and reopening database in node-sqlite3 boosts query performance

My node application needs to perform 3 queries, each depend on the results from the previous. So I put them into callbacks and a simplified code structure would be like this:

var express = require('express');
var sqlite3 = require('sqlite3').verbose();
var app = express();
var db = new sqlite3.Database('data.db');

app.get('/api/v1/data', (req, res) {
  db.all(query1, function (err, rows) {
    // get data from rows assuming has data
    db.all(query2, function (err, rows) {
      // do stuff
      db.all(query3, function (err, rows) {
        // do last bit of stuff
      });
     });
   });
});

It turns out that query2 is always way slower than the other two, even though the expected complexity is:

query1 < query2 < query3

I have rooted out indexing issues because when executed independently in sqlite3 desk top applications (DB Browser and Valentino Studio), the order is as expected.

But then I inserted a close and re-open action before executing query2, and query2 performance was boosted.

var express = require('express');
var sqlite3 = require('sqlite3').verbose();
var app = express();
var db = new sqlite3.Database('data.db');

app.get('/api/v1/data', (req, res) {
  db.all(query1, function (err, rows) {
    // get data from rows assuming has data

    // The magic "reopen" speeds up query2 significantly
    db.close();
    db = new sqlite3.Database('data.db');

    db.all(query2, function (err, rows) {
      // do stuff
      db.all(query3, function (err, rows) {
        // do last bit of stuff
      });
     });
   });
});

Does anyone have any idea why this happened?



via benjaminz

No comments:

Post a Comment