Add initial version of dijkstra backend cloudron image

node_modules/knex/scripts/stress-test/README.txt

@@ -0,0 +1,18 @@
+# Test scripts to evaluate stability of drivers / pool etc.
+
+# To run this test you need to be in this directory + have node >= 8
+# and startup docker containers with proxy and sql servers
+
+docker-compose up --no-start
+docker-compose start
+
+# Select different test script to run:
+
+node mysql2-random-hanging-every-now-and-then.js 2> /dev/null | grep -B500 -A2 -- "- STATS" 
+node mysql2-sudden-exit-without-error
+node knex-stress-test.js | grep -A 3 -- "- STATS "
+node reconnect-test-mysql-based-drivers.js 2> /dev/null | grep -A 3 -- "- STATS "
+
+# Shut down docker instances when done:
+
+docker-compose down

node_modules/knex/scripts/stress-test/docker-compose.yml

@@ -0,0 +1,47 @@
+version: '3'
+
+services:
+  toxiproxy:
+    image: shopify/toxiproxy
+    ports:
+      - "8474:8474"
+      - "23306:23306"
+      - "25432:25432"
+      - "21521:21521"
+      - "21433:21433"
+    links:
+      - "mysql"
+      - "postgresql"
+      - "oracledbxe"
+      - "mssql"
+
+  mysql:
+    image: mysql:5.7
+    ports:
+      - "33306:3306"
+    environment:
+      - TZ=UTC
+      - MYSQL_ROOT_PASSWORD=mysqlrootpassword
+
+  postgresql:
+    image: mdillon/postgis
+    ports:
+      - "35432:5432"
+    environment:
+      - POSTGRES_PASSWORD=postgresrootpassword
+      - POSTGRES_USER=postgres
+
+  oracledbxe:
+    image: wnameless/oracle-xe-11g
+    ports:
+      - "31521:1521"
+    environment:
+      - ORACLE_ALLOW_REMOTE=true
+
+  mssql:
+    image: microsoft/mssql-server-linux:2017-latest
+    ports:
+      - "31433:1433"
+    environment:
+      - ACCEPT_EULA=Y
+      - SA_PASSWORD=S0meVeryHardPassword

node_modules/knex/scripts/stress-test/knex-stress-test.js

@@ -0,0 +1,196 @@
+const Knex = require('../../lib');
+
+const Bluebird = require('bluebird');
+const toxiproxy = require('toxiproxy-node-client');
+const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
+const rp = require('request-promise-native');
+
+// init instances
+const pg = Knex({
+  client: 'pg',
+  connection:
+    'postgres://postgres:postgresrootpassword@localhost:25432/postgres',
+  pool: { max: 50 },
+});
+
+const mysql2 = Knex({
+  client: 'mysql2',
+  connection:
+    'mysql://root:mysqlrootpassword@localhost:23306/?charset=utf8&connectTimeout=500',
+  pool: { max: 50 },
+});
+
+const mysql = Knex({
+  client: 'mysql',
+  connection:
+    'mysql://root:mysqlrootpassword@localhost:23306/?charset=utf8&connectTimeout=500',
+  pool: { max: 50 },
+});
+
+const mssql = Knex({
+  client: 'mssql',
+  connection: {
+    port: 21433,
+    user: 'sa',
+    password: 'S0meVeryHardPassword',
+    server: 'localhost',
+    requestTimeout: 500,
+  },
+  pool: { max: 50 },
+});
+
+/* TODO: figure out how to nicely install oracledb node driver on osx
+const oracledb = Knex({
+  client: 'oracledb',
+  connection: {
+    user          : "travis",
+    password      : "travis",
+    connectString : "localhost/XE",
+    // https://github.com/oracle/node-oracledb/issues/525
+    stmtCacheSize : 0
+  },
+  pool: { max: 50 }
+});
+*/
+
+const counters = {};
+
+function setQueryCounters(instance, name) {
+  const counts = (counters[name] = { queries: 0, results: 0, errors: 0 });
+  instance.on('query', () => (counts.queries += 1));
+  instance.on('query-response', () => (counts.results += 1));
+  instance.on('query-error', () => (counts.errors += 1));
+}
+
+setQueryCounters(pg, 'pg');
+setQueryCounters(mysql, 'mysql');
+setQueryCounters(mysql2, 'mysql2');
+setQueryCounters(mssql, 'mssql');
+
+const _ = require('lodash');
+
+// start printing out counters
+let lastCounters = _.cloneDeep(counters);
+
+setInterval(() => {
+  const reqsPerSec = {};
+  for (let key of Object.keys(counters)) {
+    reqsPerSec[key] = {
+      queries: (counters[key].queries - lastCounters[key].queries) / 2,
+      results: (counters[key].results - lastCounters[key].results) / 2,
+      errors: (counters[key].errors - lastCounters[key].errors) / 2,
+    };
+  }
+  console.log(
+    '------------------------ STATS PER SECOND ------------------------'
+  );
+  console.dir(reqsPerSec, { colors: true });
+  console.log(
+    '------------------------------- EOS ------------------------------'
+  );
+  lastCounters = _.cloneDeep(counters);
+}, 2000);
+
+async function killConnectionsPg() {
+  return pg.raw(`SELECT pg_terminate_backend(pg_stat_activity.pid) 
+    FROM pg_stat_activity
+    WHERE pg_stat_activity.datname = 'postgres' 
+      AND pid <> pg_backend_pid()`);
+}
+
+async function killConnectionsMyslq(client) {
+  const [rows, colDefs] = await client.raw(`SHOW FULL PROCESSLIST`);
+  await Promise.all(rows.map((row) => client.raw(`KILL ${row.Id}`)));
+}
+
+async function killConnectionsMssql() {
+  const rows = await mssql('sys.dm_exec_sessions').select('session_id');
+  await Promise.all(rows.map((row) => mssql.raw(`KILL ${row.session_id}`)));
+}
+
+async function main() {
+  async function loopQueries(prefix, query) {
+    const queries = () => [...Array(50).fill(query)];
+
+    while (true) {
+      try {
+        await Promise.all(queries());
+      } catch (err) {
+        console.log(prefix, err);
+      }
+    }
+  }
+
+  async function recreateProxy(serviceName, listenPort, proxyToPort) {
+    try {
+      await rp.delete({
+        url: `${toxicli.host}/proxies/${serviceName}`,
+      });
+    } catch (err) {}
+
+    const proxy = await toxicli.createProxy({
+      name: serviceName,
+      listen: `0.0.0.0:${listenPort}`,
+      upstream: `${serviceName}:${proxyToPort}`,
+    });
+
+    // add some latency
+    await proxy.addToxic(
+      new toxiproxy.Toxic(proxy, {
+        type: 'latency',
+        attributes: { latency: 1, jitter: 1 },
+      })
+    );
+
+    // cause connections to be closed every 500 bytes
+    await proxy.addToxic(
+      new toxiproxy.Toxic(proxy, {
+        type: 'limit_data',
+        attributes: { bytes: 5000 },
+      })
+    );
+  }
+
+  // create TCP proxies for simulating bad connections etc.
+  async function recreateProxies() {
+    await recreateProxy('postgresql', 25432, 5432);
+    await recreateProxy('mysql', 23306, 3306);
+    await recreateProxy('oracledbxe', 21521, 1521);
+    await recreateProxy('mssql', 21433, 1433);
+  }
+
+  await recreateProxies();
+
+  loopQueries('PSQL:', pg.raw('select 1'));
+  loopQueries('PSQL TO:', pg.raw('select 1').timeout(20));
+
+  loopQueries('MYSQL:', mysql.raw('select 1'));
+  loopQueries('MYSQL TO:', mysql.raw('select 1').timeout(20));
+
+  // mysql2 still crashes app (without connection killer nor timeouts)
+  // https://github.com/sidorares/node-mysql2/issues/731
+  //  loopQueries('MYSQL2:', mysql2.raw('select 1'));
+  //  loopQueries('MYSQL2 TO:', mysql2.raw('select 1').timeout(20));
+
+  loopQueries('MSSQL:', mssql.raw('select 1'));
+  loopQueries('MSSQL TO:', mssql.raw('select 1').timeout(20));
+
+  setInterval(recreateProxies, 2000);
+
+  while (true) {
+    await Bluebird.delay(20); // kill everything every quite often from server side
+    try {
+      await Promise.all([
+        killConnectionsPg(),
+        killConnectionsMyslq(mysql),
+        //        killConnectionsMyslq(mysql2),
+        killConnectionsMssql(),
+      ]);
+    } catch (err) {
+      console.log('KILLER ERROR:', err);
+    }
+  }
+}
+
+process.on('exit', () => console.log('- STATS PRINT NEAR END LOGS ')); // marker for grep...
+main();

node_modules/knex/scripts/stress-test/mysql2-random-hanging-every-now-and-then.js

@@ -0,0 +1,145 @@
+/**
+ * Test case for figuring out robust way to recognize if connection is dead
+ * for mysql based drivers.
+ */
+const Bluebird = require('bluebird');
+const toxiproxy = require('toxiproxy-node-client');
+const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
+const rp = require('request-promise-native');
+const _ = require('lodash');
+
+async function stdMysqlQuery(con, sql) {
+  return new Promise((resolve, reject) => {
+    try {
+      con.query(
+        {
+          sql,
+          timeout: 500,
+        },
+        function(error, results, fields) {
+          if (error) {
+            reject(error);
+          } else {
+            resolve(results);
+          }
+        }
+      );
+    } catch (err) {
+      reject(err); // double sure...
+    }
+  });
+}
+
+const mysql2 = require('mysql2');
+let mysql2Con = { _fatalError: 'initmefirst' };
+async function mysql2Query(sql) {
+  // recreate connection on fatal error
+  if (mysql2Con._fatalError) {
+    console.log('========== Reconnecting mysql2');
+    mysql2Con = mysql2.createConnection({
+      host: 'localhost',
+      user: 'root',
+      password: 'mysqlrootpassword',
+      port: 23306,
+      connectTimeout: 500,
+      debug: true,
+    });
+    mysql2Con.on('error', (err) => {
+      console.log('- STATS Mysql2 connection died:', err);
+    });
+  }
+  console.log('================ MYSQL2 Running query ======');
+  const res = await stdMysqlQuery(mysql2Con, sql);
+  console.log('====================== done ================');
+  return res;
+}
+
+const counters = {};
+function setMysqlQueryCounters(name) {
+  const counts = (counters[name] = { queries: 0, results: 0, errors: 0 });
+}
+setMysqlQueryCounters('mysql2');
+
+// start printing out counters
+let lastCounters = _.cloneDeep(counters);
+setInterval(() => {
+  const reqsPerSec = {};
+  for (let key of Object.keys(counters)) {
+    reqsPerSec[key] = {
+      queries: counters[key].queries - lastCounters[key].queries,
+      results: counters[key].results - lastCounters[key].results,
+      errors: counters[key].errors - lastCounters[key].errors,
+    };
+  }
+  console.log(
+    '------------------------ STATS PER SECOND ------------------------'
+  );
+  console.dir(reqsPerSec, { colors: true });
+  console.log(
+    '------------------------------- EOS ------------------------------'
+  );
+  lastCounters = _.cloneDeep(counters);
+}, 1000);
+
+async function recreateProxy(serviceName, listenPort, proxyToPort) {
+  try {
+    await rp.delete({
+      url: `${toxicli.host}/proxies/${serviceName}`,
+    });
+  } catch (err) {}
+
+  const proxy = await toxicli.createProxy({
+    name: serviceName,
+    listen: `0.0.0.0:${listenPort}`,
+    upstream: `${serviceName}:${proxyToPort}`,
+  });
+
+  // add some latency
+  await proxy.addToxic(
+    new toxiproxy.Toxic(proxy, {
+      type: 'latency',
+      attributes: { latency: 1, jitter: 1 },
+    })
+  );
+
+  // cause connections to be closed every some transferred bytes
+  await proxy.addToxic(
+    new toxiproxy.Toxic(proxy, {
+      type: 'limit_data',
+      attributes: { bytes: 1000 },
+    })
+  );
+}
+
+async function main() {
+  await recreateProxy('mysql', 23306, 3306);
+  setInterval(() => recreateProxy('mysql', 23306, 3306), 2000);
+
+  async function loopQueries(prefix, query) {
+    const counts = counters[prefix];
+
+    while (true) {
+      try {
+        counts.queries += 1;
+        // without this delay we might endup to busy failure loop
+        await Bluebird.delay(0);
+        await query();
+        counts.results += 1;
+      } catch (err) {
+        counts.errors += 1;
+        console.log(prefix, err);
+      }
+    }
+  }
+
+  loopQueries('mysql2', () => mysql2Query('select 1'));
+
+  // wait forever
+  while (true) {
+    await Bluebird.delay(1000);
+  }
+}
+
+main()
+  .then(() => console.log('DONE'))
+  .catch((err) => console.log(err));

node_modules/knex/scripts/stress-test/mysql2-sudden-exit-without-error.js

@@ -0,0 +1,100 @@
+/**
+ * Test case when mysql2 driver strangely exits when one tries to send query
+ * to dead connection.
+ */
+
+const Bluebird = require('bluebird');
+const toxiproxy = require('toxiproxy-node-client');
+const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
+const rp = require('request-promise-native');
+
+// drops old toxicproxy and creates new
+async function recreateProxy(serviceName, listenPort, proxyToPort) {
+  try {
+    await rp.delete({
+      url: `${toxicli.host}/proxies/${serviceName}`,
+    });
+  } catch (err) {
+    // there was no proxy by that name... its ok
+  }
+
+  const proxy = await toxicli.createProxy({
+    name: serviceName,
+    listen: `0.0.0.0:${listenPort}`,
+    upstream: `${serviceName}:${proxyToPort}`,
+  });
+}
+
+async function insanelyParanoidQuery(con) {
+  console.log('sending query');
+  const res = await new Promise((resolve, reject) => {
+    try {
+      con.query('select 1', [], function(err, rows, fields) {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(rows);
+        }
+      });
+    } catch (err) {
+      console.log('Huh synchronous exception?! (shouldnt be possible)');
+      reject(err);
+    }
+  });
+  console.log(res);
+}
+
+async function main() {
+  // create proxy from localhost:23306 -> mysqldocker:3306
+  await recreateProxy('mysql', 23306, 3306);
+
+  // ------------- setup mysql2 db driver connection
+  const mysql2 = require('mysql2'); // with mysql this works...
+  const mysql2Con = await mysql2.createConnection({
+    host: 'localhost',
+    user: 'root',
+    password: 'mysqlrootpassword',
+    port: 23306,
+  });
+
+  mysql2Con.on('error', function(err) {
+    console.log("I'm dead", err);
+  });
+
+  console.log('Going to cut connections');
+
+  // cut connection during recreate
+  await recreateProxy('mysql', 23306, 3306);
+
+  console.log('Proxy recreated... start waiting');
+
+  // wait forever
+  while (true) {
+    await Bluebird.delay(1000);
+    try {
+      await insanelyParanoidQuery(mysql2Con);
+    } catch (err) {
+      console.log('query failed:', err);
+    }
+    await recreateProxy('mysql', 23306, 3306);
+  }
+}
+
+main()
+  .then(() => console.log('Process stopped normally'))
+  .catch((err) => {
+    console.log('Process stopped to failure', err);
+  });
+
+console.log('Waiting for eventloop to stop...');
+
+process.on('uncaughtException', function(err) {
+  console.log('uncaughtException', err);
+});
+
+process.on('exit', function() {
+  console.log(
+    'Did someone call process.exit() or wat? exitCode:',
+    process.exitCode
+  );
+});

node_modules/knex/scripts/stress-test/reconnect-test-mysql-based-drivers.js

@@ -0,0 +1,184 @@
+/**
+ * Test case for figuring out robust way to recognize if connection is dead
+ * for mysql based drivers.
+ */
+const Bluebird = require('bluebird');
+const toxiproxy = require('toxiproxy-node-client');
+const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
+const rp = require('request-promise-native');
+
+async function stdMysqlQuery(con, sql) {
+  return new Promise((resolve, reject) => {
+    try {
+      con.query(
+        {
+          sql,
+          timeout: 4000,
+        },
+        function(error, results, fields) {
+          if (error) {
+            reject(error);
+          } else {
+            resolve(results);
+          }
+        }
+      );
+    } catch (err) {
+      reject(err); // double sure...
+    }
+  });
+}
+
+// ALL THE DRIVERS HAS DIFFERENT BAG OF TRICKS TO RECOVER AND
+// RECOGNIZE WHEN CONNECTION HAS BEEN CLOSED
+
+// ------------- setup mysql db driver connection
+const mysql = require('mysql');
+let mysqlCon = { state: 'disconnected' };
+async function mysqlQuery(sql) {
+  // best way to check if connection is still alive
+  if (mysqlCon.state === 'disconnected') {
+    console.log('reconnecting mysql');
+    mysqlCon = mysql.createConnection({
+      host: 'localhost',
+      user: 'root',
+      password: 'mysqlrootpassword',
+      port: 23306,
+      connectTimeout: 500,
+    });
+    // not always triggered, if this happens during query
+    mysqlCon.on('error', (err) => {
+      console.log('- STATS Mysql connection died:', err);
+    });
+  }
+  return stdMysqlQuery(mysqlCon, sql);
+}
+
+// ------------- setup mysql2 db driver connection
+const mysql2 = require('mysql2');
+let mysql2Con = { _fatalError: 'initmefirst' };
+async function mysql2Query(sql) {
+  if (mysql2Con._fatalError) {
+    console.log('reconnecting mysql2');
+    mysql2Con = mysql2.createConnection({
+      host: 'localhost',
+      user: 'root',
+      password: 'mysqlrootpassword',
+      port: 23306,
+      connectTimeout: 500,
+    });
+    mysql2Con.on('error', (err) => {
+      console.log('- STATS Mysql2 connection died:', err);
+    });
+  }
+  console.log('================ MYSQL2 Running query....');
+  const res = await stdMysqlQuery(mysql2Con, sql);
+  console.log('=========== done');
+  return res;
+}
+
+const counters = {};
+function setMysqlQueryCounters(name) {
+  const counts = (counters[name] = { queries: 0, results: 0, errors: 0 });
+}
+setMysqlQueryCounters('mysql');
+setMysqlQueryCounters('mysql2');
+
+const _ = require('lodash');
+
+// start printing out counters
+let lastCounters = _.cloneDeep(counters);
+
+setInterval(() => {
+  const reqsPerSec = {};
+  for (let key of Object.keys(counters)) {
+    reqsPerSec[key] = {
+      queries: counters[key].queries - lastCounters[key].queries,
+      results: counters[key].results - lastCounters[key].results,
+      errors: counters[key].errors - lastCounters[key].errors,
+    };
+  }
+  console.log(
+    '------------------------ STATS PER SECOND ------------------------'
+  );
+  console.dir(reqsPerSec, { colors: true });
+  console.log(
+    '------------------------------- EOS ------------------------------'
+  );
+  lastCounters = _.cloneDeep(counters);
+
+  // if hang
+  ///if (reqsPerSec.mysql2.queries === 0) process.exit(0);
+}, 1000);
+
+async function main() {
+  async function recreateProxy(serviceName, listenPort, proxyToPort) {
+    try {
+      await rp.delete({
+        url: `${toxicli.host}/proxies/${serviceName}`,
+      });
+    } catch (err) {}
+
+    const proxy = await toxicli.createProxy({
+      name: serviceName,
+      listen: `0.0.0.0:${listenPort}`,
+      upstream: `${serviceName}:${proxyToPort}`,
+    });
+
+    // add some latency
+    await proxy.addToxic(
+      new toxiproxy.Toxic(proxy, {
+        type: 'latency',
+        attributes: { latency: 1, jitter: 1 },
+      })
+    );
+
+    // cause connections to be closed every some transferred bytes
+    await proxy.addToxic(
+      new toxiproxy.Toxic(proxy, {
+        type: 'limit_data',
+        attributes: { bytes: 1000 },
+      })
+    );
+  }
+
+  // create TCP proxies for simulating bad connections etc.
+  async function recreateProxies() {
+    console.log('----- Recreating proxies -> cutting connections completely');
+    await recreateProxy('postgresql', 25432, 5432);
+    await recreateProxy('mysql', 23306, 3306);
+    await recreateProxy('oracledbxe', 21521, 1521);
+  }
+  setInterval(() => recreateProxies(), 2000);
+
+  async function loopQueries(prefix, query) {
+    const counts = counters[prefix];
+
+    while (true) {
+      try {
+        counts.queries += 1;
+        // without this delay we endup to busy failure loop
+        await Bluebird.delay(0);
+        await query();
+        counts.results += 1;
+      } catch (err) {
+        counts.errors += 1;
+        console.log(prefix, err);
+      }
+    }
+  }
+
+  await recreateProxies();
+
+  loopQueries('mysql', () => mysqlQuery('select 1'));
+  loopQueries('mysql2', () => mysql2Query('select 1'));
+
+  // wait forever
+  while (true) {
+    await Bluebird.delay(1000);
+  }
+}
+
+main()
+  .then(() => console.log('DONE'))
+  .catch((err) => console.log(err));