How do I do a bulk insert in mySQL using node.js

Question

Bulk inserts are possible by using nested array, see the github page

Nested arrays are turned into grouped lists (for bulk inserts), e.g. [['a', 'b'], ['c', 'd']] turns into ('a', 'b'), ('c', 'd')

You just insert a nested array of elements.

An example is given in here

var mysql = require('mysql');
var conn = mysql.createConnection({
    ...
});

var sql = "INSERT INTO Test (name, email, n) VALUES ?";
var values = [
    ['demian', '[email protected]', 1],
    ['john', '[email protected]', 2],
    ['mark', '[email protected]', 3],
    ['pete', '[email protected]', 4]
];
conn.query(sql, [values], function(err) {
    if (err) throw err;
    conn.end();
});

Note: values is an array of arrays wrapped in an array

[ [ [...], [...], [...] ] ]

There is also a totally different node-msql package for bulk insertion

@Ragnar123 answer is correct, but I see a lot of people saying in the comments that it is not working. I had the same problem and it seems like you need to wrap your array in [] like this:

var pars = [
    [99, "1984-11-20", 1.1, 2.2, 200], 
    [98, "1984-11-20", 1.1, 2.2, 200], 
    [97, "1984-11-20", 1.1, 2.2, 200]
];

It needs to be passed like [pars] into the method.

I ran into this today (mysql 2.16.0) and thought I'd share my solution:

const items = [
    {name: 'alpha', description: 'describes alpha', value: 1},
    ...
];

db.query(
    'INSERT INTO my_table (name, description, value) VALUES ?',
    [items.map(item => [item.name, item.description, item.value])],
    (error, results) => {...}
);

I was looking around for an answer on bulk inserting Objects.

The answer by Ragnar123 led me to making this function:

function bulkInsert(connection, table, objectArray, callback) {
  let keys = Object.keys(objectArray[0]);
  let values = objectArray.map( obj => keys.map( key => obj[key]));
  let sql = 'INSERT INTO ' + table + ' (' + keys.join(',') + ') VALUES ?';
  connection.query(sql, [values], function (error, results, fields) {
    if (error) callback(error);
    callback(null, results);
  });
}

bulkInsert(connection, 'my_table_of_objects', objectArray, (error, response) => {
  if (error) res.send(error);
  res.json(response);
});

Hope it helps!

All props to Ragnar123 for his answer.

I just wanted to expand it after the question asked by Josh Harington to talk about inserted IDs.

These will be sequential. See this answer : Does a MySQL multi-row insert grab sequential autoincrement IDs?

Hence you can just do this (notice what I did with the result.insertId):

  var statement = 'INSERT INTO ?? (' + sKeys.join() + ') VALUES ?';
  var insertStatement = [tableName, values];
  var sql = db.connection.format(statement, insertStatement);
  db.connection.query(sql, function(err, result) {
    if (err) {
      return clb(err);
    }
    var rowIds = [];
    for (var i = result.insertId; i < result.insertId + result.affectedRows; i++) {
      rowIds.push(i);
    }
    for (var i in persistentObjects) {
      var persistentObject = persistentObjects[i];
      persistentObject[persistentObject.idAttributeName()] = rowIds[i];
    }
    clb(null, persistentObjects);
  });

(I pulled the values from an array of objects that I called persistentObjects.)

Hope this helps.

This is a fast "raw-copy-paste" snipped to push a file column in mysql with node.js >= 11

250k row in few seconds

'use strict';

const mysql = require('promise-mysql');
const fs = require('fs');
const readline = require('readline');

async function run() {
  const connection = await mysql.createConnection({
    host: '1.2.3.4',
    port: 3306,
    user: 'my-user',
    password: 'my-psw',
    database: 'my-db',
  });

  const rl = readline.createInterface({ input: fs.createReadStream('myfile.txt') });

  let total = 0;
  let buff = [];
  for await (const line of rl) {
    buff.push([line]);
    total++;
    if (buff.length % 2000 === 0) {
      await connection.query('INSERT INTO Phone (Number) VALUES ?', [buff]);
      console.log(total);
      buff = [];
    }
  }

  if (buff.length > 0) {
    await connection.query('INSERT INTO Phone (Number) VALUES ?', [buff]);
    console.log(total);
  }

  console.log('end');
  connection.close();
}

run().catch(console.log);

If Ragnar's answer doesn't work for you. Here is probably why (based on my experience) -

1. I wasn't using node-mysql package as shown my Ragnar. I was using mysql package. They're different (if you didn't notice - just like me). But I'm not sure if it has anything to do with the ? not working, since it seemed to work for many folks using the mysql package.

2. Try using a variable instead of ?

The following worked for me -

var mysql = require('node-mysql');
var conn = mysql.createConnection({
    ...
});

var sql = "INSERT INTO Test (name, email, n) VALUES :params";
var values = [
    ['demian', '[email protected]', 1],
    ['john', '[email protected]', 2],
    ['mark', '[email protected]', 3],
    ['pete', '[email protected]', 4]
];
conn.query(sql, { params: values}, function(err) {
    if (err) throw err;
    conn.end();
});

Hope this helps someone.