Batch insert MongoDB and NodeJs

Jean Dupont Source

I'm working on MongoDB 2.6.9 and NodeJs 0.10.37 and according to my previous question MongoDB calculating score from an existing fields and put it in a new field in the same collection and I have got a structured answer for doing that from chridam just like this :

var bulkUpdateOps = db.collection1.initializeUnorderedBulkOp(),
    cursor = db.collection1.find(), // cursor 
    counter = 0;

cursor.forEach(function(doc) {
    // computations
    var c1, c2, c3, c4, Field8;
    c1 = 10 + (0.03*doc.Field3);
    c2 = (doc.Field2 == 1) ? 1: 0.03;
    c3 = 7 - (doc.Field5.match(new RegExp(".", "g")) || []).length;
    c4 = (doc.Field2 == 1) ? Math.pow(doc.Field, -0.6) : 1;
    Field8 = c1*c2*c3*c4;

    bulkUpdateOps.find({ "_id": doc._id }).updateOne({
        "$set": { "Field8": Field8 }
    });

    if (counter % 500 == 0) {
        bulkUpdateOps.execute();
        bulkUpdateOps = db.collection1.initializeUnorderedBulkOp();
    }
})

if (counter % 500 != 0) { bulkUpdateOps.execute(); } 

But I still don't know from where start to achieve the above, I should create a js file which will do this Batch insert for me. From mongoose connection to this Batch insert.

node.jsmongodbmongoose

Answers

answered 2 years ago chridam #1

In order to use the underlying bulk operations API in Mongoose, you should be able to access it via the .collection property from the mongoose model. Before using the API, wait for mongoose to successfully connect to the db since Mongoose doesn't really support the "initializeOrderedBulkOp()" currently as it doesn't work with its internal buffering system. So something like the following implementation (not tested) should give you an idea:

var mongoose = require('mongoose'),
    express = require('express'),
    Schema = mongoose.Schema;

mongoose.connect('mongodb://localhost/mydb');
var collection1Schema = new Schema({},{ strict: false, collection: 'Collection1' }),    
    MyModel = mongoose.model("MyModel", collection1Schema );

mongoose.set('debug', true);

mongoose.connection.on("open", function (err) {
    if (err) throw err;  
    var bulkUpdateOps = MyModel.collection.initializeUnorderedBulkOp(), 
        counter = 0;

    MyModel.find({}).lean().exec(function (err, docs) {
        if (err) throw err; 

        docs.forEach(function (doc){
            // computations
            var c1, c2, c3, c4, Field8;
            c1 = 10 + (0.03*doc.Field3);
            c2 = (doc.Field2 == 1) ? 1: 0.03;
            c3 = 7 - (doc.Field5.match(new RegExp(".", "g")) || []).length;
            c4 = (doc.Field2 == 1) ? Math.pow(doc.Field, -0.6) : 1;
            Field8 = c1*c2*c3*c4;

            counter++;

            bulkUpdateOps.find({ "_id": doc._id }).updateOne({
                "$set": { "Field8": Field8 }
            });

            if (counter % 500 == 0 ) {
                bulkUpdateOps.execute(function(err, result) {
                    if (err) throw err;  
                    bulkUpdateOps = MyModel.collection.initializeUnorderedBulkOp();
                    console.log(result);
                });
            } 

        });     

        if (counter % 500 != 0 ) {            
            bulkUpdateOps.execute(function(err, result) {
                if (err) throw err;  
                console.log(result);
            });         
        }       
    });

    var app = express();
    app.listen(3000, function () {
        console.log('now listening on http://localhost:3000');
    });
});

comments powered by Disqus