Keeping CouchDB design docs up to date with Node.js

CouchDB views are defined typically as Javascript snippets and are part of special documents called design documents. I noticed that keeping these design documents up to date during development is pretty cumbersome and error prone. So I devised simply way to keep them updated using Node.js and Cradle couchdb driver.

Idea is to define the views in as variables in runnable js script and run that with Node each time it’s changed.

Here is the code. Copy it to e.g. cdb-views.js.

var cradle = require('cradle');

cradle.setup({ host: 'localhost',
               port: 5984,
               options: { cache:true, raw: false }});

var cclient = new (cradle.Connection)

function _createdb(dbname) {
    var db = cclient.database(dbname);
    db.exists(function(err, exists) {
        if (!exists) {
            db.create()
        }
    });
    return db;
}
var DB_SOMETHING = _createdb('somedb')

function cradle_error(err, res) {
    if (err) console.log(err)
}


function update_views( db, docpath, code ) {

    function save_doc() {
        db.save(docpath, code, function(err) {
            // view has changed, so initiate cleanup to get rid of old
            // indexes
            db.viewCleanup( cradle_error );
        });

        return true;
    }

    function compare_code( str1, str2 ) {
        var p1 = str1.split('\n');
        var p2 = str2.split('\n');

        for ( var i=0; i < p1.length || i < p2.length; i++ ) {
            var l1 = p1[i];
            var l2 = p2[i];
            l1 = l1 ? l1.trim() : '';
            l2 = l2 ? l2.trim() : '';
            if ( !l1 && !l2 ) continue;
            if ( l1 != l2 ) return true;
        }
        return false;
    }

    // compare function definitions in document and in code
    function compare_def(docdef, codedef) {
        var i = 0;

        if (!docdef && codedef) {
            console.log('creating "' + docpath +'"')
            return true;
        }
        if (!codedef && docdef) {
            console.log('removing "' + docpath +'"')
            return true;
        }
        if (!codedef && !docdef) {
            return false;
        }

        for (var u in docdef) {

            i++;
            if (codedef[u] == undefined) {
                console.log('definition of "' + u + '" removed - updating "' + docpath +'"')
                return true;
            }

            if (typeof(codedef[u]) == 'function') {
                if (!codedef[u] || compare_code( docdef[u], codedef[u].toString()) ) {
                    console.log('definition of "' + u + '" changed - updating "' + docpath +'"')
                    return true;
                }
            } else for (var f in docdef[u]) {
                i++;
                if (!codedef[u][f] || compare_code( docdef[u][f], codedef[u][f].toString()) ) {
                    console.log('definition of "' + u + '.' + f + '" changed - updating "' + docpath +'"')
                    return true;
                }

            }
        }
        // check that both doc and code have same number of functions
        for (var u in codedef) {
            i--;
            if (typeof(codedef[u]) != 'function') {
                for (var f in codedef[u]) {
                    i--;
                }
            }
        }
        if (i != 0) {
            console.log('new definitions - updating "' + docpath +'"')
            return true;
        }

        return false;
    }

    db.get(docpath, function(err, doc) {

        if (!doc) {
            console.log('not found - creating "' + docpath +'"')
            return save_doc();
        }

        if (compare_def(doc.updates, code.updates) || compare_def(doc.views, code.views)) {
            return save_doc();
        }
        console.log('"' + docpath +'" up to date')
    });
}

var EXAMPLE1_DDOC = {
    language: 'javascript',
    views: {
        active: {
            map: function (doc) {
                if (doc.lastsession) {
                    emit(parseInt(doc.lastsession / 1000), 1)
                }
            },
            reduce: function(keys, counts, rereduce) {
                return sum(counts)
            }
        },
        users: function(doc) { 
            if (doc.created) {
                emit(parseInt(doc.created / 1000), 1)
            }
        }
    }    
}

var EXAMPLE2_DDOC = {
    language: 'javascript',
    views: {
        myview: function(doc) {
            if (doc.param1 && doc.param2) {
                emit([doc.param1, doc.param2], null)
            }
        }
    }
}

update_views(DB_SOMETHING, '_design/example1', EXAMPLE1_DDOC);
update_views(DB_SOMETHING, '_design/example2', EXAMPLE2_DDOC);

The code is pretty simple.

  1. First it loads the Cradle couchdb driver and creates needed databases if they do not already exist. In this example only single database ‘somedb’ is created.
  2. The update_views is responsible of keeping the design docs up to date. It loads the design doc from defined DB and compares it to the code defined in the design doc in this file. If it has changed (or missing) it will be recreated.
  3. The example design docs (EXAMPLE1_DDOC and EXAMPLE2_DDOC) are simple design doc definitions as Javascript object. You’re familiar with CouchDB so this is self explanatory.
  4. Lastly the code just calls the update_views to update the design documents.

Now it’s possible to maintain the views in this Javascript file, the Node will make sure that the syntax is always valid.

Example output:

Views are up to date.

$ node cdb-views.js
"_design/example1" up to date
"_design/example2" up to date

Definition of view example2/myview has changed

$ node cdb-views.js
"_design/example1" up to date
definition of "myview" changed - updating "_design/example2"

Design doc example2 can not be found and is created.

$ node cdb-views.js
"_design/example1" up to date
no design doc found updating "_design/example2"

 

 

Callbacks from Threaded Node.js C++ Extension

UPDATE: this guide is bit outdated, new Node (0.6 > ) versions support easier way to access pooled worker threads so extension doesn’t need to create its own. See links in comments.

Writing threaded Node.js extension requires some care. All Javascript in Node.js is executed in single main thread, so you can not simply call the V8 engine directly from your background thread. That would cause segfault. Recommended way to do this is to spawn new thread on background and use the libev events to wake up the main thread to execute the Javascript callbacks.

Node.js framework has lots of ready stuff for implementing extensions, but there is no simple example how to implement this kind extension so here it is.

Add-on Source

Save this source to texample.cc

#include <queue>

// node headers
#include <v8.h>
#include <node.h>
#include <ev.h>
#include <pthread.h>
#include <unistd.h>
#include <string.h>

using namespace node;
using namespace v8;

// handles required for callback messages
static pthread_t texample_thread;
static ev_async eio_texample_notifier;
Persistent<String> callback_symbol;
Persistent<Object> module_handle;

// message queue
std::queue<int> cb_msg_queue = std::queue<int>();
pthread_mutex_t queue_mutex = PTHREAD_MUTEX_INITIALIZER;

// The background thread
static void* TheThread(void *)
{
    int i = 0;
    while(true) {
         // fire event every 5 seconds
        sleep(5);
       pthread_mutex_lock(&queue_mutex);
       cb_msg_queue.push(i);
       pthread_mutex_unlock(&queue_mutex);
       i++;
      // wake up callback
      ev_async_send(EV_DEFAULT_UC_ &eio_texample_notifier);
    }
    return NULL;
}

// callback that runs the javascript in main thread
static void Callback(EV_P_ ev_async *watcher, int revents)
{
    HandleScope scope;

    assert(watcher == &eio_texample_notifier);
    assert(revents == EV_ASYNC);

    // locate callback from the module context if defined by script
    // texample = require('texample')
    // texample.callback = function( ... ) { ..
    Local<Value> callback_v = module_handle->Get(callback_symbol);
    if (!callback_v->IsFunction()) {
         // callback not defined, ignore
         return;
    }
    Local<Function> callback = Local<Function>::Cast(callback_v);

    // dequeue callback message
    pthread_mutex_lock(&queue_mutex);
    int number = cb_msg_queue.front();
    cb_msg_queue.pop();
    pthread_mutex_unlock(&queue_mutex);

    TryCatch try_catch;

    // prepare arguments for the callback
    Local<Value> argv[1];
    argv[0] = Local<Value>::New(Integer::New(number));

    // call the callback and handle possible exception
    callback->Call(module_handle, 1, argv);

    if (try_catch.HasCaught()) {
        FatalException(try_catch);
    }
}

// Start the background thread
Handle<Value> Start(const Arguments &args)
{
    HandleScope scope;

    // start background thread and event handler for callback
    ev_async_init(&eio_texample_notifier, Callback);
    //ev_set_priority(&eio_texample_notifier, EV_MAXPRI);
    ev_async_start(EV_DEFAULT_UC_ &eio_texample_notifier);
    ev_unref(EV_DEFAULT_UC);
    pthread_create(&texample_thread, NULL, TheThread, 0);

    return True();
}

void Initialize(Handle<Object> target)
{
    HandleScope scope;

    NODE_SET_METHOD(target, "start", Start);

    callback_symbol = NODE_PSYMBOL("callback");
    // store handle for callback context
    module_handle = Persistent<Object>::New(target);
}

extern "C" {
static void Init(Handle<Object> target)
{
    Initialize(target);
}

NODE_MODULE(texample, Init);

Function walkthrough

  • The Init function gets called when you require('texample') the native module.
  • Initialize function defines module function start that will be called by javascript. It also stores the module handle for locating and calling the script defined callback on right context.
  • Start function initializes the libev event notifier and starts the background thread TheThread
  • Thread TheThread simply loops, sleeps and puts incremental integers to the queue and wakes up the main thread each time.
  • Callback function gets waken up the libev and it locates and calls the javascript function callback

Building

Copy this to the ‘wscript’  file.

def set_options(opt):
  opt.tool_options("compiler_cxx")

def configure(conf):
  conf.check_tool("compiler_cxx")
  conf.check_tool("node_addon")

def build(bld):
  obj = bld.new_task_gen("cxx", "shlib", "node_addon")
  obj.cxxflags = ["-g", "-D_FILE_OFFSET_BITS=64",
                  "-D_LARGEFILE_SOURCE", "-Wall"]
  obj.target = "texample"
  obj.source = "texample.cc"

Compile the code with node-waf

$ node-waf configure
$ node-waf build

Running

Start node shell and load the native add on module

$ node
> texample = require('./build/default/texample');

Define the callback function that the module will call

> texample.callback = function(i) {
... console.log('Bang: ' + i);
... }
>

Call start to kick of the background thread

> texample.start();
true
>

Wait for 5 seconds, you’ll start seeing your callback getting triggered every 5 seconds.

> Bang: 0
Bang: 1
> Bang: 2

Have fun!

Node.js TLS client example

Couldn’t find good end to end example of Node.js new raw SSL client API, so here is one. This snippet just connects to the https://encrypted.google.com and fetches the front page.

tls = require('tls');

// callback for when secure connection established
function connected(stream) {
    if (stream) {
       // socket connected
      stream.write("GET / HTTP/1.0\n\rHost: encrypted.google.com:443\n\r\n\r");  
    } else {
      console.log("Connection failed");
    }
}

// needed to keep socket variable in scope
var dummy = this;

// try to connect to the server
dummy.socket = tls.connect(443, 'encrypted.google.com', function() {
   // callback called only after successful socket connection
   dummy.connected = true;
   if (dummy.socket.authorized) {
      // authorization successful
      dummy.socket.setEncoding('utf-8');
      connected(dummy.socket);
   } else {
      // authorization failed
     console.log(dummy.socket.authorizationError);
     connected(null);
   }
});

dummy.socket.addListener('data', function(data) {
   // received data
   console.log(data);
});

dummy.socket.addListener('error', function(error) {
   if (!dummy.connected) {
     // socket was not connected, notify callback
     connected(null);
   }
   console.log("FAIL");
   console.log(error);
});

dummy.socket.addListener('close', function() {
 // do something
});

If you want to use client certificate authentication, define the options and give that as additional parameter to the tls.connect call.

var keyPem = fs.readFileSync("key-noenc.pem", encoding='ascii');
var certPem = fs.readFileSync("cert.pem", encoding='ascii');
var options = {key:keyPem, cert:certPem };

...
dummy.socket = tls.connect(443, 'some.example.com', options, function() {
....

Apple Push Notifications with Node.js

When your iPhone app backend  needs to send Apple Push Notifications, it must do this over raw SSL socket using Apple proprietary raw binary interface. Standard Web REST is not supported. This kind of sucks, because if your entire backend is web based you need to break that cleanliness with external HTTP to APN proxy. One option is to use services like Urban Airship, but you can also build the proxy by yourself.

One potential platform for this is hyped Node.js, the rising javascript engine for building ad-hoc web servers. Web is full of examples of building simple HTTP based server or proxy with Node.js, so this post is only the part where we open a secure connection to the Apple server and send push notifications with plain Node.js javascript.

Please note that Apple assumes that you pool and keep sockets open as long as you have notifications to send. So, don’t make naive implementation that makes new socket for each HTTP request. Some simple pooling and reuse is a must for real implementation.

In addition for sending the push notifications, your app also needs to poll the APNS feedback service to find out what devices have uninstalled the app and should not be pushed new notifications. See more details in post Apple Push Notification feedback service.

1. Get Certificates

Apple’s Push notification server authenticates application by SSL certificates. There is no additional authentication handshake after secure connection has been established.

First we need the PEM format certificates that you can get by exporting  them with Apple Keytool. Export also the Apple Worldwide CA certificate. See this excellent blog post (up to step 5)  for details how to acquire the PEM files: http://blog.boxedice.com/2010/06/05/how-to-renew-your-apple-push-notification-push-ssl-certificate/

Now you should have following certificate files.

  • app-cert.pem  (Application cerificate)
  • app-key-noenc.pem  (Application private key)
  • apple-worldwide-certificate-authority.cer  (Apple CA certificate)

2. Open Connection to Push Server

UPDATE:See more complete TLS example here.

Moving on the actual implementation in Node.js. This is quite simple, you just read the various certificate files as string and use them as credentials.

You must also have SSL support built in your Node.js binary.

var fs = require('fs');
var crypto = require('crypto');
var tls = require('tls');

var certPem = fs.readFileSync('app-cert.pem', encoding='ascii');
var keyPem = fs.readFileSync('app-key-noenc.pem', encoding='ascii');
var caCert = fs.readFileSync('apple-worldwide-certificate-authority.cer', encoding='ascii');
var options = { key: keyPem, cert: certPem, ca: [ caCert ] }

function connectAPN( next ) {
    var stream = tls.connect(2195, 'gateway.sandbox.push.apple.com', options, function() {
        // connected
        next( !stream.authorized, stream );
    });
}

3. Write Push Notification

After secure connection is established, you can simply write push notifications to the socket as binary data. Push notification is addressed to a device with 32 byte long push token that must be acquired by your iPhone application and sent to your backend somehow.

Easy format is simple hexadecimal string, so we define first a helper method to convert that hexadecimal string to binary buffer at server side.

function hextobin(hexstr) {
   buf = new Buffer(hexstr.length / 2);
   for(var i = 0; i < hexstr.length/2 ; i++) {
      buf[i] = (parseInt(hexstr[i * 2], 16) << 4) + (parseInt(hexstr[i * 2 + 1], 16));
   }
   return buf;
 }

Then define the data you want to send. The push payload is a serialized JSON string, that has one mandatory property ‘aps’. The JSON may contain additionally application specific custom properties.

var pushnd = { aps: { alert:'This is a test' }};
// Push token from iPhone app. 32 bytes as hexadecimal string
var hextoken = '85ab4a0cf2 ... 238adf';  

Now we can construct the actual push binary PDU (Protocol Data Unit). Note that payload length is encoded UTF-8 string length, not number of characters. This would be also good place to check the maximum payload length (255 bytes).

payload = JSON.stringify(pushnd);
var payloadlen = Buffer.byteLength(payload, 'utf-8');
var tokenlen = 32;
var buffer = new Buffer(1 +  4 + 4 + 2 + tokenlen + 2 + payloadlen);
var i = 0;
buffer[i++] = 1; // command
var msgid = 0xbeefcace; // message identifier, can be left 0
buffer[i++] = msgid >> 24 & 0xFF;
buffer[i++] = msgid >> 16 & 0xFF;
buffer[i++] = msgid >> 8 & 0xFF;
buffer[i++] = msgid > 0xFF;

// expiry in epoch seconds (1 hour)
var seconds = Math.round(new Date().getTime() / 1000) + 1*60*60;
buffer[i++] = seconds >> 24 & 0xFF;
buffer[i++] = seconds >> 16 & 0xFF;
buffer[i++] = seconds >> 8 & 0xFF;
buffer[i++] = seconds > 0xFF;

buffer[i++] = tokenlen >> 8 & 0xFF; // token length
buffer[i++] = tokenlen & 0xFF;
var token = hextobin(hextoken);
token.copy(buffer, i, 0, tokenlen)
i += tokenlen;
buffer[i++] = payloadlen >> 8 & 0xFF; // payload length
buffer[i++] = payloadlen & 0xFF;

var payload = Buffer(payload);
payload.copy(buffer, i, 0, payloadlen);

stream.write(buffer);  // write push notification

And that’s it.

4. Handling Error Messages

Apple does not return anything from the socket unless there was an error.  In that case Apple server sends you single binary error message with reason code (offending message is identified by the message id you set in push message)  and closes connection immediately after that.

To parse error message. Stream encoding is utf-8, so we get buffer instance as data argument.

stream.on('data', function(data) {
   var command = data[0] & 0x0FF;  // always 8
   var status = data[1] & 0x0FF;  // error code
   var msgid = (data[2] << 24) + (data[3] << 16) + (data[4] << 8 ) + (data[5]);
   console.log(command+':'+status+':'+msgid);
 }

This implementation assumes that all data (6 bytes) is received on single event. In theory Node.js might return data in smaller pieces.

5. Reading Apple Feedback notifications

Apple requires that you read feedback notifications daily, so you know what push tokens have expired or app was uninstalled. See this blog post Polling Apple Push Notification feedback service with Node.js for details.