Keeping CouchDB design docs up to date with Node.js

CouchDB views are defined typically as Javascript snippets and are part of special documents called design documents. I noticed that keeping these design documents up to date during development is pretty cumbersome and error prone. So I devised simply way to keep them updated using Node.js and Cradle couchdb driver.

Idea is to define the views in as variables in runnable js script and run that with Node each time it’s changed.

Here is the code. Copy it to e.g. cdb-views.js.

var cradle = require('cradle');

cradle.setup({ host: 'localhost',
               port: 5984,
               options: { cache:true, raw: false }});

var cclient = new (cradle.Connection)

function _createdb(dbname) {
    var db = cclient.database(dbname);
    db.exists(function(err, exists) {
        if (!exists) {
            db.create()
        }
    });
    return db;
}
var DB_SOMETHING = _createdb('somedb')

function cradle_error(err, res) {
    if (err) console.log(err)
}


function update_views( db, docpath, code ) {

    function save_doc() {
        db.save(docpath, code, function(err) {
            // view has changed, so initiate cleanup to get rid of old
            // indexes
            db.viewCleanup( cradle_error );
        });

        return true;
    }

    function compare_code( str1, str2 ) {
        var p1 = str1.split('\n');
        var p2 = str2.split('\n');

        for ( var i=0; i < p1.length || i < p2.length; i++ ) {
            var l1 = p1[i];
            var l2 = p2[i];
            l1 = l1 ? l1.trim() : '';
            l2 = l2 ? l2.trim() : '';
            if ( !l1 && !l2 ) continue;
            if ( l1 != l2 ) return true;
        }
        return false;
    }

    // compare function definitions in document and in code
    function compare_def(docdef, codedef) {
        var i = 0;

        if (!docdef && codedef) {
            console.log('creating "' + docpath +'"')
            return true;
        }
        if (!codedef && docdef) {
            console.log('removing "' + docpath +'"')
            return true;
        }
        if (!codedef && !docdef) {
            return false;
        }

        for (var u in docdef) {

            i++;
            if (codedef[u] == undefined) {
                console.log('definition of "' + u + '" removed - updating "' + docpath +'"')
                return true;
            }

            if (typeof(codedef[u]) == 'function') {
                if (!codedef[u] || compare_code( docdef[u], codedef[u].toString()) ) {
                    console.log('definition of "' + u + '" changed - updating "' + docpath +'"')
                    return true;
                }
            } else for (var f in docdef[u]) {
                i++;
                if (!codedef[u][f] || compare_code( docdef[u][f], codedef[u][f].toString()) ) {
                    console.log('definition of "' + u + '.' + f + '" changed - updating "' + docpath +'"')
                    return true;
                }

            }
        }
        // check that both doc and code have same number of functions
        for (var u in codedef) {
            i--;
            if (typeof(codedef[u]) != 'function') {
                for (var f in codedef[u]) {
                    i--;
                }
            }
        }
        if (i != 0) {
            console.log('new definitions - updating "' + docpath +'"')
            return true;
        }

        return false;
    }

    db.get(docpath, function(err, doc) {

        if (!doc) {
            console.log('not found - creating "' + docpath +'"')
            return save_doc();
        }

        if (compare_def(doc.updates, code.updates) || compare_def(doc.views, code.views)) {
            return save_doc();
        }
        console.log('"' + docpath +'" up to date')
    });
}

var EXAMPLE1_DDOC = {
    language: 'javascript',
    views: {
        active: {
            map: function (doc) {
                if (doc.lastsession) {
                    emit(parseInt(doc.lastsession / 1000), 1)
                }
            },
            reduce: function(keys, counts, rereduce) {
                return sum(counts)
            }
        },
        users: function(doc) { 
            if (doc.created) {
                emit(parseInt(doc.created / 1000), 1)
            }
        }
    }    
}

var EXAMPLE2_DDOC = {
    language: 'javascript',
    views: {
        myview: function(doc) {
            if (doc.param1 && doc.param2) {
                emit([doc.param1, doc.param2], null)
            }
        }
    }
}

update_views(DB_SOMETHING, '_design/example1', EXAMPLE1_DDOC);
update_views(DB_SOMETHING, '_design/example2', EXAMPLE2_DDOC);

The code is pretty simple.

  1. First it loads the Cradle couchdb driver and creates needed databases if they do not already exist. In this example only single database ‘somedb’ is created.
  2. The update_views is responsible of keeping the design docs up to date. It loads the design doc from defined DB and compares it to the code defined in the design doc in this file. If it has changed (or missing) it will be recreated.
  3. The example design docs (EXAMPLE1_DDOC and EXAMPLE2_DDOC) are simple design doc definitions as Javascript object. You’re familiar with CouchDB so this is self explanatory.
  4. Lastly the code just calls the update_views to update the design documents.

Now it’s possible to maintain the views in this Javascript file, the Node will make sure that the syntax is always valid.

Example output:

Views are up to date.

$ node cdb-views.js
"_design/example1" up to date
"_design/example2" up to date

Definition of view example2/myview has changed

$ node cdb-views.js
"_design/example1" up to date
definition of "myview" changed - updating "_design/example2"

Design doc example2 can not be found and is created.

$ node cdb-views.js
"_design/example1" up to date
no design doc found updating "_design/example2"

 

 

Callbacks from Threaded Node.js C++ Extension

UPDATE: this guide is bit outdated, new Node (0.6 > ) versions support easier way to access pooled worker threads so extension doesn’t need to create its own. See links in comments.

Writing threaded Node.js extension requires some care. All Javascript in Node.js is executed in single main thread, so you can not simply call the V8 engine directly from your background thread. That would cause segfault. Recommended way to do this is to spawn new thread on background and use the libev events to wake up the main thread to execute the Javascript callbacks.

Node.js framework has lots of ready stuff for implementing extensions, but there is no simple example how to implement this kind extension so here it is.

Add-on Source

Save this source to texample.cc

#include <queue>

// node headers
#include <v8.h>
#include <node.h>
#include <ev.h>
#include <pthread.h>
#include <unistd.h>
#include <string.h>

using namespace node;
using namespace v8;

// handles required for callback messages
static pthread_t texample_thread;
static ev_async eio_texample_notifier;
Persistent<String> callback_symbol;
Persistent<Object> module_handle;

// message queue
std::queue<int> cb_msg_queue = std::queue<int>();
pthread_mutex_t queue_mutex = PTHREAD_MUTEX_INITIALIZER;

// The background thread
static void* TheThread(void *)
{
    int i = 0;
    while(true) {
         // fire event every 5 seconds
        sleep(5);
       pthread_mutex_lock(&queue_mutex);
       cb_msg_queue.push(i);
       pthread_mutex_unlock(&queue_mutex);
       i++;
      // wake up callback
      ev_async_send(EV_DEFAULT_UC_ &eio_texample_notifier);
    }
    return NULL;
}

// callback that runs the javascript in main thread
static void Callback(EV_P_ ev_async *watcher, int revents)
{
    HandleScope scope;

    assert(watcher == &eio_texample_notifier);
    assert(revents == EV_ASYNC);

    // locate callback from the module context if defined by script
    // texample = require('texample')
    // texample.callback = function( ... ) { ..
    Local<Value> callback_v = module_handle->Get(callback_symbol);
    if (!callback_v->IsFunction()) {
         // callback not defined, ignore
         return;
    }
    Local<Function> callback = Local<Function>::Cast(callback_v);

    // dequeue callback message
    pthread_mutex_lock(&queue_mutex);
    int number = cb_msg_queue.front();
    cb_msg_queue.pop();
    pthread_mutex_unlock(&queue_mutex);

    TryCatch try_catch;

    // prepare arguments for the callback
    Local<Value> argv[1];
    argv[0] = Local<Value>::New(Integer::New(number));

    // call the callback and handle possible exception
    callback->Call(module_handle, 1, argv);

    if (try_catch.HasCaught()) {
        FatalException(try_catch);
    }
}

// Start the background thread
Handle<Value> Start(const Arguments &args)
{
    HandleScope scope;

    // start background thread and event handler for callback
    ev_async_init(&eio_texample_notifier, Callback);
    //ev_set_priority(&eio_texample_notifier, EV_MAXPRI);
    ev_async_start(EV_DEFAULT_UC_ &eio_texample_notifier);
    ev_unref(EV_DEFAULT_UC);
    pthread_create(&texample_thread, NULL, TheThread, 0);

    return True();
}

void Initialize(Handle<Object> target)
{
    HandleScope scope;

    NODE_SET_METHOD(target, "start", Start);

    callback_symbol = NODE_PSYMBOL("callback");
    // store handle for callback context
    module_handle = Persistent<Object>::New(target);
}

extern "C" {
static void Init(Handle<Object> target)
{
    Initialize(target);
}

NODE_MODULE(texample, Init);

Function walkthrough

  • The Init function gets called when you require('texample') the native module.
  • Initialize function defines module function start that will be called by javascript. It also stores the module handle for locating and calling the script defined callback on right context.
  • Start function initializes the libev event notifier and starts the background thread TheThread
  • Thread TheThread simply loops, sleeps and puts incremental integers to the queue and wakes up the main thread each time.
  • Callback function gets waken up the libev and it locates and calls the javascript function callback

Building

Copy this to the ‘wscript’  file.

def set_options(opt):
  opt.tool_options("compiler_cxx")

def configure(conf):
  conf.check_tool("compiler_cxx")
  conf.check_tool("node_addon")

def build(bld):
  obj = bld.new_task_gen("cxx", "shlib", "node_addon")
  obj.cxxflags = ["-g", "-D_FILE_OFFSET_BITS=64",
                  "-D_LARGEFILE_SOURCE", "-Wall"]
  obj.target = "texample"
  obj.source = "texample.cc"

Compile the code with node-waf

$ node-waf configure
$ node-waf build

Running

Start node shell and load the native add on module

$ node
> texample = require('./build/default/texample');

Define the callback function that the module will call

> texample.callback = function(i) {
... console.log('Bang: ' + i);
... }
>

Call start to kick of the background thread

> texample.start();
true
>

Wait for 5 seconds, you’ll start seeing your callback getting triggered every 5 seconds.

> Bang: 0
Bang: 1
> Bang: 2

Have fun!

Node.js TLS client example

Couldn’t find good end to end example of Node.js new raw SSL client API, so here is one. This snippet just connects to the https://encrypted.google.com and fetches the front page.

tls = require('tls');

// callback for when secure connection established
function connected(stream) {
    if (stream) {
       // socket connected
      stream.write("GET / HTTP/1.0\n\rHost: encrypted.google.com:443\n\r\n\r");  
    } else {
      console.log("Connection failed");
    }
}

// needed to keep socket variable in scope
var dummy = this;

// try to connect to the server
dummy.socket = tls.connect(443, 'encrypted.google.com', function() {
   // callback called only after successful socket connection
   dummy.connected = true;
   if (dummy.socket.authorized) {
      // authorization successful
      dummy.socket.setEncoding('utf-8');
      connected(dummy.socket);
   } else {
      // authorization failed
     console.log(dummy.socket.authorizationError);
     connected(null);
   }
});

dummy.socket.addListener('data', function(data) {
   // received data
   console.log(data);
});

dummy.socket.addListener('error', function(error) {
   if (!dummy.connected) {
     // socket was not connected, notify callback
     connected(null);
   }
   console.log("FAIL");
   console.log(error);
});

dummy.socket.addListener('close', function() {
 // do something
});

If you want to use client certificate authentication, define the options and give that as additional parameter to the tls.connect call.

var keyPem = fs.readFileSync("key-noenc.pem", encoding='ascii');
var certPem = fs.readFileSync("cert.pem", encoding='ascii');
var options = {key:keyPem, cert:certPem };

...
dummy.socket = tls.connect(443, 'some.example.com', options, function() {
....

Node.js Application Configuration Files

What is the best practice to make configuration file for your Node.js application? Writing property file parser or passing parameters at command line is cumbersome.

Eval

One easy way to separate configuration and application code is by using eval statement. Define your configuration as simple Javascript associative array and load and evaluage it on app startup.

Example configuration file myconfig.js

settings = {
    a: 10,
    // this is used for something
    SOME_FILE: "/tmp/something"
}

Then at start of your application

var fs = require('fs');
eval(fs.readFileSync('myconfig.js', encoding="ascii"));

Now settings object can be used as your program settings. e.g.

var mydata = fs.readFileSync(settings.SOME_FILE);
for( i = 0 ; i < settings.a ; i++) {
   // do something
}

Require

Another alternative to load configuration, as stated in comments, is to define configuration as module file and require it.

//-- configuration.js
module.exports = {
  a: 10,
  SOME_FILE: '/tmp/foo'
}

In application code then require file

var settings = require('./configuration');

This prevents other global variable creeping in global scope, but it’s hackier to do dynamic configuration reloading. If you detect that file has changed, and would want to reload it at runtime you must delete entry from require cache and re-require the file. Another minor complication is that require uses its own search path (that you can override with NODE_PATH env. variable) so it’s more work to define dynamic location for configuration file in your app. (e.g. set it from command line).

// to reload file with require
var path = require('path');
var filename = path.resolve('./configuration.js');
delete require.cache[filename];
var tools = require('./configuration');

Plain javascript as configure file has benefit (and downside) that it’s possible to run any javascript in the settings. For example.

settings = {
    started: new Date(),
    nonce: ~~(1E6 * Math.random()),
    a: 10,
    SOME_FILE: "/tmp/something"
}

Both of these methods are mostly matter of taste. Eval is bit riskier as it allows leaking variables to global namespace but you’ll never have anything “stupid” in the configuration files anyway. Right?

JSON file

I’m not fan of using JSON as configuration format as it’s cumbersome to write and most editors are not able to show syntax errors in it. JSON also does not support comments that can be a big problem in more complicated configuration files.
Example configuration file myconfig.json

{
   "a":10,
   "SOME_FILE":"/tmp/something"
}

Then at start of your application read json file and parse it to object.

var fs = require('fs');
var settings = JSON.parse(fs.readFileSync('myconfig.json', encoding="ascii"));

And then use settings as usual

var mydata = fs.readFileSync(settings.SOME_FILE);
for( i = 0 ; i < settings.a ; i++) {
   // do something
}

Merging configuration files

One way to simplify significantly configuration management is to do hierarchical configuration. For example have single base configuration file and then define overrides for developer, testing and production use.

For this we need merge function.

// merges o2 properties to o1
var merge = exports.merge = function(o1, o2) {
    for (var prop in o2) {
         var val = o2[prop];
         if (o1.hasOwnProperty(prop)) {
             if (typeof val == 'object') {
                 if (val && val.constructor != Array) { // not array
                     val = merge(o1[prop], val);
                 }
             }
         }
         o1[prop] = val; // copy and override
    }
    return o1;
}

You can use merge to combine configurations. For example, lets have these two configuration objects.

// base configuration from baseconf.js
var baseconfig = {
    a: "someval",
    env: {
        name "base",
        code: 1
    }
}

// test config from localconf.js
var localconfig = {
    env: { 
        name "test"
        db: 'localhost'
    },
    test: true
}

Now it’s possible to merge these easily

var settings = merge( baseconfig, localconfig );

console.log( settings. a ); // prints 'someval'
console.log( settings.env.name ); // prints 'test'
console.log( settings.env.code ); // prints '1'
console.log( settings.env.db ); // prints 'localhost'
console.log( settings.env.test ); // prints 'true'

Socket Pooling on Node.js

UPDATE: This post is pretty old and nowadays there are quite a few socket pooling implementations for Node.js. I recommend taking look into Jackpot (https://github.com/3rd-Eden/jackpot). It’s simple and does the job.

I was looking for connection pooling for my Apple Push notification proxy but couldn’t find proper pool implementation, just some experiments that didn’t do proper error handling or could not handle basic real life requirements. I also needed a pool that could be called by both blocking and non-blocking mode.

Basic functionality requirements of a typical connection pool

  • Ensure that connections are available and no more than maximum number of parallel connections exists
  • Do not keep unnecessary connections open for long. Less traffic, less connections.
  • Handle connection decay. For example if connection gets closed by remote peer while it’s waiting in pool.
  • Sane retry logic. Most importantly do not retry connections as fast as CPU allows in case of error.
  • Can either guarantee waiting time, or supports way of checking connection availability.

Lets define first the interface as Node.js module that exports reserve and release methods.

exports.ConnectionPool = function(factory) {
 var self = {};
 var waitlist = [] ;   // callbacks waiting for connection
 var connections = [];  // unused connections currently in pool
 var ccount = 0;  // number of current connections

 // called to reserve connection from pool. Calls callback without connection if wait is false
 self.reserve = function(callback, wait) { ... }
 // returns connection to the pool. Connection is destroyed if destroy is true
 self.release = function(connection, destroy) { ... }
}

Pool requires user provided factory that is dictionary that defines 3 functions and one property.

factory = {
create : function(callback) { ... }  // create connection and call callback with it
validate: function(validate) { .. } // return true or false for connection
destroy: function(connection) { .. }  // destroy connection
max: 5
}

Pool implementation needs several methods for house keeping. checkWaiters() is called to create new connections for waiting callbacks.

function checkWaiters() {
  if(waitlist.length > 0 && ccount < factory.max) {
     ccount++;
     factory.create(function(connection) {          
       if(!connection) {
         ccount--; // failed                    
       } else {
         if(waitlist.length > 0)
            waitlist.shift()(connection);
         else
           connections.push(connection);
       }
    });                           
  }      
 }

Then its counterpart, destroyConnection() that removes connection from pool for good. This function can be called from several places and situations so it adds “deleted” flag to the connection to avoid duplicate processing. It also tries to recreate new connection immediately (if needed) by calling checkWaiters().

function destroyConnection(connection) {
    if(connection.destroyed) {
       return;
    }
    connection.destroyed = true;
    for(var i=0; i < connections.length; i++) {
        // remove from pool if it's there                                
        if(connection == connections[i]) {
           clearTimeout(connection.timeoutid);
           connections.splice(i,1);
        }
    }
    ccount--;
    factory.destroy(connection);

    // connection was lost, we need to create new one if there are       
    // waiting requests                                                  
    checkWaiters();
 }

Then the actual reserve interface method. Function has two modes, if wait is false it returns immediately if it fails to give connection, otherwise the callback goes to the waiting list.  Connections from pool are validated with factory before they are passed to the callback.

self.reserve = function(callback, wait) {
    if (wait == undefined) {
        wait = true;
    }
    if(connections.length > 0) {  // pool has available connections
        connection = connections.shift();
        if(factory.validate(connection)) {  // is it still valid
           clearTimeout(connection.timeoutid);  // cancel the cleanup timeout
           callback(connection);
           return;
        } else {
            destroyConnection(connection);  // stale connection
        }
    }
    if(ccount >= factory.max) { // maximum number of connections created
       if(!wait) {
          callback();
       } else {
          waitlist.push(callback);
       }
       return;
    }
    ccount++;  // try to create connection
    factory.create(function(connection) {
        if(!connection) {
           ccount--; // failed                                          
           if(!wait) {
              callback();
           } else {
              waitlist.push(callback);
           }
        } else {
          callback(connection); // connection created successfully
        }
    });
 }

And the release method. Release method destroys connection if requested and forgets about it then completely. Otherwise it tries to find callback from waiting list and passes the connection for immediate reuse. In case there is nobody waiting, it puts the connection back  to pool and times the connection cleanup event in 10 seconds.

self.release = function(connection, destroy) {
    if(destroy) {
       destroyConnection(connection);
    } else {
       if(waitlist.length > 0) {
          waitlist.shift()(connection);
         return;
       }
       connections.push(connection);
       connection.timeoutid = setTimeout(function() {
           destroyConnection(connection);
       }, 10000);
    }
 }

And finally the background polling that is responsible mainly of connection retry logic. It polls the waiting list once per second, as you remember that function creates connections if there is anyone waiting for it. The connections are normally created on demand in reserve() call.

function poll() {
   checkWaiters()
   setTimeout(poll, 1000);
}
setTimeout(poll, 1000);  // start poller

How this then handles the different error cases?

  • Idle connections are cleaned by timeout call that is set on release()
  • User code can request connection delete by setting the destroy flag to true in call to release()
  • Connections that go bad while in pool are (hopefully) intercepted by user provided factory.validate()
  • Counter keeps track of maximum number of created connections and because its increased before creating connection it also limits maximum number of parallel connection attempts!
  • When connection creation fails, the callback goes to waiting list and poll per second tries to create new parallel connections.
  • User code that needs process immediately can set wait flag to false when reserving connection. Code could be also changed to timed out callback call, so wait could be defined as milliseconds instead of instant fail v.s. infinite wait as its done now.

Then how to use it.

HTTP Pool Example

var pool = require('./pool');
 var http = require('http');
var httppool = pool.ConnectionPool({
 create: function(callback) { callback(http.createClient(80, "www.google.com")); },
 validate: function(connection) { return true; /* no need to validate  */ },
 destroy : function(httpclient) {  /* nothing to destroy */},
 max: settings.couchdbmax
 });

Using the pool

 httppool.reserve(function(connection) {
   var req = connection.request("GET", "/index.html");
   req.on('error', function(error) {
     httppool.release(connection, true);
   });
   req.on('response', function(response) {
     body = '';
     response.on('data', function(data) {
     body += data;
   });
   response.on('end', function() {
     console.log(body)
     httppool.release(connection);
   });
 });

Client socket pool

var pool = require('./pool');
var net = require('net');
var apnpool = pool.ConnectionPool({
 create: function(callback) {
   function errorcb(error) {  // error handler
     require('util').puts(error.stack);
     callback();
   }
   connection = net.createConnection(12345, 'server.example.com');
   connection.once('error', errorcb); 
   connection.on('connect', function() {
     connection.removeListener('error', errorcb); // clear error handler before passing forward
     callback(connection);
   });
 },
 validate: function(connection) { return connection.writable; },
 destroy : function(connection) { connection.end(); },
 max: 5,
 });

Reuse is little bit tricky with sockets, as you need probably set and clear ‘error’ and  ‘data’ event handlers for reading the responses in each worker.

 

 

Apple Push Notifications with Node.js

When your iPhone app backend  needs to send Apple Push Notifications, it must do this over raw SSL socket using Apple proprietary raw binary interface. Standard Web REST is not supported. This kind of sucks, because if your entire backend is web based you need to break that cleanliness with external HTTP to APN proxy. One option is to use services like Urban Airship, but you can also build the proxy by yourself.

One potential platform for this is hyped Node.js, the rising javascript engine for building ad-hoc web servers. Web is full of examples of building simple HTTP based server or proxy with Node.js, so this post is only the part where we open a secure connection to the Apple server and send push notifications with plain Node.js javascript.

Please note that Apple assumes that you pool and keep sockets open as long as you have notifications to send. So, don’t make naive implementation that makes new socket for each HTTP request. Some simple pooling and reuse is a must for real implementation.

In addition for sending the push notifications, your app also needs to poll the APNS feedback service to find out what devices have uninstalled the app and should not be pushed new notifications. See more details in post Apple Push Notification feedback service.

1. Get Certificates

Apple’s Push notification server authenticates application by SSL certificates. There is no additional authentication handshake after secure connection has been established.

First we need the PEM format certificates that you can get by exporting  them with Apple Keytool. Export also the Apple Worldwide CA certificate. See this excellent blog post (up to step 5)  for details how to acquire the PEM files: http://blog.boxedice.com/2010/06/05/how-to-renew-your-apple-push-notification-push-ssl-certificate/

Now you should have following certificate files.

  • app-cert.pem  (Application cerificate)
  • app-key-noenc.pem  (Application private key)
  • apple-worldwide-certificate-authority.cer  (Apple CA certificate)

2. Open Connection to Push Server

UPDATE:See more complete TLS example here.

Moving on the actual implementation in Node.js. This is quite simple, you just read the various certificate files as string and use them as credentials.

You must also have SSL support built in your Node.js binary.

var fs = require('fs');
var crypto = require('crypto');
var tls = require('tls');

var certPem = fs.readFileSync('app-cert.pem', encoding='ascii');
var keyPem = fs.readFileSync('app-key-noenc.pem', encoding='ascii');
var caCert = fs.readFileSync('apple-worldwide-certificate-authority.cer', encoding='ascii');
var options = { key: keyPem, cert: certPem, ca: [ caCert ] }

function connectAPN( next ) {
    var stream = tls.connect(2195, 'gateway.sandbox.push.apple.com', options, function() {
        // connected
        next( !stream.authorized, stream );
    });
}

3. Write Push Notification

After secure connection is established, you can simply write push notifications to the socket as binary data. Push notification is addressed to a device with 32 byte long push token that must be acquired by your iPhone application and sent to your backend somehow.

Easy format is simple hexadecimal string, so we define first a helper method to convert that hexadecimal string to binary buffer at server side.

function hextobin(hexstr) {
   buf = new Buffer(hexstr.length / 2);
   for(var i = 0; i < hexstr.length/2 ; i++) {
      buf[i] = (parseInt(hexstr[i * 2], 16) << 4) + (parseInt(hexstr[i * 2 + 1], 16));
   }
   return buf;
 }

Then define the data you want to send. The push payload is a serialized JSON string, that has one mandatory property ‘aps’. The JSON may contain additionally application specific custom properties.

var pushnd = { aps: { alert:'This is a test' }};
// Push token from iPhone app. 32 bytes as hexadecimal string
var hextoken = '85ab4a0cf2 ... 238adf';  

Now we can construct the actual push binary PDU (Protocol Data Unit). Note that payload length is encoded UTF-8 string length, not number of characters. This would be also good place to check the maximum payload length (255 bytes).

payload = JSON.stringify(pushnd);
var payloadlen = Buffer.byteLength(payload, 'utf-8');
var tokenlen = 32;
var buffer = new Buffer(1 +  4 + 4 + 2 + tokenlen + 2 + payloadlen);
var i = 0;
buffer[i++] = 1; // command
var msgid = 0xbeefcace; // message identifier, can be left 0
buffer[i++] = msgid >> 24 & 0xFF;
buffer[i++] = msgid >> 16 & 0xFF;
buffer[i++] = msgid >> 8 & 0xFF;
buffer[i++] = msgid > 0xFF;

// expiry in epoch seconds (1 hour)
var seconds = Math.round(new Date().getTime() / 1000) + 1*60*60;
buffer[i++] = seconds >> 24 & 0xFF;
buffer[i++] = seconds >> 16 & 0xFF;
buffer[i++] = seconds >> 8 & 0xFF;
buffer[i++] = seconds > 0xFF;

buffer[i++] = tokenlen >> 8 & 0xFF; // token length
buffer[i++] = tokenlen & 0xFF;
var token = hextobin(hextoken);
token.copy(buffer, i, 0, tokenlen)
i += tokenlen;
buffer[i++] = payloadlen >> 8 & 0xFF; // payload length
buffer[i++] = payloadlen & 0xFF;

var payload = Buffer(payload);
payload.copy(buffer, i, 0, payloadlen);

stream.write(buffer);  // write push notification

And that’s it.

4. Handling Error Messages

Apple does not return anything from the socket unless there was an error.  In that case Apple server sends you single binary error message with reason code (offending message is identified by the message id you set in push message)  and closes connection immediately after that.

To parse error message. Stream encoding is utf-8, so we get buffer instance as data argument.

stream.on('data', function(data) {
   var command = data[0] & 0x0FF;  // always 8
   var status = data[1] & 0x0FF;  // error code
   var msgid = (data[2] << 24) + (data[3] << 16) + (data[4] << 8 ) + (data[5]);
   console.log(command+':'+status+':'+msgid);
 }

This implementation assumes that all data (6 bytes) is received on single event. In theory Node.js might return data in smaller pieces.

5. Reading Apple Feedback notifications

Apple requires that you read feedback notifications daily, so you know what push tokens have expired or app was uninstalled. See this blog post Polling Apple Push Notification feedback service with Node.js for details.