node-kafka/lib/producer/high-level-producer.js

/*
 * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
 *
 * Copyright (c) 2016 Blizzard Entertainment
 *
 * This software may be modified and distributed under the terms
 * of the MIT license.  See the LICENSE.txt file for details.
 */

module.exports = HighLevelProducer;

var util = require('util');
var Producer = require('../producer');
var LibrdKafkaError = require('../error');
var EventEmitter = require('events').EventEmitter;
var RefCounter = require('../tools/ref-counter');
var shallowCopy = require('../util').shallowCopy;
var isObject = require('../util').isObject;

util.inherits(HighLevelProducer, Producer);

var noopSerializer = createSerializer(function (v) { return v; });

/**
 * Create a serializer
 *
 * Method simply wraps a serializer provided by a user
 * so it adds context to the error
 *
 * @returns {function} Serialization function
 */
function createSerializer(serializer) {
  var applyFn = function serializationWrapper(v, cb) {
    try {
      return cb ? serializer(v, cb) : serializer(v);
    } catch (e) {
      var modifiedError = new Error('Could not serialize value: ' + e.message);
      modifiedError.value = v;
      modifiedError.serializer = serializer;
      throw modifiedError;
    }
  };

  // We can check how many parameters the function has and activate the asynchronous
  // operation if the number of parameters the function accepts is > 1
  return {
    apply: applyFn,
    async: serializer.length > 1
  };
}

/**
 * Producer class for sending messages to Kafka in a higher level fashion
 *
 * This is the main entry point for writing data to Kafka if you want more
 * functionality than librdkafka supports out of the box. You
 * configure this like you do any other client, with a global
 * configuration and default topic configuration.
 *
 * Once you instantiate this object, you need to connect to it first.
 * This allows you to get the metadata and make sure the connection
 * can be made before you depend on it. After that, problems with
 * the connection will by brought down by using poll, which automatically
 * runs when a transaction is made on the object.
 *
 * This has a few restrictions, so it is not for free!
 *
 * 1. You may not define opaque tokens
 *    The higher level producer is powered by opaque tokens.
 * 2. Every message ack will dispatch an event on the node thread.
 * 3. Will use a ref counter to determine if there are outgoing produces.
 *
 * This will return the new object you should use instead when doing your
 * produce calls
 *
 * @param {object} conf - Key value pairs to configure the producer
 * @param {object} topicConf - Key value pairs to create a default
 * topic configuration
 * @extends Producer
 * @constructor
 */
function HighLevelProducer(conf, topicConf) {
  if (!(this instanceof HighLevelProducer)) {
    return new HighLevelProducer(conf, topicConf);
  }

  // Force this to be true for the high level producer
  conf = shallowCopy(conf);
  conf.dr_cb = true;

  // producer is an initialized consumer object
  // @see NodeKafka::Producer::Init
  Producer.call(this, conf, topicConf);
  var self = this;

  // Add a delivery emitter to the producer
  this._hl = {
    deliveryEmitter: new EventEmitter(),
    messageId: 0,
    // Special logic for polling. We use a reference counter to know when we need
    // to be doing it and when we can stop. This means when we go into fast polling
    // mode we don't need to do multiple calls to poll since they all will yield
    // the same result
    pollingRefTimeout: null,
  };

  // Add the polling ref counter to the class which ensures we poll when we go active
  this._hl.pollingRef = new RefCounter(function() {
    self._hl.pollingRefTimeout = setInterval(function() {
      try {
        self.poll();
      } catch (e) {
        if (!self._isConnected) {
          // If we got disconnected for some reason there is no point
          // in polling anymore
          clearInterval(self._hl.pollingRefTimeout);
        }
      }
    }, 1);
  }, function() {
    clearInterval(self._hl.pollingRefTimeout);
  });

  // Default poll interval. More sophisticated polling is also done in create rule method
  this.setPollInterval(1000);

  // Listen to all delivery reports to propagate elements with a _message_id to the emitter
  this.on('delivery-report', function(err, report) {
    if (report.opaque && report.opaque.__message_id !== undefined) {
      self._hl.deliveryEmitter.emit(report.opaque.__message_id, err, report.offset);
    }
  });

  // Save old produce here since we are making some modifications for it
  this._oldProduce = this.produce;
  this.produce = this._modifiedProduce;

  // Serializer information
  this.keySerializer = noopSerializer;
  this.valueSerializer = noopSerializer;
}

/**
 * Produce a message to Kafka asynchronously.
 *
 * This is the method mainly used in this class. Use it to produce
 * a message to Kafka.
 *
 * When this is sent off, and you recieve your callback, the assurances afforded
 * to you will be equal to those provided by your ack level.
 *
 * @param {string} topic - The topic name to produce to.
 * @param {number|null} partition - The partition number to produce to.
 * @param {Buffer|null} message - The message to produce.
 * @param {string} key - The key associated with the message.
 * @param {number|null} timestamp - Timestamp to send with the message.
 * @param {object} headers - A list of custom key value pairs that provide message metadata.
 * @param {function} callback - Callback to call when the delivery report is recieved.
 * @throws {LibrdKafkaError} - Throws a librdkafka error if it failed.
 * @return {boolean} - returns an error if it failed, or true if not
 * @see Producer#produce
 */
HighLevelProducer.prototype._modifiedProduce = function(topic, partition, message, key, timestamp, headers, callback) {
  // headers are optional
  if (arguments.length === 6) {
    callback = headers;
    headers = undefined;
  }

  // Add the message id
  var opaque = {
    __message_id: this._hl.messageId++,
  };

  this._hl.pollingRef.increment();

  var self = this;

  var resolvedSerializedValue;
  var resolvedSerializedKey;
  var calledBack = false;

  // Actually do the produce with new key and value based on deserialized
  // results
  function doProduce(v, k) {
    try {
      var r = self._oldProduce(topic, partition,
        v, k,
        timestamp, opaque, headers);

      self._hl.deliveryEmitter.once(opaque.__message_id, function(err, offset) {
        self._hl.pollingRef.decrement();
        setImmediate(function() {
          // Offset must be greater than or equal to 0 otherwise it is a null offset
          // Possibly because we have acks off
          callback(err, offset >= 0 ? offset : null);
        });
      });

      return r;
    } catch (e) {
      callback(e);
    }
  }

  function produceIfComplete() {
    if (resolvedSerializedKey !== undefined && resolvedSerializedValue !== undefined) {
      doProduce(resolvedSerializedValue, resolvedSerializedKey);
    }
  }

  // To run on a promise if returned by the serializer
  function finishSerializedValue(v) {
    if (!calledBack) {
      resolvedSerializedValue = v;
      produceIfComplete();
    }
  }

  // To run on a promise of returned by the serializer
  function finishSerializedKey(k) {
    resolvedSerializedKey = k;

    if (!calledBack) {
      produceIfComplete();
    }
  }

  function failSerializedValue(err) {
    if (!calledBack) {
      calledBack = true;
      callback(err);
    }
  }

  function failSerializedKey(err) {
    if (!calledBack) {
      calledBack = true;
      callback(err);
    }
  }

  function valueSerializerCallback(err, v) {
    if (err) {
      failSerializedValue(err);
    } else {
      finishSerializedValue(v);
    }
  }

  function keySerializerCallback(err, v) {
    if (err) {
      failSerializedKey(err);
    } else {
      finishSerializedKey(v);
    }
  }

  try {
    if (this.valueSerializer.async) {
      // If this is async we need to give it a callback
      this.valueSerializer.apply(message, valueSerializerCallback);
    } else {
      var serializedValue = this.valueSerializer.apply(message);
      // Check if we were returned a promise in order to support promise behavior
      if (serializedValue &&
          typeof serializedValue.then === 'function' &&
          typeof serializedValue.catch === 'function') {
        // This is a promise. We need to hook into its then and catch
        serializedValue.then(finishSerializedValue).catch(failSerializedValue);
      } else {
        resolvedSerializedValue = serializedValue;
      }
    }

    if (this.keySerializer.async) {
      // If this is async we need to give it a callback
      this.keySerializer.apply(key, keySerializerCallback);
    } else {
      var serializedKey = this.keySerializer.apply(key);
      // Check if we were returned a promise in order to support promise behavior
      if (serializedKey &&
          typeof serializedKey.then === 'function' &&
          typeof serializedKey.catch === 'function') {
        // This is a promise. We need to hook into its then and catch
        serializedKey.then(finishSerializedKey).catch(failSerializedKey);
      } else {
        resolvedSerializedKey = serializedKey;
      }
    }

    // Only do the produce here if we are complete. That is, if the key
    // and value have been serialized.
    produceIfComplete();
  } catch (e) {
    setImmediate(function() {
      calledBack = true;
      callback(e);
    });
  }
};

/**
 * Set the key serializer
 *
 * This allows the value inside the produce call to differ from the value of the
 * value actually produced to kafka. Good if, for example, you want to serialize
 * it to a particular format.
 */
HighLevelProducer.prototype.setKeySerializer = function(serializer) {
  this.keySerializer = createSerializer(serializer);
};

/**
 * Set the value serializer
 *
 * This allows the value inside the produce call to differ from the value of the
 * value actually produced to kafka. Good if, for example, you want to serialize
 * it to a particular format.
 */
HighLevelProducer.prototype.setValueSerializer = function(serializer) {
  this.valueSerializer = createSerializer(serializer);
};