Ejemplo Topología

    TopologyBuilder builder = new TopologyBuilder();

    /*
        Setup device payload processing
    */
    KafkaSpout deviceSpout = new KafkaSpout(deviceKafkaConf);
    builder.setSpout(deviceSpoutId, deviceSpout, 1);

    builder.setBolt(
      "fetchitem",
      new FetchItemFromInventoryBolt(pythonPath),
      10).shuffleGrouping(deviceSpoutId).shuffleGrouping("cloudpayload");
    builder.setBolt(
      "updateposted",
      new UpdatePostedCollectionBolt(pythonPath),
      2).shuffleGrouping("fetchitem");
    ...

    StormSubmitter.submitTopology(
        TOPOLOGY_NAME, conf, builder.createTopology());

Multilang

"Pegamento" entre Apache Storm y otros lenguajes de programación.

Protocolo de comunicación entre procesos mediante stdin/stdout multiservidor.

ShellBolt, ShellSpout

Ejemplo ShellBolt (Java)

public class SendToMetricsBolt extends ShellBolt implements IRichBolt {

  public SendToMetricsBolt(String pythonPath) {
    super(pythonPath, "send_to_metrics_bolt.py");
  }

  @Override
  public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declare(new Fields("payload"));
  }

  @Override
  public Map<String, Object> getComponentConfiguration() {
    return null;
  }
}

Ejemplo ShellBolt (python)

"""Code sending the converted payload to metrics.
"""
__all__ = [
    'SendToMetricsBolt',
]

from serverdensity.alerts_processing.storm.metrics import MetricsDispatcher
from serverdensity.alerts_processing.storm import SDBaseBolt

class SendToMetricsBolt(SDBaseBolt):

    def initialize_bolt(self, stormconf, context):
        self.dispatcher = MetricsDispatcher(self.statsd_conn, self.logger)

    def process_payload(self, payload):

        if not self.dispatcher.send_to_metrics_service(payload):
            self.logger.error(
                'SendToMetricsBolt unable to send to metrics',
                extra={'barium': self.barium})
            return False

        return True

if __name__ == "__main__":
    SendToMetricsBolt().run()

Ejemplo ShellBolt (emits)

        

        storm_helper.emit([json.dumps(preact_payload)], "preact")
        if payload_type == 'agent':
            storm_helper.emit([json.dumps(payload)])
        elif payload_type == 'cloud':
            storm_helper.emit([json.dumps(payload)], "cloudpayload")

Ejemplo ShellBolt (helper)

    def process(self, tup):
        """Process an storm tuple.

        :param tup: an Storm Tuple.
        """
        self.barium = 'NO_BARIUM'

        timer = statsd.timer.Timer(
            'alerts_processing.{0}.storm.{1}.process'.format(
                self.host_name, self.name), self.statsd_conn)
        timer.start()
        self.statsd_counter.increment('started')
        payload = tup.values[0]
        if isinstance(payload, basestring):
            try:
                payload = json.loads(payload)
            except JSONDecodeError:
                self.logger.error(
                    "Got a JSONDecodeError decoding payload '{0}'".format(payload),
                    extra={'barium': self.barium})
                raise
        self.barium = payload['barium']
       
        if self.process_payload(payload):
            self.statsd_counter.increment('success')
            timer.stop('success')
        else:
            self.statsd_counter.increment('failure')
            timer.stop('failure')