PubSub message queue system

high-throughput

scalable

durable

distributed

# Pobieramy ze strony kafka.apache.org binarki i rozpakowujemy
> wget http://ftp.ps.pl/pub/apache/kafka/0.8.2.1/kafka_2.11-0.8.2.1.tgz
> tar -xzf kafka_2.10-0.8.2.1.tgz
> cd kafka_2.10-0.8.2.1

# domyślnie mamy jeden plik konfiguracyjny, dodajmy dwa nowe
> cp config/server.properties config/server-1.properties
> cp config/server.properties config/server-2.properties

# config/server-1.properties:
    broker.id=1
    port=9093
    log.dir=/tmp/kafka-logs-1
 
# config/server-2.properties:
    broker.id=2
    port=9094
    log.dir=/tmp/kafka-logs-2

# Najpierw uruchamiamy ZooKeeper'a w jednym oknie terminala

> bin/zookeeper-server-start.sh config/zookeeper.properties

# Następnie w kolejnych oknach brokery 

# tab 1
> bin/kafka-server-start.sh config/server.properties

# tab 2
> bin/kafka-server-start.sh config/server-1.properties

# tab 3
> bin/kafka-server-start.sh config/server-2.properties

> bin/kafka-topics.sh --create \
    --zookeeper localhost:2181 \
    --replication-factor 3     \
    --partitions 6             \
    --topic brug-topic

Created topic "brug-topic".

> bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic brug-topic
    Topic:brug-topic	PartitionCount:6	ReplicationFactor:3	Configs:
    Topic: brug-topic	Partition: 0	Leader: 2	Replicas: 2,0,1	Isr: 2,0,1
    Topic: brug-topic	Partition: 1	Leader: 0	Replicas: 0,1,2	Isr: 0,1,2
    Topic: brug-topic	Partition: 2	Leader: 1	Replicas: 1,2,0	Isr: 1,2,0
    Topic: brug-topic	Partition: 3	Leader: 2	Replicas: 2,1,0	Isr: 2,1,0
    Topic: brug-topic	Partition: 4	Leader: 0	Replicas: 0,2,1	Isr: 0,2,1
    Topic: brug-topic	Partition: 5	Leader: 1	Replicas: 1,0,2	Isr: 1,0,2

# Producer
> bin/kafka-console-producer.sh --broker-list localhost:9092 \
    --topic brug-topic
$> 

# Consumer
> bin/kafka-console-consumer.sh --zookeeper localhost:2181 \
    --from-beginning \
    --topic brug-topic

DEMO