Reakt Kafka Example
Reakt-Kakfa adapts Kafka to Reakt promises and streams.
Reakt has promise libraries for Vert.x, Netty, Guava, and Cassandra.
Using Promises with Kafka Producers
finalAsyncProducer<Long, String> producer =newAsyncProducer<>(createProducer());
...
producer.send(TOPIC, key, value)
.catchError(throwable -> {
System.err.println("Trouble sending record "+ throwable.getLocalizedMessage());
throwable.printStackTrace(System.err);
})
.then(recordMetadata -> {
System.out.printf("%d %d %s \n", recordMetadata.offset(),
recordMetadata.partition(), recordMetadata.topic());
}).invoke();
Using Streams with Kafka Consumers
finalStreamConsumer<Long, String> stream =StreamConsumer.subscribe(createConsumer(), TOPIC, result -> {
result.then(consumerRecords -> {
System.out.println("Got message "+ consumerRecords.count());
consumerRecords.forEach(record -> {
countDownLatch.countDown();
});
result.request(1); //calls commitAsync
}).catchError(throwable -> {
System.err.println("Trouble Getting record "+ throwable.getLocalizedMessage());
throwable.printStackTrace(System.err);
result.cancel();
});
});
stream.close();
Full integration test and example for Reakt Kafka
packageio.advantageous.reakt.kafka;
importorg.apache.kafka.clients.consumer.Consumer;
importorg.apache.kafka.clients.consumer.ConsumerConfig;
importorg.apache.kafka.clients.consumer.KafkaConsumer;
importorg.apache.kafka.clients.producer.KafkaProducer;
importorg.apache.kafka.clients.producer.Producer;
importorg.apache.kafka.clients.producer.ProducerConfig;
importorg.apache.kafka.common.serialization.LongDeserializer;
importorg.apache.kafka.common.serialization.LongSerializer;
importorg.apache.kafka.common.serialization.StringDeserializer;
importorg.apache.kafka.common.serialization.StringSerializer;
importorg.junit.Test;
importjava.util.Properties;
importjava.util.concurrent.CountDownLatch;
importjava.util.concurrent.ExecutorService;
importjava.util.concurrent.Executors;
importjava.util.concurrent.TimeUnit;
import staticorg.junit.Assert.assertEquals;
publicclassIntegrationTest {
privatefinalstaticStringTOPIC="my-test-topic";
privatefinalstaticStringBOOTSTRAP_SERVERS="localhost:9092";
privatefinalstaticintSEND_RECORD_COUNT=10_000;
@Test
publicvoidtest() throwsException {
finalKafkaServer kafkaServer =newKafkaServer();
System.out.println("Starting server");
Thread.sleep(10_000);
finalExecutorService executorService =Executors.newSingleThreadExecutor();
finalAsyncProducer<Long, String> producer =newAsyncProducer<>(createProducer());
executorService.execute(() -> {
for (int i =0; i <SEND_RECORD_COUNT; i++) {
if (i %1000==0) System.out.println("Sending message "+ i);
producer.send(TOPIC, 1L* i, "value "+ i)
.catchError(throwable -> {
System.err.println("Trouble sending record "+ throwable.getLocalizedMessage());
throwable.printStackTrace(System.err);
})
.then(recordMetadata -> {
if (recordMetadata.offset() %1000==0)
System.out.printf("%d %d %s \n", recordMetadata.offset(),
recordMetadata.partition(), recordMetadata.topic());
})
.invoke();
}
producer.flush();
});
finalCountDownLatch countDownLatch =newCountDownLatch(SEND_RECORD_COUNT);
finalStreamConsumer<Long, String> stream =StreamConsumer.subscribe(createConsumer(), TOPIC, result -> {
result.then(consumerRecords -> {
System.out.println("Got message "+ consumerRecords.count());
consumerRecords.forEach(record -> {
countDownLatch.countDown();
});
result.request(1);
}).catchError(throwable -> {
System.err.println("Trouble Getting record "+ throwable.getLocalizedMessage());
throwable.printStackTrace(System.err);
result.cancel();
});
});
Thread.sleep(3_000);
countDownLatch.await(10, TimeUnit.SECONDS);
assertEquals(0, countDownLatch.getCount());
stream.close();
producer.close();
executorService.shutdown();
Thread.sleep(3_000);
kafkaServer.shutdown();
Thread.sleep(3_000);
}
privatestaticProducer<Long, String>createProducer() {
Properties props =newProperties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
props.put(ProducerConfig.CLIENT_ID_CONFIG, "KafkaExampleProducer");
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName());
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
returnnewKafkaProducer<>(props);
}
privatestaticConsumer<Long, String>createConsumer() {
Properties props =newProperties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
props.put(ConsumerConfig.GROUP_ID_CONFIG, "KafkaExampleConsumer");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
LongDeserializer.class.getName());
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
StringDeserializer.class.getName());
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000);
returnnewKafkaConsumer<>(props);
}
}
Notice that we use an embedded version of Kafka.
Kafka embedded
packageio.advantageous.reakt.kafka;
importorg.apache.zookeeper.server.ServerConfig;
importorg.apache.zookeeper.server.ZooKeeperServerMain;
importorg.apache.zookeeper.server.quorum.QuorumPeerConfig;
importjava.io.IOException;
importjava.util.Properties;
importkafka.server.KafkaConfig;
importkafka.server.KafkaServerStartable;
publicclassKafkaServer {
privatefinalZooKeeperServerMain zooKeeperServer;
privatefinalKafkaServerStartable kafkaServer;
publicKafkaServer() {
finalProperties zkProperties =newProperties();
finalProperties kafkaProperties =newProperties();
try {
//load properties
kafkaProperties.load(Class.class.getResourceAsStream("/io/advantageous/reakt/kafka/kafka.properties"));
zkProperties.load(Class.class.getResourceAsStream("/io/advantageous/reakt/kafka/zookeeper.properties"));
} catch (Exception e){
thrownewRuntimeException(e);
}
QuorumPeerConfig quorumConfiguration =newQuorumPeerConfig();
try {
quorumConfiguration.parseProperties(zkProperties);
} catch(Exception e) {
thrownewRuntimeException(e);
}
zooKeeperServer =newZooKeeperServerMain();
finalServerConfig configuration =newServerConfig();
configuration.readFrom(quorumConfiguration);
newThread(() -> {
try {
zooKeeperServer.runFromConfig(configuration);
} catch (IOException e) {
e.printStackTrace(System.err);
}
}).start();
//start local kafka broker
kafkaServer =newKafkaServerStartable(newKafkaConfig(kafkaProperties));
kafkaServer.startup();
}
publicvoidshutdown() {
kafkaServer.shutdown();
}
publicstaticvoidmain(String[] args) {
newKafkaServer();
}
}