Reakt Kafka Example

Reakt-Kakfa adapts Kafka to Reakt promises and streams.

Reakt has promise libraries for Vert.x, Netty, Guava, and Cassandra.

Using Promises with Kafka Producers

finalAsyncProducer<Long, String> producer =newAsyncProducer<>(createProducer());
...
producer.send(TOPIC, key, value)
    .catchError(throwable -> {
System.err.println("Trouble sending record "+ throwable.getLocalizedMessage());
                     throwable.printStackTrace(System.err);
    })
    .then(recordMetadata -> {
System.out.printf("%d %d %s \n", recordMetadata.offset(),
                recordMetadata.partition(), recordMetadata.topic());
    }).invoke();

Using Streams with Kafka Consumers

finalStreamConsumer<Long, String> stream =StreamConsumer.subscribe(createConsumer(), TOPIC, result -> {
    result.then(consumerRecords -> {
System.out.println("Got message "+ consumerRecords.count());
        consumerRecords.forEach(record -> {
            countDownLatch.countDown();
        });
        result.request(1); //calls commitAsync
    }).catchError(throwable -> {
System.err.println("Trouble Getting record "+ throwable.getLocalizedMessage());
        throwable.printStackTrace(System.err);
        result.cancel();
    });
});

stream.close();

Full integration test and example for Reakt Kafka

packageio.advantageous.reakt.kafka;

importorg.apache.kafka.clients.consumer.Consumer;
importorg.apache.kafka.clients.consumer.ConsumerConfig;
importorg.apache.kafka.clients.consumer.KafkaConsumer;
importorg.apache.kafka.clients.producer.KafkaProducer;
importorg.apache.kafka.clients.producer.Producer;
importorg.apache.kafka.clients.producer.ProducerConfig;
importorg.apache.kafka.common.serialization.LongDeserializer;
importorg.apache.kafka.common.serialization.LongSerializer;
importorg.apache.kafka.common.serialization.StringDeserializer;
importorg.apache.kafka.common.serialization.StringSerializer;
importorg.junit.Test;

importjava.util.Properties;
importjava.util.concurrent.CountDownLatch;
importjava.util.concurrent.ExecutorService;
importjava.util.concurrent.Executors;
importjava.util.concurrent.TimeUnit;

import staticorg.junit.Assert.assertEquals;

publicclassIntegrationTest {

privatefinalstaticStringTOPIC="my-test-topic";
privatefinalstaticStringBOOTSTRAP_SERVERS="localhost:9092";
privatefinalstaticintSEND_RECORD_COUNT=10_000;


@Test
publicvoidtest() throwsException {

finalKafkaServer kafkaServer =newKafkaServer();
System.out.println("Starting server");
Thread.sleep(10_000);

finalExecutorService executorService =Executors.newSingleThreadExecutor();

finalAsyncProducer<Long, String> producer =newAsyncProducer<>(createProducer());

        executorService.execute(() -> {
for (int i =0; i <SEND_RECORD_COUNT; i++) {
if (i %1000==0) System.out.println("Sending message "+ i);
                producer.send(TOPIC, 1L* i, "value "+ i)
                        .catchError(throwable -> {
System.err.println("Trouble sending record "+ throwable.getLocalizedMessage());
                            throwable.printStackTrace(System.err);
                        })
                        .then(recordMetadata -> {
if (recordMetadata.offset() %1000==0)
System.out.printf("%d %d %s \n", recordMetadata.offset(),
                                        recordMetadata.partition(), recordMetadata.topic());
                        })
                        .invoke();
            }
            producer.flush();
        });


finalCountDownLatch countDownLatch =newCountDownLatch(SEND_RECORD_COUNT);

finalStreamConsumer<Long, String> stream =StreamConsumer.subscribe(createConsumer(), TOPIC, result -> {
            result.then(consumerRecords -> {
System.out.println("Got message "+ consumerRecords.count());
                consumerRecords.forEach(record -> {
                    countDownLatch.countDown();
                });
                result.request(1);
            }).catchError(throwable -> {
System.err.println("Trouble Getting record "+ throwable.getLocalizedMessage());
                throwable.printStackTrace(System.err);
                result.cancel();
            });
        });

Thread.sleep(3_000);

        countDownLatch.await(10, TimeUnit.SECONDS);
        assertEquals(0, countDownLatch.getCount());
        stream.close();
        producer.close();
        executorService.shutdown();

Thread.sleep(3_000);
        kafkaServer.shutdown();
Thread.sleep(3_000);

    }


privatestaticProducer<Long, String>createProducer() {
Properties props =newProperties();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
        props.put(ProducerConfig.CLIENT_ID_CONFIG, "KafkaExampleProducer");
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName());
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

returnnewKafkaProducer<>(props);
    }

privatestaticConsumer<Long, String>createConsumer() {
Properties props =newProperties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "KafkaExampleConsumer");
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
LongDeserializer.class.getName());
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
StringDeserializer.class.getName());

        props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000);
returnnewKafkaConsumer<>(props);
    }
}

Notice that we use an embedded version of Kafka.

Kafka embedded

packageio.advantageous.reakt.kafka;

importorg.apache.zookeeper.server.ServerConfig;
importorg.apache.zookeeper.server.ZooKeeperServerMain;
importorg.apache.zookeeper.server.quorum.QuorumPeerConfig;

importjava.io.IOException;
importjava.util.Properties;

importkafka.server.KafkaConfig;
importkafka.server.KafkaServerStartable;

publicclassKafkaServer {

privatefinalZooKeeperServerMain zooKeeperServer;
privatefinalKafkaServerStartable kafkaServer;

publicKafkaServer() {

finalProperties zkProperties =newProperties();
finalProperties kafkaProperties =newProperties();
try {
//load properties
            kafkaProperties.load(Class.class.getResourceAsStream("/io/advantageous/reakt/kafka/kafka.properties"));
            zkProperties.load(Class.class.getResourceAsStream("/io/advantageous/reakt/kafka/zookeeper.properties"));
        } catch (Exception e){
thrownewRuntimeException(e);
        }


QuorumPeerConfig quorumConfiguration =newQuorumPeerConfig();
try {
            quorumConfiguration.parseProperties(zkProperties);
        } catch(Exception e) {
thrownewRuntimeException(e);
        }

        zooKeeperServer =newZooKeeperServerMain();
finalServerConfig configuration =newServerConfig();
        configuration.readFrom(quorumConfiguration);


newThread(() -> {
try {
                zooKeeperServer.runFromConfig(configuration);
            } catch (IOException e) {
                e.printStackTrace(System.err);
            }
        }).start();



//start local kafka broker
        kafkaServer =newKafkaServerStartable(newKafkaConfig(kafkaProperties));
        kafkaServer.startup();


    }

publicvoidshutdown() {
        kafkaServer.shutdown();
    }

publicstaticvoidmain(String[] args) {
newKafkaServer();
    }
}