Quantcast
Channel: Sleepless Dev
Viewing all articles
Browse latest Browse all 217

Reakt Kafka Example

$
0
0

Reakt Kafka Example

Reakt-Kakfa adapts Kafka to Reakt promises and streams.
Reakt has promise libraries for Vert.x, Netty, Guava, and Cassandra.

Using Promises with Kafka Producers

finalAsyncProducer<Long, String> producer =newAsyncProducer<>(createProducer());
...
producer.send(TOPIC, key, value)
.catchError(throwable -> {
System.err.println("Trouble sending record "+ throwable.getLocalizedMessage());
throwable.printStackTrace(System.err);
})
.then(recordMetadata -> {
System.out.printf("%d %d %s \n", recordMetadata.offset(),
recordMetadata.partition(), recordMetadata.topic());
}).invoke();

Using Streams with Kafka Consumers

finalStreamConsumer<Long, String> stream =StreamConsumer.subscribe(createConsumer(), TOPIC, result -> {
result.then(consumerRecords -> {
System.out.println("Got message "+ consumerRecords.count());
consumerRecords.forEach(record -> {
countDownLatch.countDown();
});
result.request(1); //calls commitAsync
}).catchError(throwable -> {
System.err.println("Trouble Getting record "+ throwable.getLocalizedMessage());
throwable.printStackTrace(System.err);
result.cancel();
});
});

stream.close();

Full integration test and example for Reakt Kafka

packageio.advantageous.reakt.kafka;

importorg.apache.kafka.clients.consumer.Consumer;
importorg.apache.kafka.clients.consumer.ConsumerConfig;
importorg.apache.kafka.clients.consumer.KafkaConsumer;
importorg.apache.kafka.clients.producer.KafkaProducer;
importorg.apache.kafka.clients.producer.Producer;
importorg.apache.kafka.clients.producer.ProducerConfig;
importorg.apache.kafka.common.serialization.LongDeserializer;
importorg.apache.kafka.common.serialization.LongSerializer;
importorg.apache.kafka.common.serialization.StringDeserializer;
importorg.apache.kafka.common.serialization.StringSerializer;
importorg.junit.Test;

importjava.util.Properties;
importjava.util.concurrent.CountDownLatch;
importjava.util.concurrent.ExecutorService;
importjava.util.concurrent.Executors;
importjava.util.concurrent.TimeUnit;

import staticorg.junit.Assert.assertEquals;

publicclassIntegrationTest {

privatefinalstaticStringTOPIC="my-test-topic";
privatefinalstaticStringBOOTSTRAP_SERVERS="localhost:9092";
privatefinalstaticintSEND_RECORD_COUNT=10_000;


@Test
publicvoidtest() throwsException {

finalKafkaServer kafkaServer =newKafkaServer();
System.out.println("Starting server");
Thread.sleep(10_000);

finalExecutorService executorService =Executors.newSingleThreadExecutor();

finalAsyncProducer<Long, String> producer =newAsyncProducer<>(createProducer());

executorService.execute(() -> {
for (int i =0; i <SEND_RECORD_COUNT; i++) {
if (i %1000==0) System.out.println("Sending message "+ i);
producer.send(TOPIC, 1L* i, "value "+ i)
.catchError(throwable -> {
System.err.println("Trouble sending record "+ throwable.getLocalizedMessage());
throwable.printStackTrace(System.err);
})
.then(recordMetadata -> {
if (recordMetadata.offset() %1000==0)
System.out.printf("%d %d %s \n", recordMetadata.offset(),
recordMetadata.partition(), recordMetadata.topic());
})
.invoke();
}
producer.flush();
});


finalCountDownLatch countDownLatch =newCountDownLatch(SEND_RECORD_COUNT);

finalStreamConsumer<Long, String> stream =StreamConsumer.subscribe(createConsumer(), TOPIC, result -> {
result.then(consumerRecords -> {
System.out.println("Got message "+ consumerRecords.count());
consumerRecords.forEach(record -> {
countDownLatch.countDown();
});
result.request(1);
}).catchError(throwable -> {
System.err.println("Trouble Getting record "+ throwable.getLocalizedMessage());
throwable.printStackTrace(System.err);
result.cancel();
});
});

Thread.sleep(3_000);

countDownLatch.await(10, TimeUnit.SECONDS);
assertEquals(0, countDownLatch.getCount());
stream.close();
producer.close();
executorService.shutdown();

Thread.sleep(3_000);
kafkaServer.shutdown();
Thread.sleep(3_000);

}


privatestaticProducer<Long, String>createProducer() {
Properties props =newProperties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
props.put(ProducerConfig.CLIENT_ID_CONFIG, "KafkaExampleProducer");
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName());
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

returnnewKafkaProducer<>(props);
}

privatestaticConsumer<Long, String>createConsumer() {
Properties props =newProperties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
props.put(ConsumerConfig.GROUP_ID_CONFIG, "KafkaExampleConsumer");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
LongDeserializer.class.getName());
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
StringDeserializer.class.getName());

props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 1000);
returnnewKafkaConsumer<>(props);
}
}
Notice that we use an embedded version of Kafka.

Kafka embedded

packageio.advantageous.reakt.kafka;

importorg.apache.zookeeper.server.ServerConfig;
importorg.apache.zookeeper.server.ZooKeeperServerMain;
importorg.apache.zookeeper.server.quorum.QuorumPeerConfig;

importjava.io.IOException;
importjava.util.Properties;

importkafka.server.KafkaConfig;
importkafka.server.KafkaServerStartable;

publicclassKafkaServer {

privatefinalZooKeeperServerMain zooKeeperServer;
privatefinalKafkaServerStartable kafkaServer;

publicKafkaServer() {

finalProperties zkProperties =newProperties();
finalProperties kafkaProperties =newProperties();
try {
//load properties
kafkaProperties.load(Class.class.getResourceAsStream("/io/advantageous/reakt/kafka/kafka.properties"));
zkProperties.load(Class.class.getResourceAsStream("/io/advantageous/reakt/kafka/zookeeper.properties"));
} catch (Exception e){
thrownewRuntimeException(e);
}


QuorumPeerConfig quorumConfiguration =newQuorumPeerConfig();
try {
quorumConfiguration.parseProperties(zkProperties);
} catch(Exception e) {
thrownewRuntimeException(e);
}

zooKeeperServer =newZooKeeperServerMain();
finalServerConfig configuration =newServerConfig();
configuration.readFrom(quorumConfiguration);


newThread(() -> {
try {
zooKeeperServer.runFromConfig(configuration);
} catch (IOException e) {
e.printStackTrace(System.err);
}
}).start();



//start local kafka broker
kafkaServer =newKafkaServerStartable(newKafkaConfig(kafkaProperties));
kafkaServer.startup();


}

publicvoidshutdown() {
kafkaServer.shutdown();
}

publicstaticvoidmain(String[] args) {
newKafkaServer();
}
}

Viewing all articles
Browse latest Browse all 217

Trending Articles