fbpx

Redis Pipeline code in C++ and Java client

(Last Updated On: August 28, 2015)

Redis Pipeline code in C++ and Java client

I will use this for message queueing with failover which is part of Redis 3

This is for mass insert so I need to test this with the link below. I hope it can work efficiently. I also looked at Erlang with examples not touch in 4 or 5 years. No thanks

 

http://stackoverflow.com/questions/13141166/c-tutorial-or-example-of-redis-for-fastest-insert
Redis pipeline example in with hiredis 
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include "hiredis.h"
#define CHECK(X) if ( !X || X->type == REDIS_REPLY_ERROR ) { printf("Error\n"); exit(-1); }
/******************************************************************************/
/* Fill Redis with some dummy data */
void fill( redisContext *c, int n )
{
redisReply *reply;
unsigned int i,j,k;
unsigned int count = 0;
reply = (redisReply *) redisCommand(c,"SELECT %d",n);
CHECK(reply);
freeReplyObject(reply);
reply = (redisReply *) redisCommand(c,"FLUSHDB");
CHECK(reply);
freeReplyObject(reply);
for (i=0; i<100000; ++i )
{
int cmd = 0;
for (j=0; j<100; ++j )
{
if ( j % 2 == 0 )
{
++count;
redisAppendCommand(c,"SADD %d %d",i,j );
++cmd;
}
}
while ( cmd-- > 0 )
{
int r = redisGetReply(c, (void **) &reply );
if ( r == REDIS_ERR ) { printf("Error\n"); exit(-1); }
CHECK(reply);
freeReplyObject(reply);
}
}
}
/******************************************************************************/
/* Query function: the queries are batched using pipelining to improve efficiency */
void query( redisContext *c, int n )
{
redisReply *reply;
unsigned int i,j,k;
unsigned int count = 0;
unsigned int *s = 0;
unsigned int *v = 0;
unsigned int *f = 0;
unsigned int batch = 100;
/* Use this instead to remove pipelining */
/* unsigned int batch = 1; */
reply = (redisReply *) redisCommand(c,"SELECT %d",n);
CHECK(reply);
freeReplyObject(reply);
s = (unsigned int *) malloc( batch * sizeof(unsigned int) );
v = (unsigned int *) malloc( batch * sizeof(unsigned int) );
f = (unsigned int *) malloc( batch * sizeof(unsigned int) );
/* Loop on queries */
for (i=0; i<1000000/batch; ++i )
{
unsigned int cmd = 0;
/* First step: run a batch of SISMEMBER commands */
for ( j=0; j<batch; ++j )
{
s[j] = rand() % 100000;
v[j] = rand() % 100;
redisAppendCommand(c,"SISMEMBER %d %d", s[j], v[j] );
++cmd;
}
/* Read the replies (i.e. all the booleans) */
j = 0;
while ( cmd-- > 0 )
{
int r = redisGetReply(c, (void **) &reply );
if ( r == REDIS_ERR ) { printf("Error\n"); exit(-1); }
CHECK(reply);
f[j++] = reply->integer;
freeReplyObject(reply);
}
/* Second step: run a batch of SADD commands */
cmd = 0;
for ( j=0; j<batch; ++j )
{
/* printf( "%d %d %d\n", s[j], v[j], f[j] ); */
if ( f[j] )
{
redisAppendCommand(c,"SADD %d %d",s[j]-1,v[j]+1 );
++cmd;
}
}
/* Read (and ignore) the replies */
while ( cmd-- > 0 )
{
int r = redisGetReply(c, (void **) &reply );
if ( r == REDIS_ERR ) { printf("Error\n"); exit(-1); }
CHECK(reply);
freeReplyObject(reply);
}
}
free( s );
free( v );
free( f );
}
/******************************************************************************/
/* Just used to extract the number of queries done by the Redis server */
const char *CRLF="\r\n";
const char *USED="used_memory_human:";
const char *CMD ="total_commands_processed:";
void redisTop( redisContext *c )
{
redisReply *reply;
char *p1, *p2;
char bU[32], bC[32];
long nCmd = 0;
reply = (redisReply *) redisCommand(c,"INFO");
CHECK(reply);
if ( reply->type == REDIS_REPLY_STRING )
{
p1 = strstr(reply->str, USED );
if (!p1) return;
p1 += strlen(USED);
p2 = strstr(p1,CRLF);
memcpy(bU,p1,p2-p1);
bU[p2-p1] = 0;
p1 = strstr(p2, CMD );
if (!p1) return;
p1 += strlen(CMD);
p2 = strstr(p1,CRLF);
memcpy(bC,p1,p2-p1);
bC[p2-p1] = 0;
nCmd = atol(bC);
printf("%ld\n", nCmd );
freeReplyObject(reply);
}
}
/******************************************************************************/
/* Calling with first parameter = 0 will flush and create dummy data in Redis.
Calling with first parameter = 1 will run a query benchmark */
int main(int argc, char *argv[] ) {
redisContext *c;
redisReply *reply;
int n = 0;
int db = 0;
if ( argc != 2 )
{
fprintf(stderr,"Usage: %s n\n",argv[0]);
exit(-1);
}
n = atoi(argv[1]);
c = redisConnect( "localhost", 6379);
/* c = redisConnectUnix("/tmp/redis.sock"); */
if (c->err) {
printf("Connection error: %s\n", c->errstr);
exit(1);
}
reply = (redisReply *) redisCommand(c,"CONFIG RESETSTAT");
CHECK(reply);
freeReplyObject(reply);
if ( n == 0 )
fill(c,0);
else
query(c,0);
redisTop(c);
redisFree(c);
return 0;
}
/******************************************************************************/
view raw example.c hosted with ❤ by GitHub
Compilation lines:
g++ -c -O3 -I ../hiredis -g -ggdb example.cc
g++ -o example -O3 -I ../hiredis -pthread example.o -g -ggdb -L../hiredis -lhiredis -lm
view raw gistfile1.txt hosted with ❤ by GitHub
OBJ = genload.o
BINS = genload
CC=g++
OPTIMIZATION?=-O3
CFLAGS?=$(OPTIMIZATION) $(ARCH) $(PROF) -I ../hiredis
CCLINK?=-pthread
LDFLAGS?=-L../hiredis -lhiredis -lm
OBJ = example.o
BINS = example
CC=g++
OPTIMIZATION?=-O3
CFLAGS?=$(OPTIMIZATION) $(ARCH) $(PROF) -I ../hiredis
CCLINK?=-pthread
LDFLAGS?=-L../hiredis -lhiredis -lm
CCOPT= $(CFLAGS) $(CCLINK)
DEBUG?= -g -ggdb
all: ${BINS}
# Deps (use make dep to generate this)
example.o: example.cc
# Binaries:
example: $(OBJ)
$(CC) -o $@ $(CCOPT) $(OBJ) $(DEBUG) $(LDFLAGS)
.cc.o:
$(CC) -c $(CFLAGS) $(OBJARCH) $(DEBUG) $(COMPILE_TIME) $<
clean:
rm -rf $(OBJ) $(BINS)
view raw Makefile hosted with ❤ by GitHub
redis pipeline example java client java pipeline client https://github.com/xetorthio/jedis/blob/master/src/main/java/redis/clients/jedis/Pipeline.java Build options for Jedis: https://github.com/xetorthio/jedis/wiki/Getting-started
Build hiredis (C Client binding for Redis) 
https://github.com/antirez/redis <-- this seems to be best client option since it works best and easiest to set up. also written by salvatore the author of redis.

Seems to work ok so I will assume you can safely mix C code into C++ 

~/redis-client/hiredis/redis/deps/hiredis

Create c files with Makefile etc at https://gist.github.com/dspezia/1893378 (in above directory)

Create example.c in
/home/caustic/redis-client/hiredis/redis/deps/example# 
Use compile lines at bottom of above link to build 


You can stop any instance of Redis by doing:
/etc/init.d/redis-server stop
http://stackoverflow.com/questions/6910378/how-can-i-stop-redis-server

Performance metrics in C with HiRedis library on 4 GB Virtual Machine in Ubuntu Linux:
 4 gb virtual machine 1 million rows to insert in 7 seconds, 2 seconds to read via pipeline

For Jedis, refer to https://github.com/xetorthio/jedis/wiki/Getting-started
Download latest Jedis JAR with dependency as hinted. Create a new Eclipse project and add 
JAR files to project. 

**** HERE is an excellent and simple tutorial to get Jedis working within Java 
http://www.tutorialspoint.com/redis/redis_java.htm

Note you may need to add the Jedis JAR file by using (manually add to each terminal session?)

export CLASSPATH="$CLASSPATH:jedis-2.1.0-sources.jar"

So you should be able to build:  javac RedisJava.java

Join my FREE newsletter to see how I implement this into my algo trading environment 

Test C HiRedis library 

NOTE I now post my TRADING ALERTS into my personal FACEBOOK ACCOUNT and TWITTER. Don't worry as I don't post stupid cat videos or what I eat!

Subscribe For Latest Updates

Sign up to best of business news, informed analysis and opinions on what matters to you.
Invalid email address
We promise not to spam you. You can unsubscribe at any time.

NOTE!

Check NEW site on stock forex and ETF analysis and automation

Scroll to Top