diff --git a/Makefile b/Makefile
index fa3de0bbd17e20606d550c1e8fc7cb35792e414d..296ec4488ad84fb0c9f3b23eb48d5d65fbffaf3c 100644
--- a/Makefile
+++ b/Makefile
@@ -11,7 +11,8 @@ CASSANDRA_7_FILE=apache-cassandra-0.7.9-bin.tar.gz
CASSANDRA_8_DIR=db/cassandra-0.8/lib
CASSANDRA_8_FILE=apache-cassandra-0.8.7-bin.tar.gz
HBASE_DIR=db/hbase/lib
-HBASE_FILE=hbase-0.90.4.tar.gz
+HBASE_VERSION=0.90.5
+HBASE_FILE=hbase-$(HBASE_VERSION).tar.gz
INFINISPAN_DIR=db/infinispan-5.0/lib
INFINISPAN_FILE=infinispan-5.0.0.CR8-bin.zip
MONGODB_DIR=db/mongodb/lib
@@ -20,6 +21,8 @@ REDIS_DIR=db/redis/lib
REDIS_FILE=jedis-2.0.0.jar
VOLDEMORT_DIR=db/voldemort/lib
VOLDEMORT_FILE=voldemort-0.90.1.tar.gz
+MAPKEEPER_DIR=db/mapkeeper/lib
+MAPKEEPER_FILE=mapkeeper.jar
.PHONY: build
build: download-database-deps
@@ -35,6 +38,7 @@ download-database-deps: $(CASSANDRA_5_DIR)/$(CASSANDRA_5_FILE) \
$(MONGODB_DIR)/$(MONGODB_FILE) \
$(REDIS_DIR)/$(REDIS_FILE) \
$(VOLDEMORT_DIR)/$(VOLDEMORT_FILE) \
+ $(MAPKEEPER_DIR)/$(MAPKEEPER_FILE) \
$(CASSANDRA_5_DIR)/$(CASSANDRA_5_FILE) :
wget http://archive.apache.org/dist/cassandra/0.5.1/$(CASSANDRA_5_FILE)\
@@ -57,7 +61,7 @@ $(CASSANDRA_8_DIR)/$(CASSANDRA_8_FILE) :
tar -C $(CASSANDRA_8_DIR) -zxf $(CASSANDRA_8_DIR)/$(CASSANDRA_8_FILE)
$(HBASE_DIR)/$(HBASE_FILE) :
- wget http://archive.apache.org/dist/hbase/hbase-0.90.4/$(HBASE_FILE)\
+ wget http://archive.apache.org/dist/hbase/hbase-$(HBASE_VERSION)/$(HBASE_FILE)\
-O $(HBASE_DIR)/$(HBASE_FILE)
tar -C $(HBASE_DIR) -zxf $(HBASE_DIR)/$(HBASE_FILE)
@@ -79,3 +83,8 @@ $(VOLDEMORT_DIR)/$(VOLDEMORT_FILE) :
-O $(VOLDEMORT_DIR)/$(VOLDEMORT_FILE)
tar -C $(VOLDEMORT_DIR) -zxf $(VOLDEMORT_DIR)/$(VOLDEMORT_FILE)
+$(MAPKEEPER_DIR)/$(MAPKEEPER_FILE) :
+ wget https://raw.github.com/m1ch1/mapkeeper/master/lib/mapkeeper.jar \
+ -O $(MAPKEEPER_DIR)/$(MAPKEEPER_FILE)
+ wget https://raw.github.com/m1ch1/mapkeeper/master/lib/libthrift-0.6.1.jar \
+ -O $(MAPKEEPER_DIR)/libthrift-0.6.1.jar
diff --git a/build.xml b/build.xml
index 56115521fb403f0c14101d11d0d72f553d6d52f8..85002ac0a17f3826f66fb72104a08a39500d7f10 100644
--- a/build.xml
+++ b/build.xml
@@ -70,6 +70,11 @@
<antcall target="dbcompile"/>
</target>
+ <target name="dbcompile-mapkeeper" depends="compile">
+ <property name="db.dir" value="db/mapkeeper"/>
+ <antcall target="dbcompile"/>
+ </target>
+
<target name="compile">
<mkdir dir="${classes.dir}"/>
<javac includeantruntime="false" srcdir="${src.dir}" destdir="${classes.dir}" classpathref="build.classpath" deprecation="on">
diff --git a/changes b/changes
index b9afc136e5bfd074028bfaca13a4e6e5fbc6c69d..f66005efb0bf3117b349628181865e85b45f58cb 100644
--- a/changes
+++ b/changes
@@ -22,6 +22,9 @@
* gh-50 Switch unordered key generation from FNV32 to FNV64 (sears)
* gh-51 Improved Random Number Generation Performance and add Exponential distribution support (sears)
* gh-52 Mongo db fix (sears)
+ * gh-54 Add mapkeeper driver (m1ch1)
+ * gh-57 voldemort - enable nio connector (akkumar)
+ * gh-58 benchmarking with hbase 0.90.5 (akkumar)
0.1.3
* Voldemort binding (rsumbaly)
diff --git a/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java b/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java
index c2ecf75ce79e59e60b9db779ce7979e207a2accf..90b8db9a7dd96b9cabef2bdf8d3d2d930a16b8ce 100644
--- a/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java
+++ b/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java
@@ -1,18 +1,18 @@
-/**
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you
- * may not use this file except in compliance with the License. You
- * may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the License for the specific language governing
- * permissions and limitations under the License. See accompanying
- * LICENSE file.
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
*/
package com.yahoo.ycsb.db;
@@ -52,7 +52,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
{
// BFC: Change to fix broken build (with HBase 0.20.6)
//private static final Configuration config = HBaseConfiguration.create();
- private static final HBaseConfiguration config = new HBaseConfiguration();
+ private static final Configuration config = HBaseConfiguration.create(); //new HBaseConfiguration();
public boolean _debug=false;
@@ -68,32 +68,32 @@ public class HBaseClient extends com.yahoo.ycsb.DB
public static final Object tableLock = new Object();
- /**
- * Initialize any state for this DB.
- * Called once per DB instance; there is one DB instance per client thread.
- */
- public void init() throws DBException
- {
- if ( (getProperties().getProperty("debug")!=null) &&
- (getProperties().getProperty("debug").compareTo("true")==0) )
- {
- _debug=true;
- }
-
- _columnFamily = getProperties().getProperty("columnfamily");
- if (_columnFamily == null)
- {
- System.err.println("Error, must specify a columnfamily for HBase table");
- throw new DBException("No columnfamily specified");
- }
+ /**
+ * Initialize any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
+ public void init() throws DBException
+ {
+ if ( (getProperties().getProperty("debug")!=null) &&
+ (getProperties().getProperty("debug").compareTo("true")==0) )
+ {
+ _debug=true;
+ }
+
+ _columnFamily = getProperties().getProperty("columnfamily");
+ if (_columnFamily == null)
+ {
+ System.err.println("Error, must specify a columnfamily for HBase table");
+ throw new DBException("No columnfamily specified");
+ }
_columnFamilyBytes = Bytes.toBytes(_columnFamily);
}
/**
- * Cleanup any state for this DB.
- * Called once per DB instance; there is one DB instance per client thread.
- */
+ * Cleanup any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
public void cleanup() throws DBException
{
try {
@@ -119,24 +119,24 @@ public class HBaseClient extends com.yahoo.ycsb.DB
/**
* Read a record from the database. Each field/value pair from the result will be stored in a HashMap.
- *
- * @param table The name of the table
- * @param key The record key of the record to read.
- * @param fields The list of fields to read, or null for all of them
- * @param result A HashMap of field/value pairs for the result
- * @return Zero on success, a non-zero error code on error
- */
- public int read(String table, String key, Set<String> fields, HashMap<String,ByteIterator> result)
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to read.
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A HashMap of field/value pairs for the result
+ * @return Zero on success, a non-zero error code on error
+ */
+ public int read(String table, String key, Set<String> fields, HashMap<String,ByteIterator> result)
{
//if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
_hTable = null;
- try
+ try
{
getHTable(table);
_table = table;
}
- catch (IOException e)
+ catch (IOException e)
{
System.err.println("Error accessing HBase table: "+e);
return ServerError;
@@ -146,10 +146,10 @@ public class HBaseClient extends com.yahoo.ycsb.DB
Result r = null;
try
{
- if (_debug) {
- System.out.println("Doing read from HBase columnfamily "+_columnFamily);
- System.out.println("Doing read for key: "+key);
- }
+ if (_debug) {
+ System.out.println("Doing read from HBase columnfamily "+_columnFamily);
+ System.out.println("Doing read for key: "+key);
+ }
Get g = new Get(Bytes.toBytes(key));
if (fields == null) {
g.addFamily(_columnFamilyBytes);
@@ -181,30 +181,30 @@ public class HBaseClient extends com.yahoo.ycsb.DB
}
}
- return Ok;
+ return Ok;
}
/**
* Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap.
- *
- * @param table The name of the table
- * @param startkey The record key of the first record to read.
- * @param recordcount The number of records to read
- * @param fields The list of fields to read, or null for all of them
- * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
- * @return Zero on success, a non-zero error code on error
- */
+ *
+ * @param table The name of the table
+ * @param startkey The record key of the first record to read.
+ * @param recordcount The number of records to read
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
+ * @return Zero on success, a non-zero error code on error
+ */
public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,ByteIterator>> result)
{
//if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
_hTable = null;
- try
+ try
{
getHTable(table);
_table = table;
}
- catch (IOException e)
+ catch (IOException e)
{
System.err.println("Error accessing HBase table: "+e);
return ServerError;
@@ -290,12 +290,12 @@ public class HBaseClient extends com.yahoo.ycsb.DB
//if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
_hTable = null;
- try
+ try
{
getHTable(table);
_table = table;
}
- catch (IOException e)
+ catch (IOException e)
{
System.err.println("Error accessing HBase table: "+e);
return ServerError;
@@ -312,11 +312,11 @@ public class HBaseClient extends com.yahoo.ycsb.DB
if (_debug) {
System.out.println("Adding field/value " + entry.getKey() + "/"+
entry.getValue() + " to put request");
- }
+ }
p.add(_columnFamilyBytes,Bytes.toBytes(entry.getKey()),entry.getValue().toArray());
}
- try
+ try
{
_hTable.put(p);
}
@@ -327,7 +327,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
}
return ServerError;
}
- catch (ConcurrentModificationException e)
+ catch (ConcurrentModificationException e)
{
//do nothing for now...hope this is rare
return ServerError;
@@ -343,31 +343,31 @@ public class HBaseClient extends com.yahoo.ycsb.DB
* @param table The name of the table
* @param key The record key of the record to insert.
* @param values A HashMap of field/value pairs to insert in the record
- * @return Zero on success, a non-zero error code on error
- */
- public int insert(String table, String key, HashMap<String,ByteIterator> values)
+ * @return Zero on success, a non-zero error code on error
+ */
+ public int insert(String table, String key, HashMap<String,ByteIterator> values)
{
return update(table,key,values);
}
- /**
- * Delete a record from the database.
- *
- * @param table The name of the table
- * @param key The record key of the record to delete.
- * @return Zero on success, a non-zero error code on error
- */
- public int delete(String table, String key)
+ /**
+ * Delete a record from the database.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to delete.
+ * @return Zero on success, a non-zero error code on error
+ */
+ public int delete(String table, String key)
{
//if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
_hTable = null;
- try
+ try
{
getHTable(table);
_table = table;
}
- catch (IOException e)
+ catch (IOException e)
{
System.err.println("Error accessing HBase table: "+e);
return ServerError;
@@ -379,7 +379,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
}
Delete d = new Delete(Bytes.toBytes(key));
- try
+ try
{
_hTable.delete(d);
}
@@ -404,7 +404,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
final int keyspace=10000; //120000000;
- final int threadcount=Integer.parseInt(args[0]);
+ final int threadcount=Integer.parseInt(args[0]);
final String columnfamily=args[1];
@@ -415,7 +415,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
for (int i=0; i<threadcount; i++)
{
- Thread t=new Thread()
+ Thread t=new Thread()
{
public void run()
{
@@ -451,7 +451,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
HashSet<String> s = new HashSet();
s.add("field1");
s.add("field2");
-
+
rescode=cli.read("table1", key, s, result);
//rescode=cli.delete("table1",key);
rescode=cli.read("table1", key, s, result);
@@ -461,7 +461,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
scanFields.add("field3");
Vector<HashMap<String,ByteIterator>> scanResults = new Vector<HashMap<String,ByteIterator>>();
rescode = cli.scan("table1","user2",20,null,scanResults);
-
+
long en=System.currentTimeMillis();
accum+=(en-st);
diff --git a/db/mapkeeper/lib/README b/db/mapkeeper/lib/README
new file mode 100644
index 0000000000000000000000000000000000000000..a59623a18f5acb0cc98865253127e25658b39a4a
--- /dev/null
+++ b/db/mapkeeper/lib/README
@@ -0,0 +1,6 @@
+This directory should contain 2 jars; mapkeeper.jar and libthrift-0.6.1.jar.
+To get them, do:
+
+$ wget https://raw.github.com/m1ch1/mapkeeper/master/lib/mapkeeper.jar
+$ wget https://raw.github.com/m1ch1/mapkeeper/master/lib/libthrift-0.6.1.jar
+
diff --git a/db/mapkeeper/src/com/yahoo/ycsb/db/MapKeeperClient.java b/db/mapkeeper/src/com/yahoo/ycsb/db/MapKeeperClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..594c9308dce7a0dfd8c6f6610333043930f1d5cf
--- /dev/null
+++ b/db/mapkeeper/src/com/yahoo/ycsb/db/MapKeeperClient.java
@@ -0,0 +1,196 @@
+package com.yahoo.ycsb.db;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.transport.TFramedTransport;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+
+import com.yahoo.mapkeeper.BinaryResponse;
+import com.yahoo.mapkeeper.MapKeeper;
+import com.yahoo.mapkeeper.Record;
+import com.yahoo.mapkeeper.RecordListResponse;
+import com.yahoo.mapkeeper.ResponseCode;
+import com.yahoo.mapkeeper.ScanOrder;
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.DB;
+import com.yahoo.ycsb.StringByteIterator;
+import com.yahoo.ycsb.workloads.CoreWorkload;
+
+public class MapKeeperClient extends DB {
+ MapKeeper.Client c;
+ boolean writeallfields;
+ static boolean initteddb = false;
+ private synchronized static void initDB(Properties p, MapKeeper.Client c) throws TException {
+ if(!initteddb) {
+ initteddb = true;
+ c.addMap(p.getProperty(CoreWorkload.TABLENAME_PROPERTY, CoreWorkload.TABLENAME_PROPERTY_DEFAULT));
+ }
+ }
+
+ public void init() {
+ TTransport tr = new TFramedTransport(new TSocket("localhost", 9090));
+ TProtocol proto = new TBinaryProtocol(tr);
+ c = new MapKeeper.Client(proto);
+ try {
+ tr.open();
+ initDB(getProperties(), c);
+ } catch(TException e) {
+ throw new RuntimeException(e);
+ }
+ writeallfields = Boolean.parseBoolean(getProperties().getProperty(CoreWorkload.WRITE_ALL_FIELDS_PROPERTY,
+ CoreWorkload.WRITE_ALL_FIELDS_PROPERTY_DEFAULT));
+ }
+
+ ByteBuffer encode(HashMap<String, ByteIterator> values) {
+ int len = 0;
+ for(String k : values.keySet()) {
+ len += (k.length() + 1 + values.get(k).bytesLeft() + 1);
+ }
+ byte[] array = new byte[len];
+ int i = 0;
+ for(String k : values.keySet()) {
+ for(int j = 0; j < k.length(); j++) {
+ array[i] = (byte)k.charAt(j);
+ i++;
+ }
+ array[i] = '\t'; // XXX would like to use sane delimiter (null, 254, 255, ...) but java makes this nearly impossible
+ i++;
+ ByteIterator v = values.get(k);
+ i = v.nextBuf(array, i);
+ array[i] = '\t';
+ i++;
+ }
+ array[array.length-1] = 0;
+ ByteBuffer buf = ByteBuffer.wrap(array);
+ buf.rewind();
+ return buf;
+ }
+ void decode(Set<String> fields, String tups, HashMap<String, ByteIterator> tup) {
+ String[] tok = tups.split("\\t");
+ if(tok.length == 0) { throw new IllegalStateException("split returned empty array!"); }
+ for(int i = 0; i < tok.length; i+=2) {
+ if(fields == null || fields.contains(tok[i])) {
+ if(tok.length < i+2) { throw new IllegalStateException("Couldn't parse tuple <" + tups + "> at index " + i); }
+ if(tok[i] == null || tok[i+1] == null) throw new NullPointerException("Key is " + tok[i] + " val is + " + tok[i+1]);
+ tup.put(tok[i], new StringByteIterator(tok[i+1]));
+ }
+ }
+ if(tok.length == 0) {
+ System.err.println("Empty tuple: " + tups);
+ }
+ }
+
+ int ycsbThriftRet(BinaryResponse succ, ResponseCode zero, ResponseCode one) {
+ return ycsbThriftRet(succ.responseCode, zero, one);
+ }
+ int ycsbThriftRet(ResponseCode rc, ResponseCode zero, ResponseCode one) {
+ return
+ rc == zero ? 0 :
+ rc == one ? 1 : 2;
+ }
+ ByteBuffer bufStr(String str) {
+ ByteBuffer buf = ByteBuffer.wrap(str.getBytes());
+ return buf;
+ }
+ String strResponse(BinaryResponse buf) {
+ return new String(buf.value.array());
+ }
+
+ @Override
+ public int read(String table, String key, Set<String> fields,
+ HashMap<String, ByteIterator> result) {
+ try {
+ ByteBuffer buf = bufStr(key);
+
+ BinaryResponse succ = c.get(table, buf);
+
+ int ret = ycsbThriftRet(
+ succ,
+ ResponseCode.RecordExists,
+ ResponseCode.RecordNotFound);
+
+ if(ret == 0) {
+ decode(fields, strResponse(succ), result);
+ }
+ return ret;
+ } catch(TException e) {
+ e.printStackTrace();
+ return 2;
+ }
+ }
+
+ @Override
+ public int scan(String table, String startkey, int recordcount,
+ Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
+ try {
+ //XXX what to pass in for nulls / zeros?
+ RecordListResponse res = c.scan(table, ScanOrder.Ascending, bufStr(startkey), true, null, false, recordcount, 0);
+ int ret = ycsbThriftRet(res.responseCode, ResponseCode.Success, ResponseCode.ScanEnded);
+ if(ret == 0) {
+ for(Record r : res.records) {
+ HashMap<String, ByteIterator> tuple = new HashMap<String, ByteIterator>();
+ // Note: r.getKey() and r.getValue() call special helper methods that trim the buffer
+ // to an appropriate length, and memcpy it to a byte[]. Trying to manipulate the ByteBuffer
+ // directly leads to trouble.
+ tuple.put("key", new StringByteIterator(new String(r.getKey())));
+ decode(fields, new String(r.getValue())/*strBuf(r.bufferForValue())*/, tuple);
+ result.add(tuple);
+ }
+ }
+ return ret;
+ } catch(TException e) {
+ e.printStackTrace();
+ return 2;
+ }
+ }
+
+ @Override
+ public int update(String table, String key,
+ HashMap<String, ByteIterator> values) {
+ try {
+ if(!writeallfields) {
+ HashMap<String, ByteIterator> oldval = new HashMap<String, ByteIterator>();
+ read(table, key, null, oldval);
+ for(String k: values.keySet()) {
+ oldval.put(k, values.get(k));
+ }
+ values = oldval;
+ }
+ ResponseCode succ = c.update(table, bufStr(key), encode(values));
+ return ycsbThriftRet(succ, ResponseCode.RecordExists, ResponseCode.RecordNotFound);
+ } catch(TException e) {
+ e.printStackTrace();
+ return 2;
+ }
+ }
+
+ @Override
+ public int insert(String table, String key,
+ HashMap<String, ByteIterator> values) {
+ try {
+ int ret = ycsbThriftRet(c.insert(table, bufStr(key), encode(values)), ResponseCode.Success, ResponseCode.RecordExists);
+ return ret;
+ } catch(TException e) {
+ e.printStackTrace();
+ return 2;
+ }
+ }
+
+ @Override
+ public int delete(String table, String key) {
+ try {
+ return ycsbThriftRet(c.remove(table, bufStr(key)), ResponseCode.Success, ResponseCode.RecordExists);
+ } catch(TException e) {
+ e.printStackTrace();
+ return 2;
+ }
+ }
+}
diff --git a/db/voldemort/config/server.properties b/db/voldemort/config/server.properties
index a77b070ae2a782990845f0d95f7fec4a273c3cda..fce5ee43e6bbd86e2f7b19e17c21734ac32dbf04 100644
--- a/db/voldemort/config/server.properties
+++ b/db/voldemort/config/server.properties
@@ -21,6 +21,6 @@ mysql.password=3306
mysql.database=test
#NIO connector settings.
-enable.nio.connector=false
+enable.nio.connector=true
storage.configs=voldemort.store.bdb.BdbStorageConfiguration, voldemort.store.readonly.ReadOnlyStorageConfiguration