diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..fa3de0bbd17e20606d550c1e8fc7cb35792e414d
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,81 @@
+#
+# Running 'make -f Makefile.download-db-libs' will automatically download all
+# of the libraries needed to build the YCSB drivers.
+#
+CASSANDRA_5_DIR=db/cassandra-0.5/lib
+CASSANDRA_5_FILE=apache-cassandra-0.5.1-bin.tar.gz
+CASSANDRA_6_DIR=db/cassandra-0.6/lib
+CASSANDRA_6_FILE=apache-cassandra-0.6.13-bin.tar.gz
+CASSANDRA_7_DIR=db/cassandra-0.7/lib
+CASSANDRA_7_FILE=apache-cassandra-0.7.9-bin.tar.gz
+CASSANDRA_8_DIR=db/cassandra-0.8/lib
+CASSANDRA_8_FILE=apache-cassandra-0.8.7-bin.tar.gz
+HBASE_DIR=db/hbase/lib
+HBASE_FILE=hbase-0.90.4.tar.gz
+INFINISPAN_DIR=db/infinispan-5.0/lib
+INFINISPAN_FILE=infinispan-5.0.0.CR8-bin.zip
+MONGODB_DIR=db/mongodb/lib
+MONGODB_FILE=mongo-2.7.2.jar
+REDIS_DIR=db/redis/lib
+REDIS_FILE=jedis-2.0.0.jar
+VOLDEMORT_DIR=db/voldemort/lib
+VOLDEMORT_FILE=voldemort-0.90.1.tar.gz
+
+.PHONY: build
+build: download-database-deps
+ ant -q -e compile
+ grep name=\"dbcompile build.xml | perl -ne '$$_=~/name=\"(.+)\"\s+depends/; print "$$1\n"; system "ant -q -e $$1"'
+
+download-database-deps: $(CASSANDRA_5_DIR)/$(CASSANDRA_5_FILE) \
+ $(CASSANDRA_6_DIR)/$(CASSANDRA_6_FILE) \
+ $(CASSANDRA_7_DIR)/$(CASSANDRA_7_FILE) \
+ $(CASSANDRA_8_DIR)/$(CASSANDRA_8_FILE) \
+ $(HBASE_DIR)/$(HBASE_FILE) \
+ $(INFINISPAN_DIR)/$(INFINISPAN_FILE) \
+ $(MONGODB_DIR)/$(MONGODB_FILE) \
+ $(REDIS_DIR)/$(REDIS_FILE) \
+ $(VOLDEMORT_DIR)/$(VOLDEMORT_FILE) \
+
+$(CASSANDRA_5_DIR)/$(CASSANDRA_5_FILE) :
+ wget http://archive.apache.org/dist/cassandra/0.5.1/$(CASSANDRA_5_FILE)\
+ -O $(CASSANDRA_5_DIR)/$(CASSANDRA_5_FILE)
+ tar -C $(CASSANDRA_5_DIR) -zxf $(CASSANDRA_5_DIR)/$(CASSANDRA_5_FILE)
+
+$(CASSANDRA_6_DIR)/$(CASSANDRA_6_FILE) :
+ wget http://archive.apache.org/dist/cassandra/0.6.13/$(CASSANDRA_6_FILE)\
+ -O $(CASSANDRA_6_DIR)/$(CASSANDRA_6_FILE)
+ tar -C $(CASSANDRA_6_DIR) -zxf $(CASSANDRA_6_DIR)/$(CASSANDRA_6_FILE)
+
+$(CASSANDRA_7_DIR)/$(CASSANDRA_7_FILE) :
+ wget http://archive.apache.org/dist/cassandra/0.7.9/$(CASSANDRA_7_FILE)\
+ -O $(CASSANDRA_7_DIR)/$(CASSANDRA_7_FILE)
+ tar -C $(CASSANDRA_7_DIR) -zxf $(CASSANDRA_7_DIR)/$(CASSANDRA_7_FILE)
+
+$(CASSANDRA_8_DIR)/$(CASSANDRA_8_FILE) :
+ wget http://archive.apache.org/dist/cassandra/0.8.7/$(CASSANDRA_8_FILE)\
+ -O $(CASSANDRA_8_DIR)/$(CASSANDRA_8_FILE)
+ tar -C $(CASSANDRA_8_DIR) -zxf $(CASSANDRA_8_DIR)/$(CASSANDRA_8_FILE)
+
+$(HBASE_DIR)/$(HBASE_FILE) :
+ wget http://archive.apache.org/dist/hbase/hbase-0.90.4/$(HBASE_FILE)\
+ -O $(HBASE_DIR)/$(HBASE_FILE)
+ tar -C $(HBASE_DIR) -zxf $(HBASE_DIR)/$(HBASE_FILE)
+
+$(INFINISPAN_DIR)/$(INFINISPAN_FILE) :
+ wget http://iweb.dl.sourceforge.net/project/infinispan/infinispan/5.0.0.CR8/$(INFINISPAN_FILE)\
+ -O $(INFINISPAN_DIR)/$(INFINISPAN_FILE)
+ unzip -a $(INFINISPAN_DIR)/$(INFINISPAN_FILE) -d $(INFINISPAN_DIR)
+
+$(MONGODB_DIR)/$(MONGODB_FILE) :
+ wget https://github.com/downloads/mongodb/mongo-java-driver/$(MONGODB_FILE)\
+ -O $(MONGODB_DIR)/$(MONGODB_FILE)
+
+$(REDIS_DIR)/$(REDIS_FILE) :
+ wget https://github.com/downloads/xetorthio/jedis/$(REDIS_FILE)\
+ -O $(REDIS_DIR)/$(REDIS_FILE)
+
+$(VOLDEMORT_DIR)/$(VOLDEMORT_FILE) :
+ wget https://github.com/downloads/voldemort/voldemort/$(VOLDEMORT_FILE)\
+ -O $(VOLDEMORT_DIR)/$(VOLDEMORT_FILE)
+ tar -C $(VOLDEMORT_DIR) -zxf $(VOLDEMORT_DIR)/$(VOLDEMORT_FILE)
+
diff --git a/build.xml b/build.xml
index a71c34226181bc9a642f265968bf98613ba60f47..56115521fb403f0c14101d11d0d72f553d6d52f8 100644
--- a/build.xml
+++ b/build.xml
@@ -72,20 +72,20 @@
<target name="compile">
<mkdir dir="${classes.dir}"/>
- <javac srcdir="${src.dir}" destdir="${classes.dir}" classpathref="build.classpath" deprecation="on">
+ <javac includeantruntime="false" srcdir="${src.dir}" destdir="${classes.dir}" classpathref="build.classpath" deprecation="on">
<compilerarg value="-Xlint:unchecked"/>
</javac>
<antcall target="makejar"/>
</target>
- <target name="dbcompile">
+ <target name="dbcompile">
<path id="dbclasspath">
<fileset dir="${db.dir}/lib" includes="**/*.jar"/>
<fileset file="build/ycsb.jar"/>
</path>
<mkdir dir="${classes.dir}"/>
- <javac srcdir="${db.dir}/src" destdir="${classes.dir}" classpathref="dbclasspath" deprecation="on">
+ <javac includeantruntime="false" srcdir="${db.dir}/src" destdir="${classes.dir}" classpathref="dbclasspath" deprecation="on">
<compilerarg value="-Xlint:unchecked"/>
</javac>
<antcall target="makejar"/>
@@ -105,7 +105,7 @@
<javadoc destdir="${doc.dir}/javadoc" packagenames="com.yahoo.ycsb,com.yahoo.ycsb.workloads,com.yahoo.ycsb.db,com.yahoo.ycsb.generator,com.yahoo.ycsb.measurements">
<fileset dir="." defaultexcludes="yes">
<include name="src/**"/>
- <include name="db/**/src/**"/>
+ <include name="db/*/src/**"/>
</fileset>
</javadoc>
</target>
diff --git a/changes b/changes
index 93b11bfa07a7a1d39eb34725e5291af3e92b622d..b9afc136e5bfd074028bfaca13a4e6e5fbc6c69d 100644
--- a/changes
+++ b/changes
@@ -13,6 +13,15 @@
* gh-27 MongoDbClient was not working with non localhost URLs (arunxarun)
* gh-29 Add simple sharding capabilities for JDBC driver (kibab)
* gh-40 Merge Redis database interface layer (lehmannro)
+ * gh-42 Response latencies are measured in microseconds (mikewied)
+ * gh-43 Variable length fields (sears)
+ * gh-44 Constant occupancy workload (sears)
+ * gh-45 Modify DB API for efficient large object support (sears)
+ * gh-46 Fixed typo in RedisClient (Zlender)
+ * gh-49 Build fix (sears)
+ * gh-50 Switch unordered key generation from FNV32 to FNV64 (sears)
+ * gh-51 Improved Random Number Generation Performance and add Exponential distribution support (sears)
+ * gh-52 Mongo db fix (sears)
0.1.3
* Voldemort binding (rsumbaly)
diff --git a/db/cassandra-0.5/src/com/yahoo/ycsb/db/CassandraClient5.java b/db/cassandra-0.5/src/com/yahoo/ycsb/db/CassandraClient5.java
index bf8b58f8a6f923bf89072711a1569db6f89e512b..19f9f52812c147c0962faa82b7fda8dc3b3c1360 100644
--- a/db/cassandra-0.5/src/com/yahoo/ycsb/db/CassandraClient5.java
+++ b/db/cassandra-0.5/src/com/yahoo/ycsb/db/CassandraClient5.java
@@ -135,7 +135,7 @@ public class CassandraClient5 extends DB
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
- public int read(String table, String key, Set<String> fields, HashMap<String,String> result)
+ public int read(String table, String key, Set<String> fields, HashMap<String,ByteIterator> result)
{
Exception errorexception=null;
@@ -170,7 +170,7 @@ public class CassandraClient5 extends DB
for (ColumnOrSuperColumn oneresult : results)
{
Column column=oneresult.column;
- result.put(new String(column.name),new String(column.value));
+ result.put(new String(column.name),new ByteArrayByteIterator(column.value));
if (_debug)
{
@@ -214,7 +214,7 @@ public class CassandraClient5 extends DB
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error
*/
- public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,String>> result)
+ public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,ByteIterator>> result)
{
Exception errorexception=null;
@@ -248,12 +248,12 @@ public class CassandraClient5 extends DB
for (KeySlice oneresult : results)
{
- HashMap<String,String> tuple = new HashMap<String, String>();
+ HashMap<String,ByteIterator> tuple = new HashMap<String, ByteIterator>();
for (ColumnOrSuperColumn onecol : oneresult.columns)
{
Column column=onecol.column;
- tuple.put(new String(column.name),new String(column.value));
+ tuple.put(new String(column.name),new ByteArrayByteIterator(column.value));
if (_debug)
{
@@ -296,7 +296,7 @@ public class CassandraClient5 extends DB
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
- public int update(String table, String key, HashMap<String,String> values)
+ public int update(String table, String key, HashMap<String,ByteIterator> values)
{
return insert(table,key,values);
}
@@ -310,7 +310,7 @@ public class CassandraClient5 extends DB
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
- public int insert(String table, String key, HashMap<String,String> values)
+ public int insert(String table, String key, HashMap<String,ByteIterator> values)
{
Exception errorexception=null;
@@ -327,7 +327,7 @@ public class CassandraClient5 extends DB
{
for (String field : values.keySet())
{
- String val=values.get(field);
+ String val=values.get(field).toString();
Column col=new Column(field.getBytes("UTF-8"),val.getBytes("UTF-8"),timestamp);
ColumnOrSuperColumn c=new ColumnOrSuperColumn();
@@ -428,14 +428,14 @@ public class CassandraClient5 extends DB
System.exit(0);
}
- HashMap<String,String> vals=new HashMap<String,String>();
- vals.put("age","57");
- vals.put("middlename","bradley");
- vals.put("favoritecolor","blue");
+ HashMap<String,ByteIterator> vals=new HashMap<String,ByteIterator>();
+ vals.put("age",new StringByteIterator("57"));
+ vals.put("middlename",new StringByteIterator("bradley"));
+ vals.put("favoritecolor",new StringByteIterator("blue"));
int res=cli.insert("usertable","BrianFrankCooper",vals);
System.out.println("Result of insert: "+res);
- HashMap<String,String> result=new HashMap<String,String>();
+ HashMap<String,ByteIterator> result=new HashMap<String,ByteIterator>();
HashSet<String> fields=new HashSet<String>();
fields.add("middlename");
fields.add("age");
diff --git a/db/cassandra-0.6/src/com/yahoo/ycsb/db/CassandraClient6.java b/db/cassandra-0.6/src/com/yahoo/ycsb/db/CassandraClient6.java
index 8c1967ff1b65cc718947c81bd947e2cbd7c98cfa..6d4a57252ad274e93c8585600c5cd5ea9174119c 100644
--- a/db/cassandra-0.6/src/com/yahoo/ycsb/db/CassandraClient6.java
+++ b/db/cassandra-0.6/src/com/yahoo/ycsb/db/CassandraClient6.java
@@ -132,7 +132,7 @@ public class CassandraClient6 extends DB
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
- public int read(String table, String key, Set<String> fields, HashMap<String,String> result)
+ public int read(String table, String key, Set<String> fields, HashMap<String,ByteIterator> result)
{
Exception errorexception=null;
@@ -177,7 +177,7 @@ public class CassandraClient6 extends DB
for (ColumnOrSuperColumn oneresult : results)
{
Column column=oneresult.column;
- result.put(new String(column.name),new String(column.value));
+ result.put(new String(column.name),new ByteArrayByteIterator(column.value));
if (_debug)
{
@@ -221,7 +221,7 @@ public class CassandraClient6 extends DB
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error
*/
- public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,String>> result)
+ public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,ByteIterator>> result)
{
Exception errorexception=null;
@@ -261,12 +261,12 @@ public class CassandraClient6 extends DB
for (KeySlice oneresult : results)
{
- HashMap<String,String> tuple = new HashMap<String, String>();
+ HashMap<String,ByteIterator> tuple = new HashMap<String,ByteIterator>();
for (ColumnOrSuperColumn onecol : oneresult.columns)
{
Column column=onecol.column;
- tuple.put(new String(column.name),new String(column.value));
+ tuple.put(new String(column.name),new ByteArrayByteIterator(column.value));
if (_debug)
{
@@ -309,7 +309,7 @@ public class CassandraClient6 extends DB
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
- public int update(String table, String key, HashMap<String,String> values)
+ public int update(String table, String key, HashMap<String,ByteIterator> values)
{
return insert(table,key,values);
}
@@ -323,7 +323,7 @@ public class CassandraClient6 extends DB
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
- public int insert(String table, String key, HashMap<String,String> values)
+ public int insert(String table, String key, HashMap<String,ByteIterator> values)
{
Exception errorexception=null;
@@ -340,8 +340,8 @@ public class CassandraClient6 extends DB
{
for (String field : values.keySet())
{
- String val=values.get(field);
- Column col=new Column(field.getBytes("UTF-8"),val.getBytes("UTF-8"),timestamp);
+ ByteIterator val=values.get(field);
+ Column col=new Column(field.getBytes("UTF-8"),val.toArray(),timestamp);
ColumnOrSuperColumn c=new ColumnOrSuperColumn();
c.setColumn(col);
@@ -441,14 +441,14 @@ public class CassandraClient6 extends DB
System.exit(0);
}
- HashMap<String,String> vals=new HashMap<String,String>();
- vals.put("age","57");
- vals.put("middlename","bradley");
- vals.put("favoritecolor","blue");
+ HashMap<String,ByteIterator> vals=new HashMap<String,ByteIterator>();
+ vals.put("age",new StringByteIterator("57"));
+ vals.put("middlename",new StringByteIterator("bradley"));
+ vals.put("favoritecolor",new StringByteIterator("blue"));
int res=cli.insert("usertable","BrianFrankCooper",vals);
System.out.println("Result of insert: "+res);
- HashMap<String,String> result=new HashMap<String,String>();
+ HashMap<String,ByteIterator> result=new HashMap<String,ByteIterator>();
HashSet<String> fields=new HashSet<String>();
fields.add("middlename");
fields.add("age");
diff --git a/db/cassandra-0.7/src/com/yahoo/ycsb/db/CassandraClient7.java b/db/cassandra-0.7/src/com/yahoo/ycsb/db/CassandraClient7.java
index c4fac12469979684de10cff72d11d3588f2aaaa5..c5a39692b9d37216ad46e24afbb02125b8cca533 100644
--- a/db/cassandra-0.7/src/com/yahoo/ycsb/db/CassandraClient7.java
+++ b/db/cassandra-0.7/src/com/yahoo/ycsb/db/CassandraClient7.java
@@ -179,7 +179,7 @@ public class CassandraClient7 extends DB
* A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
- public int read(String table, String key, Set<String> fields, HashMap<String, String> result)
+ public int read(String table, String key, Set<String> fields, HashMap<String, ByteIterator> result)
{
if (!_table.equals(table)) {
try
@@ -224,13 +224,13 @@ public class CassandraClient7 extends DB
Column column;
String name;
- String value;
+ ByteIterator value;
for (ColumnOrSuperColumn oneresult : results)
{
column = oneresult.column;
name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining());
- value = new String(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining());
+ value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining());
result.put(name,value);
@@ -282,7 +282,7 @@ public class CassandraClient7 extends DB
* @return Zero on success, a non-zero error code on error
*/
public int scan(String table, String startkey, int recordcount, Set<String> fields,
- Vector<HashMap<String, String>> result)
+ Vector<HashMap<String, ByteIterator>> result)
{
if (!_table.equals(table)) {
try
@@ -327,19 +327,19 @@ public class CassandraClient7 extends DB
System.out.println("Scanning startkey: " + startkey);
}
- HashMap<String, String> tuple;
+ HashMap<String, ByteIterator> tuple;
for (KeySlice oneresult : results)
{
- tuple = new HashMap<String, String>();
+ tuple = new HashMap<String, ByteIterator>();
Column column;
String name;
- String value;
+ ByteIterator value;
for (ColumnOrSuperColumn onecol : oneresult.columns)
{
column = onecol.column;
name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining());
- value = new String(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining());
+ value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining());
tuple.put(name, value);
@@ -386,7 +386,7 @@ public class CassandraClient7 extends DB
* A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
- public int update(String table, String key, HashMap<String, String> values)
+ public int update(String table, String key, HashMap<String, ByteIterator> values)
{
return insert(table, key, values);
}
@@ -404,7 +404,7 @@ public class CassandraClient7 extends DB
* A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
- public int insert(String table, String key, HashMap<String, String> values)
+ public int insert(String table, String key, HashMap<String, ByteIterator> values)
{
if (!_table.equals(table)) {
try
@@ -432,11 +432,11 @@ public class CassandraClient7 extends DB
ByteBuffer wrappedKey = ByteBuffer.wrap(key.getBytes("UTF-8"));
ColumnOrSuperColumn column;
- for (Map.Entry<String, String> entry : values.entrySet())
+ for (Map.Entry<String, ByteIterator> entry : values.entrySet())
{
column = new ColumnOrSuperColumn();
column.setColumn( new Column( ByteBuffer.wrap(entry.getKey().getBytes("UTF-8")),
- ByteBuffer.wrap(entry.getValue().getBytes("UTF-8")),
+ ByteBuffer.wrap(entry.getValue().toArray()),
System.currentTimeMillis()) );
mutations.add(new Mutation().setColumn_or_supercolumn(column));
@@ -543,14 +543,14 @@ public class CassandraClient7 extends DB
System.exit(0);
}
- HashMap<String, String> vals = new HashMap<String, String>();
- vals.put("age", "57");
- vals.put("middlename", "bradley");
- vals.put("favoritecolor", "blue");
+ HashMap<String, ByteIterator> vals = new HashMap<String, ByteIterator>();
+ vals.put("age", new StringByteIterator("57"));
+ vals.put("middlename", new StringByteIterator("bradley"));
+ vals.put("favoritecolor", new StringByteIterator("blue"));
int res = cli.insert("usertable", "BrianFrankCooper", vals);
System.out.println("Result of insert: " + res);
- HashMap<String, String> result = new HashMap<String, String>();
+ HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
HashSet<String> fields = new HashSet<String>();
fields.add("middlename");
fields.add("age");
diff --git a/db/cassandra-0.8/src/com/yahoo/ycsb/db/CassandraClient8.java b/db/cassandra-0.8/src/com/yahoo/ycsb/db/CassandraClient8.java
index 9d760c59114a572ba965745203fc0cf17a9438cb..ec8abf3234eea3aec767789ba09b0275e0a6bdf5 100644
--- a/db/cassandra-0.8/src/com/yahoo/ycsb/db/CassandraClient8.java
+++ b/db/cassandra-0.8/src/com/yahoo/ycsb/db/CassandraClient8.java
@@ -179,7 +179,7 @@ public class CassandraClient8 extends DB
* A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
- public int read(String table, String key, Set<String> fields, HashMap<String, String> result)
+ public int read(String table, String key, Set<String> fields, HashMap<String, ByteIterator> result)
{
if (!_table.equals(table)) {
try
@@ -224,13 +224,13 @@ public class CassandraClient8 extends DB
Column column;
String name;
- String value;
+ ByteIterator value;
for (ColumnOrSuperColumn oneresult : results)
{
column = oneresult.column;
name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining());
- value = new String(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining());
+ value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining());
result.put(name,value);
@@ -282,7 +282,7 @@ public class CassandraClient8 extends DB
* @return Zero on success, a non-zero error code on error
*/
public int scan(String table, String startkey, int recordcount, Set<String> fields,
- Vector<HashMap<String, String>> result)
+ Vector<HashMap<String, ByteIterator>> result)
{
if (!_table.equals(table)) {
try
@@ -327,19 +327,19 @@ public class CassandraClient8 extends DB
System.out.println("Scanning startkey: " + startkey);
}
- HashMap<String, String> tuple;
+ HashMap<String, ByteIterator> tuple;
for (KeySlice oneresult : results)
{
- tuple = new HashMap<String, String>();
+ tuple = new HashMap<String, ByteIterator>();
Column column;
String name;
- String value;
+ ByteIterator value;
for (ColumnOrSuperColumn onecol : oneresult.columns)
{
column = onecol.column;
name = new String(column.name.array(), column.name.position()+column.name.arrayOffset(), column.name.remaining());
- value = new String(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining());
+ value = new ByteArrayByteIterator(column.value.array(), column.value.position()+column.value.arrayOffset(), column.value.remaining());
tuple.put(name, value);
@@ -386,7 +386,7 @@ public class CassandraClient8 extends DB
* A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
- public int update(String table, String key, HashMap<String, String> values)
+ public int update(String table, String key, HashMap<String, ByteIterator> values)
{
return insert(table, key, values);
}
@@ -404,7 +404,7 @@ public class CassandraClient8 extends DB
* A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
- public int insert(String table, String key, HashMap<String, String> values)
+ public int insert(String table, String key, HashMap<String, ByteIterator> values)
{
if (!_table.equals(table)) {
try
@@ -433,11 +433,11 @@ public class CassandraClient8 extends DB
Column col;
ColumnOrSuperColumn column;
- for (Map.Entry<String, String> entry : values.entrySet())
+ for (Map.Entry<String, ByteIterator> entry : values.entrySet())
{
col = new Column();
col.setName(ByteBuffer.wrap(entry.getKey().getBytes("UTF-8")));
- col.setValue(ByteBuffer.wrap(entry.getValue().getBytes("UTF-8")));
+ col.setValue(ByteBuffer.wrap(entry.getValue().toArray()));
col.setTimestamp(System.currentTimeMillis());
column = new ColumnOrSuperColumn();
@@ -547,14 +547,14 @@ public class CassandraClient8 extends DB
System.exit(0);
}
- HashMap<String, String> vals = new HashMap<String, String>();
- vals.put("age", "57");
- vals.put("middlename", "bradley");
- vals.put("favoritecolor", "blue");
+ HashMap<String, ByteIterator> vals = new HashMap<String, ByteIterator>();
+ vals.put("age", new StringByteIterator("57"));
+ vals.put("middlename", new StringByteIterator("bradley"));
+ vals.put("favoritecolor", new StringByteIterator("blue"));
int res = cli.insert("usertable", "BrianFrankCooper", vals);
System.out.println("Result of insert: " + res);
- HashMap<String, String> result = new HashMap<String, String>();
+ HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
HashSet<String> fields = new HashSet<String>();
fields.add("middlename");
fields.add("age");
diff --git a/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java b/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java
index 0a5c3f52ec6f51abebecfe3867abfe7de5814fee..c2ecf75ce79e59e60b9db779ce7979e207a2accf 100644
--- a/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java
+++ b/db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java
@@ -19,6 +19,9 @@ package com.yahoo.ycsb.db;
import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.ByteArrayByteIterator;
+import com.yahoo.ycsb.StringByteIterator;
import java.io.IOException;
import java.util.*;
@@ -123,7 +126,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
- public int read(String table, String key, Set<String> fields, HashMap<String,String> result)
+ public int read(String table, String key, Set<String> fields, HashMap<String,ByteIterator> result)
{
//if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
@@ -171,7 +174,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
for (KeyValue kv : r.raw()) {
result.put(
Bytes.toString(kv.getQualifier()),
- Bytes.toString(kv.getValue()));
+ new ByteArrayByteIterator(kv.getValue()));
if (_debug) {
System.out.println("Result for field: "+Bytes.toString(kv.getQualifier())+
" is: "+Bytes.toString(kv.getValue()));
@@ -191,7 +194,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error
*/
- public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,String>> result)
+ public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,ByteIterator>> result)
{
//if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
@@ -240,12 +243,12 @@ public class HBaseClient extends com.yahoo.ycsb.DB
System.out.println("Got scan result for key: "+key);
}
- HashMap<String,String> rowResult = new HashMap<String, String>();
+ HashMap<String,ByteIterator> rowResult = new HashMap<String, ByteIterator>();
for (KeyValue kv : rr.raw()) {
rowResult.put(
Bytes.toString(kv.getQualifier()),
- Bytes.toString(kv.getValue()));
+ new ByteArrayByteIterator(kv.getValue()));
}
//add rowResult to result vector
result.add(rowResult);
@@ -282,7 +285,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
- public int update(String table, String key, HashMap<String,String> values)
+ public int update(String table, String key, HashMap<String,ByteIterator> values)
{
//if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
@@ -304,13 +307,13 @@ public class HBaseClient extends com.yahoo.ycsb.DB
System.out.println("Setting up put for key: "+key);
}
Put p = new Put(Bytes.toBytes(key));
- for (Map.Entry<String, String> entry : values.entrySet())
+ for (Map.Entry<String, ByteIterator> entry : values.entrySet())
{
if (_debug) {
System.out.println("Adding field/value " + entry.getKey() + "/"+
entry.getValue() + " to put request");
}
- p.add(_columnFamilyBytes,Bytes.toBytes(entry.getKey()),Bytes.toBytes(entry.getValue()));
+ p.add(_columnFamilyBytes,Bytes.toBytes(entry.getKey()),entry.getValue().toArray());
}
try
@@ -342,7 +345,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
- public int insert(String table, String key, HashMap<String,String> values)
+ public int insert(String table, String key, HashMap<String,ByteIterator> values)
{
return update(table,key,values);
}
@@ -456,7 +459,7 @@ public class HBaseClient extends com.yahoo.ycsb.DB
HashSet<String> scanFields = new HashSet<String>();
scanFields.add("field1");
scanFields.add("field3");
- Vector<HashMap<String,String>> scanResults = new Vector<HashMap<String,String>>();
+ Vector<HashMap<String,ByteIterator>> scanResults = new Vector<HashMap<String,ByteIterator>>();
rescode = cli.scan("table1","user2",20,null,scanResults);
long en=System.currentTimeMillis();
diff --git a/db/infinispan-5.0/src/com/yahoo/ycsb/db/InfinispanClient.java b/db/infinispan-5.0/src/com/yahoo/ycsb/db/InfinispanClient.java
index b0e61dbfaf4ca85922fe1af188f3ff1c90d39b93..528ded5ca70d1a9dbf3ad311103883a96b9afa9d 100644
--- a/db/infinispan-5.0/src/com/yahoo/ycsb/db/InfinispanClient.java
+++ b/db/infinispan-5.0/src/com/yahoo/ycsb/db/InfinispanClient.java
@@ -2,6 +2,9 @@ package com.yahoo.ycsb.db;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.StringByteIterator;
+
import org.infinispan.Cache;
import org.infinispan.atomic.AtomicMap;
import org.infinispan.atomic.AtomicMapLookup;
@@ -53,7 +56,7 @@ public class InfinispanClient extends DB {
infinispanManager = null;
}
- public int read(String table, String key, Set<String> fields, HashMap<String, String> result) {
+ public int read(String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
try {
Map<String, String> row;
if (clustered) {
@@ -65,9 +68,9 @@ public class InfinispanClient extends DB {
if (row != null) {
result.clear();
if (fields == null || fields.isEmpty()) {
- result.putAll(row);
+ StringByteIterator.putAllAsByteIterators(result, row);
} else {
- for (String field : fields) result.put(field, row.get(field));
+ for (String field : fields) result.put(field, new StringByteIterator(row.get(field)));
}
}
return OK;
@@ -76,24 +79,24 @@ public class InfinispanClient extends DB {
}
}
- public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, String>> result) {
+ public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
logger.warn("Infinispan does not support scan semantics");
return OK;
}
- public int update(String table, String key, HashMap<String, String> values) {
+ public int update(String table, String key, HashMap<String, ByteIterator> values) {
try {
if (clustered) {
AtomicMap<String, String> row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key);
- row.putAll(values);
+ StringByteIterator.putAllAsStrings(row, values);
} else {
Cache<String, Map<String, String>> cache = infinispanManager.getCache(table);
Map<String, String> row = cache.get(key);
if (row == null) {
- row = values;
+ row = StringByteIterator.getStringMap(values);
cache.put(key, row);
} else {
- row.putAll(values);
+ StringByteIterator.putAllAsStrings(row, values);
}
}
@@ -103,12 +106,12 @@ public class InfinispanClient extends DB {
}
}
- public int insert(String table, String key, HashMap<String, String> values) {
+ public int insert(String table, String key, HashMap<String, ByteIterator> values) {
try {
if (clustered) {
AtomicMap<String, String> row = AtomicMapLookup.getAtomicMap(infinispanManager.getCache(table), key);
row.clear();
- row.putAll(values);
+ StringByteIterator.putAllAsStrings(row, values);
} else {
infinispanManager.getCache(table).put(key, values);
}
diff --git a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClient.java b/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClient.java
index 7bb3a1fcc06953502b0dd85ebbdd943ce8dde70f..8d1a6a7ea3411ce090cd1ded32dcd7d9268eeafa 100644
--- a/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClient.java
+++ b/db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClient.java
@@ -19,6 +19,8 @@ package com.yahoo.ycsb.db;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.StringByteIterator;
import java.sql.*;
import java.util.*;
@@ -294,7 +296,7 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
@Override
public int read(String tableName, String key, Set<String> fields,
- HashMap<String, String> result) {
+ HashMap<String, ByteIterator> result) {
if (tableName == null) {
return -1;
}
@@ -316,7 +318,7 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
if (result != null && fields != null) {
for (String field : fields) {
String value = resultSet.getString(field);
- result.put(field, value);
+ result.put(field, new StringByteIterator(value));
}
}
resultSet.close();
@@ -329,7 +331,7 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
@Override
public int scan(String tableName, String startKey, int recordcount,
- Set<String> fields, Vector<HashMap<String, String>> result) {
+ Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
if (tableName == null) {
return -1;
}
@@ -346,10 +348,10 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
ResultSet resultSet = scanStatement.executeQuery();
for (int i = 0; i < recordcount && resultSet.next(); i++) {
if (result != null && fields != null) {
- HashMap<String, String> values = new HashMap<String, String>();
+ HashMap<String, ByteIterator> values = new HashMap<String, ByteIterator>();
for (String field : fields) {
String value = resultSet.getString(field);
- values.put(field, value);
+ values.put(field, new StringByteIterator(value));
}
result.add(values);
}
@@ -363,7 +365,7 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
}
@Override
- public int update(String tableName, String key, HashMap<String, String> values) {
+ public int update(String tableName, String key, HashMap<String, ByteIterator> values) {
if (tableName == null) {
return -1;
}
@@ -378,8 +380,8 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
updateStatement = createAndCacheUpdateStatement(type, key);
}
int index = 1;
- for (Map.Entry<String, String> entry : values.entrySet()) {
- updateStatement.setString(index++, entry.getValue());
+ for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
+ updateStatement.setString(index++, entry.getValue().toString());
}
updateStatement.setString(index, key);
int result = updateStatement.executeUpdate();
@@ -392,7 +394,7 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
}
@Override
- public int insert(String tableName, String key, HashMap<String, String> values) {
+ public int insert(String tableName, String key, HashMap<String, ByteIterator> values) {
if (tableName == null) {
return -1;
}
@@ -408,8 +410,8 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
}
insertStatement.setString(1, key);
int index = 2;
- for (Map.Entry<String, String> entry : values.entrySet()) {
- String field = entry.getValue();
+ for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
+ String field = entry.getValue().toString();
insertStatement.setString(index++, field);
}
int result = insertStatement.executeUpdate();
diff --git a/db/mongodb/src/com/yahoo/ycsb/db/MongoDbClient.java b/db/mongodb/src/com/yahoo/ycsb/db/MongoDbClient.java
index ac9341892cefe1bae29c1b7914fd0b63cf48bdab..df70e148f7ba34b106ce009c6e3fcfa79ca975cf 100644
--- a/db/mongodb/src/com/yahoo/ycsb/db/MongoDbClient.java
+++ b/db/mongodb/src/com/yahoo/ycsb/db/MongoDbClient.java
@@ -14,6 +14,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Properties;
import java.util.Set;
+import java.util.Map;
import java.util.Vector;
import org.bson.types.ObjectId;
@@ -29,6 +30,8 @@ import com.mongodb.WriteConcern;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.StringByteIterator;
/**
* MongoDB client for YCSB framework.
@@ -136,7 +139,7 @@ public class MongoDbClient extends DB {
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
- public int insert(String table, String key, HashMap<String, String> values) {
+ public int insert(String table, String key, HashMap<String, ByteIterator> values) {
com.mongodb.DB db = null;
try {
db = mongo.getDB(database);
@@ -145,8 +148,9 @@ public class MongoDbClient extends DB {
DBCollection collection = db.getCollection(table);
DBObject r = new BasicDBObject().append("_id", key);
- r.putAll(values);
-
+ for(String k: values.keySet()) {
+ r.put(k, values.get(k).toString());
+ }
collection.setWriteConcern(writeConcern);
collection.insert(r);
@@ -179,7 +183,7 @@ public class MongoDbClient extends DB {
* @return Zero on success, a non-zero error code on error or "not found".
*/
public int read(String table, String key, Set<String> fields,
- HashMap<String, String> result) {
+ HashMap<String, ByteIterator> result) {
com.mongodb.DB db = null;
try {
db = mongo.getDB(database);
@@ -228,7 +232,7 @@ public class MongoDbClient extends DB {
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
- public int update(String table, String key, HashMap<String, String> values) {
+ public int update(String table, String key, HashMap<String, ByteIterator> values) {
com.mongodb.DB db = null;
try {
db = mongo.getDB(database);
@@ -243,7 +247,7 @@ public class MongoDbClient extends DB {
String tmpKey = null, tmpVal = null;
while (keys.hasNext()) {
tmpKey = keys.next();
- tmpVal = values.get(tmpKey);
+ tmpVal = values.get(tmpKey).toString();
fieldsToSet.put(tmpKey, tmpVal);
}
@@ -281,7 +285,7 @@ public class MongoDbClient extends DB {
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
public int scan(String table, String startkey, int recordcount,
- Set<String> fields, Vector<HashMap<String, String>> result) {
+ Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
com.mongodb.DB db=null;
try {
db = mongo.getDB(database);
@@ -293,7 +297,7 @@ public class MongoDbClient extends DB {
DBCursor cursor = collection.find(q).limit(recordcount);
while (cursor.hasNext()) {
//toMap() returns a Map, but result.add() expects a Map<String,String>. Hence, the suppress warnings.
- result.add((HashMap<String, String>) cursor.next().toMap());
+ result.add(StringByteIterator.getByteIteratorMap((Map<String,String>)cursor.next().toMap()));
}
return 0;
diff --git a/db/redis/src/com/yahoo/ycsb/db/RedisClient.java b/db/redis/src/com/yahoo/ycsb/db/RedisClient.java
index 650e6ca61e9eec8d37f783dfdecabb479c8b298a..0d5c3ff1efee1522b57d56688b90bf0be2e5fcba 100644
--- a/db/redis/src/com/yahoo/ycsb/db/RedisClient.java
+++ b/db/redis/src/com/yahoo/ycsb/db/RedisClient.java
@@ -8,7 +8,10 @@
package com.yahoo.ycsb.db;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.StringByteIterator;
+import java.util.Map;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -68,9 +71,9 @@ public class RedisClient extends DB {
@Override
public int read(String table, String key, Set<String> fields,
- HashMap<String, String> result) {
+ HashMap<String, ByteIterator> result) {
if (fields == null) {
- result.putAll(jedis.hgetAll(key));
+ StringByteIterator.putAllAsByteIterators(result, jedis.hgetAll(key));
}
else {
String[] fieldArray = (String[])fields.toArray(new String[fields.size()]);
@@ -80,7 +83,8 @@ public class RedisClient extends DB {
Iterator<String> valueIterator = values.iterator();
while (fieldIterator.hasNext() && valueIterator.hasNext()) {
- result.put(fieldIterator.next(), valueIterator.next());
+ result.put(fieldIterator.next(),
+ new StringByteIterator(valueIterator.next()));
}
assert !fieldIterator.hasNext() && !valueIterator.hasNext();
}
@@ -88,8 +92,8 @@ public class RedisClient extends DB {
}
@Override
- public int insert(String table, String key, HashMap<String, String> values) {
- if (jedis.hmset(key, values).equals("OK")) {
+ public int insert(String table, String key, HashMap<String, ByteIterator> values) {
+ if (jedis.hmset(key, StringByteIterator.getStringMap(values)).equals("OK")) {
jedis.zadd(INDEX_KEY, hash(key), key);
return 0;
}
@@ -104,19 +108,19 @@ public class RedisClient extends DB {
}
@Override
- public int update(String table, String key, HashMap<String, String> values) {
- return jedis.hmset(key, values).equals("OK") ? 0 : 1;
+ public int update(String table, String key, HashMap<String, ByteIterator> values) {
+ return jedis.hmset(key, StringByteIterator.getStringMap(values)).equals("OK") ? 0 : 1;
}
@Override
public int scan(String table, String startkey, int recordcount,
- Set<String> fields, Vector<HashMap<String, String>> result) {
+ Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
Set<String> keys = jedis.zrangeByScore(INDEX_KEY, hash(startkey),
Double.POSITIVE_INFINITY, 0, recordcount);
- HashMap<String, String> values;
+ HashMap<String, ByteIterator> values;
for (String key : keys) {
- values = new HashMap<String, String>();
+ values = new HashMap<String, ByteIterator>();
read(table, key, fields, values);
result.add(values);
}
diff --git a/db/voldemort/src/com/yahoo/ycsb/db/VoldemortClient.java b/db/voldemort/src/com/yahoo/ycsb/db/VoldemortClient.java
index 1454d6e819c0b6c9b3d8812daf1d7d1a2bf17abd..84093b917ddb5357f6f105667c24c6b1d1398edf 100644
--- a/db/voldemort/src/com/yahoo/ycsb/db/VoldemortClient.java
+++ b/db/voldemort/src/com/yahoo/ycsb/db/VoldemortClient.java
@@ -15,6 +15,9 @@ import voldemort.versioning.Versioned;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.StringByteIterator;
+
public class VoldemortClient extends DB {
@@ -66,17 +69,17 @@ public class VoldemortClient extends DB {
}
@Override
- public int insert(String table, String key, HashMap<String, String> values) {
+ public int insert(String table, String key, HashMap<String, ByteIterator> values) {
if ( checkStore(table) == ERROR ) {
return ERROR;
}
- storeClient.put(key, values);
+ storeClient.put(key, (HashMap<String,String>)StringByteIterator.getStringMap(values));
return OK;
}
@Override
public int read(String table, String key, Set<String> fields,
- HashMap<String, String> result) {
+ HashMap<String, ByteIterator> result) {
if ( checkStore(table) == ERROR ) {
return ERROR;
}
@@ -90,23 +93,23 @@ public class VoldemortClient extends DB {
for (String field : fields) {
String val = versionedValue.getValue().get(field);
if ( val != null )
- result.put(field, val);
+ result.put(field, new StringByteIterator(val));
}
} else {
- result.putAll(versionedValue.getValue());
+ StringByteIterator.putAllAsByteIterators(result, versionedValue.getValue());
}
return OK;
}
@Override
public int scan(String table, String startkey, int recordcount,
- Set<String> fields, Vector<HashMap<String, String>> result) {
+ Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
logger.warn("Voldemort does not support Scan semantics");
return OK;
}
@Override
- public int update(String table, String key, HashMap<String, String> values) {
+ public int update(String table, String key, HashMap<String, ByteIterator> values) {
if ( checkStore(table) == ERROR ) {
return ERROR;
}
@@ -117,12 +120,12 @@ public class VoldemortClient extends DB {
if ( versionedValue != null ) {
version = ((VectorClock) versionedValue.getVersion()).incremented(0, 1);
value = versionedValue.getValue();
- for (Entry<String, String> entry : values.entrySet()) {
- value.put(entry.getKey(), entry.getValue());
+ for (Entry<String, ByteIterator> entry : values.entrySet()) {
+ value.put(entry.getKey(), entry.getValue().toString());
}
} else {
version = new VectorClock();
- value.putAll(values);
+ StringByteIterator.putAllAsStrings(value, values);
}
storeClient.put(key, Versioned.value(value, version));
diff --git a/src/com/yahoo/ycsb/BasicDB.java b/src/com/yahoo/ycsb/BasicDB.java
index 19d626732ee04cc9bf8de651066d1c8af85064ad..9490451eff7e3be3d22ad30cbb1392d0712d8d6d 100644
--- a/src/com/yahoo/ycsb/BasicDB.java
+++ b/src/com/yahoo/ycsb/BasicDB.java
@@ -21,7 +21,6 @@ import java.util.HashMap;
import java.util.Properties;
import java.util.Set;
import java.util.Enumeration;
-import java.util.Random;
import java.util.Vector;
@@ -37,13 +36,11 @@ public class BasicDB extends DB
public static final String SIMULATE_DELAY_DEFAULT="0";
- Random random;
boolean verbose;
int todelay;
public BasicDB()
{
- random=new Random();
todelay=0;
}
@@ -54,7 +51,7 @@ public class BasicDB extends DB
{
try
{
- Thread.sleep((long)random.nextInt(todelay));
+ Thread.sleep((long)Utils.random().nextInt(todelay));
}
catch (InterruptedException e)
{
@@ -98,7 +95,7 @@ public class BasicDB extends DB
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
- public int read(String table, String key, Set<String> fields, HashMap<String,String> result)
+ public int read(String table, String key, Set<String> fields, HashMap<String,ByteIterator> result)
{
delay();
@@ -133,7 +130,7 @@ public class BasicDB extends DB
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error
*/
- public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,String>> result)
+ public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,ByteIterator>> result)
{
delay();
@@ -167,7 +164,7 @@ public class BasicDB extends DB
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
- public int update(String table, String key, HashMap<String,String> values)
+ public int update(String table, String key, HashMap<String,ByteIterator> values)
{
delay();
@@ -196,7 +193,7 @@ public class BasicDB extends DB
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
- public int insert(String table, String key, HashMap<String,String> values)
+ public int insert(String table, String key, HashMap<String,ByteIterator> values)
{
delay();
diff --git a/src/com/yahoo/ycsb/ByteArrayByteIterator.java b/src/com/yahoo/ycsb/ByteArrayByteIterator.java
new file mode 100644
index 0000000000000000000000000000000000000000..3e36f98823c07cebaadf0ffc6394a5c6c0ebfd51
--- /dev/null
+++ b/src/com/yahoo/ycsb/ByteArrayByteIterator.java
@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb;
+
+public class ByteArrayByteIterator extends ByteIterator {
+ byte[] str;
+ int off;
+ final int len;
+ public ByteArrayByteIterator(byte[] s) {
+ this.str = s;
+ this.off = 0;
+ this.len = s.length;
+ }
+
+ public ByteArrayByteIterator(byte[] s, int off, int len) {
+ this.str = s;
+ this.off = off;
+ this.len = off + len;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return off < len;
+ }
+
+ @Override
+ public byte nextByte() {
+ byte ret = str[off];
+ off++;
+ return ret;
+ }
+
+ @Override
+ public long bytesLeft() {
+ return len - off;
+ }
+
+}
diff --git a/src/com/yahoo/ycsb/ByteIterator.java b/src/com/yahoo/ycsb/ByteIterator.java
new file mode 100644
index 0000000000000000000000000000000000000000..9bb3b63e36fcb6a95dbad08da0ea02cbf43f4005
--- /dev/null
+++ b/src/com/yahoo/ycsb/ByteIterator.java
@@ -0,0 +1,92 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb;
+
+import java.util.Iterator;
+import java.util.ArrayList;
+/**
+ * YCSB-specific buffer class. ByteIterators are designed to support
+ * efficient field generation, and to allow backend drivers that can stream
+ * fields (instead of materializing them in RAM) to do so.
+ * <p>
+ * YCSB originially used String objects to represent field values. This led to
+ * two performance issues.
+ * </p><p>
+ * First, it leads to unnecessary conversions between UTF-16 and UTF-8, both
+ * during field generation, and when passing data to byte-based backend
+ * drivers.
+ * </p><p>
+ * Second, Java strings are represented internally using UTF-16, and are
+ * built by appending to a growable array type (StringBuilder or
+ * StringBuffer), then calling a toString() method. This leads to a 4x memory
+ * overhead as field values are being built, which prevented YCSB from
+ * driving large object stores.
+ * </p>
+ * The StringByteIterator class contains a number of convenience methods for
+ * backend drivers that convert between Map<String,String> and
+ * Map<String,ByteBuffer>.
+ *
+ * @author sears
+ */
+public abstract class ByteIterator implements Iterator<Byte> {
+
+ @Override
+ public abstract boolean hasNext();
+
+ @Override
+ public Byte next() {
+ throw new UnsupportedOperationException();
+ //return nextByte();
+ }
+
+ public abstract byte nextByte();
+ /** @return byte offset immediately after the last valid byte */
+ public int nextBuf(byte[] buf, int buf_off) {
+ int sz = buf_off;
+ while(sz < buf.length && hasNext()) {
+ buf[sz] = nextByte();
+ sz++;
+ }
+ return sz;
+ }
+
+ public abstract long bytesLeft();
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** Consumes remaining contents of this object, and returns them as a string. */
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ while(this.hasNext()) { sb.append((char)nextByte()); }
+ return sb.toString();
+ }
+ /** Consumes remaining contents of this object, and returns them as a byte array. */
+ public byte[] toArray() {
+ long left = bytesLeft();
+ if(left != (int)left) { throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!"); }
+ byte[] ret = new byte[(int)left];
+ int off = 0;
+ while(off < ret.length) {
+ off = nextBuf(ret, off);
+ }
+ return ret;
+ }
+
+}
diff --git a/src/com/yahoo/ycsb/Client.java b/src/com/yahoo/ycsb/Client.java
index 9da86041a6c2c601737b9681fb5e53bca2ed0894..64585f81a02276d04f0ba9142a9d7bf7375889f3 100644
--- a/src/com/yahoo/ycsb/Client.java
+++ b/src/com/yahoo/ycsb/Client.java
@@ -137,8 +137,6 @@ class StatusThread extends Thread
*/
class ClientThread extends Thread
{
- static Random random=new Random();
-
DB _db;
boolean _dotransactions;
Workload _workload;
@@ -215,7 +213,7 @@ class ClientThread extends Thread
//and the sleep() doesn't make sense for granularities < 1 ms anyway
if ( (_target>0) && (_target<=1.0) )
{
- sleep(random.nextInt((int)(1.0/_target)));
+ sleep(Utils.random().nextInt((int)(1.0/_target)));
}
}
catch (InterruptedException e)
diff --git a/src/com/yahoo/ycsb/CommandLine.java b/src/com/yahoo/ycsb/CommandLine.java
index c6fb13ded0dd9588a43dee8bd33fd02fc9309d3f..c177426beb5436f1dc120dfece7e531968ed3e40 100644
--- a/src/com/yahoo/ycsb/CommandLine.java
+++ b/src/com/yahoo/ycsb/CommandLine.java
@@ -293,10 +293,10 @@ public class CommandLine
}
}
- HashMap<String,String> result=new HashMap<String,String>();
+ HashMap<String,ByteIterator> result=new HashMap<String,ByteIterator>();
int ret=db.read(table,tokens[1],fields,result);
System.out.println("Return code: "+ret);
- for (Map.Entry<String,String> ent : result.entrySet())
+ for (Map.Entry<String,ByteIterator> ent : result.entrySet())
{
System.out.println(ent.getKey()+"="+ent.getValue());
}
@@ -322,7 +322,7 @@ public class CommandLine
}
}
- Vector<HashMap<String,String>> results=new Vector<HashMap<String,String>>();
+ Vector<HashMap<String,ByteIterator>> results=new Vector<HashMap<String,ByteIterator>>();
int ret=db.scan(table,tokens[1],Integer.parseInt(tokens[2]),fields,results);
System.out.println("Return code: "+ret);
int record=0;
@@ -334,10 +334,10 @@ public class CommandLine
{
System.out.println("--------------------------------");
}
- for (HashMap<String,String> result : results)
+ for (HashMap<String,ByteIterator> result : results)
{
System.out.println("Record "+(record++));
- for (Map.Entry<String,String> ent : result.entrySet())
+ for (Map.Entry<String,ByteIterator> ent : result.entrySet())
{
System.out.println(ent.getKey()+"="+ent.getValue());
}
@@ -353,12 +353,12 @@ public class CommandLine
}
else
{
- HashMap<String,String> values=new HashMap<String,String>();
+ HashMap<String,ByteIterator> values=new HashMap<String,ByteIterator>();
for (int i=2; i<tokens.length; i++)
{
String[] nv=tokens[i].split("=");
- values.put(nv[0],nv[1]);
+ values.put(nv[0],new StringByteIterator(nv[1]));
}
int ret=db.update(table,tokens[1],values);
@@ -373,12 +373,12 @@ public class CommandLine
}
else
{
- HashMap<String,String> values=new HashMap<String,String>();
+ HashMap<String,ByteIterator> values=new HashMap<String,ByteIterator>();
for (int i=2; i<tokens.length; i++)
{
String[] nv=tokens[i].split("=");
- values.put(nv[0],nv[1]);
+ values.put(nv[0],new StringByteIterator(nv[1]));
}
int ret=db.insert(table,tokens[1],values);
diff --git a/src/com/yahoo/ycsb/DB.java b/src/com/yahoo/ycsb/DB.java
index d790d131e2e2c2aa119a79c91435c33703b41331..91a695f140623729b7a7dbd8b8ebf087ff91ba9f 100644
--- a/src/com/yahoo/ycsb/DB.java
+++ b/src/com/yahoo/ycsb/DB.java
@@ -90,7 +90,7 @@ public abstract class DB
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error or "not found".
*/
- public abstract int read(String table, String key, Set<String> fields, HashMap<String,String> result);
+ public abstract int read(String table, String key, Set<String> fields, HashMap<String,ByteIterator> result);
/**
* Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap.
@@ -102,7 +102,7 @@ public abstract class DB
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
- public abstract int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,String>> result);
+ public abstract int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,ByteIterator>> result);
/**
* Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified
@@ -113,7 +113,7 @@ public abstract class DB
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
- public abstract int update(String table, String key, HashMap<String,String> values);
+ public abstract int update(String table, String key, HashMap<String,ByteIterator> values);
/**
* Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified
@@ -124,7 +124,7 @@ public abstract class DB
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
- public abstract int insert(String table, String key, HashMap<String,String> values);
+ public abstract int insert(String table, String key, HashMap<String,ByteIterator> values);
/**
* Delete a record from the database.
diff --git a/src/com/yahoo/ycsb/DBWrapper.java b/src/com/yahoo/ycsb/DBWrapper.java
index 8ca7a89fbc5c7197dea71abfba0d26ea7e1fd395..4516e7b0adf39e8fa733a7e7cb20aa708fbda0e8 100644
--- a/src/com/yahoo/ycsb/DBWrapper.java
+++ b/src/com/yahoo/ycsb/DBWrapper.java
@@ -81,12 +81,12 @@ public class DBWrapper extends DB
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
- public int read(String table, String key, Set<String> fields, HashMap<String,String> result)
+ public int read(String table, String key, Set<String> fields, HashMap<String,ByteIterator> result)
{
- long st=System.currentTimeMillis();
+ long st=System.nanoTime();
int res=_db.read(table,key,fields,result);
- long en=System.currentTimeMillis();
- _measurements.measure("READ",(int)(en-st));
+ long en=System.nanoTime();
+ _measurements.measure("READ",(int)((en-st)/1000));
_measurements.reportReturnCode("READ",res);
return res;
}
@@ -101,12 +101,12 @@ public class DBWrapper extends DB
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error
*/
- public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,String>> result)
+ public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String,ByteIterator>> result)
{
- long st=System.currentTimeMillis();
+ long st=System.nanoTime();
int res=_db.scan(table,startkey,recordcount,fields,result);
- long en=System.currentTimeMillis();
- _measurements.measure("SCAN",(int)(en-st));
+ long en=System.nanoTime();
+ _measurements.measure("SCAN",(int)((en-st)/1000));
_measurements.reportReturnCode("SCAN",res);
return res;
}
@@ -120,12 +120,12 @@ public class DBWrapper extends DB
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
- public int update(String table, String key, HashMap<String,String> values)
+ public int update(String table, String key, HashMap<String,ByteIterator> values)
{
- long st=System.currentTimeMillis();
+ long st=System.nanoTime();
int res=_db.update(table,key,values);
- long en=System.currentTimeMillis();
- _measurements.measure("UPDATE",(int)(en-st));
+ long en=System.nanoTime();
+ _measurements.measure("UPDATE",(int)((en-st)/1000));
_measurements.reportReturnCode("UPDATE",res);
return res;
}
@@ -139,12 +139,12 @@ public class DBWrapper extends DB
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
- public int insert(String table, String key, HashMap<String,String> values)
+ public int insert(String table, String key, HashMap<String,ByteIterator> values)
{
- long st=System.currentTimeMillis();
+ long st=System.nanoTime();
int res=_db.insert(table,key,values);
- long en=System.currentTimeMillis();
- _measurements.measure("INSERT",(int)(en-st));
+ long en=System.nanoTime();
+ _measurements.measure("INSERT",(int)((en-st)/1000));
_measurements.reportReturnCode("INSERT",res);
return res;
}
@@ -158,10 +158,10 @@ public class DBWrapper extends DB
*/
public int delete(String table, String key)
{
- long st=System.currentTimeMillis();
+ long st=System.nanoTime();
int res=_db.delete(table,key);
- long en=System.currentTimeMillis();
- _measurements.measure("DELETE",(int)(en-st));
+ long en=System.nanoTime();
+ _measurements.measure("DELETE",(int)((en-st)/1000));
_measurements.reportReturnCode("DELETE",res);
return res;
}
diff --git a/src/com/yahoo/ycsb/InputStreamByteIterator.java b/src/com/yahoo/ycsb/InputStreamByteIterator.java
new file mode 100644
index 0000000000000000000000000000000000000000..5e90496cb3abff047bc5c15e888c466d1f4cfa8f
--- /dev/null
+++ b/src/com/yahoo/ycsb/InputStreamByteIterator.java
@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb;
+
+import java.io.InputStream;
+
+public class InputStreamByteIterator extends ByteIterator {
+ long len;
+ InputStream ins;
+ long off;
+
+ public InputStreamByteIterator(InputStream ins, long len) {
+ this.len = len;
+ this.ins = ins;
+ off = 0;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return off < len;
+ }
+
+ @Override
+ public byte nextByte() {
+ int ret;
+ try {
+ ret = ins.read();
+ } catch(Exception e) {
+ throw new IllegalStateException(e);
+ }
+ if(ret == -1) { throw new IllegalStateException("Past EOF!"); }
+ off++;
+ return (byte)ret;
+ }
+
+ @Override
+ public long bytesLeft() {
+ return len - off;
+ }
+
+}
diff --git a/src/com/yahoo/ycsb/RandomByteIterator.java b/src/com/yahoo/ycsb/RandomByteIterator.java
new file mode 100644
index 0000000000000000000000000000000000000000..99239f859fb8011e663af50acaa985b09a6d0586
--- /dev/null
+++ b/src/com/yahoo/ycsb/RandomByteIterator.java
@@ -0,0 +1,84 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb;
+
+public class RandomByteIterator extends ByteIterator {
+ long len;
+ long off;
+ int buf_off;
+ byte[] buf;
+
+ @Override
+ public boolean hasNext() {
+ return (off + buf_off) < len;
+ }
+
+ private void fillBytesImpl(byte[] buf, int base) {
+ int bytes = Utils.random().nextInt();
+ try {
+ buf[base+0] = (byte)(((bytes ) & 31) + ' ');
+ buf[base+1] = (byte)(((bytes >> 5 ) & 31) + ' ');
+ buf[base+2] = (byte)(((bytes >> 10) & 31) + ' ');
+ buf[base+3] = (byte)(((bytes >> 15) & 31) + ' ');
+ buf[base+4] = (byte)(((bytes >> 20) & 31) + ' ');
+ buf[base+5] = (byte)(((bytes >> 25) & 31) + ' ');
+ } catch (ArrayIndexOutOfBoundsException e) { /* ignore it */ }
+ }
+
+ private void fillBytes() {
+ if(buf_off == buf.length) {
+ fillBytesImpl(buf, 0);
+ buf_off = 0;
+ off += buf.length;
+ }
+ }
+
+ public RandomByteIterator(long len) {
+ this.len = len;
+ this.buf = new byte[6];
+ this.buf_off = buf.length;
+ fillBytes();
+ this.off = 0;
+ }
+
+ public byte nextByte() {
+ fillBytes();
+ buf_off++;
+ return buf[buf_off-1];
+ }
+ @Override
+ public int nextBuf(byte[] b, int buf_off) {
+ int ret;
+ if(len - off < b.length - buf_off) {
+ ret = (int)(len - off);
+ } else {
+ ret = b.length - buf_off;
+ }
+ int i;
+ for(i = 0; i < ret; i+=6) {
+ fillBytesImpl(b, i+buf_off);
+ }
+ off+=ret;
+ return ret + buf_off;
+ }
+
+
+ @Override
+ public long bytesLeft() {
+ return len - off;
+ }
+}
diff --git a/src/com/yahoo/ycsb/StringByteIterator.java b/src/com/yahoo/ycsb/StringByteIterator.java
new file mode 100644
index 0000000000000000000000000000000000000000..84465cba3ef0482bea716c3daf2cf7ca45463961
--- /dev/null
+++ b/src/com/yahoo/ycsb/StringByteIterator.java
@@ -0,0 +1,108 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb;
+
+import java.util.Map;
+import java.util.HashMap;
+
+public class StringByteIterator extends ByteIterator {
+ String str;
+ int off;
+
+ /**
+ * Put all of the entries of one map into the other, converting
+ * String values into ByteIterators.
+ */
+ public static void putAllAsByteIterators(Map<String, ByteIterator> out, Map<String, String> in) {
+ for(String s: in.keySet()) { out.put(s, new StringByteIterator(in.get(s))); }
+ }
+
+ /**
+ * Put all of the entries of one map into the other, converting
+ * ByteIterator values into Strings.
+ */
+ public static void putAllAsStrings(Map<String, String> out, Map<String, ByteIterator> in) {
+ for(String s: in.keySet()) { out.put(s, in.get(s).toString()); }
+ }
+
+ /**
+ * Create a copy of a map, converting the values from Strings to
+ * StringByteIterators.
+ */
+ public static HashMap<String, ByteIterator> getByteIteratorMap(Map<String, String> m) {
+ HashMap<String, ByteIterator> ret =
+ new HashMap<String,ByteIterator>();
+
+ for(String s: m.keySet()) {
+ ret.put(s, new StringByteIterator(m.get(s)));
+ }
+ return ret;
+ }
+
+ /**
+ * Create a copy of a map, converting the values from
+ * StringByteIterators to Strings.
+ */
+ public static HashMap<String, String> getStringMap(Map<String, ByteIterator> m) {
+ HashMap<String, String> ret = new HashMap<String,String>();
+
+ for(String s: m.keySet()) {
+ ret.put(s, m.get(s).toString());;
+ }
+ return ret;
+ }
+
+ public StringByteIterator(String s) {
+ this.str = s;
+ this.off = 0;
+ }
+ @Override
+ public boolean hasNext() {
+ return off < str.length();
+ }
+
+ @Override
+ public byte nextByte() {
+ byte ret = (byte)str.charAt(off);
+ off++;
+ return ret;
+ }
+
+ @Override
+ public long bytesLeft() {
+ return str.length() - off;
+ }
+
+ /**
+ * Specialization of general purpose toString() to avoid unnecessary
+ * copies.
+ * <p>
+ * Creating a new StringByteIterator, then calling toString()
+ * yields the original String object, and does not perform any copies
+ * or String conversion operations.
+ * </p>
+ */
+ @Override
+ public String toString() {
+ if(off > 0) {
+ return super.toString();
+ } else {
+ return str;
+ }
+ }
+}
diff --git a/src/com/yahoo/ycsb/Utils.java b/src/com/yahoo/ycsb/Utils.java
index f94ebc0d2d84d63d5122b5567d9b94ba617189de..f49bc0f5c03cd1b3cd7993d9d97ba619cc5cde37 100644
--- a/src/com/yahoo/ycsb/Utils.java
+++ b/src/com/yahoo/ycsb/Utils.java
@@ -24,8 +24,17 @@ import java.util.Random;
*/
public class Utils
{
- static Random random=new Random();
-
+ private static final Random rand = new Random();
+ private static final ThreadLocal<Random> rng = new ThreadLocal<Random>();
+
+ public static Random random() {
+ Random ret = rng.get();
+ if(ret == null) {
+ ret = new Random(rand.nextLong());
+ rng.set(ret);
+ }
+ return ret;
+ }
/**
* Generate a random ASCII string of a given length.
*/
@@ -34,7 +43,7 @@ public class Utils
int interval='~'-' '+1;
byte []buf = new byte[length];
- random.nextBytes(buf);
+ random().nextBytes(buf);
for (int i = 0; i < length; i++) {
if (buf[i] < 0) {
buf[i] = (byte)((-buf[i] % interval) + ' ');
@@ -48,9 +57,9 @@ public class Utils
/**
* Hash an integer value.
*/
- public static int hash(int val)
+ public static long hash(long val)
{
- return FNVhash32(val);
+ return FNVhash64(val);
}
public static final int FNV_offset_basis_32=0x811c9dc5;
diff --git a/src/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java b/src/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java
new file mode 100644
index 0000000000000000000000000000000000000000..94dd3a6ae414f0ef40d20ba5f87d448a2af30a27
--- /dev/null
+++ b/src/com/yahoo/ycsb/generator/ConstantIntegerGenerator.java
@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb.generator;
+
+/**
+ * A trivial integer generator that always returns the same value.
+ *
+ * @author sears
+ *
+ */
+public class ConstantIntegerGenerator extends IntegerGenerator {
+ private final int i;
+ /**
+ * @param i The integer that this generator will always return.
+ */
+ public ConstantIntegerGenerator(int i) {
+ this.i = i;
+ }
+
+ @Override
+ public int nextInt() {
+ return i;
+ }
+
+ @Override
+ public double mean() {
+ return i;
+ }
+
+}
diff --git a/src/com/yahoo/ycsb/generator/CounterGenerator.java b/src/com/yahoo/ycsb/generator/CounterGenerator.java
index 70ce1b045075b209f14b2298d8437e8d01b2d68c..df37b2398c5f0f46e886d7df7367398f68d86b90 100644
--- a/src/com/yahoo/ycsb/generator/CounterGenerator.java
+++ b/src/com/yahoo/ycsb/generator/CounterGenerator.java
@@ -17,33 +17,41 @@
package com.yahoo.ycsb.generator;
+import java.util.concurrent.atomic.AtomicInteger;
+
/**
* Generates a sequence of integers 0, 1, ...
*/
public class CounterGenerator extends IntegerGenerator
{
- int counter;
+ final AtomicInteger counter;
/**
* Create a counter that starts at countstart
*/
public CounterGenerator(int countstart)
{
- counter=countstart;
- setLastInt(countstart-1);
+ counter=new AtomicInteger(countstart);
+ setLastInt(counter.get()-1);
}
/**
* If the generator returns numeric (integer) values, return the next value as an int. Default is to return -1, which
* is appropriate for generators that do not return numeric values.
*/
- public synchronized int nextInt()
+ public int nextInt()
{
- int lastint=counter;
- counter++;
- setLastInt(lastint);
- return lastint;
+ int ret = counter.getAndIncrement();
+ setLastInt(ret);
+ return ret;
+ }
+ @Override
+ public int lastInt()
+ {
+ return counter.get() - 1;
+ }
+ @Override
+ public double mean() {
+ throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!");
}
-
-
}
diff --git a/src/com/yahoo/ycsb/generator/DiscreteGenerator.java b/src/com/yahoo/ycsb/generator/DiscreteGenerator.java
index 087312c03519bbc50ef69f04f63288125c3c0b97..c28f3ae771e1bbe667902e925bf0a16f0bdf8fe3 100644
--- a/src/com/yahoo/ycsb/generator/DiscreteGenerator.java
+++ b/src/com/yahoo/ycsb/generator/DiscreteGenerator.java
@@ -20,6 +20,7 @@ package com.yahoo.ycsb.generator;
import java.util.Vector;
import java.util.Random;
+import com.yahoo.ycsb.Utils;
import com.yahoo.ycsb.WorkloadException;
/**
@@ -40,13 +41,11 @@ public class DiscreteGenerator extends Generator
}
Vector<Pair> _values;
- Random _random;
String _lastvalue;
public DiscreteGenerator()
{
_values=new Vector<Pair>();
- _random=new Random();
_lastvalue=null;
}
@@ -62,7 +61,7 @@ public class DiscreteGenerator extends Generator
sum+=p._weight;
}
- double val=_random.nextDouble();
+ double val=Utils.random().nextDouble();
for (Pair p : _values)
{
diff --git a/src/com/yahoo/ycsb/generator/ExponentialGenerator.java b/src/com/yahoo/ycsb/generator/ExponentialGenerator.java
new file mode 100644
index 0000000000000000000000000000000000000000..de1a2760c4b02b308666dbc7ceaefb9cc8805cfd
--- /dev/null
+++ b/src/com/yahoo/ycsb/generator/ExponentialGenerator.java
@@ -0,0 +1,102 @@
+/**
+ * Copyright (c) 2011 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.generator;
+
+import java.util.Random;
+
+import com.yahoo.ycsb.Utils;
+
+/**
+ * A generator of an exponential distribution. It produces a sequence
+ * of time intervals (integers) according to an exponential
+ * distribution. Smaller intervals are more frequent than larger
+ * ones, and there is no bound on the length of an interval. When you
+ * construct an instance of this class, you specify a parameter gamma,
+ * which corresponds to the rate at which events occur.
+ * Alternatively, 1/gamma is the average length of an interval.
+ */
+public class ExponentialGenerator extends IntegerGenerator
+{
+ // What percentage of the readings should be within the most recent exponential.frac portion of the dataset?
+ public static final String EXPONENTIAL_PERCENTILE_PROPERTY="exponential.percentile";
+ public static final String EXPONENTIAL_PERCENTILE_DEFAULT="95";
+
+ // What fraction of the dataset should be accessed exponential.percentile of the time?
+ public static final String EXPONENTIAL_FRAC_PROPERTY = "exponential.frac";
+ public static final String EXPONENTIAL_FRAC_DEFAULT = "0.8571428571"; // 1/7
+
+ /**
+ * The exponential constant to use.
+ */
+ double _gamma;
+
+ /******************************* Constructors **************************************/
+
+ /**
+ * Create an exponential generator with a mean arrival rate of
+ * gamma. (And half life of 1/gamma).
+ */
+ public ExponentialGenerator(double mean)
+ {
+ _gamma = 1.0/mean;
+ }
+ public ExponentialGenerator(double percentile, double range)
+ {
+ _gamma = -Math.log(1.0-percentile/100.0) / range; //1.0/mean;
+ }
+
+ /****************************************************************************************/
+
+ /**
+ * Generate the next item. this distribution will be skewed toward lower integers; e.g. 0 will
+ * be the most popular, 1 the next most popular, etc.
+ * @param itemcount The number of items in the distribution.
+ * @return The next item in the sequence.
+ */
+ @Override
+ public int nextInt()
+ {
+ return (int)nextLong();
+ }
+
+ /**
+ * Generate the next item as a long.
+ *
+ * @param itemcount The number of items in the distribution.
+ * @return The next item in the sequence.
+ */
+ public long nextLong()
+ {
+ return (long) (-Math.log(Utils.random().nextDouble()) / _gamma);
+ }
+
+ @Override
+ public double mean() {
+ return 1.0/_gamma;
+ }
+ public static void main(String args[]) {
+ ExponentialGenerator e = new ExponentialGenerator(90, 100);
+ int j = 0;
+ for(int i = 0; i < 1000; i++) {
+ if(e.nextInt() < 100) {
+ j++;
+ }
+ }
+ System.out.println("Got " + j + " hits. Expect 900");
+ }
+}
diff --git a/src/com/yahoo/ycsb/generator/HistogramGenerator.java b/src/com/yahoo/ycsb/generator/HistogramGenerator.java
new file mode 100644
index 0000000000000000000000000000000000000000..aba42c47b0c060a92b667c7d6808f3b608c37dac
--- /dev/null
+++ b/src/com/yahoo/ycsb/generator/HistogramGenerator.java
@@ -0,0 +1,114 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb.generator;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Random;
+
+import com.yahoo.ycsb.Utils;
+import com.yahoo.ycsb.generator.IntegerGenerator;
+
+/**
+ * Generate integers according to a histogram distribution. The histogram
+ * buckets are of width one, but the values are multiplied by a block size.
+ * Therefore, instead of drawing sizes uniformly at random within each
+ * bucket, we always draw the largest value in the current bucket, so the value
+ * drawn is always a multiple of block_size.
+ *
+ * The minimum value this distribution returns is block_size (not zero).
+ *
+ * Modified Nov 19 2010 by sears
+ *
+ * @author snjones
+ *
+ */
+public class HistogramGenerator extends IntegerGenerator {
+
+ long block_size;
+ long[] buckets;
+ long area;
+ long weighted_area = 0;
+ double mean_size = 0;
+
+ public HistogramGenerator(String histogramfile) throws IOException {
+ BufferedReader in = new BufferedReader(new FileReader(histogramfile));
+ String str;
+ String[] line;
+
+ ArrayList<Integer> a = new ArrayList<Integer>();
+
+ str = in.readLine();
+ if(str == null) {
+ throw new IOException("Empty input file!\n");
+ }
+ line = str.split("\t");
+ if(line[0].compareTo("BlockSize") != 0) {
+ throw new IOException("First line of histogram is not the BlockSize!\n");
+ }
+ block_size = Integer.parseInt(line[1]);
+
+ while((str = in.readLine()) != null){
+ // [0] is the bucket, [1] is the value
+ line = str.split("\t");
+
+ a.add(Integer.parseInt(line[0]), Integer.parseInt(line[1]));
+ }
+ buckets = new long[a.size()];
+ for(int i = 0; i < a.size(); i++) {
+ buckets[i] = a.get(i);
+ }
+
+ in.close();
+ init();
+ }
+
+ public HistogramGenerator(long[] buckets, int block_size) {
+ this.block_size = block_size;
+ this.buckets = buckets;
+ init();
+ }
+ private void init() {
+ for(int i = 0; i < buckets.length; i++) {
+ area += buckets[i];
+ weighted_area = i * buckets[i];
+ }
+ // calculate average file size
+ mean_size = ((double)block_size) * ((double)weighted_area) / (double)(area);
+ }
+
+ @Override
+ public int nextInt() {
+ int number = Utils.random().nextInt((int)area);
+ int i;
+
+ for(i = 0; i < (buckets.length - 1); i++){
+ number -= buckets[i];
+ if(number <= 0){
+ return (int)((i+1)*block_size);
+ }
+ }
+
+ return (int)(i * block_size);
+ }
+
+ @Override
+ public double mean() {
+ return mean_size;
+ }
+}
diff --git a/src/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java b/src/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java
index dd12db7acece45083a800dbd57d4815eb486ce12..969ae7706d781b48e50f2289db602eb666a761d9 100644
--- a/src/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java
+++ b/src/com/yahoo/ycsb/generator/HotspotIntegerGenerator.java
@@ -18,6 +18,8 @@ package com.yahoo.ycsb.generator;
import java.util.Random;
+import com.yahoo.ycsb.Utils;
+
/**
* Generate integers resembling a hotspot distribution where x% of operations
* access y% of data items. The parameters specify the bounds for the numbers,
@@ -37,7 +39,6 @@ public class HotspotIntegerGenerator extends IntegerGenerator {
private final int coldInterval;
private final double hotsetFraction;
private final double hotOpnFraction;
- private final Random random;
/**
* Create a generator for Hotspot distributions.
@@ -71,12 +72,12 @@ public class HotspotIntegerGenerator extends IntegerGenerator {
this.hotInterval = (int)(interval * hotsetFraction);
this.coldInterval = interval - hotInterval;
this.hotOpnFraction = hotOpnFraction;
- random = new Random();
}
@Override
public int nextInt() {
int value = 0;
+ Random random = Utils.random();
if (random.nextDouble() < hotOpnFraction) {
// Choose a value from the hot set.
value = lowerBound + random.nextInt(hotInterval);
@@ -115,5 +116,9 @@ public class HotspotIntegerGenerator extends IntegerGenerator {
public double getHotOpnFraction() {
return hotOpnFraction;
}
-
+ @Override
+ public double mean() {
+ return hotOpnFraction * (lowerBound + hotInterval/2.0)
+ + (1 - hotOpnFraction) * (lowerBound + hotInterval + coldInterval/2.0);
+ }
}
diff --git a/src/com/yahoo/ycsb/generator/IntegerGenerator.java b/src/com/yahoo/ycsb/generator/IntegerGenerator.java
index 8268f14cfd7a79663bc23f1304289fc89ba12b9c..13d36cd1f1f423e7cd97b36789e2e02152d1ef09 100644
--- a/src/com/yahoo/ycsb/generator/IntegerGenerator.java
+++ b/src/com/yahoo/ycsb/generator/IntegerGenerator.java
@@ -31,7 +31,7 @@ public abstract class IntegerGenerator extends Generator
* Set the last value generated. IntegerGenerator subclasses must use this call
* to properly set the last string value, or the lastString() and lastInt() calls won't work.
*/
- public void setLastInt(int last)
+ protected void setLastInt(int last)
{
lastint=last;
}
@@ -54,9 +54,10 @@ public abstract class IntegerGenerator extends Generator
* Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet
* been called, lastString() should return something reasonable.
*/
+ @Override
public String lastString()
{
- return ""+lastint;
+ return ""+lastInt();
}
/**
@@ -67,4 +68,8 @@ public abstract class IntegerGenerator extends Generator
{
return lastint;
}
+ /**
+ * Return the expected value (mean) of the values this generator will return.
+ */
+ public abstract double mean();
}
diff --git a/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java b/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java
index 0f5731b3e25d2f6ab0b43efa39b0b881527272fb..8b62c4c2bacef2c2c094c9c54b6a55e8ed8e7af9 100644
--- a/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java
+++ b/src/com/yahoo/ycsb/generator/ScrambledZipfianGenerator.java
@@ -126,4 +126,12 @@ public class ScrambledZipfianGenerator extends IntegerGenerator
System.out.println(""+gen.nextInt());
}
}
+
+ /**
+ * since the values are scrambled (hopefully uniformly), the mean is simply the middle of the range.
+ */
+ @Override
+ public double mean() {
+ return ((double)(((long)_min) +(long)_max))/2.0;
+ }
}
diff --git a/src/com/yahoo/ycsb/generator/SkewedLatestGenerator.java b/src/com/yahoo/ycsb/generator/SkewedLatestGenerator.java
index 1b86097ebf996f02e2d62282ee2ac3f54de1e304..376eb7775c6b7cf6c44549728d4a03a4d47916b6 100644
--- a/src/com/yahoo/ycsb/generator/SkewedLatestGenerator.java
+++ b/src/com/yahoo/ycsb/generator/SkewedLatestGenerator.java
@@ -28,7 +28,7 @@ public class SkewedLatestGenerator extends IntegerGenerator
public SkewedLatestGenerator(CounterGenerator basis)
{
_basis=basis;
- _zipfian=new ZipfianGenerator(Integer.parseInt(_basis.lastString()));
+ _zipfian=new ZipfianGenerator(_basis.lastInt());
nextInt();
}
@@ -37,7 +37,7 @@ public class SkewedLatestGenerator extends IntegerGenerator
*/
public int nextInt()
{
- int max=Integer.parseInt(_basis.lastString());
+ int max=_basis.lastInt();
int nextint=max-_zipfian.nextInt(max);
setLastInt(nextint);
return nextint;
@@ -53,4 +53,9 @@ public class SkewedLatestGenerator extends IntegerGenerator
}
+ @Override
+ public double mean() {
+ throw new UnsupportedOperationException("Can't compute mean of non-stationary distribution!");
+ }
+
}
diff --git a/src/com/yahoo/ycsb/generator/UniformIntegerGenerator.java b/src/com/yahoo/ycsb/generator/UniformIntegerGenerator.java
index 74ebba09616e37c0cb8020d8ac0922f7bd762382..fa6af1316ca3d156653d9b362f1f59debd451a97 100644
--- a/src/com/yahoo/ycsb/generator/UniformIntegerGenerator.java
+++ b/src/com/yahoo/ycsb/generator/UniformIntegerGenerator.java
@@ -19,12 +19,13 @@ package com.yahoo.ycsb.generator;
import java.util.Random;
+import com.yahoo.ycsb.Utils;
+
/**
* Generates integers randomly uniform from an interval.
*/
public class UniformIntegerGenerator extends IntegerGenerator
{
- Random _random;
int _lb,_ub,_interval;
/**
@@ -35,7 +36,6 @@ public class UniformIntegerGenerator extends IntegerGenerator
*/
public UniformIntegerGenerator(int lb, int ub)
{
- _random=new Random();
_lb=lb;
_ub=ub;
_interval=_ub-_lb+1;
@@ -44,10 +44,14 @@ public class UniformIntegerGenerator extends IntegerGenerator
@Override
public int nextInt()
{
- int ret=_random.nextInt(_interval)+_lb;
+ int ret=Utils.random().nextInt(_interval)+_lb;
setLastInt(ret);
return ret;
}
+ @Override
+ public double mean() {
+ return ((double)((long)(_lb + (long)_ub))) / 2.0;
+ }
}
diff --git a/src/com/yahoo/ycsb/generator/ZipfianGenerator.java b/src/com/yahoo/ycsb/generator/ZipfianGenerator.java
index cc0ade83dedda42fbd7f3cd5dea1c1fb37ffa244..8a70f0869a16c7d2776072c1b6c4ed2e94e38e27 100644
--- a/src/com/yahoo/ycsb/generator/ZipfianGenerator.java
+++ b/src/com/yahoo/ycsb/generator/ZipfianGenerator.java
@@ -19,6 +19,8 @@ package com.yahoo.ycsb.generator;
import java.util.Random;
+import com.yahoo.ycsb.Utils;
+
/**
* A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than others, according
* to a zipfian distribution. When you construct an instance of this class, you specify the number of items in the set to draw from, either
@@ -61,8 +63,6 @@ public class ZipfianGenerator extends IntegerGenerator
*/
double alpha,zetan,eta,theta,zeta2theta;
- Random random;
-
/**
* The number of items used to compute zetan the last time.
*/
@@ -136,8 +136,6 @@ public class ZipfianGenerator extends IntegerGenerator
base=min;
zipfianconstant=_zipfianconstant;
- random=new Random();
-
theta=zipfianconstant;
zeta2theta=zeta(2,theta);
@@ -273,7 +271,7 @@ public class ZipfianGenerator extends IntegerGenerator
}
}
- double u=random.nextDouble();
+ double u=Utils.random().nextDouble();
double uz=u*zetan;
if (uz<1.0)
@@ -316,4 +314,12 @@ public class ZipfianGenerator extends IntegerGenerator
{
new ZipfianGenerator(ScrambledZipfianGenerator.ITEM_COUNT);
}
+
+ /**
+ * @todo Implement ZipfianGenerator.mean()
+ */
+ @Override
+ public double mean() {
+ throw new UnsupportedOperationException("@todo implement ZipfianGenerator.mean()");
+ }
}
diff --git a/src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java b/src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java
index a97d925dc533d801b4f12522fa14ad8a14d5254b..2cba955a2a3c926f1d7b719c26b9696562522a08 100644
--- a/src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java
+++ b/src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java
@@ -86,13 +86,13 @@ public class OneMeasurementHistogram extends OneMeasurement
*/
public synchronized void measure(int latency)
{
- if (latency>=_buckets)
+ if (latency/1000>=_buckets)
{
histogramoverflow++;
}
else
{
- histogram[latency]++;
+ histogram[latency/1000]++;
}
operations++;
totallatency+=latency;
@@ -115,9 +115,9 @@ public class OneMeasurementHistogram extends OneMeasurement
public void exportMeasurements(MeasurementsExporter exporter) throws IOException
{
exporter.write(getName(), "Operations", operations);
- exporter.write(getName(), "AverageLatency(ms)", (((double)totallatency)/((double)operations)));
- exporter.write(getName(), "MinLatency(ms)", min);
- exporter.write(getName(), "MaxLatency(ms)", max);
+ exporter.write(getName(), "AverageLatency(us)", (((double)totallatency)/((double)operations)));
+ exporter.write(getName(), "MinLatency(us)", min);
+ exporter.write(getName(), "MaxLatency(us)", max);
int opcounter=0;
boolean done95th=false;
@@ -159,7 +159,7 @@ public class OneMeasurementHistogram extends OneMeasurement
double report=((double)windowtotallatency)/((double)windowoperations);
windowtotallatency=0;
windowoperations=0;
- return "["+getName()+" AverageLatency(ms)="+d.format(report)+"]";
+ return "["+getName()+" AverageLatency(us)="+d.format(report)+"]";
}
}
diff --git a/src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java b/src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java
index 19415ba1863085a9ea7deebf7b738c33937ea26a..25c77b76740feaed3bf134c78b249e4e9146a827 100644
--- a/src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java
+++ b/src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java
@@ -132,9 +132,9 @@ public class OneMeasurementTimeSeries extends OneMeasurement
checkEndOfUnit(true);
exporter.write(getName(), "Operations", operations);
- exporter.write(getName(), "AverageLatency(ms)", (((double)totallatency)/((double)operations)));
- exporter.write(getName(), "MinLatency(ms)", min);
- exporter.write(getName(), "MaxLatency(ms)", max);
+ exporter.write(getName(), "AverageLatency(us)", (((double)totallatency)/((double)operations)));
+ exporter.write(getName(), "MinLatency(us)", min);
+ exporter.write(getName(), "MaxLatency(us)", max);
//TODO: 95th and 99th percentile latency
@@ -173,7 +173,7 @@ public class OneMeasurementTimeSeries extends OneMeasurement
double report=((double)windowtotallatency)/((double)windowoperations);
windowtotallatency=0;
windowoperations=0;
- return "["+getName()+" AverageLatency(ms)="+d.format(report)+"]";
+ return "["+getName()+" AverageLatency(us)="+d.format(report)+"]";
}
}
diff --git a/src/com/yahoo/ycsb/workloads/ConstantOccupancyWorkload.java b/src/com/yahoo/ycsb/workloads/ConstantOccupancyWorkload.java
new file mode 100644
index 0000000000000000000000000000000000000000..6a5e7d025d93e21be5ebb099965437abda3c7cd8
--- /dev/null
+++ b/src/com/yahoo/ycsb/workloads/ConstantOccupancyWorkload.java
@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb.workloads;
+
+import java.util.Properties;
+
+import com.yahoo.ycsb.WorkloadException;
+import com.yahoo.ycsb.Client;
+import com.yahoo.ycsb.generator.IntegerGenerator;
+
+/**
+ * A disk-fragmenting workload.
+ * <p>
+ * Properties to control the client:
+ * </p>
+ * <UL>
+ * <LI><b>disksize</b>: how many bytes of storage can the disk store? (default 100,000,000)
+ * <LI><b>occupancy</b>: what fraction of the available storage should be used? (default 0.9)
+ * <LI><b>requestdistribution</b>: what distribution should be used to select the records to operate on - uniform, zipfian or latest (default: histogram)
+ * </ul>
+ *
+ *
+ * <p> See also:
+ * Russell Sears, Catharine van Ingen.
+ * <a href='https://database.cs.wisc.edu/cidr/cidr2007/papers/cidr07p34.pdf'>Fragmentation in Large Object Repositories</a>,
+ * CIDR 2006. [<a href='https://database.cs.wisc.edu/cidr/cidr2007/slides/p34-sears.ppt'>Presentation</a>]
+ * </p>
+ *
+ *
+ * @author sears
+ *
+ */
+public class ConstantOccupancyWorkload extends CoreWorkload {
+ long disksize;
+ long storageages;
+ IntegerGenerator objectsizes;
+ double occupancy;
+
+ long object_count;
+
+ public static final String STORAGE_AGE_PROPERTY = "storageages";
+ public static final long STORAGE_AGE_PROPERTY_DEFAULT = 10;
+
+ public static final String DISK_SIZE_PROPERTY = "disksize";
+ public static final long DISK_SIZE_PROPERTY_DEFAULT = 100 * 1000 * 1000;
+
+ public static final String OCCUPANCY_PROPERTY = "occupancy";
+ public static final double OCCUPANCY_PROPERTY_DEFAULT = 0.9;
+
+ @Override
+ public void init(Properties p) throws WorkloadException
+ {
+ disksize = Long.parseLong( p.getProperty(DISK_SIZE_PROPERTY, DISK_SIZE_PROPERTY_DEFAULT+""));
+ storageages = Long.parseLong( p.getProperty(STORAGE_AGE_PROPERTY, STORAGE_AGE_PROPERTY_DEFAULT+""));
+ occupancy = Double.parseDouble(p.getProperty(OCCUPANCY_PROPERTY, OCCUPANCY_PROPERTY_DEFAULT+""));
+
+ if(p.getProperty(Client.RECORD_COUNT_PROPERTY) != null ||
+ p.getProperty(Client.INSERT_COUNT_PROPERTY) != null ||
+ p.getProperty(Client.OPERATION_COUNT_PROPERTY) != null) {
+ System.err.println("Warning: record, insert or operation count was set prior to initting ConstantOccupancyWorkload. Overriding old values.");
+ }
+ IntegerGenerator g = CoreWorkload.getFieldLengthGenerator(p);
+ double fieldsize = g.mean();
+ int fieldcount = Integer.parseInt(p.getProperty(FIELD_COUNT_PROPERTY, FIELD_COUNT_PROPERTY_DEFAULT));
+
+ object_count = (long)(occupancy * ((double)disksize / (fieldsize * (double)fieldcount)));
+ if(object_count == 0) {
+ throw new IllegalStateException("Object count was zero. Perhaps disksize is too low?");
+ }
+ p.setProperty(Client.RECORD_COUNT_PROPERTY, object_count+"");
+ p.setProperty(Client.OPERATION_COUNT_PROPERTY, (storageages*object_count)+"");
+ p.setProperty(Client.INSERT_COUNT_PROPERTY, object_count+"");
+
+ super.init(p);
+ }
+
+}
diff --git a/src/com/yahoo/ycsb/workloads/CoreWorkload.java b/src/com/yahoo/ycsb/workloads/CoreWorkload.java
index dd1dbd064dc1442c197ecfb133f5964b823ab17c..2d17889396f870cda804249973d53f74b64c2415 100644
--- a/src/com/yahoo/ycsb/workloads/CoreWorkload.java
+++ b/src/com/yahoo/ycsb/workloads/CoreWorkload.java
@@ -21,8 +21,11 @@ import java.util.Properties;
import com.yahoo.ycsb.*;
import com.yahoo.ycsb.generator.CounterGenerator;
import com.yahoo.ycsb.generator.DiscreteGenerator;
+import com.yahoo.ycsb.generator.ExponentialGenerator;
import com.yahoo.ycsb.generator.Generator;
+import com.yahoo.ycsb.generator.ConstantIntegerGenerator;
import com.yahoo.ycsb.generator.HotspotIntegerGenerator;
+import com.yahoo.ycsb.generator.HistogramGenerator;
import com.yahoo.ycsb.generator.IntegerGenerator;
import com.yahoo.ycsb.generator.ScrambledZipfianGenerator;
import com.yahoo.ycsb.generator.SkewedLatestGenerator;
@@ -30,6 +33,7 @@ import com.yahoo.ycsb.generator.UniformIntegerGenerator;
import com.yahoo.ycsb.generator.ZipfianGenerator;
import com.yahoo.ycsb.measurements.Measurements;
+import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Vector;
@@ -84,18 +88,41 @@ public class CoreWorkload extends Workload
int fieldcount;
+ /**
+ * The name of the property for the field length distribution. Options are "uniform", "zipfian" (favoring short records), "constant", and "histogram".
+ *
+ * If "uniform", "zipfian" or "constant", the maximum field length will be that specified by the fieldlength property. If "histogram", then the
+ * histogram will be read from the filename specified in the "fieldlengthhistogram" property.
+ */
+ public static final String FIELD_LENGTH_DISTRIBUTION_PROPERTY="fieldlengthdistribution";
+ /**
+ * The default field length distribution.
+ */
+ public static final String FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT = "constant";
+
/**
* The name of the property for the length of a field in bytes.
*/
public static final String FIELD_LENGTH_PROPERTY="fieldlength";
-
/**
- * The default length of a field in bytes.
+ * The default maximum length of a field in bytes.
*/
public static final String FIELD_LENGTH_PROPERTY_DEFAULT="100";
- int fieldlength;
+ /**
+ * The name of a property that specifies the filename containing the field length histogram (only used if fieldlengthdistribution is "histogram").
+ */
+ public static final String FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY = "fieldlengthhistogram";
+ /**
+ * The default filename containing a field length histogram.
+ */
+ public static final String FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY_DEFAULT = "hist.txt";
+ /**
+ * Generator object that produces field lengths. The value of this depends on the properties that start with "FIELD_LENGTH_".
+ */
+ IntegerGenerator fieldlengthgenerator;
+
/**
* The name of the property for deciding whether to read one field (false) or all fields (true) of a record.
*/
@@ -247,6 +274,29 @@ public class CoreWorkload extends Workload
int recordcount;
+ protected static IntegerGenerator getFieldLengthGenerator(Properties p) throws WorkloadException{
+ IntegerGenerator fieldlengthgenerator;
+ String fieldlengthdistribution = p.getProperty(FIELD_LENGTH_DISTRIBUTION_PROPERTY, FIELD_LENGTH_DISTRIBUTION_PROPERTY_DEFAULT);
+ int fieldlength=Integer.parseInt(p.getProperty(FIELD_LENGTH_PROPERTY,FIELD_LENGTH_PROPERTY_DEFAULT));
+ String fieldlengthhistogram = p.getProperty(FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY, FIELD_LENGTH_HISTOGRAM_FILE_PROPERTY_DEFAULT);
+ if(fieldlengthdistribution.compareTo("constant") == 0) {
+ fieldlengthgenerator = new ConstantIntegerGenerator(fieldlength);
+ } else if(fieldlengthdistribution.compareTo("uniform") == 0) {
+ fieldlengthgenerator = new UniformIntegerGenerator(1, fieldlength);
+ } else if(fieldlengthdistribution.compareTo("zipfian") == 0) {
+ fieldlengthgenerator = new ZipfianGenerator(1, fieldlength);
+ } else if(fieldlengthdistribution.compareTo("histogram") == 0) {
+ try {
+ fieldlengthgenerator = new HistogramGenerator(fieldlengthhistogram);
+ } catch(IOException e) {
+ throw new WorkloadException("Couldn't read field length histogram file: "+fieldlengthhistogram, e);
+ }
+ } else {
+ throw new WorkloadException("Unknown field length distribution \""+fieldlengthdistribution+"\"");
+ }
+ return fieldlengthgenerator;
+ }
+
/**
* Initialize the scenario.
* Called once, in the main client thread, before any operations are started.
@@ -254,8 +304,10 @@ public class CoreWorkload extends Workload
public void init(Properties p) throws WorkloadException
{
table = p.getProperty(TABLENAME_PROPERTY,TABLENAME_PROPERTY_DEFAULT);
+
fieldcount=Integer.parseInt(p.getProperty(FIELD_COUNT_PROPERTY,FIELD_COUNT_PROPERTY_DEFAULT));
- fieldlength=Integer.parseInt(p.getProperty(FIELD_LENGTH_PROPERTY,FIELD_LENGTH_PROPERTY_DEFAULT));
+ fieldlengthgenerator = CoreWorkload.getFieldLengthGenerator(p);
+
double readproportion=Double.parseDouble(p.getProperty(READ_PROPORTION_PROPERTY,READ_PROPORTION_PROPERTY_DEFAULT));
double updateproportion=Double.parseDouble(p.getProperty(UPDATE_PROPORTION_PROPERTY,UPDATE_PROPORTION_PROPERTY_DEFAULT));
double insertproportion=Double.parseDouble(p.getProperty(INSERT_PROPORTION_PROPERTY,INSERT_PROPORTION_PROPERTY_DEFAULT));
@@ -275,6 +327,14 @@ public class CoreWorkload extends Workload
{
orderedinserts=false;
}
+ else if (requestdistrib.compareTo("exponential")==0)
+ {
+ double percentile = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_PERCENTILE_PROPERTY,
+ ExponentialGenerator.EXPONENTIAL_PERCENTILE_DEFAULT));
+ double frac = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_FRAC_PROPERTY,
+ ExponentialGenerator.EXPONENTIAL_FRAC_DEFAULT));
+ keychooser = new ExponentialGenerator(percentile, recordcount*frac);
+ }
else
{
orderedinserts=true;
@@ -341,7 +401,7 @@ public class CoreWorkload extends Workload
}
else
{
- throw new WorkloadException("Unknown distribution \""+requestdistrib+"\"");
+ throw new WorkloadException("Unknown request distribution \""+requestdistrib+"\"");
}
fieldchooser=new UniformIntegerGenerator(0,fieldcount-1);
@@ -360,6 +420,33 @@ public class CoreWorkload extends Workload
}
}
+ public String buildKeyName(long keynum) {
+ if (!orderedinserts)
+ {
+ keynum=Utils.hash(keynum);
+ }
+ return "user"+keynum;
+ }
+ HashMap<String, ByteIterator> buildValues() {
+ HashMap<String,ByteIterator> values=new HashMap<String,ByteIterator>();
+
+ for (int i=0; i<fieldcount; i++)
+ {
+ String fieldkey="field"+i;
+ ByteIterator data= new RandomByteIterator(fieldlengthgenerator.nextInt());
+ values.put(fieldkey,data);
+ }
+ return values;
+ }
+ HashMap<String, ByteIterator> buildUpdate() {
+ //update a random field
+ HashMap<String, ByteIterator> values=new HashMap<String,ByteIterator>();
+ String fieldname="field"+fieldchooser.nextString();
+ ByteIterator data = new RandomByteIterator(fieldlengthgenerator.nextInt());
+ values.put(fieldname,data);
+ return values;
+ }
+
/**
* Do one insert operation. Because it will be called concurrently from multiple client threads, this
* function must be thread safe. However, avoid synchronized, or the threads will block waiting for each
@@ -369,18 +456,8 @@ public class CoreWorkload extends Workload
public boolean doInsert(DB db, Object threadstate)
{
int keynum=keysequence.nextInt();
- if (!orderedinserts)
- {
- keynum=Utils.hash(keynum);
- }
- String dbkey="user"+keynum;
- HashMap<String,String> values=new HashMap<String,String>();
- for (int i=0; i<fieldcount; i++)
- {
- String fieldkey="field"+i;
- String data=Utils.ASCIIString(fieldlength);
- values.put(fieldkey,data);
- }
+ String dbkey = buildKeyName(keynum);
+ HashMap<String, ByteIterator> values = buildValues();
if (db.insert(table,dbkey,values) == 0)
return true;
else
@@ -421,22 +498,31 @@ public class CoreWorkload extends Workload
return true;
}
+ int nextKeynum() {
+ int keynum;
+ if(keychooser instanceof ExponentialGenerator) {
+ do
+ {
+ keynum=transactioninsertkeysequence.lastInt() - keychooser.nextInt();
+ }
+ while(keynum < 0);
+ } else {
+ do
+ {
+ keynum=keychooser.nextInt();
+ }
+ while (keynum > transactioninsertkeysequence.lastInt());
+ }
+ return keynum;
+ }
+
public void doTransactionRead(DB db)
{
//choose a random key
- int keynum;
- do
- {
- keynum=keychooser.nextInt();
- }
- while (keynum>transactioninsertkeysequence.lastInt());
+ int keynum = nextKeynum();
+
+ String keyname = buildKeyName(keynum);
- if (!orderedinserts)
- {
- keynum=Utils.hash(keynum);
- }
- String keyname="user"+keynum;
-
HashSet<String> fields=null;
if (!readallfields)
@@ -448,24 +534,15 @@ public class CoreWorkload extends Workload
fields.add(fieldname);
}
- db.read(table,keyname,fields,new HashMap<String,String>());
+ db.read(table,keyname,fields,new HashMap<String,ByteIterator>());
}
public void doTransactionReadModifyWrite(DB db)
{
//choose a random key
- int keynum;
- do
- {
- keynum=keychooser.nextInt();
- }
- while (keynum>transactioninsertkeysequence.lastInt());
-
- if (!orderedinserts)
- {
- keynum=Utils.hash(keynum);
- }
- String keyname="user"+keynum;
+ int keynum = nextKeynum();
+
+ String keyname = buildKeyName(keynum);
HashSet<String> fields=null;
@@ -478,31 +555,24 @@ public class CoreWorkload extends Workload
fields.add(fieldname);
}
- HashMap<String,String> values=new HashMap<String,String>();
+ HashMap<String,ByteIterator> values;
if (writeallfields)
{
//new data for all the fields
- for (int i=0; i<fieldcount; i++)
- {
- String fieldname="field"+i;
- String data=Utils.ASCIIString(fieldlength);
- values.put(fieldname,data);
- }
+ values = buildValues();
}
else
{
//update a random field
- String fieldname="field"+fieldchooser.nextString();
- String data=Utils.ASCIIString(fieldlength);
- values.put(fieldname,data);
+ values = buildUpdate();
}
//do the transaction
long st=System.currentTimeMillis();
- db.read(table,keyname,fields,new HashMap<String,String>());
+ db.read(table,keyname,fields,new HashMap<String,ByteIterator>());
db.update(table,keyname,values);
@@ -514,18 +584,9 @@ public class CoreWorkload extends Workload
public void doTransactionScan(DB db)
{
//choose a random key
- int keynum;
- do
- {
- keynum=keychooser.nextInt();
- }
- while (keynum>transactioninsertkeysequence.lastInt());
+ int keynum = nextKeynum();
- if (!orderedinserts)
- {
- keynum=Utils.hash(keynum);
- }
- String startkeyname="user"+keynum;
+ String startkeyname = buildKeyName(keynum);
//choose a random scan length
int len=scanlength.nextInt();
@@ -541,43 +602,27 @@ public class CoreWorkload extends Workload
fields.add(fieldname);
}
- db.scan(table,startkeyname,len,fields,new Vector<HashMap<String,String>>());
+ db.scan(table,startkeyname,len,fields,new Vector<HashMap<String,ByteIterator>>());
}
public void doTransactionUpdate(DB db)
{
//choose a random key
- int keynum;
- do
- {
- keynum=keychooser.nextInt();
- }
- while (keynum>transactioninsertkeysequence.lastInt());
+ int keynum = nextKeynum();
- if (!orderedinserts)
- {
- keynum=Utils.hash(keynum);
- }
- String keyname="user"+keynum;
+ String keyname=buildKeyName(keynum);
- HashMap<String,String> values=new HashMap<String,String>();
+ HashMap<String,ByteIterator> values;
if (writeallfields)
{
//new data for all the fields
- for (int i=0; i<fieldcount; i++)
- {
- String fieldname="field"+i;
- String data=Utils.ASCIIString(fieldlength);
- values.put(fieldname,data);
- }
+ values = buildValues();
}
else
{
//update a random field
- String fieldname="field"+fieldchooser.nextString();
- String data=Utils.ASCIIString(fieldlength);
- values.put(fieldname,data);
+ values = buildUpdate();
}
db.update(table,keyname,values);
@@ -587,19 +632,10 @@ public class CoreWorkload extends Workload
{
//choose the next key
int keynum=transactioninsertkeysequence.nextInt();
- if (!orderedinserts)
- {
- keynum=Utils.hash(keynum);
- }
- String dbkey="user"+keynum;
-
- HashMap<String,String> values=new HashMap<String,String>();
- for (int i=0; i<fieldcount; i++)
- {
- String fieldkey="field"+i;
- String data=Utils.ASCIIString(fieldlength);
- values.put(fieldkey,data);
- }
+
+ String dbkey = buildKeyName(keynum);
+
+ HashMap<String, ByteIterator> values = buildValues();
db.insert(table,dbkey,values);
}
}