diff --git a/.gitignore b/.gitignore
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4d8f35166bba186ca62fd8ab46dabef2bf134317 100644
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1 @@
+/elasticsearch/target/
diff --git a/CHANGELOG b/CHANGELOG
index a5d40f0db85bf3c20e076f41e8e42fec396b64a1..469901e1a403ff6dfaffcb2877e7ea9a86aa4275 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,7 +1,10 @@
+- gh-95 Bump MongoDB version to 2.9.0 (allanbank)
- gh-67 Use checkstyle (m1ch1)
- gh-76 Implemented OrientDB client (lvca)
- gh-88 YCSB client for Amazon DynamoDB (jananin)
- gh-89 Patch for YCSB Cassandra Client version 1.0.6 (jananin)
+- gh-93 New ElasticSearch Database Implementation (saden1)
+- gh-97 Bug fixes in dynamodb plugin (jananin)
0.1.4 - 2/22/12
diff --git a/bin/ycsb b/bin/ycsb
index a819be8bc38c7fac843f75e46020c4d48852c17f..7323a413f669b51a1cf68cfc516e50931237aea7 100755
--- a/bin/ycsb
+++ b/bin/ycsb
@@ -29,6 +29,7 @@ DATABASES = {
"cassandra-8" : "com.yahoo.ycsb.db.CassandraClient8",
"cassandra-10" : "com.yahoo.ycsb.db.CassandraClient10",
"dynamodb" : "com.yahoo.ycsb.db.DynamoDBClient",
+ "elasticsearch": "com.yahoo.ycsb.db.ElasticSearchClient",
"gemfire" : "com.yahoo.ycsb.db.GemFireClient",
"hbase" : "com.yahoo.ycsb.db.HBaseClient",
"hypertable" : "com.yahoo.ycsb.db.HypertableClient",
@@ -106,7 +107,7 @@ database = sys.argv[2]
db_classname = DATABASES[database]
options = sys.argv[3:]
-ycsb_command = ["java", "-cp", ":".join(find_jars(ycsb_home, database)), \
+ycsb_command = ["java", "-cp", os.pathsep.join(find_jars(ycsb_home, database)), \
COMMANDS[sys.argv[1]]["main"], "-db", db_classname] + options
if command:
ycsb_command.append(command)
diff --git a/core/src/main/java/com/yahoo/ycsb/DBWrapper.java b/core/src/main/java/com/yahoo/ycsb/DBWrapper.java
index 4516e7b0adf39e8fa733a7e7cb20aa708fbda0e8..57a9648648d739cb2325756ed837cb2702f0fc18 100644
--- a/core/src/main/java/com/yahoo/ycsb/DBWrapper.java
+++ b/core/src/main/java/com/yahoo/ycsb/DBWrapper.java
@@ -69,7 +69,10 @@ public class DBWrapper extends DB
*/
public void cleanup() throws DBException
{
+ long st=System.nanoTime();
_db.cleanup();
+ long en=System.nanoTime();
+ _measurements.measure("CLEANUP", (int)((en-st)/1000));
}
/**
diff --git a/dynamodb/README b/dynamodb/README
index 798d8cf3c7b49e271588142ba2b546928aec5cba..42f84e7228c72ea625372f24b0078c1e41fa5dd9 100644
--- a/dynamodb/README
+++ b/dynamodb/README
@@ -3,10 +3,13 @@ CONFIGURE
YCSB_HOME - YCSB home directory
DYNAMODB_HOME - Amazon DynamoDB package files
+Please refer to https://github.com/brianfrankcooper/YCSB/wiki/Using-the-Database-Libraries
+for more information on setup.
+
BENCHMARK
-$YCSB_HOME/bin/ycsb load dynamodb -P worklaods/workloada -P dynamodb.properties
-$YCSB_HOME/bin/ycsb run dynamodb -P worklaods/workloada -P dynamodb.properties
+$YCSB_HOME/bin/ycsb load dynamodb -P workloads/workloada -P dynamodb.properties
+$YCSB_HOME/bin/ycsb run dynamodb -P workloads/workloada -P dynamodb.properties
PROPERTIES
diff --git a/dynamodb/conf/dynamodb.properties b/dynamodb/conf/dynamodb.properties
index 762e55fbce49a56ef71b8af21189146db1991c1d..a510e4c9df53df68ff5c4393c142017d5cbb1f1d 100644
--- a/dynamodb/conf/dynamodb.properties
+++ b/dynamodb/conf/dynamodb.properties
@@ -17,7 +17,7 @@
#dynamodb.endpoint = http://dynamodb.us-east-1.amazonaws.com
# Strongly recommended to set to uniform.Refer FAQs in README
-#requestdistribution = uniform
+#requestdistribution = uniform
# Enable/disable debug messages.Defaults to false
# "true" or "false"
diff --git a/dynamodb/src/main/java/com/yahoo/ycsb/db/DynamoDBClient.java b/dynamodb/src/main/java/com/yahoo/ycsb/db/DynamoDBClient.java
index 9c2f601f01b277d9d2e88349a7338c04100000d0..56808b8ccfa07d8a8070e10acd4bf75cd62aec94 100644
--- a/dynamodb/src/main/java/com/yahoo/ycsb/db/DynamoDBClient.java
+++ b/dynamodb/src/main/java/com/yahoo/ycsb/db/DynamoDBClient.java
@@ -200,7 +200,7 @@ public class DynamoDBClient extends DB {
}
startKey = res.getLastEvaluatedKey();
- }
+ }
return OK;
}
@@ -249,7 +249,7 @@ public class DynamoDBClient extends DB {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
- return res.getConsumedCapacityUnits().intValue();
+ return OK;
}
@Override
@@ -267,7 +267,7 @@ public class DynamoDBClient extends DB {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
- return res.getConsumedCapacityUnits().intValue();
+ return OK;
}
private static Map<String, AttributeValue> createAttributes(
diff --git a/elasticsearch/README.md b/elasticsearch/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..047a9ea3f27817737128f764a4726358e94fd101
--- /dev/null
+++ b/elasticsearch/README.md
@@ -0,0 +1,65 @@
+## Quick Start
+
+This section describes how to run YCSB on ElasticSearch running locally.
+
+### 1. Set Up YCSB
+
+Clone the YCSB git repository and compile:
+
+ git clone git://github.com/brianfrankcooper/YCSB.git
+ cd YCSB
+ mvn clean package
+
+### 2. Run YCSB
+
+Now you are ready to run! First, load the data:
+
+ ./bin/ycsb load elasticsearch -s -P workloads/workloada
+
+Then, run the workload:
+
+ ./bin/ycsb run elasticsearch -s -P workloads/workloada
+
+For further configuration see below:
+
+### Defaults Configuration
+The default setting for the ElasticSearch node that is created is as follows:
+
+ cluster.name=es.ycsb.cluster
+ node.local=true
+ path.data=$TEMP_DIR/esdata
+ discovery.zen.ping.multicast.enabled=false
+ index.mapping._id.indexed=true
+ index.gateway.type=none
+ gateway.type=none
+ index.number_of_shards=1
+ index.number_of_replicas=0
+ es.index.key=es.ycsb
+
+### Custom Configuration
+If you wish to customize the settings used to create the ElasticSerach node
+you can created a new property file that contains your desired ElasticSearch
+node settings and pass it in via the parameter to 'bin/ycsb' script. Note that
+the default properties will be kept if you don't explicitly overwrite them.
+
+Assuming that we have a properties file named "myproperties.data" that contains
+custom ElasticSearch node configuration you can execute the following to
+pass it into the ElasticSearch client:
+
+
+ ./bin/ycsb run elasticsearch -P workloads/workloada -P myproperties.data -s
+
+
+If you wish to use a in-memory store type rather than the default disk store add
+the following properties to your custom properties file. For a large number of
+insert operations insure that you have sufficient memory on your test system
+otherwise you will run out of memory.
+
+ index.store.type=memory
+ index.store.fs.memory.enabled=true
+ cache.memory.small_buffer_size=4mb
+ cache.memory.large_cache_size=1024mb
+
+If you wish to change the default index name you can set the following property:
+
+ es.index.key=my_index_key
diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..15a38e2371c51f1d301b86254bd38c0b234dec60
--- /dev/null
+++ b/elasticsearch/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>root</artifactId>
+ <version>0.1.4</version>
+ </parent>
+
+ <artifactId>elasticsearch-binding</artifactId>
+ <name>ElasticSearch Binding</name>
+ <packaging>jar</packaging>
+ <properties>
+ <elasticsearch-version>0.19.8</elasticsearch-version>
+ </properties>
+ <repositories>
+ <repository>
+ <id>sonatype-nexus-snapshots</id>
+ <name>Sonatype Nexus Snapshots</name>
+ <url>https://oss.sonatype.org/content/repositories/releases</url>
+ </repository>
+ </repositories>
+ <dependencies>
+ <dependency>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.1.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-nop</artifactId>
+ <version>1.6.4</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticSearchClient.java b/elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticSearchClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..aff2a81dced5b588a7cb72760852f684a4eb2f35
--- /dev/null
+++ b/elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticSearchClient.java
@@ -0,0 +1,266 @@
+package com.yahoo.ycsb.db;
+
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.DB;
+import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.StringByteIterator;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
+import static org.elasticsearch.common.settings.ImmutableSettings.*;
+import org.elasticsearch.common.settings.ImmutableSettings.Builder;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import static org.elasticsearch.common.xcontent.XContentFactory.*;
+import static org.elasticsearch.index.query.FilterBuilders.*;
+import static org.elasticsearch.index.query.QueryBuilders.*;
+import org.elasticsearch.index.query.RangeFilterBuilder;
+import org.elasticsearch.node.Node;
+import static org.elasticsearch.node.NodeBuilder.*;
+import org.elasticsearch.search.SearchHit;
+
+/**
+ * ElasticSearch client for YCSB framework.
+ *
+ * <p>Default properties to set:</p> <ul> <li>es.cluster.name = es.ycsb.cluster
+ * <li>es.client = true <li>es.index.key = es.ycsb</ul>
+ *
+ * @author Sharmarke Aden
+ *
+ */
+public class ElasticSearchClient extends DB {
+
+ public static final String DEFAULT_CLUSTER_NAME = "es.ycsb.cluster";
+ public static final String DEFAULT_INDEX_KEY = "es.ycsb";
+ private Node node;
+ private Client client;
+ private String indexKey;
+
+ /**
+ * Initialize any state for this DB. Called once per DB instance; there is
+ * one DB instance per client thread.
+ */
+ @Override
+ public void init() throws DBException {
+ // initialize OrientDB driver
+ Properties props = getProperties();
+ this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY);
+ String clusterName = props.getProperty("cluster.name", DEFAULT_CLUSTER_NAME);
+ Boolean newdb = Boolean.parseBoolean(props.getProperty("elasticsearch.newdb", "false"));
+ Builder settings = settingsBuilder()
+ .put("node.local", "true")
+ .put("path.data", System.getProperty("java.io.tmpdir") + "/esdata")
+ .put("discovery.zen.ping.multicast.enabled", "false")
+ .put("index.mapping._id.indexed", "true")
+ .put("index.gateway.type", "none")
+ .put("gateway.type", "none")
+ .put("index.number_of_shards", "1")
+ .put("index.number_of_replicas", "0");
+
+
+ //if properties file contains elasticsearch user defined properties
+ //add it to the settings file (will overwrite the defaults).
+ settings.put(props);
+ System.out.println("ElasticSearch starting node = " + settings.get("cluster.name"));
+ System.out.println("ElasticSearch node data path = " + settings.get("path.data"));
+
+ node = nodeBuilder().clusterName(clusterName).settings(settings).node();
+ node.start();
+ client = node.client();
+
+ if (newdb) {
+ client.admin().indices().prepareDelete(indexKey).execute().actionGet();
+ client.admin().indices().prepareCreate(indexKey).execute().actionGet();
+ } else {
+ boolean exists = client.admin().indices().exists(Requests.indicesExistsRequest(indexKey)).actionGet().isExists();
+ if (!exists) {
+ client.admin().indices().prepareCreate(indexKey).execute().actionGet();
+ }
+ }
+ }
+
+ @Override
+ public void cleanup() throws DBException {
+ if (!node.isClosed()) {
+ client.close();
+ node.stop();
+ node.close();
+ }
+ }
+
+ /**
+ * Insert a record in the database. Any field/value pairs in the specified
+ * values HashMap will be written into the record with the specified record
+ * key.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to insert.
+ * @param values A HashMap of field/value pairs to insert in the record
+ * @return Zero on success, a non-zero error code on error. See this class's
+ * description for a discussion of error codes.
+ */
+ @Override
+ public int insert(String table, String key, HashMap<String, ByteIterator> values) {
+ try {
+ final XContentBuilder doc = jsonBuilder().startObject();
+
+ for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
+ doc.field(entry.getKey(), entry.getValue());
+ }
+
+ doc.endObject();
+
+ client.prepareIndex(indexKey, table, key)
+ .setSource(doc)
+ .execute()
+ .actionGet();
+
+ return 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ /**
+ * Delete a record from the database.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to delete.
+ * @return Zero on success, a non-zero error code on error. See this class's
+ * description for a discussion of error codes.
+ */
+ @Override
+ public int delete(String table, String key) {
+ try {
+ client.prepareDelete(indexKey, table, key)
+ .execute()
+ .actionGet();
+ return 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ /**
+ * Read a record from the database. Each field/value pair from the result
+ * will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to read.
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A HashMap of field/value pairs for the result
+ * @return Zero on success, a non-zero error code on error or "not found".
+ */
+ @Override
+ public int read(String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
+ try {
+ final GetResponse response = client.prepareGet(indexKey, table, key)
+ .execute()
+ .actionGet();
+
+ if (response.isExists()) {
+ if (fields != null) {
+ for (String field : fields) {
+ result.put(field, new StringByteIterator((String) response.getSource().get(field)));
+ }
+ } else {
+ for (String field : response.getSource().keySet()) {
+ result.put(field, new StringByteIterator((String) response.getSource().get(field)));
+ }
+ }
+ return 0;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ /**
+ * Update a record in the database. Any field/value pairs in the specified
+ * values HashMap will be written into the record with the specified record
+ * key, overwriting any existing values with the same field name.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to write.
+ * @param values A HashMap of field/value pairs to update in the record
+ * @return Zero on success, a non-zero error code on error. See this class's
+ * description for a discussion of error codes.
+ */
+ @Override
+ public int update(String table, String key, HashMap<String, ByteIterator> values) {
+ try {
+ final GetResponse response = client.prepareGet(indexKey, table, key)
+ .execute()
+ .actionGet();
+
+ if (response.isExists()) {
+ for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
+ response.getSource().put(entry.getKey(), entry.getValue());
+ }
+
+ client.prepareIndex(indexKey, table, key)
+ .setSource(response.getSource())
+ .execute()
+ .actionGet();
+
+ return 0;
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ /**
+ * Perform a range scan for a set of records in the database. Each
+ * field/value pair from the result will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param startkey The record key of the first record to read.
+ * @param recordcount The number of records to read
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A Vector of HashMaps, where each HashMap is a set
+ * field/value pairs for one record
+ * @return Zero on success, a non-zero error code on error. See this class's
+ * description for a discussion of error codes.
+ */
+ @Override
+ public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
+ try {
+ final RangeFilterBuilder filter = rangeFilter("_id").gte(startkey);
+ final SearchResponse response = client.prepareSearch(indexKey)
+ .setTypes(table)
+ .setQuery(matchAllQuery())
+ .setFilter(filter)
+ .setSize(recordcount)
+ .execute()
+ .actionGet();
+
+ HashMap<String, ByteIterator> entry;
+
+ for (SearchHit hit : response.getHits()) {
+ entry = new HashMap<String, ByteIterator>(fields.size());
+
+ for (String field : fields) {
+ entry.put(field, new StringByteIterator((String) hit.getSource().get(field)));
+ }
+
+ result.add(entry);
+ }
+
+ return 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+}
diff --git a/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticSearchClientTest.java b/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticSearchClientTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..3ef8ed9ce9baa79e0a0a157379940361f08c6904
--- /dev/null
+++ b/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticSearchClientTest.java
@@ -0,0 +1,137 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package com.yahoo.ycsb.db;
+
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.StringByteIterator;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Vector;
+import static org.testng.AssertJUnit.*;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ *
+ * @author saden
+ */
+public class ElasticSearchClientTest {
+
+ protected final static ElasticSearchClient instance = new ElasticSearchClient();
+ protected final static HashMap<String, ByteIterator> MOCK_DATA;
+ protected final static String MOCK_TABLE = "MOCK_TABLE";
+ protected final static String MOCK_KEY0 = "0";
+ protected final static String MOCK_KEY1 = "1";
+ protected final static String MOCK_KEY2 = "2";
+
+ static {
+ MOCK_DATA = new HashMap<String, ByteIterator>(10);
+ for (int i = 1; i <= 10; i++) {
+ MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
+ }
+ }
+
+ @BeforeClass
+ public static void setUpClass() throws DBException {
+ instance.init();
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws DBException {
+ instance.cleanup();
+ }
+
+ @BeforeMethod
+ public void setUp() {
+ instance.insert(MOCK_TABLE, MOCK_KEY1, MOCK_DATA);
+ instance.insert(MOCK_TABLE, MOCK_KEY2, MOCK_DATA);
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ instance.delete(MOCK_TABLE, MOCK_KEY1);
+ instance.delete(MOCK_TABLE, MOCK_KEY2);
+ }
+
+ /**
+ * Test of insert method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testInsert() {
+ System.out.println("insert");
+ int expResult = 0;
+ int result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA);
+ assertEquals(expResult, result);
+ }
+
+ /**
+ * Test of delete method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testDelete() {
+ System.out.println("delete");
+ int expResult = 0;
+ int result = instance.delete(MOCK_TABLE, MOCK_KEY1);
+ assertEquals(expResult, result);
+ }
+
+ /**
+ * Test of read method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testRead() {
+ System.out.println("read");
+ Set<String> fields = MOCK_DATA.keySet();
+ HashMap<String, ByteIterator> resultParam = new HashMap<String, ByteIterator>(10);
+ int expResult = 0;
+ int result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam);
+ assertEquals(expResult, result);
+ }
+
+ /**
+ * Test of update method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testUpdate() {
+ System.out.println("update");
+ int i;
+ HashMap<String, ByteIterator> newValues = new HashMap<String, ByteIterator>(10);
+
+ for (i = 1; i <= 10; i++) {
+ newValues.put("field" + i, new StringByteIterator("newvalue" + i));
+ }
+
+ int expResult = 0;
+ int result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues);
+ assertEquals(expResult, result);
+
+ //validate that the values changed
+ HashMap<String, ByteIterator> resultParam = new HashMap<String, ByteIterator>(10);
+ instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);
+
+ for (i = 1; i <= 10; i++) {
+ assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
+ }
+
+ }
+
+ /**
+ * Test of scan method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testScan() {
+ System.out.println("scan");
+ int recordcount = 10;
+ Set<String> fields = MOCK_DATA.keySet();
+ Vector<HashMap<String, ByteIterator>> resultParam = new Vector<HashMap<String, ByteIterator>>(10);
+ int expResult = 0;
+ int result = instance.scan(MOCK_TABLE, MOCK_KEY1, recordcount, fields, resultParam);
+ assertEquals(expResult, result);
+ }
+}
diff --git a/hbase/src/main/java/com/yahoo/ycsb/db/HBaseClient.java b/hbase/src/main/java/com/yahoo/ycsb/db/HBaseClient.java
index 90b8db9a7dd96b9cabef2bdf8d3d2d930a16b8ce..53f0c79701210db491bd8da1478a32c738229f2c 100644
--- a/hbase/src/main/java/com/yahoo/ycsb/db/HBaseClient.java
+++ b/hbase/src/main/java/com/yahoo/ycsb/db/HBaseClient.java
@@ -30,6 +30,7 @@ import java.util.*;
//import java.util.Set;
//import java.util.Vector;
+import com.yahoo.ycsb.measurements.Measurements;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
@@ -96,10 +97,16 @@ public class HBaseClient extends com.yahoo.ycsb.DB
*/
public void cleanup() throws DBException
{
+ // Get the measurements instance as this is the only client that should
+ // count clean up time like an update since autoflush is off.
+ Measurements _measurements = Measurements.getMeasurements();
try {
+ long st=System.nanoTime();
if (_hTable != null) {
_hTable.flushCommits();
}
+ long en=System.nanoTime();
+ _measurements.measure("UPDATE", (int)((en-st)/1000));
} catch (IOException e) {
throw new DBException(e);
}
diff --git a/jdbc/src/main/conf/db.properties b/jdbc/src/main/conf/db.properties
index c985d57bc4c329971764bff1906e30d799853611..dac28e236d9270dacb2b6ecf5b6b59b7f2f8a4ac 100644
--- a/jdbc/src/main/conf/db.properties
+++ b/jdbc/src/main/conf/db.properties
@@ -1,6 +1,7 @@
# Properties file that contains database connection information.
jdbc.driver=org.h2.Driver
+# jdbc.fetchsize=20
db.url=jdbc:h2:tcp://foo.com:9092/~/h2/ycsb
db.user=sa
db.passwd=
diff --git a/jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClient.java b/jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClient.java
index 8d1a6a7ea3411ce090cd1ded32dcd7d9268eeafa..c800ff0ae07cb3b5e8a17011b5334ead46138b32 100644
--- a/jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClient.java
+++ b/jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClient.java
@@ -55,6 +55,7 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
private ArrayList<Connection> conns;
private boolean initialized = false;
private Properties props;
+ private Integer jdbcFetchSize;
private static final String DEFAULT_PROP = "";
private ConcurrentMap<StatementType, PreparedStatement> cachedStatements;
@@ -176,6 +177,19 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
String passwd = props.getProperty(CONNECTION_PASSWD, DEFAULT_PROP);
String driver = props.getProperty(DRIVER_CLASS);
+ String jdbcFetchSizeStr = props.getProperty(JDBC_FETCH_SIZE);
+ if (jdbcFetchSizeStr != null) {
+ try {
+ this.jdbcFetchSize = Integer.parseInt(jdbcFetchSizeStr);
+ } catch (NumberFormatException nfe) {
+ System.err.println("Invalid JDBC fetch size specified: " + jdbcFetchSizeStr);
+ throw new DBException(nfe);
+ }
+ }
+
+ String autoCommitStr = props.getProperty(JDBC_AUTO_COMMIT, Boolean.TRUE.toString());
+ Boolean autoCommit = Boolean.parseBoolean(autoCommitStr);
+
try {
if (driver != null) {
Class.forName(driver);
@@ -185,9 +199,13 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
for (String url: urls.split(",")) {
System.out.println("Adding shard node URL: " + url);
Connection conn = DriverManager.getConnection(url, user, passwd);
- // Since there is no explicit commit method in the DB interface, all
- // operations should auto commit.
- conn.setAutoCommit(true);
+
+ // Since there is no explicit commit method in the DB interface, all
+ // operations should auto commit, except when explicitly told not to
+ // (this is necessary in cases such as for PostgreSQL when running a
+ // scan workload with fetchSize)
+ conn.setAutoCommit(autoCommit);
+
shardCount++;
conns.add(conn);
}
@@ -289,6 +307,7 @@ public class JdbcDBClient extends DB implements JdbcDBClientConstants {
select.append(" >= ");
select.append("?;");
PreparedStatement scanStatement = getShardConnectionByKey(key).prepareStatement(select.toString());
+ if (this.jdbcFetchSize != null) scanStatement.setFetchSize(this.jdbcFetchSize);
PreparedStatement stmt = cachedStatements.putIfAbsent(scanType, scanStatement);
if (stmt == null) return scanStatement;
else return stmt;
diff --git a/jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClientConstants.java b/jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClientConstants.java
index 79e0525e2a20f6c6ba809fa1b54a73e55865293d..120cb6452758ab21f8c425fc11b4c96f1bed59ea 100644
--- a/jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClientConstants.java
+++ b/jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClientConstants.java
@@ -35,7 +35,13 @@ public interface JdbcDBClientConstants {
/** The password to use for establishing the connection. */
public static final String CONNECTION_PASSWD = "db.passwd";
-
+
+ /** The JDBC fetch size hinted to the driver. */
+ public static final String JDBC_FETCH_SIZE = "jdbc.fetchsize";
+
+ /** The JDBC connection auto-commit property for the driver. */
+ public static final String JDBC_AUTO_COMMIT = "jdbc.autocommit";
+
/** The name of the property for the number of fields in a record. */
public static final String FIELD_COUNT_PROPERTY="fieldcount";
diff --git a/mongodb/README.md b/mongodb/README.md
index 63d0158aa6cd2154ac77c96d841dd90c203a10dd..8f10c65795f7a99863236303a5d71c0a94be43d6 100644
--- a/mongodb/README.md
+++ b/mongodb/README.md
@@ -40,3 +40,5 @@ See the next section for the list of configuration parameters for MongoDB.
### `mongodb.database` (default: `ycsb`)
### `mongodb.writeConcern` (default `safe`)
+
+### `mongodb.maxconnections` (default `10`)
diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java
index 7c8df19af43655211130c26be6650e3389dfa8d4..3082e4c9ae8fd6f1390ed91b4db5a21c26c0fd3d 100644
--- a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java
+++ b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java
@@ -11,10 +11,11 @@ package com.yahoo.ycsb.db;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.Map;
import java.util.Vector;
+import java.util.concurrent.atomic.AtomicInteger;
import com.mongodb.BasicDBObject;
import com.mongodb.DBAddress;
@@ -22,102 +23,133 @@ import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
+import com.mongodb.MongoOptions;
import com.mongodb.WriteConcern;
import com.mongodb.WriteResult;
-
+import com.yahoo.ycsb.ByteArrayByteIterator;
+import com.yahoo.ycsb.ByteIterator;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.DBException;
-import com.yahoo.ycsb.ByteIterator;
-import com.yahoo.ycsb.StringByteIterator;
/**
* MongoDB client for YCSB framework.
- *
+ *
* Properties to set:
- *
- * mongodb.url=mongodb://localhost:27017
- * mongodb.database=ycsb
+ *
+ * mongodb.url=mongodb://localhost:27017 mongodb.database=ycsb
* mongodb.writeConcern=normal
- *
+ *
* @author ypai
- *
*/
public class MongoDbClient extends DB {
- private Mongo mongo;
- private WriteConcern writeConcern;
- private String database;
+ /** Used to include a field in a response. */
+ protected static final Integer INCLUDE = Integer.valueOf(1);
+
+ /** A singleton Mongo instance. */
+ private static Mongo mongo;
+
+ /** The default write concern for the test. */
+ private static WriteConcern writeConcern;
+
+ /** The database to access. */
+ private static String database;
+
+ /** Count the number of times initialized to teardown on the last {@link #cleanup()}. */
+ private static final AtomicInteger initCount = new AtomicInteger(0);
- @Override
/**
* Initialize any state for this DB.
* Called once per DB instance; there is one DB instance per client thread.
*/
+ @Override
public void init() throws DBException {
- // initialize MongoDb driver
- Properties props = getProperties();
- String url = props.getProperty("mongodb.url", "mongodb://localhost:27017");
- database = props.getProperty("mongodb.database", "ycsb");
- String writeConcernType = props.getProperty("mongodb.writeConcern", "safe").toLowerCase();
-
- if ("none".equals(writeConcernType)) {
- writeConcern = WriteConcern.NONE;
- } else if ("safe".equals(writeConcernType)) {
- writeConcern = WriteConcern.SAFE;
- } else if ("normal".equals(writeConcernType)) {
- writeConcern = WriteConcern.NORMAL;
- } else if ("fsync_safe".equals(writeConcernType)) {
- writeConcern = WriteConcern.FSYNC_SAFE;
- } else if ("replicas_safe".equals(writeConcernType)) {
- writeConcern = WriteConcern.REPLICAS_SAFE;
- } else {
- System.err.println("ERROR: Invalid writeConcern: '" + writeConcernType + "'. " +
- "Must be [ none | safe | normal | fsync_safe | replicas_safe ]");
- System.exit(1);
- }
+ initCount.incrementAndGet();
+ synchronized (INCLUDE) {
+ if (mongo != null) {
+ return;
+ }
- try {
- // strip out prefix since Java driver doesn't currently support
- // standard connection format URL yet
- // http://www.mongodb.org/display/DOCS/Connections
- if (url.startsWith("mongodb://")) {
- url = url.substring(10);
+ // initialize MongoDb driver
+ Properties props = getProperties();
+ String url = props.getProperty("mongodb.url",
+ "mongodb://localhost:27017");
+ database = props.getProperty("mongodb.database", "ycsb");
+ String writeConcernType = props.getProperty("mongodb.writeConcern",
+ "safe").toLowerCase();
+ final String maxConnections = props.getProperty(
+ "mongodb.maxconnections", "10");
+
+ if ("none".equals(writeConcernType)) {
+ writeConcern = WriteConcern.NONE;
+ }
+ else if ("safe".equals(writeConcernType)) {
+ writeConcern = WriteConcern.SAFE;
+ }
+ else if ("normal".equals(writeConcernType)) {
+ writeConcern = WriteConcern.NORMAL;
+ }
+ else if ("fsync_safe".equals(writeConcernType)) {
+ writeConcern = WriteConcern.FSYNC_SAFE;
+ }
+ else if ("replicas_safe".equals(writeConcernType)) {
+ writeConcern = WriteConcern.REPLICAS_SAFE;
+ }
+ else {
+ System.err
+ .println("ERROR: Invalid writeConcern: '"
+ + writeConcernType
+ + "'. "
+ + "Must be [ none | safe | normal | fsync_safe | replicas_safe ]");
+ System.exit(1);
}
- // need to append db to url.
- url += "/"+database;
- System.out.println("new database url = "+url);
- mongo = new Mongo(new DBAddress(url));
- System.out.println("mongo connection created with "+url);
- } catch (Exception e1) {
- System.err.println(
- "Could not initialize MongoDB connection pool for Loader: "
- + e1.toString());
- e1.printStackTrace();
- return;
- }
+ try {
+ // strip out prefix since Java driver doesn't currently support
+ // standard connection format URL yet
+ // http://www.mongodb.org/display/DOCS/Connections
+ if (url.startsWith("mongodb://")) {
+ url = url.substring(10);
+ }
+ // need to append db to url.
+ url += "/" + database;
+ System.out.println("new database url = " + url);
+ MongoOptions options = new MongoOptions();
+ options.connectionsPerHost = Integer.parseInt(maxConnections);
+ mongo = new Mongo(new DBAddress(url), options);
+
+ System.out.println("mongo connection created with " + url);
+ }
+ catch (Exception e1) {
+ System.err
+ .println("Could not initialize MongoDB connection pool for Loader: "
+ + e1.toString());
+ e1.printStackTrace();
+ return;
+ }
+ }
}
-
+
+ /**
+ * Cleanup any state for this DB.
+ * Called once per DB instance; there is one DB instance per client thread.
+ */
@Override
- /**
- * Cleanup any state for this DB.
- * Called once per DB instance; there is one DB instance per client thread.
- */
- public void cleanup() throws DBException
- {
- try {
- mongo.close();
- } catch (Exception e1) {
- System.err.println(
- "Could not close MongoDB connection pool: "
- + e1.toString());
- e1.printStackTrace();
- return;
+ public void cleanup() throws DBException {
+ if (initCount.decrementAndGet() <= 0) {
+ try {
+ mongo.close();
+ }
+ catch (Exception e1) {
+ System.err.println("Could not close MongoDB connection pool: "
+ + e1.toString());
+ e1.printStackTrace();
+ return;
+ }
}
- }
+ }
- @Override
/**
* Delete a record from the database.
*
@@ -125,8 +157,9 @@ public class MongoDbClient extends DB {
* @param key The record key of the record to delete.
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
+ @Override
public int delete(String table, String key) {
- com.mongodb.DB db=null;
+ com.mongodb.DB db = null;
try {
db = mongo.getDB(database);
db.requestStart();
@@ -134,20 +167,18 @@ public class MongoDbClient extends DB {
DBObject q = new BasicDBObject().append("_id", key);
WriteResult res = collection.remove(q, writeConcern);
return res.getN() == 1 ? 0 : 1;
- } catch (Exception e) {
+ }
+ catch (Exception e) {
System.err.println(e.toString());
return 1;
}
- finally
- {
- if (db!=null)
- {
+ finally {
+ if (db != null) {
db.requestDone();
}
}
}
- @Override
/**
* Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified
* record key.
@@ -157,7 +188,9 @@ public class MongoDbClient extends DB {
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
- public int insert(String table, String key, HashMap<String, ByteIterator> values) {
+ @Override
+ public int insert(String table, String key,
+ HashMap<String, ByteIterator> values) {
com.mongodb.DB db = null;
try {
db = mongo.getDB(database);
@@ -166,24 +199,23 @@ public class MongoDbClient extends DB {
DBCollection collection = db.getCollection(table);
DBObject r = new BasicDBObject().append("_id", key);
- for(String k: values.keySet()) {
- r.put(k, values.get(k).toArray());
- }
- WriteResult res = collection.insert(r,writeConcern);
+ for (String k : values.keySet()) {
+ r.put(k, values.get(k).toArray());
+ }
+ WriteResult res = collection.insert(r, writeConcern);
return res.getError() == null ? 0 : 1;
- } catch (Exception e) {
+ }
+ catch (Exception e) {
e.printStackTrace();
return 1;
- } finally {
- if (db!=null)
- {
+ }
+ finally {
+ if (db != null) {
db.requestDone();
}
}
}
- @Override
- @SuppressWarnings("unchecked")
/**
* Read a record from the database. Each field/value pair from the result will be stored in a HashMap.
*
@@ -193,6 +225,8 @@ public class MongoDbClient extends DB {
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error or "not found".
*/
+ @Override
+ @SuppressWarnings("unchecked")
public int read(String table, String key, Set<String> fields,
HashMap<String, ByteIterator> result) {
com.mongodb.DB db = null;
@@ -204,16 +238,16 @@ public class MongoDbClient extends DB {
DBCollection collection = db.getCollection(table);
DBObject q = new BasicDBObject().append("_id", key);
DBObject fieldsToReturn = new BasicDBObject();
- boolean returnAllFields = fields == null;
DBObject queryResult = null;
- if (!returnAllFields) {
+ if (fields != null) {
Iterator<String> iter = fields.iterator();
while (iter.hasNext()) {
- fieldsToReturn.put(iter.next(), 1);
+ fieldsToReturn.put(iter.next(), INCLUDE);
}
queryResult = collection.findOne(q, fieldsToReturn);
- } else {
+ }
+ else {
queryResult = collection.findOne(q);
}
@@ -221,19 +255,18 @@ public class MongoDbClient extends DB {
result.putAll(queryResult.toMap());
}
return queryResult != null ? 0 : 1;
- } catch (Exception e) {
+ }
+ catch (Exception e) {
System.err.println(e.toString());
return 1;
- } finally {
- if (db!=null)
- {
+ }
+ finally {
+ if (db != null) {
db.requestDone();
}
}
}
-
- @Override
/**
* Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified
* record key, overwriting any existing values with the same field name.
@@ -243,7 +276,9 @@ public class MongoDbClient extends DB {
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
- public int update(String table, String key, HashMap<String, ByteIterator> values) {
+ @Override
+ public int update(String table, String key,
+ HashMap<String, ByteIterator> values) {
com.mongodb.DB db = null;
try {
db = mongo.getDB(database);
@@ -264,19 +299,18 @@ public class MongoDbClient extends DB {
WriteResult res = collection.update(q, u, false, false,
writeConcern);
return res.getN() == 1 ? 0 : 1;
- } catch (Exception e) {
+ }
+ catch (Exception e) {
System.err.println(e.toString());
return 1;
- } finally {
- if (db!=null)
- {
+ }
+ finally {
+ if (db != null) {
db.requestDone();
}
}
}
- @Override
- @SuppressWarnings("unchecked")
/**
* Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap.
*
@@ -287,9 +321,10 @@ public class MongoDbClient extends DB {
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
*/
+ @Override
public int scan(String table, String startkey, int recordcount,
Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
- com.mongodb.DB db=null;
+ com.mongodb.DB db = null;
try {
db = mongo.getDB(database);
db.requestStart();
@@ -299,23 +334,44 @@ public class MongoDbClient extends DB {
DBObject q = new BasicDBObject().append("_id", scanRange);
DBCursor cursor = collection.find(q).limit(recordcount);
while (cursor.hasNext()) {
- //toMap() returns a Map, but result.add() expects a Map<String,String>. Hence, the suppress warnings.
- result.add(StringByteIterator.getByteIteratorMap((Map<String,String>)cursor.next().toMap()));
+ // toMap() returns a Map, but result.add() expects a
+ // Map<String,String>. Hence, the suppress warnings.
+ HashMap<String, ByteIterator> resultMap = new HashMap<String, ByteIterator>();
+
+ DBObject obj = cursor.next();
+ fillMap(resultMap, obj);
+
+ result.add(resultMap);
}
return 0;
- } catch (Exception e) {
+ }
+ catch (Exception e) {
System.err.println(e.toString());
return 1;
}
- finally
- {
- if (db!=null)
- {
+ finally {
+ if (db != null) {
db.requestDone();
}
}
}
-}
+ /**
+ * TODO - Finish
+ *
+ * @param resultMap
+ * @param obj
+ */
+ @SuppressWarnings("unchecked")
+ protected void fillMap(HashMap<String, ByteIterator> resultMap, DBObject obj) {
+ Map<String, Object> objMap = obj.toMap();
+ for (Map.Entry<String, Object> entry : objMap.entrySet()) {
+ if (entry.getValue() instanceof byte[]) {
+ resultMap.put(entry.getKey(), new ByteArrayByteIterator(
+ (byte[]) entry.getValue()));
+ }
+ }
+ }
+}
diff --git a/pom.xml b/pom.xml
index 78542d7ef6968baf810592dda54f378cb0f18150..cfc2f938c8e0ffd122cf970e961b1ff6736af2fe 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,7 +49,7 @@
<infinispan.version>7.1.0.CR1</infinispan.version>
<openjpa.jdbc.version>2.1.1</openjpa.jdbc.version>
<mapkeeper.version>1.0</mapkeeper.version>
- <mongodb.version>2.8.0</mongodb.version>
+ <mongodb.version>2.11.2</mongodb.version>
<orientdb.version>1.0.1</orientdb.version>
<redis.version>2.0.0</redis.version>
<voldemort.version>0.81</voldemort.version>
@@ -60,12 +60,13 @@
<modules>
<!--module>build-tools</module-->
+ <module>cassandra</module>
<module>core</module>
<module>hbase</module>
<module>hypertable</module>
- <module>cassandra</module>
<module>dynamodb</module>
- <module>gemfire</module>
+ <module>elasticsearch</module>
+ <!--<module>gemfire</module>-->
<module>infinispan</module>
<module>jdbc</module>
<module>mapkeeper</module>