Merge remote-tracking branch 'upstream/master' into splitrun

Adding upstream changes to this branch

accumulo/pom.xml

@@ -22,7 +22,7 @@ LICENSE file.
   <parent>
     <groupId>com.yahoo.ycsb</groupId>
     <artifactId>binding-parent</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
     <relativePath>../binding-parent</relativePath>
   </parent>
   <artifactId>accumulo-binding</artifactId>

aerospike/pom.xml

@@ -21,7 +21,7 @@ LICENSE file.
   <parent>
     <groupId>com.yahoo.ycsb</groupId>
     <artifactId>binding-parent</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
     <relativePath>../binding-parent</relativePath>
   </parent>
 

bin/ycsb

@@ -58,9 +58,11 @@ DATABASES = {
     "cassandra-cql": "com.yahoo.ycsb.db.CassandraCQLClient",
     "cassandra2-cql": "com.yahoo.ycsb.db.CassandraCQLClient",
     "couchbase"    : "com.yahoo.ycsb.db.CouchbaseClient",
+    "couchbase2"   : "com.yahoo.ycsb.db.couchbase2.Couchbase2Client",
     "dynamodb"     : "com.yahoo.ycsb.db.DynamoDBClient",
     "elasticsearch": "com.yahoo.ycsb.db.ElasticsearchClient",
     "geode"        : "com.yahoo.ycsb.db.GeodeClient",
+    "googlebigtable"  : "com.yahoo.ycsb.db.GoogleBigtableClient",
     "googledatastore" : "com.yahoo.ycsb.db.GoogleDatastoreClient",
     "hbase094"     : "com.yahoo.ycsb.db.HBaseClient",
     "hbase098"     : "com.yahoo.ycsb.db.HBaseClient",

binding-parent/datastore-specific-descriptor/pom.xml

@@ -21,7 +21,7 @@ LICENSE file.
   <parent>
     <groupId>com.yahoo.ycsb</groupId>
     <artifactId>root</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
     <relativePath>../../</relativePath>
   </parent>
 

binding-parent/pom.xml

@@ -21,7 +21,7 @@ LICENSE file.
   <parent>
     <groupId>com.yahoo.ycsb</groupId>
     <artifactId>root</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>binding-parent</artifactId>

cassandra/pom.xml

@@ -21,7 +21,7 @@ LICENSE file.
   <parent>
      <groupId>com.yahoo.ycsb</groupId>
      <artifactId>binding-parent</artifactId>
-     <version>0.8.0-SNAPSHOT</version>
+     <version>0.9.0-SNAPSHOT</version>
      <relativePath>../binding-parent</relativePath>
   </parent>
   

cassandra/src/main/java/com/yahoo/ycsb/db/CassandraClient10.java

@@ -453,6 +453,10 @@ public class CassandraClient10 extends DB {
     }
 
     for (int i = 0; i < operationRetries; i++) {
+      mutations.clear();
+      mutationMap.clear();
+      record.clear();
+
       if (debug) {
         System.out.println("Inserting key: " + key);
       }
@@ -479,10 +483,6 @@ public class CassandraClient10 extends DB {
 
         client.batch_mutate(record, writeConsistencyLevel);
 
-        mutations.clear();
-        mutationMap.clear();
-        record.clear();
-
         if (debug) {
           System.out
               .println("ConsistencyLevel=" + writeConsistencyLevel.toString());

cassandra/src/main/java/com/yahoo/ycsb/db/CassandraClient7.java

@@ -418,6 +418,10 @@ public class CassandraClient7 extends DB {
     }
 
     for (int i = 0; i < operationRetries; i++) {
+      mutations.clear();
+      mutationMap.clear();
+      record.clear();
+
       if (debug) {
         System.out.println("Inserting key: " + key);
       }
@@ -444,10 +448,6 @@ public class CassandraClient7 extends DB {
 
         client.batch_mutate(record, ConsistencyLevel.ONE);
 
-        mutations.clear();
-        mutationMap.clear();
-        record.clear();
-
         return Status.OK;
       } catch (Exception e) {
         errorexception = e;

cassandra/src/main/java/com/yahoo/ycsb/db/CassandraClient8.java

@@ -397,6 +397,10 @@ public class CassandraClient8 extends DB {
     }
 
     for (int i = 0; i < operationRetries; i++) {
+      mutations.clear();
+      mutationMap.clear();
+      record.clear();
+
       if (debug) {
         System.out.println("Inserting key: " + key);
       }
@@ -423,10 +427,6 @@ public class CassandraClient8 extends DB {
 
         client.batch_mutate(record, ConsistencyLevel.ONE);
 
-        mutations.clear();
-        mutationMap.clear();
-        record.clear();
-
         return Status.OK;
       } catch (Exception e) {
         errorexception = e;

cassandra2/pom.xml

@@ -23,7 +23,7 @@ LICENSE file.
   <parent>
     <groupId>com.yahoo.ycsb</groupId>
     <artifactId>binding-parent</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
     <relativePath>../binding-parent</relativePath>
   </parent>
 
@@ -31,6 +31,11 @@ LICENSE file.
   <name>Cassandra 2.1+ DB Binding</name>
   <packaging>jar</packaging>
 
+  <properties>
+    <!-- Skip tests by default. will be activated by jdk8 profile -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <!-- CQL driver -->
     <dependency>
@@ -46,8 +51,9 @@ LICENSE file.
     </dependency>
     <dependency>
       <groupId>org.cassandraunit</groupId>
-      <artifactId>cassandra-unit-shaded</artifactId>
-      <version>2.1.9.2</version>
+      <artifactId>cassandra-unit</artifactId>
+      <version>3.0.0.1</version>
+      <classifier>shaded</classifier>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -57,4 +63,19 @@ LICENSE file.
       <scope>test</scope>
     </dependency>
   </dependencies>
+
+  <profiles>
+    <!-- Cassandra 2.2+ requires JDK8 to run, so none of our tests
+         will work unless we're using jdk8.
+      -->
+    <profile>
+      <id>jdk8-tests</id>
+      <activation>
+        <jdk>1.8</jdk>
+      </activation>
+      <properties>
+        <skipTests>false</skipTests>
+      </properties>
+    </profile>
+  </profiles>
 </project>

cassandra2/src/test/java/com/yahoo/ycsb/db/CassandraCQLClientTest.java

@@ -63,11 +63,12 @@ public class CassandraCQLClientTest {
   private Session session;
 
   @ClassRule
-  public static CassandraCQLUnit cassandraUnit =
-      new CassandraCQLUnit(new ClassPathCQLDataSet("ycsb.cql", "ycsb"));
+  public static CassandraCQLUnit cassandraUnit = new CassandraCQLUnit(new ClassPathCQLDataSet("ycsb.cql", "ycsb"));
 
   @Before
-  public void setUpClient() throws Exception {
+  public void setUp() throws Exception {
+    session = cassandraUnit.getSession();
+
     Properties p = new Properties();
     p.setProperty("hosts", HOST);
     p.setProperty("port", Integer.toString(PORT));
@@ -81,14 +82,11 @@ public class CassandraCQLClientTest {
     client.init();
   }
 
-  @Before
-  public void setSession() {
-    session = cassandraUnit.getSession();
-  }
-
   @After
   public void tearDownClient() throws Exception {
-    client.cleanup();
+    if (client != null) {
+      client.cleanup();
+    }
     client = null;
   }
 
@@ -96,7 +94,9 @@ public class CassandraCQLClientTest {
   public void clearTable() throws Exception {
     // Clear the table so that each test starts fresh.
     final Statement truncate = QueryBuilder.truncate(TABLE);
-    cassandraUnit.getSession().execute(truncate);
+    if (cassandraUnit != null) {
+      cassandraUnit.getSession().execute(truncate);
+    }
   }
 
   @Test

core/pom.xml

@@ -21,7 +21,7 @@ LICENSE file.
   <parent>
     <groupId>com.yahoo.ycsb</groupId>
     <artifactId>root</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
   </parent>
   
   <artifactId>core</artifactId>

couchbase/pom.xml

@@ -22,7 +22,7 @@ LICENSE file.
     <parent>
         <groupId>com.yahoo.ycsb</groupId>
         <artifactId>binding-parent</artifactId>
-        <version>0.8.0-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
         <relativePath>../binding-parent</relativePath>
     </parent>
 

couchbase2/README.md

+<!--
+Copyright (c) 2015 - 2016 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+# Couchbase (SDK 2.x) Driver for YCSB
+This driver is a binding for the YCSB facilities to operate against a Couchbase Server cluster. It uses the official
+Couchbase Java SDK (version 2.x) and provides a rich set of configuration options, including support for the N1QL
+query language.
+
+## Quickstart
+
+### 1. Start Couchbase Server
+You need to start a single node or a cluster to point the client at. Please see [http://couchbase.com](couchbase.com)
+for more details and instructions.
+
+### 2. Set up YCSB
+You can either download the release zip and run it, or just clone from master.
+
+```
+git clone git://github.com/brianfrankcooper/YCSB.git
+cd YCSB
+mvn clean package
+```
+
+### 3. Run the Workload
+Before you can actually run the workload, you need to "load" the data first.
+
+```
+bin/ycsb load couchbase2 -s -P workloads/workloada
+```
+
+Then, you can run the workload:
+
+```
+bin/ycsb run couchbase2 -s -P workloads/workloada
+```
+
+Please see the general instructions in the `doc` folder if you are not sure how it all works. You can apply a property
+(as seen in the next section) like this:
+
+```
+bin/ycsb run couchbase -s -P workloads/workloada -p couchbase.epoll=true
+```
+
+## N1QL Index Setup
+In general, every time N1QL is used (either implicitly through using `workloade` or through setting `kv=false`) some
+kind of index must be present to make it work. Depending on the workload and data size, choosing the right index is
+crucial at runtime in order to get the best performance. If in doubt, please ask at the
+[forums](http://forums.couchbase.com) or get in touch with our team at Couchbase.
+
+For `workloade` and the default `readallfields=true` we recommend creating the following index, and if using Couchbase
+Server 4.5 or later with the "Memory Optimized Index" setting on the bucket.
+
+```
+CREATE INDEX wle_idx ON `bucketname`(meta().id);
+```
+
+For other workloads, different index setups might be even more performant.
+
+## Performance Considerations
+As it is with any benchmark, there are lot of knobs to tune in order to get great or (if you are reading
+this and trying to write a competitor benchmark ;-)) bad performance.
+
+The first setting you should consider, if you are running on Linux 64bit is setting `-p couchbase.epoll=true`. This will
+then turn on the Epoll IO mechanisms in the underlying Netty library which provides better performance since it has less
+synchronization to do than the NIO default. This only works on Linux, but you are benchmarking on the OS you are
+deploying to, right?
+
+The second option, `boost`, sounds more magic than it actually is. By default this benchmark trades CPU for throughput,
+but this can be disabled by setting `-p couchbase.boost=0`. This defaults to 3, and 3 is the number of event loops run
+in the IO layer. 3 is a reasonable default but you should set it to the number of **physical** cores you have available
+on the machine if you only plan to run one YCSB instance. Make sure (using profiling) to max out your cores, but don't
+overdo it.
+
+## Sync vs Async
+By default, since YCSB is sync the code will always wait for the operation to complete. In some cases it can be useful
+to just "drive load" and disable the waiting. Note that when the "-p couchbase.syncMutationResponse=false" option is
+used, the measured results by YCSB can basically be thrown away. Still helpful sometimes during load phases to speed
+them up :)
+
+## Configuration Options
+Since no setup is the same and the goal of YCSB is to deliver realistic benchmarks, here are some setups that you can
+tune. Note that if you need more flexibility (let's say a custom transcoder), you still need to extend this driver and
+implement the facilities on your own.
+
+You can set the following properties (with the default settings applied):
+
+ - couchbase.host=127.0.0.1: The hostname from one server.
+ - couchbase.bucket=default: The bucket name to use.
+ - couchbase.password=: The password of the bucket.
+ - couchbase.syncMutationResponse=true: If mutations should wait for the response to complete.
+ - couchbase.persistTo=0: Persistence durability requirement
+ - couchbase.replicateTo=0: Replication durability requirement
+ - couchbase.upsert=false: Use upsert instead of insert or replace.
+ - couchbase.adhoc=false: If set to true, prepared statements are not used.
+ - couchbase.kv=true: If set to false, mutation operations will also be performed through N1QL.
+ - couchbase.maxParallelism=1: The server parallelism for all n1ql queries.
+ - couchbase.kvEndpoints=1: The number of KV sockets to open per server.
+ - couchbase.queryEndpoints=5: The number of N1QL Query sockets to open per server.
+ - couchbase.epoll=false: If Epoll instead of NIO should be used (only available for linux.
+ - couchbase.boost=3: If > 0 trades CPU for higher throughput. N is the number of event loops, ideally
+   set to the number of physical cores. Setting higher than that will likely degrade performance.
\ No newline at end of file

couchbase2/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Copyright (c) 2015 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.ycsb</groupId>
+        <artifactId>binding-parent</artifactId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../binding-parent</relativePath>
+    </parent>
+
+    <artifactId>couchbase2-binding</artifactId>
+    <name>Couchbase Java SDK 2.x Binding</name>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.couchbase.client</groupId>
+            <artifactId>java-client</artifactId>
+            <version>${couchbase2.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.ycsb</groupId>
+            <artifactId>core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+</project>

couchbase2/src/main/java/com/yahoo/ycsb/db/couchbase2/package-info.java

+/*
+ * Copyright (c) 2015 - 2016 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB binding for <a href="http://www.couchbase.com/">Couchbase</a>, new driver.
+ */
+package com.yahoo.ycsb.db.couchbase2;
+

distribution/pom.xml

@@ -21,7 +21,7 @@ LICENSE file.
   <parent>
     <groupId>com.yahoo.ycsb</groupId>
     <artifactId>root</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>ycsb</artifactId>
@@ -59,6 +59,11 @@ LICENSE file.
       <artifactId>couchbase-binding</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.yahoo.ycsb</groupId>
+      <artifactId>couchbase2-binding</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.yahoo.ycsb</groupId>
       <artifactId>dynamodb-binding</artifactId>
@@ -79,6 +84,11 @@ LICENSE file.
       <artifactId>googledatastore-binding</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.yahoo.ycsb</groupId>
+      <artifactId>googlebigtable-binding</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.yahoo.ycsb</groupId>
       <artifactId>hbase094-binding</artifactId>

dynamodb/pom.xml

@@ -21,7 +21,7 @@ LICENSE file.
   <parent>
     <groupId>com.yahoo.ycsb</groupId>
     <artifactId>binding-parent</artifactId>
-    <version>0.8.0-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
     <relativePath>../binding-parent</relativePath>
   </parent>
 

elasticsearch/README.md

@@ -43,15 +43,12 @@ For further configuration see below:
 The default setting for the Elasticsearch node that is created is as follows:
 
     cluster.name=es.ycsb.cluster
-    node.local=true
-    path.data=$TEMP_DIR/esdata
-    discovery.zen.ping.multicast.enabled=false
-    index.mapping._id.indexed=true
-    index.gateway.type=none
-    gateway.type=none
-    index.number_of_shards=1
-    index.number_of_replicas=0
     es.index.key=es.ycsb
+    es.number_of_shards=1
+    es.number_of_replicas=0
+    es.remote=false
+    es.newdb=false
+    es.hosts.list=localhost:9200 (only applies if es.remote=true)
 
 ### Custom Configuration
 If you wish to customize the settings used to create the Elasticsearch node
@@ -66,25 +63,17 @@ pass it into the Elasticsearch client:
 
     ./bin/ycsb run elasticsearch -P workloads/workloada -P myproperties.data -s
 
-
-If you wish to use a in-memory store type rather than the default disk store add 
-the following properties to your custom properties file. For a large number of 
-insert operations insure that you have sufficient memory on your test system 
-otherwise you will run out of memory.
-
-    index.store.type=memory
-    index.store.fs.memory.enabled=true
-    cache.memory.small_buffer_size=4mb
-    cache.memory.large_cache_size=1024mb
-
 If you wish to change the default index name you can set the following property:
 
     es.index.key=my_index_key
 
-### Troubleshoot
-If you encounter error messages such as :
-"Primary shard is not active or isn't assigned is a known node."
+If you wish to run against a remote cluster you can set the following property:
+
+    es.remote=true
+
+By default this will use localhost:9300 as a seed node to discover the cluster.
+You can also specify
 
-Try removing /tmp/esdata/ folder. 
-    rm -rf /tmp/esdata
+    es.hosts.list=(\w+:\d+)+
 
+(a comma-separated list of host/port pairs) to change this.