From c3cfc98e57c20c9a467378ced3148c5897c5001b Mon Sep 17 00:00:00 2001
From: Kevin Risden <risdenk@avalonconsult.com>
Date: Mon, 6 Feb 2017 21:34:50 -0500
Subject: [PATCH] [elasticsearch,elasticsearch5] Add Elasticsearch 5.x binding
---
bin/bindings.properties | 3 +-
distribution/pom.xml | 5 +
.../ycsb/db/ElasticsearchClientTest.java | 31 +-
elasticsearch5/README.md | 94 ++++++
elasticsearch5/pom.xml | 78 +++++
.../elasticsearch5/ElasticsearchClient.java | 274 ++++++++++++++++++
.../ElasticsearchRestClient.java | 218 ++++++++++++++
.../ycsb/db/elasticsearch5/package-info.java | 23 ++
.../src/main/resources/log4j2.properties | 6 +
.../ElasticsearchClientTest.java | 146 ++++++++++
hypertable/pom.xml | 4 +-
jdbc/pom.xml | 4 +-
pom.xml | 38 +--
13 files changed, 883 insertions(+), 41 deletions(-)
create mode 100644 elasticsearch5/README.md
create mode 100644 elasticsearch5/pom.xml
create mode 100644 elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClient.java
create mode 100644 elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchRestClient.java
create mode 100644 elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/package-info.java
create mode 100644 elasticsearch5/src/main/resources/log4j2.properties
create mode 100644 elasticsearch5/src/test/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClientTest.java
diff --git a/bin/bindings.properties b/bin/bindings.properties
index 24ffa09f..095bf9e5 100644
--- a/bin/bindings.properties
+++ b/bin/bindings.properties
@@ -30,15 +30,16 @@ aerospike:com.yahoo.ycsb.db.AerospikeClient
asynchbase:com.yahoo.ycsb.db.AsyncHBaseClient
arangodb:com.yahoo.ycsb.db.ArangoDBClient
arangodb3:com.yahoo.ycsb.db.arangodb.ArangoDB3Client
+azuredocumentdb:com.yahoo.ycsb.db.azuredocumentdb.AzureDocumentDBClient
azuretablestorage:com.yahoo.ycsb.db.azuretablestorage.AzureClient
basic:com.yahoo.ycsb.BasicDB
cassandra-cql:com.yahoo.ycsb.db.CassandraCQLClient
cassandra2-cql:com.yahoo.ycsb.db.CassandraCQLClient
couchbase:com.yahoo.ycsb.db.CouchbaseClient
couchbase2:com.yahoo.ycsb.db.couchbase2.Couchbase2Client
-azuredocumentdb:com.yahoo.ycsb.db.azuredocumentdb.AzureDocumentDBClient
dynamodb:com.yahoo.ycsb.db.DynamoDBClient
elasticsearch:com.yahoo.ycsb.db.ElasticsearchClient
+elasticsearch5:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient
geode:com.yahoo.ycsb.db.GeodeClient
googlebigtable:com.yahoo.ycsb.db.GoogleBigtableClient
googledatastore:com.yahoo.ycsb.db.GoogleDatastoreClient
diff --git a/distribution/pom.xml b/distribution/pom.xml
index c2071adf..57d3fa8c 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -94,6 +94,11 @@ LICENSE file.
<artifactId>elasticsearch-binding</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>elasticsearch5-binding</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>com.yahoo.ycsb</groupId>
<artifactId>geode-binding</artifactId>
diff --git a/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticsearchClientTest.java b/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticsearchClientTest.java
index d1ad64d1..a14f1a23 100644
--- a/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticsearchClientTest.java
+++ b/elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticsearchClientTest.java
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2012 YCSB contributors. All rights reserved.
+ * Copyright (c) 2012-2017 YCSB contributors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
@@ -30,7 +30,6 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -44,15 +43,15 @@ import static org.junit.Assert.assertEquals;
public class ElasticsearchClientTest {
@ClassRule public final static TemporaryFolder temp = new TemporaryFolder();
- protected final static ElasticsearchClient instance = new ElasticsearchClient();
- protected final static HashMap<String, ByteIterator> MOCK_DATA;
- protected final static String MOCK_TABLE = "MOCK_TABLE";
- protected final static String MOCK_KEY0 = "0";
- protected final static String MOCK_KEY1 = "1";
- protected final static String MOCK_KEY2 = "2";
+ private final static ElasticsearchClient instance = new ElasticsearchClient();
+ private final static HashMap<String, ByteIterator> MOCK_DATA;
+ private final static String MOCK_TABLE = "MOCK_TABLE";
+ private final static String MOCK_KEY0 = "0";
+ private final static String MOCK_KEY1 = "1";
+ private final static String MOCK_KEY2 = "2";
static {
- MOCK_DATA = new HashMap<String, ByteIterator>(10);
+ MOCK_DATA = new HashMap<>(10);
for (int i = 1; i <= 10; i++) {
MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
}
@@ -88,7 +87,6 @@ public class ElasticsearchClientTest {
*/
@Test
public void testInsert() {
- System.out.println("insert");
Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA);
assertEquals(Status.OK, result);
}
@@ -98,7 +96,6 @@ public class ElasticsearchClientTest {
*/
@Test
public void testDelete() {
- System.out.println("delete");
Status result = instance.delete(MOCK_TABLE, MOCK_KEY1);
assertEquals(Status.OK, result);
}
@@ -108,9 +105,8 @@ public class ElasticsearchClientTest {
*/
@Test
public void testRead() {
- System.out.println("read");
Set<String> fields = MOCK_DATA.keySet();
- HashMap<String, ByteIterator> resultParam = new HashMap<String, ByteIterator>(10);
+ HashMap<String, ByteIterator> resultParam = new HashMap<>(10);
Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam);
assertEquals(Status.OK, result);
}
@@ -120,9 +116,8 @@ public class ElasticsearchClientTest {
*/
@Test
public void testUpdate() {
- System.out.println("update");
int i;
- HashMap<String, ByteIterator> newValues = new HashMap<String, ByteIterator>(10);
+ HashMap<String, ByteIterator> newValues = new HashMap<>(10);
for (i = 1; i <= 10; i++) {
newValues.put("field" + i, new StringByteIterator("newvalue" + i));
@@ -132,13 +127,12 @@ public class ElasticsearchClientTest {
assertEquals(Status.OK, result);
//validate that the values changed
- HashMap<String, ByteIterator> resultParam = new HashMap<String, ByteIterator>(10);
+ HashMap<String, ByteIterator> resultParam = new HashMap<>(10);
instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);
for (i = 1; i <= 10; i++) {
assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
}
-
}
/**
@@ -146,10 +140,9 @@ public class ElasticsearchClientTest {
*/
@Test
public void testScan() {
- System.out.println("scan");
int recordcount = 10;
Set<String> fields = MOCK_DATA.keySet();
- Vector<HashMap<String, ByteIterator>> resultParam = new Vector<HashMap<String, ByteIterator>>(10);
+ Vector<HashMap<String, ByteIterator>> resultParam = new Vector<>(10);
Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, recordcount, fields, resultParam);
assertEquals(Status.OK, result);
}
diff --git a/elasticsearch5/README.md b/elasticsearch5/README.md
new file mode 100644
index 00000000..b87fc874
--- /dev/null
+++ b/elasticsearch5/README.md
@@ -0,0 +1,94 @@
+<!--
+Copyright (c) 2017 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+## Quick Start
+
+This section describes how to run YCSB on Elasticsearch 5.x running locally.
+
+### 1. Set Up YCSB
+
+Clone the YCSB git repository and compile:
+
+ git clone git://github.com/brianfrankcooper/YCSB.git
+ cd YCSB
+ mvn clean package
+
+### 2. Run YCSB
+
+Now you are ready to run! First, load the data:
+
+ ./bin/ycsb load elasticsearch5 -s -P workloads/workloada -p path.home=<path>
+
+Then, run the workload:
+
+ ./bin/ycsb run elasticsearch5 -s -P workloads/workloada -p path.home=<path>
+
+Note that the `<path>` specified in each execution should be the same.
+
+The Elasticsearch 5 binding has two modes of operation, embedded mode and remote
+mode. In embedded mode, the client creates an embedded instance of
+Elasticsearch that uses the specified `<path>` to persist data between
+executions.
+
+In remote mode, the client will hit a standalone instance of Elasticsearch. To
+use remote mode, add the flags `-p es.remote=true` and specify a hosts list via
+`-p es.hosts.list=<hostname1:port1>,...,<hostnamen:portn>`.
+
+ ./bin/ycsb run elasticsearch5 -s -P workloads/workloada -p es.remote=true \
+ -p es.hosts.list=<hostname1:port1>,...,<hostnamen:portn>`
+
+Note that `es.hosts.list` defaults to `localhost:9300`. For further
+configuration see below:
+
+### Defaults Configuration
+The default setting for the Elasticsearch node that is created is as follows:
+
+ cluster.name=es.ycsb.cluster
+ es.index.key=es.ycsb
+ es.number_of_shards=1
+ es.number_of_replicas=0
+ es.remote=false
+ es.newdb=false
+ es.hosts.list=localhost:9300 (only applies if es.remote=true)
+
+### Custom Configuration
+If you wish to customize the settings used to create the Elasticsearch node
+you can created a new property file that contains your desired Elasticsearch
+node settings and pass it in via the parameter to 'bin/ycsb' script. Note that
+the default properties will be kept if you don't explicitly overwrite them.
+
+Assuming that we have a properties file named "myproperties.data" that contains
+custom Elasticsearch node configuration you can execute the following to
+pass it into the Elasticsearch client:
+
+
+ ./bin/ycsb run elasticsearch5 -P workloads/workloada -P myproperties.data -s
+
+If you wish to change the default index name you can set the following property:
+
+ es.index.key=my_index_key
+
+If you wish to run against a remote cluster you can set the following property:
+
+ es.remote=true
+
+By default this will use localhost:9300 as a seed node to discover the cluster.
+You can also specify
+
+ es.hosts.list=(\w+:\d+)+
+
+(a comma-separated list of host/port pairs) to change this.
diff --git a/elasticsearch5/pom.xml b/elasticsearch5/pom.xml
new file mode 100644
index 00000000..3eb38808
--- /dev/null
+++ b/elasticsearch5/pom.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Copyright (c) 2017 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>binding-parent</artifactId>
+ <version>0.13.0-SNAPSHOT</version>
+ <relativePath>../binding-parent</relativePath>
+ </parent>
+
+ <properties>
+ <!-- Skip tests by default. will be activated by jdk8 profile -->
+ <skipTests>true</skipTests>
+ </properties>
+
+ <artifactId>elasticsearch5-binding</artifactId>
+ <name>Elasticsearch 5.x Binding</name>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <!-- jna is supported in ES and will be used when provided
+ otherwise a fallback is used -->
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ <version>4.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>transport</artifactId>
+ <version>${elasticsearch5-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>rest</artifactId>
+ <version>${elasticsearch5-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>2.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.7</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClient.java b/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClient.java
new file mode 100644
index 00000000..00cba31b
--- /dev/null
+++ b/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClient.java
@@ -0,0 +1,274 @@
+/**
+ * Copyright (c) 2017 YCSB contributors. All rights reserved.
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.db.elasticsearch5;
+
+import com.yahoo.ycsb.*;
+import org.apache.http.HttpHost;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+
+import static org.elasticsearch.common.settings.Settings.Builder;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ * Elasticsearch client for YCSB framework.
+ */
+public class ElasticsearchClient extends DB {
+
+ private static final String DEFAULT_CLUSTER_NAME = "es.ycsb.cluster";
+ private static final String DEFAULT_INDEX_KEY = "es.ycsb";
+ private static final String DEFAULT_REMOTE_HOST = "localhost:9300";
+ private static final int NUMBER_OF_SHARDS = 1;
+ private static final int NUMBER_OF_REPLICAS = 0;
+ private Node node;
+ private Client client;
+ private String indexKey;
+ private Boolean remoteMode;
+
+ /**
+ * Initialize any state for this DB. Called once per DB instance; there is one
+ * DB instance per client thread.
+ */
+ @Override
+ public void init() throws DBException {
+ final Properties props = getProperties();
+
+ // Check if transport client needs to be used (To connect to multiple elasticsearch nodes)
+ remoteMode = Boolean.parseBoolean(props.getProperty("es.remote", "false"));
+
+ final String pathHome = props.getProperty("path.home");
+
+ // when running in embedded mode, require path.home
+ if (!remoteMode && (pathHome == null || pathHome.isEmpty())) {
+ throw new IllegalArgumentException("path.home must be specified when running in embedded mode");
+ }
+
+ this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY);
+
+ int numberOfShards = parseIntegerProperty(props, "es.number_of_shards", NUMBER_OF_SHARDS);
+ int numberOfReplicas = parseIntegerProperty(props, "es.number_of_replicas", NUMBER_OF_REPLICAS);
+
+ Boolean newdb = Boolean.parseBoolean(props.getProperty("es.newdb", "false"));
+ Builder settings = Settings.builder()
+ .put("cluster.name", DEFAULT_CLUSTER_NAME)
+ .put("path.home", pathHome);
+
+ // if properties file contains elasticsearch user defined properties
+ // add it to the settings file (will overwrite the defaults).
+ settings.put(props);
+ final String clusterName = settings.get("cluster.name");
+ System.err.println("Elasticsearch starting node = " + clusterName);
+ System.err.println("Elasticsearch node path.home = " + settings.get("path.home"));
+ System.err.println("Elasticsearch Remote Mode = " + remoteMode);
+ // Remote mode support for connecting to remote elasticsearch cluster
+ if(remoteMode) {
+ RestClient restClient = RestClient.builder(
+ new HttpHost("localhost", 9200, "http")).build();
+ }
+ if (remoteMode) {
+ settings.put("client.transport.sniff", true)
+ .put("client.transport.ignore_cluster_name", false)
+ .put("client.transport.ping_timeout", "30s")
+ .put("client.transport.nodes_sampler_interval", "30s");
+ // Default it to localhost:9300
+ String[] nodeList = props.getProperty("es.hosts.list", DEFAULT_REMOTE_HOST).split(",");
+ System.out.println("Elasticsearch Remote Hosts = " + props.getProperty("es.hosts.list", DEFAULT_REMOTE_HOST));
+ TransportClient tClient = new PreBuiltTransportClient(settings.build());
+ for (String h : nodeList) {
+ String[] nodes = h.split(":");
+ try {
+ tClient.addTransportAddress(new InetSocketTransportAddress(
+ InetAddress.getByName(nodes[0]),
+ Integer.parseInt(nodes[1])
+ ));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Unable to parse port number.", e);
+ } catch (UnknownHostException e) {
+ throw new IllegalArgumentException("Unable to Identify host.", e);
+ }
+ }
+ client = tClient;
+ } else { // Start node only if transport client mode is disabled
+ settings.put("transport.type", "local");
+ settings.put("http.enabled", "false");
+ node = new Node(settings.build());
+ try {
+ node.start();
+ } catch (NodeValidationException e) {
+ throw new DBException(e);
+ }
+ client = node.client();
+ }
+
+ final boolean exists =
+ client.admin().indices()
+ .exists(Requests.indicesExistsRequest(indexKey)).actionGet()
+ .isExists();
+ if (exists && newdb) {
+ client.admin().indices().prepareDelete(indexKey).execute().actionGet();
+ }
+ if (!exists || newdb) {
+ client.admin().indices().create(
+ new CreateIndexRequest(indexKey)
+ .settings(
+ Settings.builder()
+ .put("index.number_of_shards", numberOfShards)
+ .put("index.number_of_replicas", numberOfReplicas)
+ )).actionGet();
+ }
+ client.admin().cluster().health(new ClusterHealthRequest().waitForGreenStatus()).actionGet();
+ }
+
+ private int parseIntegerProperty(Properties properties, String key, int defaultValue) {
+ String value = properties.getProperty(key);
+ return value == null ? defaultValue : Integer.parseInt(value);
+ }
+
+ @Override
+ public void cleanup() throws DBException {
+ if (client != null) {
+ client.close();
+ client = null;
+ }
+
+ if (!remoteMode && node != null && !node.isClosed()) {
+ try {
+ node.close();
+ node = null;
+ } catch (IOException e) {
+ throw new DBException(e);
+ }
+ }
+ }
+
+ @Override
+ public Status insert(String table, String key, HashMap<String, ByteIterator> values) {
+ try {
+ final XContentBuilder doc = jsonBuilder().startObject();
+
+ for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
+ doc.field(entry.getKey(), entry.getValue());
+ }
+
+ doc.endObject();
+
+ client.prepareIndex(indexKey, table, key).setSource(doc).execute().actionGet();
+
+ return Status.OK;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return Status.ERROR;
+ }
+ }
+
+ @Override
+ public Status delete(String table, String key) {
+ try {
+ DeleteResponse response = client.prepareDelete(indexKey, table, key).execute().actionGet();
+ if (response.status().equals(RestStatus.NOT_FOUND)) {
+ return Status.NOT_FOUND;
+ } else {
+ return Status.OK;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ return Status.ERROR;
+ }
+ }
+
+ @Override
+ public Status read(String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
+ try {
+ final GetResponse response = client.prepareGet(indexKey, table, key).execute().actionGet();
+
+ if (response.isExists()) {
+ if (fields != null) {
+ for (String field : fields) {
+ result.put(field, new StringByteIterator(
+ (String) response.getSource().get(field)));
+ }
+ } else {
+ for (String field : response.getSource().keySet()) {
+ result.put(field, new StringByteIterator(
+ (String) response.getSource().get(field)));
+ }
+ }
+ return Status.OK;
+ } else {
+ return Status.NOT_FOUND;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ return Status.ERROR;
+ }
+ }
+
+ @Override
+ public Status update(String table, String key, HashMap<String, ByteIterator> values) {
+ try {
+ final GetResponse response = client.prepareGet(indexKey, table, key).execute().actionGet();
+
+ if (response.isExists()) {
+ for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
+ response.getSource().put(entry.getKey(), entry.getValue());
+ }
+
+ client.prepareIndex(indexKey, table, key).setSource(response.getSource()).execute().actionGet();
+
+ return Status.OK;
+ } else {
+ return Status.NOT_FOUND;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ return Status.ERROR;
+ }
+ }
+
+ @Override
+ public Status scan(
+ String table,
+ String startkey,
+ int recordcount,
+ Set<String> fields,
+ Vector<HashMap<String, ByteIterator>> result) {
+ return Status.NOT_IMPLEMENTED;
+ }
+}
diff --git a/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchRestClient.java b/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchRestClient.java
new file mode 100644
index 00000000..e4f3c9e5
--- /dev/null
+++ b/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchRestClient.java
@@ -0,0 +1,218 @@
+/**
+ * Copyright (c) 2017 YCSB contributors. All rights reserved.
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.db.elasticsearch5;
+
+import com.yahoo.ycsb.*;
+import org.apache.http.HttpHost;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.settings.Settings;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.elasticsearch.common.settings.Settings.Builder;
+
+/**
+ * Elasticsearch REST client for YCSB framework.
+ */
+public class ElasticsearchRestClient extends DB {
+
+ private static final String DEFAULT_CLUSTER_NAME = "es.ycsb.cluster";
+ private static final String DEFAULT_INDEX_KEY = "es.ycsb";
+ private static final String DEFAULT_REMOTE_HOST = "localhost:9200";
+ private static final int NUMBER_OF_SHARDS = 1;
+ private static final int NUMBER_OF_REPLICAS = 0;
+ private String indexKey;
+ private RestClient restClient;
+
+ @Override
+ public void init() throws DBException {
+ final Properties props = getProperties();
+
+ this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY);
+
+ int numberOfShards = Integer.valueOf(props.getProperty("es.number_of_shards",
+ String.valueOf(NUMBER_OF_SHARDS)));
+ int numberOfReplicas = Integer.valueOf(props.getProperty("es.number_of_replicas",
+ String.valueOf(NUMBER_OF_REPLICAS)));
+
+ Boolean newdb = Boolean.parseBoolean(props.getProperty("es.newdb", "false"));
+ Builder settings = Settings.builder().put("cluster.name", DEFAULT_CLUSTER_NAME);
+
+ // if properties file contains elasticsearch user defined properties
+ // add it to the settings file (will overwrite the defaults).
+ settings.put(props);
+ final String clusterName = settings.get("cluster.name");
+ System.err.println("Elasticsearch starting node = " + clusterName);
+
+ String[] nodeList = props.getProperty("es.hosts.list", DEFAULT_REMOTE_HOST).split(",");
+ System.out.println("Elasticsearch Remote Hosts = " + props.getProperty("es.hosts.list", DEFAULT_REMOTE_HOST));
+
+ List<HttpHost> esHttpHosts = new ArrayList<>(nodeList.length);
+ for (String h : nodeList) {
+ String[] nodes = h.split(":");
+ esHttpHosts.add(new HttpHost(nodes[0], Integer.valueOf(nodes[1]), "http"));
+ }
+
+ restClient = RestClient.builder(esHttpHosts.toArray(new HttpHost[esHttpHosts.size()])).build();
+
+// final boolean exists =
+// client.admin().indices()
+// .exists(Requests.indicesExistsRequest(indexKey)).actionGet()
+// .isExists();
+// if (exists && newdb) {
+// client.admin().indices().prepareDelete(indexKey).execute().actionGet();
+// }
+// if (!exists || newdb) {
+// client.admin().indices().create(
+// new CreateIndexRequest(indexKey)
+// .settings(
+// Settings.builder()
+// .put("index.number_of_shards", numberOfShards)
+// .put("index.number_of_replicas", numberOfReplicas)
+// )).actionGet();
+// }
+// client.admin().cluster().health(new ClusterHealthRequest().waitForGreenStatus()).actionGet();
+ }
+
+ @Override
+ public void cleanup() throws DBException {
+ if (restClient != null) {
+ try {
+ restClient.close();
+ restClient = null;
+ } catch (IOException e) {
+ throw new DBException(e);
+ }
+ }
+ }
+
+ @Override
+ public Status insert(String table, String key, HashMap<String, ByteIterator> values) {
+ try {
+ Map<String, String> data = StringByteIterator.getStringMap(values);
+
+ Response response = restClient.performRequest(
+ HttpPut.METHOD_NAME,
+ "/" + indexKey + "/" + table + "/",
+ Collections.<String, String>emptyMap(),
+ new NStringEntity(new ObjectMapper().writeValueAsString(data), ContentType.APPLICATION_JSON));
+
+ if(response.getStatusLine().getStatusCode() == 200) {
+ return Status.OK;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return Status.ERROR;
+ }
+
+ @Override
+ public Status delete(String table, String key) {
+ try {
+ Response response = restClient.performRequest(
+ HttpDelete.METHOD_NAME,
+ "/" + indexKey + "/" + table + "/" + key);
+
+ if(response.getStatusLine().getStatusCode() == 200) {
+ return Status.OK;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return Status.ERROR;
+ }
+
+ @Override
+ public Status read(String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
+ try {
+ Response response = restClient.performRequest(HttpGet.METHOD_NAME, "/");
+
+ if(response.getStatusLine().getStatusCode() == 200) {
+ return Status.OK;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return Status.ERROR;
+
+// try {
+// final GetResponse response = client.prepareGet(indexKey, table, key).execute().actionGet();
+//
+// if (response.isExists()) {
+// if (fields != null) {
+// for (String field : fields) {
+// result.put(field, new StringByteIterator(
+// (String) response.getSource().get(field)));
+// }
+// } else {
+// for (String field : response.getSource().keySet()) {
+// result.put(field, new StringByteIterator(
+// (String) response.getSource().get(field)));
+// }
+// }
+// return Status.OK;
+// } else {
+// return Status.NOT_FOUND;
+// }
+// } catch (Exception e) {
+// e.printStackTrace();
+// return Status.ERROR;
+// }
+ }
+
+ @Override
+ public Status update(String table, String key, HashMap<String, ByteIterator> values) {
+// try {
+// final GetResponse response = client.prepareGet(indexKey, table, key).execute().actionGet();
+//
+// if (response.isExists()) {
+// for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
+// response.getSource().put(entry.getKey(), entry.getValue());
+// }
+//
+// client.prepareIndex(indexKey, table, key).setSource(response.getSource()).execute().actionGet();
+//
+// return Status.OK;
+// } else {
+// return Status.NOT_FOUND;
+// }
+// } catch (Exception e) {
+// e.printStackTrace();
+// return Status.ERROR;
+// }
+ return Status.NOT_IMPLEMENTED;
+ }
+
+ @Override
+ public Status scan(
+ String table,
+ String startkey,
+ int recordcount,
+ Set<String> fields,
+ Vector<HashMap<String, ByteIterator>> result) {
+ return Status.NOT_IMPLEMENTED;
+ }
+}
diff --git a/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/package-info.java b/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/package-info.java
new file mode 100644
index 00000000..6fef0914
--- /dev/null
+++ b/elasticsearch5/src/main/java/com/yahoo/ycsb/db/elasticsearch5/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2017 YCSB Contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB binding for
+ * <a href="https://www.elastic.co/products/elasticsearch">Elasticsearch</a>.
+ */
+package com.yahoo.ycsb.db.elasticsearch5;
+
diff --git a/elasticsearch5/src/main/resources/log4j2.properties b/elasticsearch5/src/main/resources/log4j2.properties
new file mode 100644
index 00000000..fced116e
--- /dev/null
+++ b/elasticsearch5/src/main/resources/log4j2.properties
@@ -0,0 +1,6 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
\ No newline at end of file
diff --git a/elasticsearch5/src/test/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClientTest.java b/elasticsearch5/src/test/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClientTest.java
new file mode 100644
index 00000000..174c42ce
--- /dev/null
+++ b/elasticsearch5/src/test/java/com/yahoo/ycsb/db/elasticsearch5/ElasticsearchClientTest.java
@@ -0,0 +1,146 @@
+/**
+ * Copyright (c) 2017 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.db.elasticsearch5;
+
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.Status;
+import com.yahoo.ycsb.StringByteIterator;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.HashMap;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+
+import static org.junit.Assert.assertEquals;
+
+public class ElasticsearchClientTest {
+
+ @ClassRule public final static TemporaryFolder temp = new TemporaryFolder();
+ private final static ElasticsearchClient instance = new ElasticsearchClient();
+ private final static HashMap<String, ByteIterator> MOCK_DATA;
+ private final static String MOCK_TABLE = "MOCK_TABLE";
+ private final static String MOCK_KEY0 = "0";
+ private final static String MOCK_KEY1 = "1";
+ private final static String MOCK_KEY2 = "2";
+
+ static {
+ MOCK_DATA = new HashMap<>(10);
+ for (int i = 1; i <= 10; i++) {
+ MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
+ }
+ }
+
+ @BeforeClass
+ public static void setUpClass() throws DBException {
+ final Properties props = new Properties();
+ props.put("path.home", temp.getRoot().toString());
+ instance.setProperties(props);
+ instance.init();
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws DBException {
+ instance.cleanup();
+ }
+
+ @Before
+ public void setUp() {
+ instance.insert(MOCK_TABLE, MOCK_KEY1, MOCK_DATA);
+ instance.insert(MOCK_TABLE, MOCK_KEY2, MOCK_DATA);
+ }
+
+ @After
+ public void tearDown() {
+ instance.delete(MOCK_TABLE, MOCK_KEY1);
+ instance.delete(MOCK_TABLE, MOCK_KEY2);
+ }
+
+ /**
+ * Test of insert method, of class ElasticsearchClient.
+ */
+ @Test
+ public void testInsert() {
+ Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA);
+ assertEquals(Status.OK, result);
+ }
+
+ /**
+ * Test of delete method, of class ElasticsearchClient.
+ */
+ @Test
+ public void testDelete() {
+ Status result = instance.delete(MOCK_TABLE, MOCK_KEY1);
+ assertEquals(Status.OK, result);
+ }
+
+ /**
+ * Test of read method, of class ElasticsearchClient.
+ */
+ @Test
+ public void testRead() {
+ Set<String> fields = MOCK_DATA.keySet();
+ HashMap<String, ByteIterator> resultParam = new HashMap<>(10);
+ Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam);
+ assertEquals(Status.OK, result);
+ }
+
+ /**
+ * Test of update method, of class ElasticsearchClient.
+ */
+ @Test
+ public void testUpdate() {
+ int i;
+ HashMap<String, ByteIterator> newValues = new HashMap<>(10);
+
+ for (i = 1; i <= 10; i++) {
+ newValues.put("field" + i, new StringByteIterator("newvalue" + i));
+ }
+
+ Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues);
+ assertEquals(Status.OK, result);
+
+ //validate that the values changed
+ HashMap<String, ByteIterator> resultParam = new HashMap<>(10);
+ instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);
+
+ for (i = 1; i <= 10; i++) {
+ assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
+ }
+
+ }
+
+ /**
+ * Test of scan method, of class ElasticsearchClient.
+ */
+ @Test
+ public void testScan() {
+ int recordcount = 10;
+ Set<String> fields = MOCK_DATA.keySet();
+ Vector<HashMap<String, ByteIterator>> resultParam = new Vector<>(10);
+ Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, recordcount, fields, resultParam);
+ assertEquals(Status.NOT_IMPLEMENTED, result);
+ }
+}
diff --git a/hypertable/pom.xml b/hypertable/pom.xml
index aa05f141..2f4bf01d 100644
--- a/hypertable/pom.xml
+++ b/hypertable/pom.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
-Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
+Copyright (c) 2012 - 2017 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
@@ -40,7 +40,7 @@ LICENSE file.
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>${thrift.version}</version>
+ <version>0.8.0</version>
</dependency>
<dependency>
<groupId>org.hypertable</groupId>
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 9f6fdd10..f33bbd9e 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
-Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
+Copyright (c) 2012 - 2017 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
@@ -33,7 +33,7 @@ LICENSE file.
<dependency>
<groupId>org.apache.openjpa</groupId>
<artifactId>openjpa-jdbc</artifactId>
- <version>${openjpa.jdbc.version}</version>
+ <version>2.1.1</version>
</dependency>
<dependency>
<groupId>com.yahoo.ycsb</groupId>
diff --git a/pom.xml b/pom.xml
index 7693d224..7a3a5f05 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
-Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
+Copyright (c) 2012 - 2017 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
@@ -66,40 +66,43 @@ LICENSE file.
<!-- Properties Management -->
<properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+
<maven.assembly.version>2.5.5</maven.assembly.version>
<maven.dependency.version>2.10</maven.dependency.version>
+
+ <!-- Binding Versions -->
+ <accumulo.version>1.6.0</accumulo.version>
+ <aerospike.version>3.1.2</aerospike.version>
+ <arangodb.version>2.7.3</arangodb.version>
+ <arangodb3.version>4.1.7</arangodb3.version>
<asynchbase.version>1.7.1</asynchbase.version>
+ <azuredocumentdb.version>1.8.1</azuredocumentdb.version>
+ <azurestorage.version>4.0.0</azurestorage.version>
+ <cassandra.cql.version>3.0.0</cassandra.cql.version>
+ <couchbase.version>1.4.10</couchbase.version>
+ <couchbase2.version>2.3.1</couchbase2.version>
<hbase094.version>0.94.27</hbase094.version>
<hbase098.version>0.98.14-hadoop2</hbase098.version>
<hbase10.version>1.0.2</hbase10.version>
- <accumulo.version>1.6.0</accumulo.version>
- <cassandra.cql.version>3.0.0</cassandra.cql.version>
+ <hypertable.version>0.9.5.6</hypertable.version>
+ <elasticsearch-version>2.4.4</elasticsearch-version>
+ <elasticsearch5-version>5.2.0</elasticsearch5-version>
<geode.version>1.0.0-incubating.M3</geode.version>
- <azuredocumentdb.version>1.8.1</azuredocumentdb.version>
<googlebigtable.version>0.2.3</googlebigtable.version>
<infinispan.version>7.2.2.Final</infinispan.version>
<kudu.version>1.1.0</kudu.version>
- <openjpa.jdbc.version>2.1.1</openjpa.jdbc.version>
<!--<mapkeeper.version>1.0</mapkeeper.version>-->
<mongodb.version>3.0.3</mongodb.version>
<mongodb.async.version>2.0.1</mongodb.async.version>
<orientdb.version>2.2.10</orientdb.version>
<redis.version>2.0.0</redis.version>
- <s3.version>1.10.20</s3.version>
- <voldemort.version>0.81</voldemort.version>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <thrift.version>0.8.0</thrift.version>
- <hypertable.version>0.9.5.6</hypertable.version>
- <couchbase.version>1.4.10</couchbase.version>
- <couchbase2.version>2.3.1</couchbase2.version>
- <tarantool.version>1.6.5</tarantool.version>
<riak.version>2.0.5</riak.version>
- <aerospike.version>3.1.2</aerospike.version>
+ <s3.version>1.10.20</s3.version>
<solr.version>5.5.3</solr.version>
<solr6.version>6.4.1</solr6.version>
- <arangodb.version>2.7.3</arangodb.version>
- <arangodb3.version>4.1.7</arangodb3.version>
- <azurestorage.version>4.0.0</azurestorage.version>
+ <tarantool.version>1.6.5</tarantool.version>
+ <voldemort.version>0.81</voldemort.version>
</properties>
<modules>
@@ -120,6 +123,7 @@ LICENSE file.
<module>azuredocumentdb</module>
<module>dynamodb</module>
<module>elasticsearch</module>
+ <module>elasticsearch5</module>
<module>geode</module>
<module>googlebigtable</module>
<module>googledatastore</module>
--
GitLab