From d55ac2bb1dfb66ab1d2a67a7d67e2f33790c7f5f Mon Sep 17 00:00:00 2001
From: Hobo Chen <hobochen96@live.com>
Date: Tue, 4 Jun 2019 11:09:18 +0800
Subject: [PATCH] [tablestore] Add binding for Alibaba Cloud's TableStore
(#1286)
---
bin/bindings.properties | 2 +-
bin/ycsb | 1 +
distribution/pom.xml | 5 +
pom.xml | 2 +
tablestore/README.md | 61 ++++
tablestore/conf/tablestore.properties | 33 +++
tablestore/pom.xml | 34 +++
.../ycsb/db/tablestore/TableStoreClient.java | 276 ++++++++++++++++++
.../ycsb/db/tablestore/package-info.java | 22 ++
.../src/main/resources/log4j.properties | 25 ++
10 files changed, 460 insertions(+), 1 deletion(-)
create mode 100644 tablestore/README.md
create mode 100644 tablestore/conf/tablestore.properties
create mode 100755 tablestore/pom.xml
create mode 100755 tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/TableStoreClient.java
create mode 100644 tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/package-info.java
create mode 100644 tablestore/src/main/resources/log4j.properties
diff --git a/bin/bindings.properties b/bin/bindings.properties
index cd60158c..ec6262b4 100644
--- a/bin/bindings.properties
+++ b/bin/bindings.properties
@@ -78,4 +78,4 @@ s3:com.yahoo.ycsb.db.S3Client
solr:com.yahoo.ycsb.db.solr.SolrClient
solr6:com.yahoo.ycsb.db.solr6.SolrClient
tarantool:com.yahoo.ycsb.db.TarantoolClient
-
+tablestore:com.yahoo.ycsb.db.tablestore.TableStoreClient
diff --git a/bin/ycsb b/bin/ycsb
index dfb105cd..263f4f6f 100755
--- a/bin/ycsb
+++ b/bin/ycsb
@@ -106,6 +106,7 @@ DATABASES = {
"solr" : "com.yahoo.ycsb.db.solr.SolrClient",
"solr6" : "com.yahoo.ycsb.db.solr6.SolrClient",
"tarantool" : "com.yahoo.ycsb.db.TarantoolClient",
+ "tablestore" : "com.yahoo.ycsb.db.tablestore.TableStoreClient"
}
OPTIONS = {
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 69f02b88..3a4003a2 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -269,6 +269,11 @@ LICENSE file.
<artifactId>tarantool-binding</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>tablestore-binding</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!--
<dependency>
<groupId>com.yahoo.ycsb</groupId>
diff --git a/pom.xml b/pom.xml
index 3a347332..96f955b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -108,6 +108,7 @@ LICENSE file.
<tarantool.version>1.6.5</tarantool.version>
<thrift.version>0.8.0</thrift.version>
<voldemort.version>0.81</voldemort.version>
+ <tablestore.version>4.8.0</tablestore.version>
</properties>
<modules>
@@ -163,6 +164,7 @@ LICENSE file.
<module>solr</module>
<module>solr6</module>
<module>tarantool</module>
+ <module>tablestore</module>
<!--<module>voldemort</module>-->
</modules>
diff --git a/tablestore/README.md b/tablestore/README.md
new file mode 100644
index 00000000..c356df10
--- /dev/null
+++ b/tablestore/README.md
@@ -0,0 +1,61 @@
+<!--
+Copyright (c) 2018 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+# Alibaba Cloud TableStore Driver for YCSB
+
+This driver provides a YCSB workload for Alibaba's hosted TableStore, a fully managed NoSQL cloud database service that enables storage of a massive amount of structured and semi-structured data.
+
+This binding is based on the Java SDK for TableStore.
+
+## Quick Start
+
+### Setup a TableStore Instance
+
+Login to the Alibaba Cloud Console and follow the [Create Instance](https://www.alibabacloud.com/help/doc-detail/55211.htm?spm=a2c63.p38356.b99.17.6822642crAxqTI).
+Make a note of the instance name.
+
+### Create a Table for YCSB Testing
+
+Follow the [Create Table](https://www.alibabacloud.com/help/doc-detail/55212.htm?spm=a2c63.p38356.b99.18.1e4e50b9dXCcmC).
+
+The primary key must be exactly one column; the type is 'String'.
+Make a note of the table name and primary key name.
+
+### `tablestore.properties`
+
+tablestore.properties is the file specifying information needed by TableStore binding.
+The minimal tablestore.properties should be look like:
+
+```
+alibaba.cloud.tablestore.access_id = $access_id
+alibaba.cloud.tablestore.access_key = $access_key
+# the Instance Access URL in 'instance details'
+alibaba.cloud.tablestore.end_point = $end_point
+alibaba.cloud.tablestore.instance_name = $instance_name
+alibaba.cloud.tablestore.primary_key = $primary_key
+```
+
+For access_id and access_key, please refer to [Java SDK: Configure an AccessKey](https://www.alibabacloud.com/help/doc-detail/43009.htm?spm=a2c63.p38356.b99.134.728966fcMpTYD1).
+
+### Load and Run a Workload
+
+```zsh
+table_name='table_name'
+
+bin/ycsb load tablestore -P workloads/workloada -P tablestore/conf/tablestore.properties -p table=$table_name -threads 2
+bin/ycsb run tablestore -P workloads/workloada -P tablestore/conf/tablestore.properties -p table=$table_name -threads 2
+```
diff --git a/tablestore/conf/tablestore.properties b/tablestore/conf/tablestore.properties
new file mode 100644
index 00000000..f0a9806b
--- /dev/null
+++ b/tablestore/conf/tablestore.properties
@@ -0,0 +1,33 @@
+# Copyright (c) 2018 YCSB contributors. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+
+alibaba.cloud.tablestore.access_id =
+alibaba.cloud.tablestore.access_key =
+alibaba.cloud.tablestore.end_point =
+alibaba.cloud.tablestore.instance_name =
+alibaba.cloud.tablestore.primary_key =
+
+# alibaba.cloud.tablestore.max_version = 1
+
+# the following parameters provided by Java SDK can be tuned
+
+# max timeout in millisecond for open a http connection, default value in SDK is 15000
+# alibaba.cloud.tablestore.connection_timeout =
+
+# max timeout in millisecond for transfering data with a connection, default value in SDK is 15000
+# alibaba.cloud.tablestore.socket_timeout =
+
+# max http connections can be open, default value in SDK is 300
+# alibaba.cloud.tablestore.max_connections =
diff --git a/tablestore/pom.xml b/tablestore/pom.xml
new file mode 100755
index 00000000..4b2c7adc
--- /dev/null
+++ b/tablestore/pom.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>binding-parent</artifactId>
+ <version>0.16.0-SNAPSHOT</version>
+ <relativePath>../binding-parent</relativePath>
+ </parent>
+
+ <artifactId>tablestore-binding</artifactId>
+ <name>TableStore DB Binding</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.aliyun.openservices</groupId>
+ <artifactId>tablestore</artifactId>
+ <version>${tablestore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.17</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/TableStoreClient.java b/tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/TableStoreClient.java
new file mode 100755
index 00000000..a635fd99
--- /dev/null
+++ b/tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/TableStoreClient.java
@@ -0,0 +1,276 @@
+/*
+ * Copyright 2018 YCSB Contributors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.db.tablestore;
+
+import java.util.*;
+import java.util.function.*;
+
+import com.yahoo.ycsb.*;
+
+import com.alicloud.openservices.tablestore.*;
+import com.alicloud.openservices.tablestore.model.*;
+
+import org.apache.log4j.Logger;
+
+/**
+ * TableStore Client for YCSB.
+ */
+
+public class TableStoreClient extends DB {
+
+ private static SyncClient client;
+
+ private int maxVersions = 1;
+ private String primaryKeyName;
+
+ private static final Logger LOGGER = Logger.getLogger(TableStoreClient.class);
+
+ // nasty here as currently there is no support of JEP218
+ private void setIntegerProperty(
+ Properties properties,
+ String propertyName,
+ ClientConfiguration clientConfiguration,
+ Function<Integer, Boolean> qualifyFunction,
+ BiConsumer<ClientConfiguration, Integer> setFunction) throws DBException {
+ String propertyString = properties.getProperty(propertyName);
+ if (propertyString != null) {
+ Integer propertyInteger = new Integer(propertyString);
+ if (qualifyFunction.apply(propertyInteger).booleanValue()) {
+ setFunction.accept(clientConfiguration, propertyInteger);
+ } else {
+ String errorMessage = "Illegal argument." + propertyName + ":" + propertyString;
+ LOGGER.error(errorMessage);
+ throw new DBException(errorMessage);
+ }
+ }
+ }
+
+ @Override
+ public void init() throws DBException {
+
+ Properties properties = getProperties();
+ String accessID = properties.getProperty("alibaba.cloud.tablestore.access_id");
+ String accessKey = properties.getProperty("alibaba.cloud.tablestore.access_key");
+ String endPoint = properties.getProperty("alibaba.cloud.tablestore.end_point");
+ String instanceName = properties.getProperty("alibaba.cloud.tablestore.instance_name");
+ String maxVersion = properties.getProperty("alibaba.cloud.tablestore.max_version", "1");
+
+ maxVersions = Integer.parseInt(maxVersion);
+ primaryKeyName = properties.getProperty("alibaba.cloud.tablestore.primary_key", "");
+
+ ClientConfiguration clientConfiguration = new ClientConfiguration();
+
+ setIntegerProperty(
+ properties,
+ "alibaba.cloud.tablestore.connection_timeout",
+ clientConfiguration,
+ (Integer t) -> t > 0,
+ (ClientConfiguration c, Integer t) -> c.setConnectionTimeoutInMillisecond(t.intValue()));
+
+ setIntegerProperty(
+ properties,
+ "alibaba.cloud.tablestore.socket_timeout",
+ clientConfiguration,
+ (Integer t) -> t > 0,
+ (ClientConfiguration c, Integer t) -> c.setSocketTimeoutInMillisecond(t.intValue()));
+
+ setIntegerProperty(
+ properties,
+ "alibaba.cloud.tablestore.max_connections",
+ clientConfiguration,
+ (Integer t) -> t > 0,
+ (ClientConfiguration c, Integer t) -> c.setMaxConnections(t.intValue()));
+
+ try {
+ synchronized (TableStoreClient.class) {
+ if (client == null) {
+ client = new SyncClient(endPoint, accessID, accessKey, instanceName, clientConfiguration);
+ LOGGER.info("new tablestore sync client\tendpoint:" + endPoint + "\tinstanceName:" + instanceName);
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ throw new DBException("Illegal argument passed in. Check the format of your parameters.", e);
+ }
+ }
+
+ private void setResult(Set<String> fields, Map<String, ByteIterator> result, Row row) {
+ if (row != null) {
+ if (fields != null) {
+ for (String field : fields) {
+ result.put(field, new StringByteIterator((row.getColumn(field).toString())));
+ }
+ } else {
+ for (Column column : row.getColumns()) {
+ result.put(column.getName(), new StringByteIterator(column.getValue().asString()));
+ }
+ }
+ }
+ }
+
+ private Status dealWithTableStoreException(TableStoreException e) {
+ if (e.getErrorCode().contains("OTSRowOperationConflict")) {
+ return Status.ERROR;
+ }
+ LOGGER.error(e);
+ return Status.ERROR;
+ }
+
+ @Override
+ public Status read(String table, String key, Set<String> fields, Map<String, ByteIterator> result) {
+ try {
+ // set primary key
+ PrimaryKeyColumn[] primaryKeyColumns = new PrimaryKeyColumn[1];
+ primaryKeyColumns[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(key));
+ PrimaryKey primaryKey = new PrimaryKey(primaryKeyColumns);
+ // set table_name
+ SingleRowQueryCriteria singleRowQueryCriteria = new SingleRowQueryCriteria(table, primaryKey);
+ singleRowQueryCriteria.setMaxVersions(maxVersions);
+ // set columns
+ if (fields != null) {
+ singleRowQueryCriteria.addColumnsToGet(fields.toArray(new String[0]));
+ }
+ // set get_row request
+ GetRowRequest getRowRequest = new GetRowRequest();
+ getRowRequest.setRowQueryCriteria(singleRowQueryCriteria);
+ // operate
+ GetRowResponse getRowResponse = client.getRow(getRowRequest);
+ // set the result
+ setResult(fields, result, getRowResponse.getRow());
+ return Status.OK;
+ } catch (TableStoreException e) {
+ return dealWithTableStoreException(e);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ return Status.ERROR;
+ }
+ }
+
+ @Override
+ public Status scan(String table, String startkey, int recordcount,
+ Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
+ try {
+ // set primary key
+ PrimaryKeyColumn[] startKey = new PrimaryKeyColumn[1];
+ startKey[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(startkey));
+
+ PrimaryKeyColumn[] endKey = new PrimaryKeyColumn[1];
+ endKey[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.INF_MAX);
+
+ RangeRowQueryCriteria criteria = new RangeRowQueryCriteria(table);
+ criteria.setInclusiveStartPrimaryKey(new PrimaryKey(startKey));
+ criteria.setExclusiveEndPrimaryKey(new PrimaryKey(endKey));
+ criteria.setMaxVersions(maxVersions);
+ // set columns
+ if (fields != null) {
+ criteria.addColumnsToGet(fields.toArray(new String[0]));
+ }
+ // set limit
+ criteria.setLimit(recordcount);
+ // set the request
+ GetRangeRequest getRangeRequest = new GetRangeRequest();
+ getRangeRequest.setRangeRowQueryCriteria(criteria);
+ GetRangeResponse getRangeResponse = client.getRange(getRangeRequest);
+ // set the result
+ List<Row> rows = getRangeResponse.getRows();
+ for (Row row : rows) {
+ HashMap<String, ByteIterator> values = new HashMap<>();
+ setResult(fields, values, row);
+ result.add(values);
+ }
+ return Status.OK;
+ } catch (TableStoreException e) {
+ return dealWithTableStoreException(e);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ return Status.ERROR;
+ }
+ }
+
+ @Override
+ public Status update(String table, String key,
+ Map<String, ByteIterator> values) {
+ try {
+ PrimaryKeyColumn[] primaryKeyColumns = new PrimaryKeyColumn[1];
+ primaryKeyColumns[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(key));
+ PrimaryKey primaryKey = new PrimaryKey(primaryKeyColumns);
+ RowUpdateChange rowUpdateChange = new RowUpdateChange(table, primaryKey);
+
+ for (Map.Entry<String, ByteIterator> entry: values.entrySet()) {
+ rowUpdateChange.put(entry.getKey(), ColumnValue.fromString(entry.getValue().toString()));
+ }
+
+ UpdateRowRequest updateRowRequest = new UpdateRowRequest();
+ updateRowRequest.setRowChange(rowUpdateChange);
+ client.updateRow(updateRowRequest);
+ return Status.OK;
+ } catch (TableStoreException e) {
+ return dealWithTableStoreException(e);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ return Status.ERROR;
+ }
+ }
+
+ @Override
+ public Status insert(String table, String key, Map<String, ByteIterator> values) {
+ try {
+ // set the primary key
+ PrimaryKeyColumn[] primaryKeyColumns = new PrimaryKeyColumn[1];
+ primaryKeyColumns[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(key));
+ PrimaryKey primaryKey = new PrimaryKey(primaryKeyColumns);
+ RowPutChange rowPutChange = new RowPutChange(table, primaryKey);
+ // set the columns
+ for (Map.Entry<String, ByteIterator> entry: values.entrySet()) {
+ rowPutChange.addColumn(entry.getKey(), ColumnValue.fromString(entry.getValue().toString()));
+ }
+ // set the putRow request
+ PutRowRequest putRowRequest = new PutRowRequest();
+ putRowRequest.setRowChange(rowPutChange);
+ // operate
+ client.putRow(putRowRequest);
+ return Status.OK;
+ } catch (TableStoreException e) {
+ return dealWithTableStoreException(e);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ return Status.ERROR;
+ }
+ }
+
+ @Override
+ public Status delete(String table, String key) {
+ try {
+ PrimaryKeyColumn[] primaryKeyColumns = new PrimaryKeyColumn[1];
+ primaryKeyColumns[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(key));
+ PrimaryKey primaryKey = new PrimaryKey(primaryKeyColumns);
+
+ RowDeleteChange rowDeleteChange = new RowDeleteChange(table, primaryKey);
+
+ DeleteRowRequest deleteRowRequest = new DeleteRowRequest();
+ deleteRowRequest.setRowChange(rowDeleteChange);
+ client.deleteRow(deleteRowRequest);
+ return Status.OK;
+ } catch (TableStoreException e) {
+ return dealWithTableStoreException(e);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ return Status.ERROR;
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/package-info.java b/tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/package-info.java
new file mode 100644
index 00000000..381044fa
--- /dev/null
+++ b/tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2018 YCSB Contributors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB binding for <a href="https://www.alibabacloud.com/product/table-store">TableStore</a>.
+ */
+package com.yahoo.ycsb.db.tablestore;
+
diff --git a/tablestore/src/main/resources/log4j.properties b/tablestore/src/main/resources/log4j.properties
new file mode 100644
index 00000000..1f49c0ae
--- /dev/null
+++ b/tablestore/src/main/resources/log4j.properties
@@ -0,0 +1,25 @@
+# Copyright (c) 2018 YCSB contributors. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+
+#define the console appender
+log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender
+
+# now define the layout for the appender
+log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout
+log4j.appender.consoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x -%m%n
+
+# now map our console appender as a root logger, means all log messages will go
+# to this appender
+log4j.rootLogger = INFO, consoleAppender
--
GitLab