[tablestore] Add binding for Alibaba Cloud's TableStore (#1286)

bin/bindings.properties

@@ -78,4 +78,4 @@ s3:com.yahoo.ycsb.db.S3Client
 solr:com.yahoo.ycsb.db.solr.SolrClient
 solr6:com.yahoo.ycsb.db.solr6.SolrClient
 tarantool:com.yahoo.ycsb.db.TarantoolClient
-
+tablestore:com.yahoo.ycsb.db.tablestore.TableStoreClient

bin/ycsb

@@ -106,6 +106,7 @@ DATABASES = {
     "solr"         : "com.yahoo.ycsb.db.solr.SolrClient",
     "solr6"        : "com.yahoo.ycsb.db.solr6.SolrClient",
     "tarantool"    : "com.yahoo.ycsb.db.TarantoolClient",
+    "tablestore"   : "com.yahoo.ycsb.db.tablestore.TableStoreClient"
 }
 
 OPTIONS = {

distribution/pom.xml

@@ -269,6 +269,11 @@ LICENSE file.
       <artifactId>tarantool-binding</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+        <groupId>com.yahoo.ycsb</groupId>
+        <artifactId>tablestore-binding</artifactId>
+        <version>${project.version}</version>
+    </dependency>
 <!--
     <dependency>
       <groupId>com.yahoo.ycsb</groupId>

pom.xml

@@ -108,6 +108,7 @@ LICENSE file.
     <tarantool.version>1.6.5</tarantool.version>
     <thrift.version>0.8.0</thrift.version>
     <voldemort.version>0.81</voldemort.version>
+    <tablestore.version>4.8.0</tablestore.version>
   </properties>
 
   <modules>
@@ -163,6 +164,7 @@ LICENSE file.
     <module>solr</module>
     <module>solr6</module>
     <module>tarantool</module>
+    <module>tablestore</module>
     <!--<module>voldemort</module>-->
   </modules>
 

tablestore/README.md

+<!--
+Copyright (c) 2018 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+# Alibaba Cloud TableStore Driver for YCSB
+
+This driver provides a YCSB workload for Alibaba's hosted TableStore, a fully managed NoSQL cloud database service that enables storage of a massive amount of structured and semi-structured data.
+
+This binding is based on the Java SDK for TableStore.
+
+## Quick Start
+
+### Setup a TableStore Instance
+
+Login to the Alibaba Cloud Console and follow the [Create Instance](https://www.alibabacloud.com/help/doc-detail/55211.htm?spm=a2c63.p38356.b99.17.6822642crAxqTI).
+Make a note of the instance name.
+
+### Create a Table for YCSB Testing
+
+Follow the [Create Table](https://www.alibabacloud.com/help/doc-detail/55212.htm?spm=a2c63.p38356.b99.18.1e4e50b9dXCcmC).
+
+The primary key must be exactly one column; the type is 'String'.
+Make a note of the table name and primary key name.
+
+### `tablestore.properties`
+
+tablestore.properties is the file specifying information needed by TableStore binding.
+The minimal tablestore.properties should be look like:
+
+```
+alibaba.cloud.tablestore.access_id = $access_id
+alibaba.cloud.tablestore.access_key = $access_key
+# the Instance Access URL in 'instance details'
+alibaba.cloud.tablestore.end_point = $end_point
+alibaba.cloud.tablestore.instance_name = $instance_name
+alibaba.cloud.tablestore.primary_key = $primary_key
+```
+
+For access_id and access_key, please refer to [Java SDK: Configure an AccessKey](https://www.alibabacloud.com/help/doc-detail/43009.htm?spm=a2c63.p38356.b99.134.728966fcMpTYD1).
+
+### Load and Run a Workload
+
+```zsh
+table_name='table_name'
+
+bin/ycsb load tablestore -P workloads/workloada -P tablestore/conf/tablestore.properties -p table=$table_name -threads 2
+bin/ycsb run tablestore -P workloads/workloada -P tablestore/conf/tablestore.properties -p table=$table_name -threads 2 
+```

tablestore/conf/tablestore.properties

+# Copyright (c) 2018 YCSB contributors. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+
+alibaba.cloud.tablestore.access_id = 
+alibaba.cloud.tablestore.access_key = 
+alibaba.cloud.tablestore.end_point = 
+alibaba.cloud.tablestore.instance_name = 
+alibaba.cloud.tablestore.primary_key = 
+
+# alibaba.cloud.tablestore.max_version = 1
+
+# the following parameters provided by Java SDK can be tuned
+
+# max timeout in millisecond for open a http connection, default value in SDK is 15000
+# alibaba.cloud.tablestore.connection_timeout = 
+
+# max timeout in millisecond for transfering data with a connection, default value in SDK is 15000
+# alibaba.cloud.tablestore.socket_timeout =
+
+# max http connections can be open, default value in SDK is 300
+# alibaba.cloud.tablestore.max_connections =

tablestore/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.ycsb</groupId>
+        <artifactId>binding-parent</artifactId>
+        <version>0.16.0-SNAPSHOT</version>
+        <relativePath>../binding-parent</relativePath>
+    </parent>
+
+    <artifactId>tablestore-binding</artifactId>
+    <name>TableStore DB Binding</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.yahoo.ycsb</groupId>
+            <artifactId>core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.aliyun.openservices</groupId>
+            <artifactId>tablestore</artifactId>
+            <version>${tablestore.version}</version>
+            </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+        </dependency>
+    </dependencies>
+
+</project>

tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/TableStoreClient.java

+/*
+ * Copyright 2018 YCSB Contributors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.db.tablestore;
+
+import java.util.*;
+import java.util.function.*;
+
+import com.yahoo.ycsb.*;
+
+import com.alicloud.openservices.tablestore.*;
+import com.alicloud.openservices.tablestore.model.*;
+
+import org.apache.log4j.Logger;
+
+/**
+ * TableStore Client for YCSB.
+ */
+
+public class TableStoreClient extends DB {
+
+  private static SyncClient client;
+
+  private int maxVersions = 1;
+  private String primaryKeyName;
+
+  private static final Logger LOGGER = Logger.getLogger(TableStoreClient.class);
+
+  // nasty here as currently there is no support of JEP218
+  private void setIntegerProperty(
+      Properties properties,
+      String propertyName,
+      ClientConfiguration clientConfiguration,
+      Function<Integer, Boolean> qualifyFunction,
+      BiConsumer<ClientConfiguration, Integer> setFunction) throws DBException {
+    String propertyString = properties.getProperty(propertyName);
+    if (propertyString != null) {
+      Integer propertyInteger = new Integer(propertyString);
+      if (qualifyFunction.apply(propertyInteger).booleanValue()) {
+        setFunction.accept(clientConfiguration, propertyInteger);
+      } else {
+        String errorMessage = "Illegal argument." + propertyName + ":" + propertyString;
+        LOGGER.error(errorMessage);
+        throw new DBException(errorMessage);
+      }
+    }
+  }
+
+  @Override
+  public void init() throws DBException {
+
+    Properties properties = getProperties();
+    String accessID = properties.getProperty("alibaba.cloud.tablestore.access_id");
+    String accessKey = properties.getProperty("alibaba.cloud.tablestore.access_key");
+    String endPoint = properties.getProperty("alibaba.cloud.tablestore.end_point");
+    String instanceName = properties.getProperty("alibaba.cloud.tablestore.instance_name");
+    String maxVersion = properties.getProperty("alibaba.cloud.tablestore.max_version", "1");
+
+    maxVersions = Integer.parseInt(maxVersion);
+    primaryKeyName = properties.getProperty("alibaba.cloud.tablestore.primary_key", "");
+
+    ClientConfiguration clientConfiguration = new ClientConfiguration();
+
+    setIntegerProperty(
+        properties,
+        "alibaba.cloud.tablestore.connection_timeout",
+        clientConfiguration,
+        (Integer t) -> t > 0,
+        (ClientConfiguration c, Integer t) -> c.setConnectionTimeoutInMillisecond(t.intValue()));
+
+    setIntegerProperty(
+        properties,
+        "alibaba.cloud.tablestore.socket_timeout",
+        clientConfiguration,
+        (Integer t) -> t > 0,
+        (ClientConfiguration c, Integer t) -> c.setSocketTimeoutInMillisecond(t.intValue()));
+
+    setIntegerProperty(
+        properties,
+        "alibaba.cloud.tablestore.max_connections",
+        clientConfiguration,
+        (Integer t) -> t > 0,
+        (ClientConfiguration c, Integer t) -> c.setMaxConnections(t.intValue()));
+
+    try {
+      synchronized (TableStoreClient.class) {
+        if (client == null) {
+          client = new SyncClient(endPoint, accessID, accessKey, instanceName, clientConfiguration);
+          LOGGER.info("new tablestore sync client\tendpoint:" + endPoint + "\tinstanceName:" + instanceName);
+        }
+      }
+    } catch (IllegalArgumentException e) {
+      throw new DBException("Illegal argument passed in. Check the format of your parameters.", e);
+    }
+  }
+
+  private void setResult(Set<String> fields, Map<String, ByteIterator> result, Row row) {
+    if (row != null) {
+      if (fields != null) {
+        for (String field : fields) {
+          result.put(field, new StringByteIterator((row.getColumn(field).toString())));
+        }
+      } else {
+        for (Column column : row.getColumns()) {
+          result.put(column.getName(), new StringByteIterator(column.getValue().asString()));
+        }
+      }
+    }
+  }
+
+  private Status dealWithTableStoreException(TableStoreException e) {
+    if (e.getErrorCode().contains("OTSRowOperationConflict")) {
+      return Status.ERROR;
+    }
+    LOGGER.error(e);
+    return Status.ERROR;
+  }
+
+  @Override
+  public Status read(String table, String key, Set<String> fields, Map<String, ByteIterator> result) {
+    try {
+      // set primary key
+      PrimaryKeyColumn[] primaryKeyColumns = new PrimaryKeyColumn[1];
+      primaryKeyColumns[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(key));
+      PrimaryKey primaryKey = new PrimaryKey(primaryKeyColumns);
+      // set table_name
+      SingleRowQueryCriteria singleRowQueryCriteria = new SingleRowQueryCriteria(table, primaryKey);
+      singleRowQueryCriteria.setMaxVersions(maxVersions);
+      // set columns
+      if (fields != null) {
+        singleRowQueryCriteria.addColumnsToGet(fields.toArray(new String[0]));
+      }
+      // set get_row request
+      GetRowRequest getRowRequest = new GetRowRequest();
+      getRowRequest.setRowQueryCriteria(singleRowQueryCriteria);
+      // operate
+      GetRowResponse getRowResponse = client.getRow(getRowRequest);
+      // set the result
+      setResult(fields, result, getRowResponse.getRow());
+      return Status.OK;
+    } catch (TableStoreException e) {
+      return dealWithTableStoreException(e);
+    } catch (Exception e) {
+      LOGGER.error(e);
+      return Status.ERROR;
+    }
+  }
+
+  @Override
+  public Status scan(String table, String startkey, int recordcount,
+                     Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
+    try {
+      // set primary key
+      PrimaryKeyColumn[] startKey = new PrimaryKeyColumn[1];
+      startKey[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(startkey));
+
+      PrimaryKeyColumn[] endKey = new PrimaryKeyColumn[1];
+      endKey[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.INF_MAX);
+
+      RangeRowQueryCriteria criteria = new RangeRowQueryCriteria(table);
+      criteria.setInclusiveStartPrimaryKey(new PrimaryKey(startKey));
+      criteria.setExclusiveEndPrimaryKey(new PrimaryKey(endKey));
+      criteria.setMaxVersions(maxVersions);
+      // set columns
+      if (fields != null) {
+        criteria.addColumnsToGet(fields.toArray(new String[0]));
+      }
+      // set limit
+      criteria.setLimit(recordcount);
+      // set the request
+      GetRangeRequest getRangeRequest = new GetRangeRequest();
+      getRangeRequest.setRangeRowQueryCriteria(criteria);
+      GetRangeResponse getRangeResponse = client.getRange(getRangeRequest);
+      // set the result
+      List<Row> rows = getRangeResponse.getRows();
+      for (Row row : rows) {
+        HashMap<String, ByteIterator> values = new HashMap<>();
+        setResult(fields, values, row);
+        result.add(values);
+      }
+      return Status.OK;
+    } catch (TableStoreException e) {
+      return dealWithTableStoreException(e);
+    } catch (Exception e) {
+      LOGGER.error(e);
+      return Status.ERROR;
+    }
+  }
+
+  @Override
+  public Status update(String table, String key,
+                       Map<String, ByteIterator> values) {
+    try {
+      PrimaryKeyColumn[] primaryKeyColumns = new PrimaryKeyColumn[1];
+      primaryKeyColumns[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(key));
+      PrimaryKey primaryKey = new PrimaryKey(primaryKeyColumns);
+      RowUpdateChange rowUpdateChange = new RowUpdateChange(table, primaryKey);
+
+      for (Map.Entry<String, ByteIterator> entry: values.entrySet()) {
+        rowUpdateChange.put(entry.getKey(), ColumnValue.fromString(entry.getValue().toString()));
+      }
+
+      UpdateRowRequest updateRowRequest = new UpdateRowRequest();
+      updateRowRequest.setRowChange(rowUpdateChange);
+      client.updateRow(updateRowRequest);
+      return Status.OK;
+    } catch (TableStoreException e) {
+      return dealWithTableStoreException(e);
+    } catch (Exception e) {
+      LOGGER.error(e);
+      return Status.ERROR;
+    }
+  }
+
+  @Override
+  public Status insert(String table, String key, Map<String, ByteIterator> values) {
+    try {
+      // set the primary key
+      PrimaryKeyColumn[] primaryKeyColumns = new PrimaryKeyColumn[1];
+      primaryKeyColumns[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(key));
+      PrimaryKey primaryKey = new PrimaryKey(primaryKeyColumns);
+      RowPutChange rowPutChange = new RowPutChange(table, primaryKey);
+      // set the columns
+      for (Map.Entry<String, ByteIterator> entry: values.entrySet()) {
+        rowPutChange.addColumn(entry.getKey(), ColumnValue.fromString(entry.getValue().toString()));
+      }
+      // set the putRow request
+      PutRowRequest putRowRequest = new PutRowRequest();
+      putRowRequest.setRowChange(rowPutChange);
+      // operate
+      client.putRow(putRowRequest);
+      return Status.OK;
+    } catch (TableStoreException e) {
+      return dealWithTableStoreException(e);
+    } catch (Exception e) {
+      LOGGER.error(e);
+      return Status.ERROR;
+    }
+  }
+
+  @Override
+  public Status delete(String table, String key) {
+    try {
+      PrimaryKeyColumn[] primaryKeyColumns = new PrimaryKeyColumn[1];
+      primaryKeyColumns[0] = new PrimaryKeyColumn(primaryKeyName, PrimaryKeyValue.fromString(key));
+      PrimaryKey primaryKey = new PrimaryKey(primaryKeyColumns);
+
+      RowDeleteChange rowDeleteChange = new RowDeleteChange(table, primaryKey);
+
+      DeleteRowRequest deleteRowRequest = new DeleteRowRequest();
+      deleteRowRequest.setRowChange(rowDeleteChange);
+      client.deleteRow(deleteRowRequest);
+      return Status.OK;
+    } catch (TableStoreException e) {
+      return dealWithTableStoreException(e);
+    } catch (Exception e) {
+      LOGGER.error(e);
+      return Status.ERROR;
+    }
+  }
+
+}
\ No newline at end of file

tablestore/src/main/java/com/yahoo/ycsb/db/tablestore/package-info.java

+/*
+ * Copyright 2018 YCSB Contributors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB binding for <a href="https://www.alibabacloud.com/product/table-store">TableStore</a>.
+ */
+package com.yahoo.ycsb.db.tablestore;
+

tablestore/src/main/resources/log4j.properties

+# Copyright (c) 2018 YCSB contributors. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+
+#define the console appender
+log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender
+
+# now define the layout for the appender
+log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout
+log4j.appender.consoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x -%m%n
+
+# now map our console appender as a root logger, means all log messages will go
+# to this appender
+log4j.rootLogger = INFO, consoleAppender