[foundationdb] add a new client for FoundationDB (#1162)

bin/bindings.properties

@@ -46,6 +46,7 @@ dynamodb:com.yahoo.ycsb.db.DynamoDBClient
 elasticsearch:com.yahoo.ycsb.db.ElasticsearchClient
 elasticsearch5:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient
 elasticsearch5-rest:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchRestClient
+foundationdb:com.yahoo.ycsb.db.foundationdb.FoundationDBClient
 geode:com.yahoo.ycsb.db.GeodeClient
 googlebigtable:com.yahoo.ycsb.db.GoogleBigtableClient
 googledatastore:com.yahoo.ycsb.db.GoogleDatastoreClient

bin/ycsb

@@ -71,6 +71,7 @@ DATABASES = {
     "dynamodb"     : "com.yahoo.ycsb.db.DynamoDBClient",
     "elasticsearch": "com.yahoo.ycsb.db.ElasticsearchClient",
     "elasticsearch5": "com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient",
+    "foundationdb" : "com.yahoo.ycsb.db.foundationdb.FoundationDBClient",
     "geode"        : "com.yahoo.ycsb.db.GeodeClient",
     "googlebigtable"  : "com.yahoo.ycsb.db.GoogleBigtableClient",
     "googledatastore" : "com.yahoo.ycsb.db.GoogleDatastoreClient",

distribution/pom.xml

@@ -114,6 +114,11 @@ LICENSE file.
       <artifactId>elasticsearch5-binding</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.yahoo.ycsb</groupId>
+      <artifactId>foundationdb-binding</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.yahoo.ycsb</groupId>
       <artifactId>geode-binding</artifactId>

foundationdb/README.md

+<!--
+Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+## Quick Start
+
+This section describes how to run YCSB on FoundationDB running locally. 
+
+### 1. Set Up YCSB
+
+Clone the YCSB git repository and compile:
+
+    git clone https://github.com/brianfrankcooper/YCSB.git
+    cd YCSB
+    mvn clean package
+
+### 2. Run YCSB
+    
+Now you are ready to run! First, load the data:
+
+    ./bin/ycsb load foundationdb -s -P workloads/workloada
+
+Then, run the workload:
+
+    ./bin/ycsb run foundationdb -s -P workloads/workloada
+
+See the next section for the list of configuration parameters for FoundationDB.
+
+## FoundationDB Configuration Parameters
+
+* ```foundationdb.apiversion``` - TThe FoundationDB API version.
+  * Default: ```520```
+* ```foundationdb.clusterfile``` - The cluster file path.
+  * Default: ```./fdb.cluster```
+* ```foundationdb.dbname``` - The database name.
+  * Default: ```DB```
+* ```foundationdb.batchsize``` - The number of rows to be batched before commit.
+  * Default: ```0```

foundationdb/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.yahoo.ycsb</groupId>
+    <artifactId>binding-parent</artifactId>
+    <version>0.15.0-SNAPSHOT</version>
+    <relativePath>../binding-parent</relativePath>
+  </parent>
+
+  <artifactId>foundationdb-binding</artifactId>
+  <name>FoundationDB Binding</name>
+
+  <packaging>jar</packaging>
+  <properties>
+    <!-- Tests do not run on jdk9,10,11 -->
+    <skipJDK9Tests>true</skipJDK9Tests>
+    <skipJDK10Tests>true</skipJDK10Tests>
+    <skipJDK11Tests>true</skipJDK11Tests>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>com.yahoo.ycsb</groupId>
+      <artifactId>core</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.foundationdb</groupId>
+      <artifactId>fdb-java</artifactId>
+      <version>${foundationdb.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.12</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>1.7.10</version>
+    </dependency>
+  </dependencies>
+</project>

foundationdb/src/main/java/com/yahoo/ycsb/db/foundationdb/FoundationDBClient.java

+/**
+ * Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.db.foundationdb;
+
+import com.apple.foundationdb.*;
+import com.apple.foundationdb.async.AsyncIterable;
+import com.apple.foundationdb.tuple.Tuple;
+
+import com.yahoo.ycsb.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.helpers.MessageFormatter;
+
+import java.util.*;
+
+/**
+ * FoundationDB client for YCSB framework.
+ */
+
+public class FoundationDBClient extends DB {
+  private FDB fdb;
+  private Database db;
+  private String dbName;
+  private int batchSize;
+  private int batchCount;
+  private static final String API_VERSION          = "foundationdb.apiversion";
+  private static final String API_VERSION_DEFAULT  = "520";
+  private static final String CLUSTER_FILE         = "foundationdb.clusterfile";
+  private static final String CLUSTER_FILE_DEFAULT = "./fdb.cluster";
+  private static final String DB_NAME              = "foundationdb.dbname";
+  private static final String DB_NAME_DEFAULT      = "DB";
+  private static final String DB_BATCH_SIZE_DEFAULT = "0";
+  private static final String DB_BATCH_SIZE         = "foundationdb.batchsize";
+
+  private Vector<String> batchKeys;
+  private Vector<Map<String, ByteIterator>> batchValues;
+
+  private static Logger logger = LoggerFactory.getLogger(FoundationDBClient.class);
+
+  /**
+   * Initialize any state for this DB. Called once per DB instance; there is one DB instance per client thread.
+   */
+  @Override
+  public void init() throws DBException {
+    // initialize FoundationDB driver
+    final Properties props = getProperties();
+    String apiVersion = props.getProperty(API_VERSION, API_VERSION_DEFAULT);
+    String clusterFile = props.getProperty(CLUSTER_FILE, CLUSTER_FILE_DEFAULT);
+    String dbBatchSize = props.getProperty(DB_BATCH_SIZE, DB_BATCH_SIZE_DEFAULT);
+    dbName = props.getProperty(DB_NAME, DB_NAME_DEFAULT);
+
+    logger.info("API Version: {}", apiVersion);
+    logger.info("Cluster File: {}\n", clusterFile);
+
+    try {
+      fdb = FDB.selectAPIVersion(Integer.parseInt(apiVersion.trim()));
+      db = fdb.open(clusterFile);
+      batchSize = Integer.parseInt(dbBatchSize);
+      batchCount = 0;
+      batchKeys = new Vector<String>(batchSize+1);
+      batchValues = new Vector<Map<String, ByteIterator>>(batchSize+1);
+    } catch (FDBException e) {
+      logger.error(MessageFormatter.format("Error in database operation: {}", "init").getMessage(), e);
+      throw new DBException(e);
+    } catch (NumberFormatException e) {
+      logger.error(MessageFormatter.format("Invalid value for apiversion property: {}", apiVersion).getMessage(), e);
+      throw new DBException(e);
+    }
+  }
+
+  @Override
+  public void cleanup() throws DBException {
+    if (batchCount > 0) {
+      batchInsert();
+      batchCount = 0;
+    }
+    try {
+      db.close();
+    } catch (FDBException e) {
+      logger.error(MessageFormatter.format("Error in database operation: {}", "cleanup").getMessage(), e);
+      throw new DBException(e);
+    }
+  }
+
+  private static String getRowKey(String db, String table, String key) {
+    //return key + ":" + table + ":" + db;
+    return db + ":" + table + ":" + key;
+  }
+
+  private static String getEndRowKey(String table) {
+    return table + ";";
+  }
+
+  private Status convTupleToMap(Tuple tuple, Set<String> fields, Map<String, ByteIterator> result) {
+    for (int i = 0; i < tuple.size(); i++) {
+      Tuple v = tuple.getNestedTuple(i);
+      String field = v.getString(0);
+      String value = v.getString(1);
+      //System.err.println(field + " : " + value);
+      result.put(field, new StringByteIterator(value));
+    }
+    if (fields != null) {
+      for (String field : fields) {
+        if (result.get(field) == null) {
+          logger.debug("field not fount: {}", field);
+          return Status.NOT_FOUND;
+        }
+      }
+    }
+    return Status.OK;
+  }
+
+  private void batchInsert() {
+    try {
+      db.run(tr -> {
+          for (int i = 0; i < batchCount; ++i) {
+            Tuple t = new Tuple();
+            for (Map.Entry<String, String> entry : StringByteIterator.getStringMap(batchValues.get(i)).entrySet()) {
+              Tuple v = new Tuple();
+              v = v.add(entry.getKey());
+              v = v.add(entry.getValue());
+              t = t.add(v);
+            }
+            tr.set(Tuple.from(batchKeys.get(i)).pack(), t.pack());
+          }
+          return null;
+        });
+    } catch (FDBException e) {
+      for (int i = 0; i < batchCount; ++i) {
+        logger.error(MessageFormatter.format("Error batch inserting key {}", batchKeys.get(i)).getMessage(), e);
+      }
+      e.printStackTrace();
+    } catch (Throwable e) {
+      for (int i = 0; i < batchCount; ++i) {
+        logger.error(MessageFormatter.format("Error batch inserting key {}", batchKeys.get(i)).getMessage(), e);
+      }
+      e.printStackTrace();
+    } finally {
+      batchKeys.clear();
+      batchValues.clear();
+    }
+  }
+
+  @Override
+  public Status insert(String table, String key, Map<String, ByteIterator> values) {
+    String rowKey = getRowKey(dbName, table, key);
+    logger.debug("insert key = {}", rowKey);
+    try {
+      batchKeys.addElement(rowKey);
+      batchValues.addElement(new HashMap<String, ByteIterator>(values));
+      batchCount++;
+      if (batchSize == 0 || batchSize == batchCount) {
+        batchInsert();
+        batchCount = 0;
+      }
+      return Status.OK;
+    } catch (Throwable e) {
+      logger.error(MessageFormatter.format("Error inserting key: {}", rowKey).getMessage(), e);
+      e.printStackTrace();
+    }
+    return Status.ERROR;
+  }
+
+  @Override
+  public Status delete(String table, String key) {
+    String rowKey = getRowKey(dbName, table, key);
+    logger.debug("delete key = {}", rowKey);
+    try {
+      db.run(tr -> {
+          tr.clear(Tuple.from(rowKey).pack());
+          return null;
+        });
+      return Status.OK;
+    } catch (FDBException e) {
+      logger.error(MessageFormatter.format("Error deleting key: {}", rowKey).getMessage(), e);
+      e.printStackTrace();
+    } catch (Exception e) {
+      logger.error(MessageFormatter.format("Error deleting key: {}", rowKey).getMessage(), e);
+      e.printStackTrace();
+    }
+    return Status.ERROR;
+  }
+
+  @Override
+  public Status read(String table, String key, Set<String> fields, Map<String, ByteIterator> result) {
+    String rowKey = getRowKey(dbName, table, key);
+    logger.debug("read key = {}", rowKey);
+    try {
+      byte[] row = db.run(tr -> {
+          byte[] r = tr.get(Tuple.from(rowKey).pack()).join();
+          return r;
+        });
+      Tuple t = Tuple.fromBytes(row);
+      if (t.size() == 0) {
+        logger.debug("key not fount: {}", rowKey);
+        return Status.NOT_FOUND;
+      }
+      return convTupleToMap(t, fields, result);
+    } catch (FDBException e) {
+      logger.error(MessageFormatter.format("Error reading key: {}", rowKey).getMessage(), e);
+      e.printStackTrace();
+    } catch (Exception e) {
+      logger.error(MessageFormatter.format("Error reading key: {}", rowKey).getMessage(), e);
+      e.printStackTrace();
+    }
+    return Status.ERROR;
+  }
+
+  @Override
+  public Status update(String table, String key, Map<String, ByteIterator> values) {
+    String rowKey = getRowKey(dbName, table, key);
+    logger.debug("update key = {}", rowKey);
+    try {
+      Status s = db.run(tr -> {
+          byte[] row = tr.get(Tuple.from(rowKey).pack()).join();
+          Tuple o = Tuple.fromBytes(row);
+          if (o.size() == 0) {
+            logger.debug("key not fount: {}", rowKey);
+            return Status.NOT_FOUND;
+          }
+          HashMap<String, ByteIterator> result = new HashMap<>();
+          if (convTupleToMap(o, null, result) != Status.OK) {
+            return Status.ERROR;
+          }
+          for (String k : values.keySet()) {
+            if (result.containsKey(k)) {
+              result.put(k, values.get(k));
+            } else {
+              logger.debug("field not fount: {}", k);
+              return Status.NOT_FOUND;
+            }
+          }
+          Tuple t = new Tuple();
+          for (Map.Entry<String, String> entry : StringByteIterator.getStringMap(result).entrySet()) {
+            Tuple v = new Tuple();
+            v = v.add(entry.getKey());
+            v = v.add(entry.getValue());
+            t = t.add(v);
+          }
+          tr.set(Tuple.from(rowKey).pack(), t.pack());
+          return Status.OK;
+        });
+      return s;
+    } catch (FDBException e) {
+      logger.error(MessageFormatter.format("Error updating key: {}", rowKey).getMessage(), e);
+      e.printStackTrace();
+    } catch (Exception e) {
+      logger.error(MessageFormatter.format("Error updating key: {}", rowKey).getMessage(), e);
+      e.printStackTrace();
+    }
+    return Status.ERROR;
+  }
+
+  @Override
+  public Status scan(String table, String startkey, int recordcount, Set<String> fields,
+      Vector<HashMap<String, ByteIterator>> result) {
+    String startRowKey = getRowKey(dbName, table, startkey);
+    String endRowKey = getEndRowKey(table);
+    logger.debug("scan key from {} to {} limit {} ", startkey, endRowKey, recordcount);
+    try (Transaction tr = db.createTransaction()) {
+      tr.options().setReadYourWritesDisable();
+      AsyncIterable<KeyValue> entryList = tr.getRange(Tuple.from(startRowKey).pack(), Tuple.from(endRowKey).pack(),
+          recordcount > 0 ? recordcount : 0);
+      List<KeyValue> entries = entryList.asList().join();
+      for (int i = 0; i < entries.size(); ++i) {
+        final HashMap<String, ByteIterator> map = new HashMap<>();
+        Tuple value = Tuple.fromBytes(entries.get(i).getValue());
+        if (convTupleToMap(value, fields, map) == Status.OK) {
+          result.add(map);
+        } else {
+          logger.error("Error scanning keys: from {} to {} limit {} ", startRowKey, endRowKey, recordcount);
+          return Status.ERROR;
+        }
+      }
+      return Status.OK;
+    } catch (FDBException e) {
+      logger.error(MessageFormatter.format("Error scanning keys: from {} to {} ",
+          startRowKey, endRowKey).getMessage(), e);
+      e.printStackTrace();
+    } catch (Exception e) {
+      logger.error(MessageFormatter.format("Error scanning keys: from {} to {} ",
+          startRowKey, endRowKey).getMessage(), e);
+      e.printStackTrace();
+    }
+    return Status.ERROR;
+  }
+}

foundationdb/src/main/java/com/yahoo/ycsb/db/foundationdb/package-info.java

+/*
+ * Copyright (c) 2015 - 2016, Yahoo!, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+/**
+ * The YCSB binding for <a href="https://www.foundationdb.org">FoundationDB</a>.
+ */
+package com.yahoo.ycsb.db.foundationdb;
+

pom.xml

@@ -79,6 +79,7 @@ LICENSE file.
     <couchbase.version>1.4.10</couchbase.version>
     <couchbase2.version>2.3.1</couchbase2.version>
     <elasticsearch5-version>5.5.1</elasticsearch5-version>
+    <foundationdb.version>5.2.5</foundationdb.version>
     <geode.version>1.2.0</geode.version>
     <googlebigtable.version>1.3.0</googlebigtable.version>
     <hbase098.version>0.98.14-hadoop2</hbase098.version>
@@ -129,6 +130,7 @@ LICENSE file.
     <module>dynamodb</module>
     <module>elasticsearch</module>
     <module>elasticsearch5</module>
+    <module>foundationdb</module>
     <module>geode</module>
     <module>googlebigtable</module>
     <module>googledatastore</module>