diff --git a/bin/bindings.properties b/bin/bindings.properties index 0298165ddcef4d4d5bea0977f33dc6c5e7b561c7..e3d1d429103ec258888bb595b5595cfbd24652bd 100644 --- a/bin/bindings.properties +++ b/bin/bindings.properties @@ -46,6 +46,7 @@ dynamodb:com.yahoo.ycsb.db.DynamoDBClient elasticsearch:com.yahoo.ycsb.db.ElasticsearchClient elasticsearch5:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient elasticsearch5-rest:com.yahoo.ycsb.db.elasticsearch5.ElasticsearchRestClient +foundationdb:com.yahoo.ycsb.db.foundationdb.FoundationDBClient geode:com.yahoo.ycsb.db.GeodeClient googlebigtable:com.yahoo.ycsb.db.GoogleBigtableClient googledatastore:com.yahoo.ycsb.db.GoogleDatastoreClient diff --git a/bin/ycsb b/bin/ycsb index d80bd034893cc54809bec7c821261f012cbbad7e..1181725ef8b1d04984002187d31c1f06c3c18ecb 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -71,6 +71,7 @@ DATABASES = { "dynamodb" : "com.yahoo.ycsb.db.DynamoDBClient", "elasticsearch": "com.yahoo.ycsb.db.ElasticsearchClient", "elasticsearch5": "com.yahoo.ycsb.db.elasticsearch5.ElasticsearchClient", + "foundationdb" : "com.yahoo.ycsb.db.foundationdb.FoundationDBClient", "geode" : "com.yahoo.ycsb.db.GeodeClient", "googlebigtable" : "com.yahoo.ycsb.db.GoogleBigtableClient", "googledatastore" : "com.yahoo.ycsb.db.GoogleDatastoreClient", diff --git a/distribution/pom.xml b/distribution/pom.xml index 18158db0f696a5c2a31890af22bce352d60ea100..1d0671ec9e9a8c7cb7850cc1f0ef895cfe98ddab 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -114,6 +114,11 @@ LICENSE file. <artifactId>elasticsearch5-binding</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>foundationdb-binding</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>com.yahoo.ycsb</groupId> <artifactId>geode-binding</artifactId> diff --git a/foundationdb/README.md b/foundationdb/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6dc193dc063353653da23f494e6b272ea01bee13 --- /dev/null +++ b/foundationdb/README.md @@ -0,0 +1,51 @@ +<!-- +Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you +may not use this file except in compliance with the License. You +may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. See accompanying +LICENSE file. +--> + +## Quick Start + +This section describes how to run YCSB on FoundationDB running locally. + +### 1. Set Up YCSB + +Clone the YCSB git repository and compile: + + git clone https://github.com/brianfrankcooper/YCSB.git + cd YCSB + mvn clean package + +### 2. Run YCSB + +Now you are ready to run! First, load the data: + + ./bin/ycsb load foundationdb -s -P workloads/workloada + +Then, run the workload: + + ./bin/ycsb run foundationdb -s -P workloads/workloada + +See the next section for the list of configuration parameters for FoundationDB. + +## FoundationDB Configuration Parameters + +* ```foundationdb.apiversion``` - TThe FoundationDB API version. + * Default: ```520``` +* ```foundationdb.clusterfile``` - The cluster file path. + * Default: ```./fdb.cluster``` +* ```foundationdb.dbname``` - The database name. + * Default: ```DB``` +* ```foundationdb.batchsize``` - The number of rows to be batched before commit. + * Default: ```0``` diff --git a/foundationdb/pom.xml b/foundationdb/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..cba25a0502376230b77c4ae4940cb735f9c1c507 --- /dev/null +++ b/foundationdb/pom.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you +may not use this file except in compliance with the License. You +may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. See accompanying +LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>binding-parent</artifactId> + <version>0.15.0-SNAPSHOT</version> + <relativePath>../binding-parent</relativePath> + </parent> + + <artifactId>foundationdb-binding</artifactId> + <name>FoundationDB Binding</name> + + <packaging>jar</packaging> + <properties> + <!-- Tests do not run on jdk9,10,11 --> + <skipJDK9Tests>true</skipJDK9Tests> + <skipJDK10Tests>true</skipJDK10Tests> + <skipJDK11Tests>true</skipJDK11Tests> + </properties> + <dependencies> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.foundationdb</groupId> + <artifactId>fdb-java</artifactId> + <version>${foundationdb.version}</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>1.7.10</version> + </dependency> + </dependencies> +</project> diff --git a/foundationdb/src/main/java/com/yahoo/ycsb/db/foundationdb/FoundationDBClient.java b/foundationdb/src/main/java/com/yahoo/ycsb/db/foundationdb/FoundationDBClient.java new file mode 100644 index 0000000000000000000000000000000000000000..fb61069f6f59e5a95398a900ce2ca4b8945670e1 --- /dev/null +++ b/foundationdb/src/main/java/com/yahoo/ycsb/db/foundationdb/FoundationDBClient.java @@ -0,0 +1,302 @@ +/** + * Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.foundationdb; + +import com.apple.foundationdb.*; +import com.apple.foundationdb.async.AsyncIterable; +import com.apple.foundationdb.tuple.Tuple; + +import com.yahoo.ycsb.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.helpers.MessageFormatter; + +import java.util.*; + +/** + * FoundationDB client for YCSB framework. + */ + +public class FoundationDBClient extends DB { + private FDB fdb; + private Database db; + private String dbName; + private int batchSize; + private int batchCount; + private static final String API_VERSION = "foundationdb.apiversion"; + private static final String API_VERSION_DEFAULT = "520"; + private static final String CLUSTER_FILE = "foundationdb.clusterfile"; + private static final String CLUSTER_FILE_DEFAULT = "./fdb.cluster"; + private static final String DB_NAME = "foundationdb.dbname"; + private static final String DB_NAME_DEFAULT = "DB"; + private static final String DB_BATCH_SIZE_DEFAULT = "0"; + private static final String DB_BATCH_SIZE = "foundationdb.batchsize"; + + private Vector<String> batchKeys; + private Vector<Map<String, ByteIterator>> batchValues; + + private static Logger logger = LoggerFactory.getLogger(FoundationDBClient.class); + + /** + * Initialize any state for this DB. Called once per DB instance; there is one DB instance per client thread. + */ + @Override + public void init() throws DBException { + // initialize FoundationDB driver + final Properties props = getProperties(); + String apiVersion = props.getProperty(API_VERSION, API_VERSION_DEFAULT); + String clusterFile = props.getProperty(CLUSTER_FILE, CLUSTER_FILE_DEFAULT); + String dbBatchSize = props.getProperty(DB_BATCH_SIZE, DB_BATCH_SIZE_DEFAULT); + dbName = props.getProperty(DB_NAME, DB_NAME_DEFAULT); + + logger.info("API Version: {}", apiVersion); + logger.info("Cluster File: {}\n", clusterFile); + + try { + fdb = FDB.selectAPIVersion(Integer.parseInt(apiVersion.trim())); + db = fdb.open(clusterFile); + batchSize = Integer.parseInt(dbBatchSize); + batchCount = 0; + batchKeys = new Vector<String>(batchSize+1); + batchValues = new Vector<Map<String, ByteIterator>>(batchSize+1); + } catch (FDBException e) { + logger.error(MessageFormatter.format("Error in database operation: {}", "init").getMessage(), e); + throw new DBException(e); + } catch (NumberFormatException e) { + logger.error(MessageFormatter.format("Invalid value for apiversion property: {}", apiVersion).getMessage(), e); + throw new DBException(e); + } + } + + @Override + public void cleanup() throws DBException { + if (batchCount > 0) { + batchInsert(); + batchCount = 0; + } + try { + db.close(); + } catch (FDBException e) { + logger.error(MessageFormatter.format("Error in database operation: {}", "cleanup").getMessage(), e); + throw new DBException(e); + } + } + + private static String getRowKey(String db, String table, String key) { + //return key + ":" + table + ":" + db; + return db + ":" + table + ":" + key; + } + + private static String getEndRowKey(String table) { + return table + ";"; + } + + private Status convTupleToMap(Tuple tuple, Set<String> fields, Map<String, ByteIterator> result) { + for (int i = 0; i < tuple.size(); i++) { + Tuple v = tuple.getNestedTuple(i); + String field = v.getString(0); + String value = v.getString(1); + //System.err.println(field + " : " + value); + result.put(field, new StringByteIterator(value)); + } + if (fields != null) { + for (String field : fields) { + if (result.get(field) == null) { + logger.debug("field not fount: {}", field); + return Status.NOT_FOUND; + } + } + } + return Status.OK; + } + + private void batchInsert() { + try { + db.run(tr -> { + for (int i = 0; i < batchCount; ++i) { + Tuple t = new Tuple(); + for (Map.Entry<String, String> entry : StringByteIterator.getStringMap(batchValues.get(i)).entrySet()) { + Tuple v = new Tuple(); + v = v.add(entry.getKey()); + v = v.add(entry.getValue()); + t = t.add(v); + } + tr.set(Tuple.from(batchKeys.get(i)).pack(), t.pack()); + } + return null; + }); + } catch (FDBException e) { + for (int i = 0; i < batchCount; ++i) { + logger.error(MessageFormatter.format("Error batch inserting key {}", batchKeys.get(i)).getMessage(), e); + } + e.printStackTrace(); + } catch (Throwable e) { + for (int i = 0; i < batchCount; ++i) { + logger.error(MessageFormatter.format("Error batch inserting key {}", batchKeys.get(i)).getMessage(), e); + } + e.printStackTrace(); + } finally { + batchKeys.clear(); + batchValues.clear(); + } + } + + @Override + public Status insert(String table, String key, Map<String, ByteIterator> values) { + String rowKey = getRowKey(dbName, table, key); + logger.debug("insert key = {}", rowKey); + try { + batchKeys.addElement(rowKey); + batchValues.addElement(new HashMap<String, ByteIterator>(values)); + batchCount++; + if (batchSize == 0 || batchSize == batchCount) { + batchInsert(); + batchCount = 0; + } + return Status.OK; + } catch (Throwable e) { + logger.error(MessageFormatter.format("Error inserting key: {}", rowKey).getMessage(), e); + e.printStackTrace(); + } + return Status.ERROR; + } + + @Override + public Status delete(String table, String key) { + String rowKey = getRowKey(dbName, table, key); + logger.debug("delete key = {}", rowKey); + try { + db.run(tr -> { + tr.clear(Tuple.from(rowKey).pack()); + return null; + }); + return Status.OK; + } catch (FDBException e) { + logger.error(MessageFormatter.format("Error deleting key: {}", rowKey).getMessage(), e); + e.printStackTrace(); + } catch (Exception e) { + logger.error(MessageFormatter.format("Error deleting key: {}", rowKey).getMessage(), e); + e.printStackTrace(); + } + return Status.ERROR; + } + + @Override + public Status read(String table, String key, Set<String> fields, Map<String, ByteIterator> result) { + String rowKey = getRowKey(dbName, table, key); + logger.debug("read key = {}", rowKey); + try { + byte[] row = db.run(tr -> { + byte[] r = tr.get(Tuple.from(rowKey).pack()).join(); + return r; + }); + Tuple t = Tuple.fromBytes(row); + if (t.size() == 0) { + logger.debug("key not fount: {}", rowKey); + return Status.NOT_FOUND; + } + return convTupleToMap(t, fields, result); + } catch (FDBException e) { + logger.error(MessageFormatter.format("Error reading key: {}", rowKey).getMessage(), e); + e.printStackTrace(); + } catch (Exception e) { + logger.error(MessageFormatter.format("Error reading key: {}", rowKey).getMessage(), e); + e.printStackTrace(); + } + return Status.ERROR; + } + + @Override + public Status update(String table, String key, Map<String, ByteIterator> values) { + String rowKey = getRowKey(dbName, table, key); + logger.debug("update key = {}", rowKey); + try { + Status s = db.run(tr -> { + byte[] row = tr.get(Tuple.from(rowKey).pack()).join(); + Tuple o = Tuple.fromBytes(row); + if (o.size() == 0) { + logger.debug("key not fount: {}", rowKey); + return Status.NOT_FOUND; + } + HashMap<String, ByteIterator> result = new HashMap<>(); + if (convTupleToMap(o, null, result) != Status.OK) { + return Status.ERROR; + } + for (String k : values.keySet()) { + if (result.containsKey(k)) { + result.put(k, values.get(k)); + } else { + logger.debug("field not fount: {}", k); + return Status.NOT_FOUND; + } + } + Tuple t = new Tuple(); + for (Map.Entry<String, String> entry : StringByteIterator.getStringMap(result).entrySet()) { + Tuple v = new Tuple(); + v = v.add(entry.getKey()); + v = v.add(entry.getValue()); + t = t.add(v); + } + tr.set(Tuple.from(rowKey).pack(), t.pack()); + return Status.OK; + }); + return s; + } catch (FDBException e) { + logger.error(MessageFormatter.format("Error updating key: {}", rowKey).getMessage(), e); + e.printStackTrace(); + } catch (Exception e) { + logger.error(MessageFormatter.format("Error updating key: {}", rowKey).getMessage(), e); + e.printStackTrace(); + } + return Status.ERROR; + } + + @Override + public Status scan(String table, String startkey, int recordcount, Set<String> fields, + Vector<HashMap<String, ByteIterator>> result) { + String startRowKey = getRowKey(dbName, table, startkey); + String endRowKey = getEndRowKey(table); + logger.debug("scan key from {} to {} limit {} ", startkey, endRowKey, recordcount); + try (Transaction tr = db.createTransaction()) { + tr.options().setReadYourWritesDisable(); + AsyncIterable<KeyValue> entryList = tr.getRange(Tuple.from(startRowKey).pack(), Tuple.from(endRowKey).pack(), + recordcount > 0 ? recordcount : 0); + List<KeyValue> entries = entryList.asList().join(); + for (int i = 0; i < entries.size(); ++i) { + final HashMap<String, ByteIterator> map = new HashMap<>(); + Tuple value = Tuple.fromBytes(entries.get(i).getValue()); + if (convTupleToMap(value, fields, map) == Status.OK) { + result.add(map); + } else { + logger.error("Error scanning keys: from {} to {} limit {} ", startRowKey, endRowKey, recordcount); + return Status.ERROR; + } + } + return Status.OK; + } catch (FDBException e) { + logger.error(MessageFormatter.format("Error scanning keys: from {} to {} ", + startRowKey, endRowKey).getMessage(), e); + e.printStackTrace(); + } catch (Exception e) { + logger.error(MessageFormatter.format("Error scanning keys: from {} to {} ", + startRowKey, endRowKey).getMessage(), e); + e.printStackTrace(); + } + return Status.ERROR; + } +} diff --git a/foundationdb/src/main/java/com/yahoo/ycsb/db/foundationdb/package-info.java b/foundationdb/src/main/java/com/yahoo/ycsb/db/foundationdb/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..ec6cc5685c1ac04c9eda6c646c58bcfdf964668b --- /dev/null +++ b/foundationdb/src/main/java/com/yahoo/ycsb/db/foundationdb/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2015 - 2016, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for <a href="https://www.foundationdb.org">FoundationDB</a>. + */ +package com.yahoo.ycsb.db.foundationdb; + diff --git a/pom.xml b/pom.xml index 6945a52b93f7ed5464653e20aee909cefea3eaac..bdae0cfdd47bf1634d372ecbe18b85239aa8f488 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ LICENSE file. <couchbase.version>1.4.10</couchbase.version> <couchbase2.version>2.3.1</couchbase2.version> <elasticsearch5-version>5.5.1</elasticsearch5-version> + <foundationdb.version>5.2.5</foundationdb.version> <geode.version>1.2.0</geode.version> <googlebigtable.version>1.3.0</googlebigtable.version> <hbase098.version>0.98.14-hadoop2</hbase098.version> @@ -129,6 +130,7 @@ LICENSE file. <module>dynamodb</module> <module>elasticsearch</module> <module>elasticsearch5</module> + <module>foundationdb</module> <module>geode</module> <module>googlebigtable</module> <module>googledatastore</module>