From fb85b598b73fea26b66180ca9ddb71e0abd2e113 Mon Sep 17 00:00:00 2001 From: Rohan Jayaraj <rohanjayaraj@users.noreply.github.com> Date: Fri, 11 May 2018 16:04:43 -0700 Subject: [PATCH] [maprdb maprjsondb] Added clients for MapR-DB and MapR-JSONDB (#1063) * Adding MapR Binary & JSON YCSB Clients * MapR binaries will be picked from the classpath --- bin/ycsb | 2 + distribution/pom.xml | 10 + maprdb/README.md | 44 ++++ maprdb/pom.xml | 77 ++++++ .../com/yahoo/ycsb/db/mapr/MapRDBClient.java | 25 ++ .../com/yahoo/ycsb/db/mapr/package-info.java | 22 ++ maprjsondb/README.md | 43 ++++ maprjsondb/pom.xml | 65 +++++ .../yahoo/ycsb/db/mapr/MapRJSONDBClient.java | 224 ++++++++++++++++++ .../yahoo/ycsb/db/mapr/ValueByteIterator.java | 58 +++++ .../com/yahoo/ycsb/db/mapr/package-info.java | 22 ++ pom.xml | 3 + 12 files changed, 595 insertions(+) create mode 100644 maprdb/README.md create mode 100644 maprdb/pom.xml create mode 100644 maprdb/src/main/java/com/yahoo/ycsb/db/mapr/MapRDBClient.java create mode 100644 maprdb/src/main/java/com/yahoo/ycsb/db/mapr/package-info.java create mode 100644 maprjsondb/README.md create mode 100644 maprjsondb/pom.xml create mode 100644 maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/MapRJSONDBClient.java create mode 100644 maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/ValueByteIterator.java create mode 100644 maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/package-info.java diff --git a/bin/ycsb b/bin/ycsb index d3d95f7c..b0242f42 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -85,6 +85,8 @@ DATABASES = { "jdbc" : "com.yahoo.ycsb.db.JdbcDBClient", "kudu" : "com.yahoo.ycsb.db.KuduYCSBClient", "memcached" : "com.yahoo.ycsb.db.MemcachedClient", + "maprdb" : "com.yahoo.ycsb.db.mapr.MapRDBClient", + "maprjsondb" : "com.yahoo.ycsb.db.mapr.MapRJSONDBClient", "mongodb" : "com.yahoo.ycsb.db.MongoDbClient", "mongodb-async": "com.yahoo.ycsb.db.AsyncMongoDbClient", "nosqldb" : "com.yahoo.ycsb.db.NoSqlDbClient", diff --git a/distribution/pom.xml b/distribution/pom.xml index a0ec3c36..16a4592b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -184,6 +184,16 @@ LICENSE file. <artifactId>memcached-binding</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>maprdb-binding</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>maprjsondb-binding</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>com.yahoo.ycsb</groupId> <artifactId>mongodb-binding</artifactId> diff --git a/maprdb/README.md b/maprdb/README.md new file mode 100644 index 00000000..9d2ed36a --- /dev/null +++ b/maprdb/README.md @@ -0,0 +1,44 @@ +<!-- +Copyright (c) 2012 - 2017 YCSB contributors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you +may not use this file except in compliance with the License. You +may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. See accompanying +LICENSE file. +--> + +## Quick Start + +This section describes how to run YCSB on a MapR Cluster against MapR-DB (Binary). + +### 1. Set Up YCSB + +Clone the YCSB git repository and compile: + + git clone https://github.com/brianfrankcooper/YCSB.git + cd YCSB + mvn clean package + +### 2. Create MapR-DB Binary Table + + maprcli volume create -name tables -path /tables + maprcli table create -path /tables/myTable + maprcli table cf create -path /tables/myTable -cfname cf0 + +### 3. Run YCSB + +###### 3.1. Load Table + + ./bin/ycsb load maprdb -P workloads/workloada -cp $(mapr classpath) -p table=/tables/myTable -p columnfamily=cf0 + +###### 3.2. Run workload + + ./bin/ycsb run maprdb -P workloads/workloadb -cp $(mapr classpath) -p table=/tables/myTable -p columnfamily=cf0 diff --git a/maprdb/pom.xml b/maprdb/pom.xml new file mode 100644 index 00000000..33e272a5 --- /dev/null +++ b/maprdb/pom.xml @@ -0,0 +1,77 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. Licensed + under the Apache License, Version 2.0 (the "License"); you may not use this + file except in compliance with the License. You may obtain a copy of the + License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by + applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the specific language + governing permissions and limitations under the License. See accompanying + LICENSE file. --> + +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>binding-parent</artifactId> + <version>0.14.0-SNAPSHOT</version> + <relativePath>../binding-parent</relativePath> + </parent> + + <artifactId>maprdb-binding</artifactId> + <name>MapRDB Binding</name> + <packaging>jar</packaging> + <repositories> + <repository> + <id>mapr-releases</id> + <url>http://repository.mapr.com/maven/</url> + <snapshots> + <enabled>false</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + </repositories> + <properties> + <!-- Tests do not run on jdk9 --> + <skipJDK9Tests>true</skipJDK9Tests> + </properties> + <dependencies> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${maprhbase.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>hbase10-binding</artifactId> + <version>${project.version}</version> + <!-- Should match all compile scoped dependencies --> + <exclusions> + <exclusion> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>1.7.10</version> + </dependency> + </dependencies> +</project> diff --git a/maprdb/src/main/java/com/yahoo/ycsb/db/mapr/MapRDBClient.java b/maprdb/src/main/java/com/yahoo/ycsb/db/mapr/MapRDBClient.java new file mode 100644 index 00000000..7a0c293a --- /dev/null +++ b/maprdb/src/main/java/com/yahoo/ycsb/db/mapr/MapRDBClient.java @@ -0,0 +1,25 @@ +/** + * Copyright (c) 2017 Yahoo! Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.mapr; + +/** + * MapR-DB(binary) client for YCSB framework. + * + */ +public class MapRDBClient extends com.yahoo.ycsb.db.HBaseClient10 { +} diff --git a/maprdb/src/main/java/com/yahoo/ycsb/db/mapr/package-info.java b/maprdb/src/main/java/com/yahoo/ycsb/db/mapr/package-info.java new file mode 100644 index 00000000..939880be --- /dev/null +++ b/maprdb/src/main/java/com/yahoo/ycsb/db/mapr/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for <a href="http://mapr.com/maprdb/">MapR-DB</a>. + */ +package com.yahoo.ycsb.db.mapr; + diff --git a/maprjsondb/README.md b/maprjsondb/README.md new file mode 100644 index 00000000..559465f5 --- /dev/null +++ b/maprjsondb/README.md @@ -0,0 +1,43 @@ +<!-- +Copyright (c) 2012 - 2017 YCSB contributors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you +may not use this file except in compliance with the License. You +may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. See accompanying +LICENSE file. +--> + +## Quick Start + +This section describes how to run YCSB on a MapR Cluster against MapR-DB (JSON). + +### 1. Set Up YCSB + +Clone the YCSB git repository and compile: + + git clone https://github.com/brianfrankcooper/YCSB.git + cd YCSB + mvn clean package + +### 2. Create MapR-DB JSON Table + + maprcli volume create -name tables -path /tables + maprcli table create -path /tables/docTable -tabletype json + +### 3. Run YCSB + +###### 3.1. Load Table + + ./bin/ycsb load maprjsondb -P workloads/workloada -cp $(mapr clientclasspath) -p table=/tables/docTable + +###### 3.2. Run workload + + ./bin/ycsb run maprjsondb -P workloads/workloadb -cp $(mapr clientclasspath) -p table=/tables/docTable diff --git a/maprjsondb/pom.xml b/maprjsondb/pom.xml new file mode 100644 index 00000000..5ab35db7 --- /dev/null +++ b/maprjsondb/pom.xml @@ -0,0 +1,65 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. Licensed + under the Apache License, Version 2.0 (the "License"); you may not use this + file except in compliance with the License. You may obtain a copy of the + License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by + applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the specific language + governing permissions and limitations under the License. See accompanying + LICENSE file. --> + +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>binding-parent</artifactId> + <version>0.14.0-SNAPSHOT</version> + <relativePath>../binding-parent</relativePath> + </parent> + + <artifactId>maprjsondb-binding</artifactId> + <name>MapR JSON DB Binding</name> + <packaging>jar</packaging> + <repositories> + <repository> + <id>mapr-releases</id> + <url>http://repository.mapr.com/maven/</url> + <snapshots> + <enabled>false</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + </repositories> + <properties> + <!-- Tests do not run on jdk9 --> + <skipJDK9Tests>true</skipJDK9Tests> + </properties> + <dependencies> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.ojai</groupId> + <artifactId>ojai</artifactId> + <version>2.0-mapr-1710</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>1.7.10</version> + </dependency> + </dependencies> +</project> diff --git a/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/MapRJSONDBClient.java b/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/MapRJSONDBClient.java new file mode 100644 index 00000000..b339ee88 --- /dev/null +++ b/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/MapRJSONDBClient.java @@ -0,0 +1,224 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.mapr; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.Vector; + +import org.ojai.Document; +import org.ojai.DocumentConstants; +import org.ojai.DocumentStream; +import org.ojai.Value; +import org.ojai.store.Connection; +import org.ojai.store.DocumentMutation; +import org.ojai.store.DocumentStore; +import org.ojai.store.Driver; +import org.ojai.store.DriverManager; +import org.ojai.store.Query; +import org.ojai.store.QueryCondition; +import org.ojai.store.QueryCondition.Op; + +import com.yahoo.ycsb.ByteIterator; +import com.yahoo.ycsb.Status; + +/** + * MapR-DB(json) client for YCSB framework. + * + */ +public class MapRJSONDBClient extends com.yahoo.ycsb.DB { + + private Connection connection = null; + private DocumentStore documentStore = null; + private Driver driver = null; + + @Override + public void init() { + connection = DriverManager.getConnection("ojai:mapr:"); + driver = connection.getDriver(); + } + + @Override + public void cleanup() { + documentStore.close(); + connection.close(); + } + + @Override + public Status read(String table, String key, Set<String> fields, + Map<String, ByteIterator> result) { + try { + DocumentStore docStore = getTable(table); + Document doc = docStore.findById(key, getFieldPaths(fields)); + buildRowResult(doc, result); + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + + @Override + public Status scan(String table, String startkey, int recordcount, + Set<String> fields, Vector<HashMap<String, ByteIterator>> result) { + try { + DocumentStore docStore = getTable(table); + QueryCondition condition = driver.newCondition() + .is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, startkey) + .build(); + Query query = driver.newQuery() + .select(getFieldPaths(fields)) + .where(condition) + .build(); + + try (DocumentStream stream = + docStore.findQuery(query)) { + int numResults = 0; + for (Document record : stream) { + result.add(buildRowResult(record)); + numResults++; + if (numResults >= recordcount) { + break; + } + } + } + return Status.OK; + } catch (Exception e) { + e.printStackTrace(); + return Status.ERROR; + } + } + + @Override + public Status update(String table, String key, + Map<String, ByteIterator> values) { + try { + DocumentStore docStore = getTable(table); + docStore.update(key, newMutation(values)); + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + + @Override + public Status insert(String table, String key, + Map<String, ByteIterator> values) { + try { + DocumentStore docStore = getTable(table); + docStore.insertOrReplace(key, newDocument(values)); + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + + @Override + public Status delete(String table, String key) { + try { + DocumentStore docStore = getTable(table); + docStore.delete(key); + return Status.OK; + } catch (Exception e) { + return Status.ERROR; + } + } + + /** + * Get the OJAI DocumentStore instance for a given table. + * + * @param tableName + * @return + */ + private DocumentStore getTable(String tableName) { + if (documentStore == null) { + documentStore = connection.getStore(tableName); + } + return documentStore; + } + + /** + * Construct a Document object from the map of OJAI values. + * + * @param values + * @return + */ + private Document newDocument(Map<String, ByteIterator> values) { + Document document = driver.newDocument(); + for (Map.Entry<String, ByteIterator> entry : values.entrySet()) { + document.set(entry.getKey(), entry.getValue().toArray()); + } + return document; + } + + /** + * Build a DocumentMutation object for the values specified. + * @param values + * @return + */ + private DocumentMutation newMutation(Map<String, ByteIterator> values) { + DocumentMutation mutation = driver.newMutation(); + for (Map.Entry<String, ByteIterator> entry : values.entrySet()) { + mutation.setOrReplace(entry.getKey(), + ByteBuffer.wrap(entry.getValue().toArray())); + } + return mutation; + } + + /** + * Get field path array from the set. + * + * @param fields + * @return + */ + private String[] getFieldPaths(Set<String> fields) { + if (fields != null) { + return fields.toArray(new String[fields.size()]); + } + return new String[0]; + } + + /** + * Build result the map from the Document passed. + * + * @param document + * @return + */ + private HashMap<String, ByteIterator> buildRowResult(Document document) { + return buildRowResult(document, null); + } + + /** + * Build result the map from the Document passed. + * + * @param document + * @param result + * @return + */ + private HashMap<String, ByteIterator> buildRowResult(Document document, + Map<String, ByteIterator> result) { + if (document != null) { + if (result == null) { + result = new HashMap<String, ByteIterator>(); + } + for (Map.Entry<String, Value> kv : document) { + result.put(kv.getKey(), new ValueByteIterator(kv.getValue())); + } + } + return (HashMap<String, ByteIterator>)result; + } +} diff --git a/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/ValueByteIterator.java b/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/ValueByteIterator.java new file mode 100644 index 00000000..fb516ae7 --- /dev/null +++ b/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/ValueByteIterator.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.mapr; + +import org.ojai.Value; +import org.ojai.util.Values; + +import com.yahoo.ycsb.ByteIterator; + +/** + * OJAI Value byte iterator. + * + * Used for parsing the document fetched MapR JSON DB + */ +public class ValueByteIterator extends ByteIterator { + + private Value value; + + public ValueByteIterator(Value value) { + this.value = value; + } + + @Override + public boolean hasNext() { + return false; + } + + @Override + public byte nextByte() { + return 0; + } + + @Override + public long bytesLeft() { + return 0; + } + + @Override + public String toString() { + return Values.asJsonString(value); + } + +} diff --git a/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/package-info.java b/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/package-info.java new file mode 100644 index 00000000..db61f71a --- /dev/null +++ b/maprjsondb/src/main/java/com/yahoo/ycsb/db/mapr/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for <a href="http://mapr.com/maprjsondb/">MapR JSON DB</a>. + */ +package com.yahoo.ycsb.db.mapr; + diff --git a/pom.xml b/pom.xml index 9f9fc1a9..2d730d4d 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ LICENSE file. <hypertable.version>0.9.5.6</hypertable.version> <infinispan.version>7.2.2.Final</infinispan.version> <kudu.version>1.1.0</kudu.version> + <maprhbase.version>1.1.8-mapr-1710</maprhbase.version> <!--<mapkeeper.version>1.0</mapkeeper.version>--> <mongodb.version>3.6.3</mongodb.version> <mongodb.async.version>2.0.1</mongodb.async.version> @@ -145,6 +146,8 @@ LICENSE file. <module>jdbc</module> <module>kudu</module> <!--<module>mapkeeper</module>--> + <module>maprdb</module> + <module>maprjsondb</module> <module>memcached</module> <module>mongodb</module> <module>nosqldb</module> -- GitLab