diff --git a/bin/bindings.properties b/bin/bindings.properties index eecc6a2e0fcfbce18d5c2af077a0152f435fec9f..c735566f0bc70f542e0133d4496c029f37aa8c58 100644 --- a/bin/bindings.properties +++ b/bin/bindings.properties @@ -52,6 +52,7 @@ googledatastore:com.yahoo.ycsb.db.GoogleDatastoreClient hbase098:com.yahoo.ycsb.db.HBaseClient hbase10:com.yahoo.ycsb.db.HBaseClient10 hbase12:com.yahoo.ycsb.db.hbase12.HBaseClient12 +hbase14:com.yahoo.ycsb.db.hbase14.HBaseClient14 hypertable:com.yahoo.ycsb.db.HypertableClient infinispan-cs:com.yahoo.ycsb.db.InfinispanRemoteClient infinispan:com.yahoo.ycsb.db.InfinispanClient diff --git a/bin/ycsb b/bin/ycsb index 5d624d23f4fbb9efb153a6439e9f94066c319e6f..946108853f06184711afe868ba36be7188eb5ef5 100755 --- a/bin/ycsb +++ b/bin/ycsb @@ -77,6 +77,7 @@ DATABASES = { "hbase098" : "com.yahoo.ycsb.db.HBaseClient", "hbase10" : "com.yahoo.ycsb.db.HBaseClient10", "hbase12" : "com.yahoo.ycsb.db.hbase12.HBaseClient12", + "hbase14" : "com.yahoo.ycsb.db.hbase14.HBaseClient14", "hypertable" : "com.yahoo.ycsb.db.HypertableClient", "infinispan-cs": "com.yahoo.ycsb.db.InfinispanRemoteClient", "infinispan" : "com.yahoo.ycsb.db.InfinispanClient", diff --git a/distribution/pom.xml b/distribution/pom.xml index 965decb1de14466e6c28ea68dba85ba0dc035b43..ea5222c73bbd1e457b195933aef657b7aca83beb 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -149,6 +149,11 @@ LICENSE file. <artifactId>hbase12-binding</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>hbase14-binding</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>com.yahoo.ycsb</groupId> <artifactId>hypertable-binding</artifactId> diff --git a/hbase14/README.md b/hbase14/README.md new file mode 100644 index 0000000000000000000000000000000000000000..01ab0ed2b7ef1f7ae15ba7f53fed82fe7cd16465 --- /dev/null +++ b/hbase14/README.md @@ -0,0 +1,27 @@ +<!-- +Copyright (c) 2015-2017 YCSB contributors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you +may not use this file except in compliance with the License. You +may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. See accompanying +LICENSE file. +--> + +# HBase (1.4+) Driver for YCSB +This driver is a binding for the YCSB facilities to operate against a HBase 1.4+ Server cluster, using a shaded client that tries to avoid leaking third party libraries. + +See `hbase098/README.md` for a quickstart to setup HBase for load testing and common configuration details. + +## Configuration Options +In addition to those options available for the `hbase098` binding, the following options are available for the `hbase14` binding: + +* `durability`: Whether or not writes should be appended to the WAL. Bypassing the WAL can improve throughput but data cannot be recovered in the event of a crash. The default is true. + diff --git a/hbase14/pom.xml b/hbase14/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..1a95c68dc6a919d019b4b3c9cb94a9dcfbd1ddf3 --- /dev/null +++ b/hbase14/pom.xml @@ -0,0 +1,85 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you +may not use this file except in compliance with the License. You +may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. See accompanying +LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>binding-parent</artifactId> + <version>0.14.0-SNAPSHOT</version> + <relativePath>../binding-parent/</relativePath> + </parent> + + <artifactId>hbase14-binding</artifactId> + <name>HBase 1.4 DB Binding</name> + + <properties> + <!-- Tests do not run on jdk9 --> + <skipJDK9Tests>true</skipJDK9Tests> + <!-- Tests can't run without a shaded hbase testing util. + See HBASE-15666, which blocks us. + For now, we rely on the HBase 1.0 binding and manual testing. + --> + <maven.test.skip>true</maven.test.skip> + </properties> + <dependencies> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>hbase10-binding</artifactId> + <version>${project.version}</version> + <!-- Should match all compile scoped dependencies --> + <exclusions> + <exclusion> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-shaded-client</artifactId> + <version>${hbase14.version}</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> +<!-- blocked on HBASE-15666 + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-testing-util</artifactId> + <version>${hbase14.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>jdk.tools</groupId> + <artifactId>jdk.tools</artifactId> + </exclusion> + </exclusions> + </dependency> +--> + </dependencies> +</project> diff --git a/hbase14/src/main/java/com/yahoo/ycsb/db/hbase14/HBaseClient14.java b/hbase14/src/main/java/com/yahoo/ycsb/db/hbase14/HBaseClient14.java new file mode 100644 index 0000000000000000000000000000000000000000..c98c5fdd4e951925e71924c7bfb672570b027c6a --- /dev/null +++ b/hbase14/src/main/java/com/yahoo/ycsb/db/hbase14/HBaseClient14.java @@ -0,0 +1,28 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.hbase14; + +/** + * HBase 1.4 client for YCSB framework. + * + * A modified version of HBaseClient (which targets HBase v1.4) utilizing the + * shaded client. + * + * It should run equivalent to following the hbase098 binding README. + * + */ +public class HBaseClient14 extends com.yahoo.ycsb.db.HBaseClient10 { +} diff --git a/hbase14/src/main/java/com/yahoo/ycsb/db/hbase14/package-info.java b/hbase14/src/main/java/com/yahoo/ycsb/db/hbase14/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..388b693375c39588b13f73c0fdcfb8040120c35b --- /dev/null +++ b/hbase14/src/main/java/com/yahoo/ycsb/db/hbase14/package-info.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +/** + * The YCSB binding for <a href="https://hbase.apache.org/">HBase</a> + * using the HBase 1.4+ shaded API. + */ +package com.yahoo.ycsb.db.hbase14; + diff --git a/hbase14/src/test/java/com/yahoo/ycsb/db/hbase14/HBaseClient14Test.java b/hbase14/src/test/java/com/yahoo/ycsb/db/hbase14/HBaseClient14Test.java new file mode 100644 index 0000000000000000000000000000000000000000..e76e5a30ce7e66d77539164ca4c5cea183dffc9e --- /dev/null +++ b/hbase14/src/test/java/com/yahoo/ycsb/db/hbase14/HBaseClient14Test.java @@ -0,0 +1,213 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.yahoo.ycsb.db.hbase14; + +import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY; +import static com.yahoo.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; + +import com.yahoo.ycsb.ByteIterator; +import com.yahoo.ycsb.Status; +import com.yahoo.ycsb.StringByteIterator; +import com.yahoo.ycsb.measurements.Measurements; +import com.yahoo.ycsb.workloads.CoreWorkload; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; +import java.util.Vector; + +/** + * Integration tests for the YCSB HBase client 1.4, using an HBase minicluster. + */ +public class HBaseClient14Test { + + private final static String COLUMN_FAMILY = "cf"; + + private static HBaseTestingUtility testingUtil; + private HBaseClient14 client; + private Table table = null; + private String tableName; + + private static boolean isWindows() { + final String os = System.getProperty("os.name"); + return os.startsWith("Windows"); + } + + /** + * Creates a mini-cluster for use in these tests. + * + * This is a heavy-weight operation, so invoked only once for the test class. + */ + @BeforeClass + public static void setUpClass() throws Exception { + // Minicluster setup fails on Windows with an UnsatisfiedLinkError. + // Skip if windows. + assumeTrue(!isWindows()); + testingUtil = HBaseTestingUtility.createLocalHTU(); + testingUtil.startMiniCluster(); + } + + /** + * Tears down mini-cluster. + */ + @AfterClass + public static void tearDownClass() throws Exception { + if (testingUtil != null) { + testingUtil.shutdownMiniCluster(); + } + } + + /** + * Sets up the mini-cluster for testing. + * + * We re-create the table for each test. + */ + @Before + public void setUp() throws Exception { + client = new HBaseClient14(); + client.setConfiguration(new Configuration(testingUtil.getConfiguration())); + + Properties p = new Properties(); + p.setProperty("columnfamily", COLUMN_FAMILY); + + Measurements.setProperties(p); + final CoreWorkload workload = new CoreWorkload(); + workload.init(p); + + tableName = p.getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); + table = testingUtil.createTable(TableName.valueOf(tableName), Bytes.toBytes(COLUMN_FAMILY)); + + client.setProperties(p); + client.init(); + } + + @After + public void tearDown() throws Exception { + table.close(); + testingUtil.deleteTable(tableName); + } + + @Test + public void testRead() throws Exception { + final String rowKey = "row1"; + final Put p = new Put(Bytes.toBytes(rowKey)); + p.addColumn(Bytes.toBytes(COLUMN_FAMILY), + Bytes.toBytes("column1"), Bytes.toBytes("value1")); + p.addColumn(Bytes.toBytes(COLUMN_FAMILY), + Bytes.toBytes("column2"), Bytes.toBytes("value2")); + table.put(p); + + final HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>(); + final Status status = client.read(tableName, rowKey, null, result); + assertEquals(Status.OK, status); + assertEquals(2, result.size()); + assertEquals("value1", result.get("column1").toString()); + assertEquals("value2", result.get("column2").toString()); + } + + @Test + public void testReadMissingRow() throws Exception { + final HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>(); + final Status status = client.read(tableName, "Missing row", null, result); + assertEquals(Status.NOT_FOUND, status); + assertEquals(0, result.size()); + } + + @Test + public void testScan() throws Exception { + // Fill with data + final String colStr = "row_number"; + final byte[] col = Bytes.toBytes(colStr); + final int n = 10; + final List<Put> puts = new ArrayList<Put>(n); + for(int i = 0; i < n; i++) { + final byte[] key = Bytes.toBytes(String.format("%05d", i)); + final byte[] value = java.nio.ByteBuffer.allocate(4).putInt(i).array(); + final Put p = new Put(key); + p.addColumn(Bytes.toBytes(COLUMN_FAMILY), col, value); + puts.add(p); + } + table.put(puts); + + // Test + final Vector<HashMap<String, ByteIterator>> result = + new Vector<HashMap<String, ByteIterator>>(); + + // Scan 5 records, skipping the first + client.scan(tableName, "00001", 5, null, result); + + assertEquals(5, result.size()); + for(int i = 0; i < 5; i++) { + final HashMap<String, ByteIterator> row = result.get(i); + assertEquals(1, row.size()); + assertTrue(row.containsKey(colStr)); + final byte[] bytes = row.get(colStr).toArray(); + final ByteBuffer buf = ByteBuffer.wrap(bytes); + final int rowNum = buf.getInt(); + assertEquals(i + 1, rowNum); + } + } + + @Test + public void testUpdate() throws Exception{ + final String key = "key"; + final HashMap<String, String> input = new HashMap<String, String>(); + input.put("column1", "value1"); + input.put("column2", "value2"); + final Status status = client.insert(tableName, key, StringByteIterator.getByteIteratorMap(input)); + assertEquals(Status.OK, status); + + // Verify result + final Get get = new Get(Bytes.toBytes(key)); + final Result result = this.table.get(get); + assertFalse(result.isEmpty()); + assertEquals(2, result.size()); + for(final java.util.Map.Entry<String, String> entry : input.entrySet()) { + assertEquals(entry.getValue(), + new String(result.getValue(Bytes.toBytes(COLUMN_FAMILY), + Bytes.toBytes(entry.getKey())))); + } + } + + @Test + @Ignore("Not yet implemented") + public void testDelete() { + fail("Not yet implemented"); + } +} + diff --git a/hbase14/src/test/resources/hbase-site.xml b/hbase14/src/test/resources/hbase-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..a8b29e451f440ad7c09b3b2f25eebaf56f07e6bb --- /dev/null +++ b/hbase14/src/test/resources/hbase-site.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Copyright (c) 2016 YCSB contributors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you +may not use this file except in compliance with the License. You +may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing +permissions and limitations under the License. See accompanying +LICENSE file. +--> + +<configuration> + <property> + <name>hbase.master.info.port</name> + <value>-1</value> + <description>The port for the hbase master web UI + Set to -1 if you do not want the info server to run. + </description> + </property> + <property> + <name>hbase.regionserver.info.port</name> + <value>-1</value> + <description>The port for the hbase regionserver web UI + Set to -1 if you do not want the info server to run. + </description> + </property> +</configuration> diff --git a/hbase14/src/test/resources/log4j.properties b/hbase14/src/test/resources/log4j.properties new file mode 100644 index 0000000000000000000000000000000000000000..a9df32e044b9374097b9c110a79f35ff34b5a793 --- /dev/null +++ b/hbase14/src/test/resources/log4j.properties @@ -0,0 +1,28 @@ +# +# Copyright (c) 2015 YCSB contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. See accompanying +# LICENSE file. +# + +# Root logger option +log4j.rootLogger=WARN, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n + +# Suppress messages from ZKTableStateManager: Creates a large number of table +# state change messages. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKTableStateManager=ERROR diff --git a/pom.xml b/pom.xml index f9b24b535f00da5c773a0ea21b5a0c666a1d823c..e67012d47d60e2c5a8117aa4e3c8a52ee1d81d62 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ LICENSE file. <hbase098.version>0.98.14-hadoop2</hbase098.version> <hbase10.version>1.0.2</hbase10.version> <hbase12.version>1.2.5</hbase12.version> + <hbase14.version>1.4.2</hbase14.version> <hypertable.version>0.9.5.6</hypertable.version> <infinispan.version>7.2.2.Final</infinispan.version> <kudu.version>1.1.0</kudu.version> @@ -136,6 +137,7 @@ LICENSE file. <module>hbase098</module> <module>hbase10</module> <module>hbase12</module> + <module>hbase14</module> <module>hypertable</module> <module>infinispan</module> <module>jdbc</module>