Skip to content
Snippets Groups Projects
Commit 4e7295c2 authored by allanbank's avatar allanbank
Browse files

Merge pull request #488 from allanbank/kudu-cleanup 

[kudu] Checkstyle updates for the Kudu binding.
parents add8e70b 23f8f300
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
kudu/pom.xml
+ 24
− 0
View file @ 4e7295c2
...@@ -42,6 +42,30 @@ LICENSE file. ...@@ -42,6 +42,30 @@ LICENSE file.
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
</dependencies> </dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>2.15</version>
<configuration>
<consoleOutput>true</consoleOutput>
<configLocation>../checkstyle.xml</configLocation>
<failOnViolation>true</failOnViolation>
<failsOnError>true</failsOnError>
</configuration>
<executions>
<execution>
<id>validate</id>
<phase>validate</phase>
<goals>
<goal>checkstyle</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<repositories> <repositories>
<repository> <repository>
<releases> <releases>
......
kudu/src/main/java/com/yahoo/ycsb/db/KuduYCSBClient.java
+ 84
− 55
View file @ 4e7295c2
...@@ -37,27 +37,40 @@ import java.util.Vector; ...@@ -37,27 +37,40 @@ import java.util.Vector;
import static org.kududb.Type.STRING; import static org.kududb.Type.STRING;
/** /**
* Kudu client for YCSB framework * Kudu client for YCSB framework. Example to load: <blockquote>
* Example to load: *
* $ ./bin/ycsb load kudu -P workloads/workloada -threads 5 * <pre>
* Example to run: * <code>
* $ ./bin/ycsb load kudu -P workloads/workloada -threads 5
* </code>
* </pre>
*
* </blockquote> Example to run: <blockquote>
*
* <pre>
* <code>
* ./bin/ycsb run kudu -P workloads/workloada -p kudu_sync_ops=true -threads 5 * ./bin/ycsb run kudu -P workloads/workloada -p kudu_sync_ops=true -threads 5
* * </code>
* </pre>
*
* </blockquote>
*/ */
public class KuduYCSBClient extends com.yahoo.ycsb.DB { public class KuduYCSBClient extends com.yahoo.ycsb.DB {
public static final String KEY = "key"; public static final String KEY = "key";
public static final Status TIMEOUT = new Status("TIMEOUT", "The operation timed out."); public static final Status TIMEOUT =
new Status("TIMEOUT", "The operation timed out.");
public static final int MAX_TABLETS = 9000; public static final int MAX_TABLETS = 9000;
public static final long DEFAULT_SLEEP = 60000; public static final long DEFAULT_SLEEP = 60000;
private static final String SYNC_OPS_OPT = "kudu_sync_ops"; private static final String SYNC_OPS_OPT = "kudu_sync_ops";
private static final String DEBUG_OPT = "kudu_debug"; private static final String DEBUG_OPT = "kudu_debug";
private static final String PRINT_ROW_ERRORS_OPT = "kudu_print_row_errors"; private static final String PRINT_ROW_ERRORS_OPT = "kudu_print_row_errors";
private static final String PRE_SPLIT_NUM_TABLETS_OPT = "kudu_pre_split_num_tablets"; private static final String PRE_SPLIT_NUM_TABLETS_OPT =
"kudu_pre_split_num_tablets";
private static final String TABLE_NUM_REPLICAS = "kudu_table_num_replicas"; private static final String TABLE_NUM_REPLICAS = "kudu_table_num_replicas";
private static final String BLOCK_SIZE_OPT = "kudu_block_size"; private static final String BLOCK_SIZE_OPT = "kudu_block_size";
private static final String MASTER_ADDRESSES_OPT = "kudu_master_addresses"; private static final String MASTER_ADDRESSES_OPT = "kudu_master_addresses";
private static final int BLOCK_SIZE_DEFAULT = 4096; private static final int BLOCK_SIZE_DEFAULT = 4096;
private static final List<String> columnNames = new ArrayList<String>(); private static final List<String> COLUMN_NAMES = new ArrayList<String>();
private static KuduClient client; private static KuduClient client;
private static Schema schema; private static Schema schema;
private static int fieldCount; private static int fieldCount;
...@@ -65,7 +78,7 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -65,7 +78,7 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
private boolean printErrors = false; private boolean printErrors = false;
private String tableName; private String tableName;
private KuduSession session; private KuduSession session;
private KuduTable table; private KuduTable kuduTable;
@Override @Override
public void init() throws DBException { public void init() throws DBException {
...@@ -73,16 +86,18 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -73,16 +86,18 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
this.debug = getProperties().getProperty(DEBUG_OPT).equals("true"); this.debug = getProperties().getProperty(DEBUG_OPT).equals("true");
} }
if (getProperties().getProperty(PRINT_ROW_ERRORS_OPT) != null) { if (getProperties().getProperty(PRINT_ROW_ERRORS_OPT) != null) {
this.printErrors = getProperties().getProperty(PRINT_ROW_ERRORS_OPT).equals("true"); this.printErrors =
getProperties().getProperty(PRINT_ROW_ERRORS_OPT).equals("true");
} }
if (getProperties().getProperty(PRINT_ROW_ERRORS_OPT) != null) { if (getProperties().getProperty(PRINT_ROW_ERRORS_OPT) != null) {
this.printErrors = getProperties().getProperty(PRINT_ROW_ERRORS_OPT).equals("true"); this.printErrors =
getProperties().getProperty(PRINT_ROW_ERRORS_OPT).equals("true");
} }
this.tableName = com.yahoo.ycsb.workloads.CoreWorkload.table; this.tableName = com.yahoo.ycsb.workloads.CoreWorkload.table;
initClient(debug, tableName, getProperties()); initClient(debug, tableName, getProperties());
this.session = client.newSession(); this.session = client.newSession();
if (getProperties().getProperty(SYNC_OPS_OPT) != null && if (getProperties().getProperty(SYNC_OPS_OPT) != null
getProperties().getProperty(SYNC_OPS_OPT).equals("false")) { && getProperties().getProperty(SYNC_OPS_OPT).equals("false")) {
this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
this.session.setMutationBufferSpace(100); this.session.setMutationBufferSpace(100);
} else { } else {
...@@ -90,15 +105,17 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -90,15 +105,17 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
} }
try { try {
this.table = client.openTable(tableName); this.kuduTable = client.openTable(tableName);
} catch (Exception e) { } catch (Exception e) {
throw new DBException("Could not open a table because of:", e); throw new DBException("Could not open a table because of:", e);
} }
} }
private synchronized static void initClient(boolean debug, String tableName, Properties prop) private static synchronized void initClient(boolean debug, String tableName,
throws DBException { Properties prop) throws DBException {
if (client != null) return; if (client != null) {
return;
}
String masterAddresses = prop.getProperty(MASTER_ADDRESSES_OPT); String masterAddresses = prop.getProperty(MASTER_ADDRESSES_OPT);
if (masterAddresses == null) { if (masterAddresses == null) {
...@@ -107,8 +124,8 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -107,8 +124,8 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
int numTablets = getIntFromProp(prop, PRE_SPLIT_NUM_TABLETS_OPT, 4); int numTablets = getIntFromProp(prop, PRE_SPLIT_NUM_TABLETS_OPT, 4);
if (numTablets > MAX_TABLETS) { if (numTablets > MAX_TABLETS) {
throw new DBException("Specified number of tablets (" + numTablets + ") must be equal " + throw new DBException("Specified number of tablets (" + numTablets
"or below " + MAX_TABLETS); + ") must be equal " + "or below " + MAX_TABLETS);
} }
int numReplicas = getIntFromProp(prop, TABLE_NUM_REPLICAS, 3); int numReplicas = getIntFromProp(prop, TABLE_NUM_REPLICAS, 3);
...@@ -117,8 +134,7 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -117,8 +134,7 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
client = new KuduClient.KuduClientBuilder(masterAddresses) client = new KuduClient.KuduClientBuilder(masterAddresses)
.defaultSocketReadTimeoutMs(DEFAULT_SLEEP) .defaultSocketReadTimeoutMs(DEFAULT_SLEEP)
.defaultOperationTimeoutMs(DEFAULT_SLEEP) .defaultOperationTimeoutMs(DEFAULT_SLEEP).build();
.build();
if (debug) { if (debug) {
System.out.println("Connecting to the masters at " + masterAddresses); System.out.println("Connecting to the masters at " + masterAddresses);
} }
...@@ -129,17 +145,14 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -129,17 +145,14 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
List<ColumnSchema> columns = new ArrayList<ColumnSchema>(fieldCount + 1); List<ColumnSchema> columns = new ArrayList<ColumnSchema>(fieldCount + 1);
ColumnSchema keyColumn = new ColumnSchema.ColumnSchemaBuilder(KEY, STRING) ColumnSchema keyColumn = new ColumnSchema.ColumnSchemaBuilder(KEY, STRING)
.key(true) .key(true).desiredBlockSize(blockSize).build();
.desiredBlockSize(blockSize)
.build();
columns.add(keyColumn); columns.add(keyColumn);
columnNames.add(KEY); COLUMN_NAMES.add(KEY);
for (int i = 0; i < fieldCount; i++) { for (int i = 0; i < fieldCount; i++) {
String name = "field" + i; String name = "field" + i;
columnNames.add(name); COLUMN_NAMES.add(name);
columns.add(new ColumnSchema.ColumnSchemaBuilder(name, STRING) columns.add(new ColumnSchema.ColumnSchemaBuilder(name, STRING)
.desiredBlockSize(blockSize) .desiredBlockSize(blockSize).build());
.build());
} }
schema = new Schema(columns); schema = new Schema(columns);
...@@ -164,8 +177,8 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -164,8 +177,8 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
} }
} }
private static int getIntFromProp(Properties prop, String propName, int defaultValue) private static int getIntFromProp(Properties prop, String propName,
throws DBException { int defaultValue) throws DBException {
String intStr = prop.getProperty(propName); String intStr = prop.getProperty(propName);
if (intStr == null) { if (intStr == null) {
return defaultValue; return defaultValue;
...@@ -173,7 +186,8 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -173,7 +186,8 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
try { try {
return Integer.valueOf(intStr); return Integer.valueOf(intStr);
} catch (NumberFormatException ex) { } catch (NumberFormatException ex) {
throw new DBException("Provided number for " + propName + " isn't a valid integer"); throw new DBException(
"Provided number for " + propName + " isn't a valid integer");
} }
} }
} }
...@@ -189,23 +203,29 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -189,23 +203,29 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
@Override @Override
public Status read(String table, String key, Set<String> fields, public Status read(String table, String key, Set<String> fields,
HashMap<String,ByteIterator> result) { HashMap<String, ByteIterator> result) {
Vector<HashMap<String, ByteIterator>> results = new Vector<HashMap<String, ByteIterator>>(); Vector<HashMap<String, ByteIterator>> results =
new Vector<HashMap<String, ByteIterator>>();
final Status status = scan(table, key, 1, fields, results); final Status status = scan(table, key, 1, fields, results);
if (!status.equals(Status.OK)) return status; if (!status.equals(Status.OK)) {
if (results.size() != 1) return Status.NOT_FOUND; return status;
}
if (results.size() != 1) {
return Status.NOT_FOUND;
}
result.putAll(results.firstElement()); result.putAll(results.firstElement());
return Status.OK; return Status.OK;
} }
@Override @Override
public Status scan(String table, String startkey, int recordcount, Set<String> fields, public Status scan(String table, String startkey, int recordcount,
Vector<HashMap<String, ByteIterator>> result) { Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
try { try {
KuduScanner.KuduScannerBuilder scannerBuilder = client.newScannerBuilder(this.table); KuduScanner.KuduScannerBuilder scannerBuilder =
client.newScannerBuilder(this.kuduTable);
List<String> querySchema; List<String> querySchema;
if (fields == null) { if (fields == null) {
querySchema = columnNames; querySchema = COLUMN_NAMES;
// No need to set the projected columns with the whole schema. // No need to set the projected columns with the whole schema.
} else { } else {
querySchema = new ArrayList<String>(fields); querySchema = new ArrayList<String>(fields);
...@@ -222,20 +242,22 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -222,20 +242,22 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
scannerBuilder.exclusiveUpperBound(upperBound); scannerBuilder.exclusiveUpperBound(upperBound);
} }
KuduScanner scanner = scannerBuilder KuduScanner scanner = scannerBuilder.limit(recordcount) // currently noop
.limit(recordcount) // currently noop
.build(); .build();
while (scanner.hasMoreRows()) { while (scanner.hasMoreRows()) {
RowResultIterator data = scanner.nextRows(); RowResultIterator data = scanner.nextRows();
addAllRowsToResult(data, recordcount, querySchema, result); addAllRowsToResult(data, recordcount, querySchema, result);
if (recordcount == result.size()) break; if (recordcount == result.size()) {
break;
}
} }
RowResultIterator closer = scanner.close(); RowResultIterator closer = scanner.close();
addAllRowsToResult(closer, recordcount, querySchema, result); addAllRowsToResult(closer, recordcount, querySchema, result);
} catch (TimeoutException te) { } catch (TimeoutException te) {
if (printErrors) { if (printErrors) {
System.err.println("Waited too long for a scan operation with start key=" + startkey); System.err.println(
"Waited too long for a scan operation with start key=" + startkey);
} }
return TIMEOUT; return TIMEOUT;
} catch (Exception e) { } catch (Exception e) {
...@@ -247,14 +269,18 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -247,14 +269,18 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
} }
private void addAllRowsToResult(RowResultIterator it, int recordcount, private void addAllRowsToResult(RowResultIterator it, int recordcount,
List<String> querySchema, List<String> querySchema, Vector<HashMap<String, ByteIterator>> result)
Vector<HashMap<String, ByteIterator>> result) throws Exception {
throws Exception {
RowResult row; RowResult row;
HashMap<String, ByteIterator> rowResult = new HashMap<String, ByteIterator>(querySchema.size()); HashMap<String, ByteIterator> rowResult =
if (it == null) return; new HashMap<String, ByteIterator>(querySchema.size());
if (it == null) {
return;
}
while (it.hasNext()) { while (it.hasNext()) {
if (result.size() == recordcount) return; if (result.size() == recordcount) {
return;
}
row = it.next(); row = it.next();
int colIdx = 0; int colIdx = 0;
for (String col : querySchema) { for (String col : querySchema) {
...@@ -266,8 +292,9 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -266,8 +292,9 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
} }
@Override @Override
public Status update(String table, String key, HashMap<String, ByteIterator> values) { public Status update(String table, String key,
Update update = this.table.newUpdate(); HashMap<String, ByteIterator> values) {
Update update = this.kuduTable.newUpdate();
PartialRow row = update.getRow(); PartialRow row = update.getRow();
row.addString(KEY, key); row.addString(KEY, key);
for (int i = 1; i < schema.getColumnCount(); i++) { for (int i = 1; i < schema.getColumnCount(); i++) {
...@@ -282,12 +309,14 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -282,12 +309,14 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
} }
@Override @Override
public Status insert(String table, String key, HashMap<String, ByteIterator> values) { public Status insert(String table, String key,
Insert insert = this.table.newInsert(); HashMap<String, ByteIterator> values) {
Insert insert = this.kuduTable.newInsert();
PartialRow row = insert.getRow(); PartialRow row = insert.getRow();
row.addString(KEY, key); row.addString(KEY, key);
for (int i = 1; i < schema.getColumnCount(); i++) { for (int i = 1; i < schema.getColumnCount(); i++) {
row.addString(i, new String(values.get(schema.getColumnByIndex(i).getName()).toArray())); row.addString(i, new String(
values.get(schema.getColumnByIndex(i).getName()).toArray()));
} }
apply(insert); apply(insert);
return Status.OK; return Status.OK;
...@@ -295,7 +324,7 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB { ...@@ -295,7 +324,7 @@ public class KuduYCSBClient extends com.yahoo.ycsb.DB {
@Override @Override
public Status delete(String table, String key) { public Status delete(String table, String key) {
Delete delete = this.table.newDelete(); Delete delete = this.kuduTable.newDelete();
PartialRow row = delete.getRow(); PartialRow row = delete.getRow();
row.addString(KEY, key); row.addString(KEY, key);
apply(delete); apply(delete);
......
kudu/src/main/java/com/yahoo/ycsb/db/package-info.java 0 → 100644
+ 22
− 0
View file @ 4e7295c2
/*
* Copyright (c) 2014, Yahoo!, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
/**
* The YCSB binding for <a href="http://getkudu.io/">Kudu</a>.
*/
package com.yahoo.ycsb.db;
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment