From 931321b2aea2df4223a76cda5d721056fd301039 Mon Sep 17 00:00:00 2001
From: Andy Kruth <kruthar@gmail.com>
Date: Tue, 2 Feb 2016 20:52:48 -0600
Subject: [PATCH] [couchbase] cleaned up scan operation
Adapted from #172
---
couchbase/README.md | 13 ++-
couchbase/pom.xml | 7 --
.../com/yahoo/ycsb/db/CouchbaseClient.java | 99 +++++++------------
3 files changed, 47 insertions(+), 72 deletions(-)
diff --git a/couchbase/README.md b/couchbase/README.md
index 4004cb7f..efe16bac 100644
--- a/couchbase/README.md
+++ b/couchbase/README.md
@@ -1,5 +1,5 @@
<!--
-Copyright (c) 2015 YCSB contributors. All rights reserved.
+Copyright (c) 2015 - 2016 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
@@ -51,6 +51,13 @@ Please see the general instructions in the `doc` folder if you are not sure how
bin/ycsb run couchbase -s -P workloads/workloada -p couchbase.useJson=false
```
+## Scans in the CouchbaseClient
+The scan operation in the CouchbaseClient requires a Couchbase View to be created manually. To do this:
+1. Go to the Couchbase UI, then to Views
+2. Create a new development view, specify a ddoc and view name, use these in your YCSB properties. See Configuration Options below.
+3. The default map code is sufficient.
+4. Save, and publish this View.
+
## Configuration Options
Since no setup is the same and the goal of YCSB is to deliver realistic benchmarks, here are some setups that you can tune. Note that if you need more flexibility (let's say a custom transcoder), you still need to extend this driver and implement the facilities on your own.
@@ -59,9 +66,11 @@ You can set the following properties (with the default settings applied):
- couchbase.url=http://127.0.0.1:8091/pools => The connection URL from one server.
- couchbase.bucket=default => The bucket name to use.
- couchbase.password= => The password of the bucket.
-
- couchbase.checkFutures=true => If the futures should be inspected (makes ops sync).
- couchbase.persistTo=0 => Observe Persistence ("PersistTo" constraint).
- couchbase.replicateTo=0 => Observe Replication ("ReplicateTo" constraint).
+ - couchbase.ddoc => The ddoc name used for scanning
+ - couchbase.view => The view name used for scanning
+ - couchbase.stale => How to deal with stale values in View Query for scanning. (OK, FALSE, UPDATE_AFTER)
- couchbase.json=true => Use json or java serialization as target format.
diff --git a/couchbase/pom.xml b/couchbase/pom.xml
index 10dfced3..a7d231a7 100644
--- a/couchbase/pom.xml
+++ b/couchbase/pom.xml
@@ -54,13 +54,6 @@ LICENSE file.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
- <!-- Gson: Java to Json conversion -->
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.2.4</version>
- <scope>compile</scope>
- </dependency>
</dependencies>
<repositories>
diff --git a/couchbase/src/main/java/com/yahoo/ycsb/db/CouchbaseClient.java b/couchbase/src/main/java/com/yahoo/ycsb/db/CouchbaseClient.java
index bf133879..1eb3aeb3 100644
--- a/couchbase/src/main/java/com/yahoo/ycsb/db/CouchbaseClient.java
+++ b/couchbase/src/main/java/com/yahoo/ycsb/db/CouchbaseClient.java
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2013 Yahoo! Inc. All rights reserved.
+ * Copyright (c) 2013 - 2016 YCSB contributors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.gson.*;
import com.yahoo.ycsb.ByteIterator;
import com.yahoo.ycsb.DB;
import com.yahoo.ycsb.DBException;
@@ -39,7 +38,6 @@ import org.slf4j.LoggerFactory;
import java.io.StringWriter;
import java.io.Writer;
-import java.lang.reflect.Type;
import java.net.URI;
import java.util.Arrays;
import java.util.HashMap;
@@ -76,6 +74,13 @@ public class CouchbaseClient extends DB {
public static final String PERSIST_PROPERTY = "couchbase.persistTo";
public static final String REPLICATE_PROPERTY = "couchbase.replicateTo";
public static final String JSON_PROPERTY = "couchbase.json";
+ public static final String DESIGN_DOC_PROPERTY = "couchbase.ddoc";
+ public static final String VIEW_PROPERTY = "couchbase.view";
+ public static final String STALE_PROPERTY = "couchbase.stale";
+ public static final String SCAN_PROPERTY = "scanproportion";
+
+ public static final String STALE_PROPERTY_DEFAULT = Stale.OK.name();
+ public static final String SCAN_PROPERTY_DEFAULT = "0.0";
protected static final ObjectMapper JSON_MAPPER = new ObjectMapper();
@@ -84,44 +89,11 @@ public class CouchbaseClient extends DB {
private ReplicateTo replicateTo;
private boolean checkFutures;
private boolean useJson;
+ private String designDoc;
+ private String viewName;
+ private Stale stale;
+ private View view;
private final Logger log = LoggerFactory.getLogger(getClass());
- private volatile Stale stale;
- public final ThreadLocal<Gson> gson = new ThreadLocal<Gson>() {
- public Gson get() {
- return GSON_BUILDER
- .create();
- }
- };
-
- /**
- * {@link Gson} instance builder
- */
- private static GsonBuilder GSON_BUILDER = new GsonBuilder().registerTypeAdapter(
- ByteIterator.class,
- new JsonSerializer<ByteIterator>() {
-
- @Override
- public JsonElement serialize(
- ByteIterator arg0,
- Type arg1,
- JsonSerializationContext arg2) {
- return new JsonPrimitive(
- arg0.toString());
- }
- })
- .registerTypeAdapter(
- ByteIterator.class,
- new JsonDeserializer<ByteIterator>() {
- @Override
- public ByteIterator deserialize(
- JsonElement arg0,
- Type arg1,
- JsonDeserializationContext arg2)
- throws JsonParseException {
- return new StringByteIterator(
- arg0.toString());
- }
- });
@Override
public void init() throws DBException {
@@ -137,6 +109,12 @@ public class CouchbaseClient extends DB {
persistTo = parsePersistTo(props.getProperty(PERSIST_PROPERTY, "0"));
replicateTo = parseReplicateTo(props.getProperty(REPLICATE_PROPERTY, "0"));
+ designDoc = getProperties().getProperty(DESIGN_DOC_PROPERTY);
+ viewName = getProperties().getProperty(VIEW_PROPERTY);
+ stale = Stale.valueOf(getProperties().getProperty(STALE_PROPERTY, STALE_PROPERTY_DEFAULT).toUpperCase());
+
+ Double scanproportion = Double.valueOf(props.getProperty(SCAN_PROPERTY, SCAN_PROPERTY_DEFAULT));
+
Properties systemProperties = System.getProperties();
systemProperties.put("net.spy.log.LoggerImpl", "net.spy.memcached.compat.log.SLF4JLogger");
System.setProperties(systemProperties);
@@ -150,6 +128,15 @@ public class CouchbaseClient extends DB {
} catch (Exception e) {
throw new DBException("Could not create CouchbaseClient object.", e);
}
+
+ if (scanproportion > 0) {
+ try {
+ view = client.getView(designDoc, viewName);
+ } catch (Exception e) {
+ throw new DBException(String.format("%s=%s and %s=%s provided, unable to connect to view.",
+ DESIGN_DOC_PROPERTY, designDoc, VIEW_PROPERTY, viewName), e.getCause());
+ }
+ }
}
/**
@@ -234,38 +221,24 @@ public class CouchbaseClient extends DB {
* @return Status.ERROR, because not implemented yet.
*/
@Override
- public Status scan(final String table, final String startkey, final int recordcount,
- final Set<String> fields, final Vector<HashMap<String, ByteIterator>> result) {
- String designDoc = getProperties().getProperty("ddoc");
- String viewName = getProperties().getProperty("view");
-
- if (designDoc == null || viewName == null) {
- System.err.println("Scan requires [ddoc, view] params");
- return Status.ERROR;
- }
-
+ public Status scan(final String table, final String startkey, final int recordcount, final Set<String> fields,
+ final Vector<HashMap<String, ByteIterator>> result) {
try {
- final View view = client.getView(designDoc, viewName);
Query query = new Query().setRangeStart(startkey)
- .setLimit(recordcount).setIncludeDocs(Boolean.TRUE)
+ .setLimit(recordcount)
+ .setIncludeDocs(true)
.setStale(stale);
-
ViewResponse response = client.query(view, query);
- HashMap<String, ByteIterator> resultMap = new HashMap<String, ByteIterator>();
for (ViewRow row : response) {
- Object obj = row.getDocument();
- if (obj == null) {
- continue;
- }
- ByteIterator recVal = gson.get().fromJson(obj.toString(),
- ByteIterator.class);
- resultMap.put(row.getKey(), recVal);
+ HashMap<String, ByteIterator> rowMap = new HashMap();
+ decode(row.getDocument(), fields, rowMap);
+ result.add(rowMap);
}
- result.add(resultMap);
+
return Status.OK;
} catch (Exception e) {
- System.err.println(e.getMessage());
+ log.error(e.getMessage());
}
return Status.ERROR;
--
GitLab