Merge pull request #411 from cmccoy/hbase-scan-pagefilter

Use a PageFilter in HBase scans

hbase098/README.md

@@ -71,3 +71,6 @@ Following options can be configurable using `-p`.
 
 * `columnfamily`: The HBase column family to target.
 * `debug` : If true, debugging logs are activated. The default is false.
+* `hbase.usepagefilter` : If true, HBase
+  [PageFilter](https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/PageFilter.html)s
+  are used to limit the number of records consumed in a scan operation. The default is true.

hbase098/src/main/java/com/yahoo/ycsb/db/HBaseClient.java

@@ -17,32 +17,25 @@
 
 package com.yahoo.ycsb.db;
 
-
 import com.yahoo.ycsb.DBException;
 import com.yahoo.ycsb.ByteIterator;
 import com.yahoo.ycsb.ByteArrayByteIterator;
+import com.yahoo.ycsb.measurements.Measurements;
 
 import java.io.IOException;
 import java.util.*;
-//import java.util.HashMap;
-//import java.util.Properties;
-//import java.util.Set;
-//import java.util.Vector;
 
-import com.yahoo.ycsb.measurements.Measurements;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HTable;
-//import org.apache.hadoop.hbase.client.Scanner;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
-//import org.apache.hadoop.hbase.io.Cell;
-//import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.filter.PageFilter;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 
@@ -63,6 +56,8 @@ public class HBaseClient extends com.yahoo.ycsb.DB
     public byte _columnFamilyBytes[];
     public boolean _clientSideBuffering = false;
     public long _writeBufferSize = 1024 * 1024 * 12;
+    /** Whether or not a page filter should be used to limit scan length. */
+    public boolean _usePageFilter = true;
 
     public static final int Ok=0;
     public static final int ServerError=-1;
@@ -91,6 +86,9 @@ public class HBaseClient extends com.yahoo.ycsb.DB
         {
             _writeBufferSize = Long.parseLong(getProperties().getProperty("writebuffersize"));
         }
+        if ("false".equals(getProperties().getProperty("hbase.usepagefilter", "true"))) {
+          _usePageFilter = false;
+        }
 
         _columnFamily = getProperties().getProperty("columnfamily");
         if (_columnFamily == null)
@@ -246,6 +244,9 @@ public class HBaseClient extends com.yahoo.ycsb.DB
         //HBase has no record limit.  Here, assume recordcount is small enough to bring back in one call.
         //We get back recordcount records
         s.setCaching(recordcount);
+        if (this._usePageFilter) {
+          s.setFilter(new PageFilter(recordcount));
+        }
 
         //add specified fields or else all fields
         if (fields == null)
@@ -284,6 +285,9 @@ public class HBaseClient extends com.yahoo.ycsb.DB
                 //add rowResult to result vector
                 result.add(rowResult);
                 numResults++;
+
+                // PageFilter does not guarantee that the number of results is <= pageSize, so this
+                // break is required.
                 if (numResults >= recordcount) //if hit recordcount, bail out
                 {
                     break;

hbase10/src/main/java/com/yahoo/ycsb/db/HBaseClient10.java

@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.filter.PageFilter;
 import org.apache.hadoop.hbase.util.Bytes;
 
 import java.io.IOException;
@@ -81,6 +82,9 @@ public class HBaseClient10 extends com.yahoo.ycsb.DB
      */
     public Durability _durability = Durability.USE_DEFAULT;
 
+    /** Whether or not a page filter should be used to limit scan length. */
+    public boolean _usePageFilter = true;
+
     /**
      * If true, buffer mutations on the client.
      * This is the default behavior for HBaseClient. For measuring
@@ -124,6 +128,10 @@ public class HBaseClient10 extends com.yahoo.ycsb.DB
             _debug=true;
         }
 
+        if ("false".equals(getProperties().getProperty("hbase.usepagefilter", "true"))) {
+          _usePageFilter = false;
+        }
+
         _columnFamily = getProperties().getProperty("columnfamily");
         if (_columnFamily == null)
         {
@@ -291,6 +299,9 @@ public class HBaseClient10 extends com.yahoo.ycsb.DB
         //HBase has no record limit.  Here, assume recordcount is small enough to bring back in one call.
         //We get back recordcount records
         s.setCaching(recordcount);
+        if (this._usePageFilter) {
+          s.setFilter(new PageFilter(recordcount));
+        }
 
         //add specified fields or else all fields
         if (fields == null)
@@ -332,6 +343,9 @@ public class HBaseClient10 extends com.yahoo.ycsb.DB
                 //add rowResult to result vector
                 result.add(rowResult);
                 numResults++;
+
+                // PageFilter does not guarantee that the number of results is <= pageSize, so this
+                // break is required.
                 if (numResults >= recordcount) //if hit recordcount, bail out
                 {
                     break;