[Core] Add retry for insertion logic to make the load process more robust

The default behavior is NO RETRY, which is the current behavior.

But optionally user can enable retry if they are working with a DB backend that is better tolerated by allowing a few retries.

core/src/main/java/com/yahoo/ycsb/workloads/CoreWorkload.java

@@ -296,6 +296,16 @@ public class CoreWorkload extends Workload {
    */
   public static final String HOTSPOT_OPN_FRACTION_DEFAULT = "0.8";
 
+  /**
+   * How many times to retry when insertion of a single item to a DB fails.
+   */
+  public static final String INSERTION_RETRY_LIMIT_DEFAULT = "0";
+
+  /**
+   * On average, how long to wait between the retries, in seconds.
+   */
+  public static final String INSERTION_RETRY_INTERVAL_DEFAULT = "3";
+
   IntegerGenerator keysequence;
 
   DiscreteGenerator operationchooser;
@@ -312,6 +322,9 @@ public class CoreWorkload extends Workload {
 
   int recordcount;
 
+  int insertionRetryLimit;
+  int insertionRetryInterval;
+
   private Measurements _measurements = Measurements.getMeasurements();
 
   protected static IntegerGenerator getFieldLengthGenerator(Properties p) throws WorkloadException {
@@ -475,6 +488,12 @@ public class CoreWorkload extends Workload {
       throw new WorkloadException(
           "Distribution \"" + scanlengthdistrib + "\" not allowed for scan length");
     }
+
+    insertionRetryLimit = Integer.parseInt(p.getProperty(
+        "insertionretrylimit", INSERTION_RETRY_LIMIT_DEFAULT));
+
+    insertionRetryInterval = Integer.parseInt(p.getProperty(
+        "insertionretryinterval", INSERTION_RETRY_INTERVAL_DEFAULT));
   }
 
   public String buildKeyName(long keynum) {
@@ -550,12 +569,37 @@ public class CoreWorkload extends Workload {
     int keynum = keysequence.nextInt();
     String dbkey = buildKeyName(keynum);
     HashMap<String, ByteIterator> values = buildValues(dbkey);
-    if (db.insert(table, dbkey, values).equals(Status.OK))
-      return true;
-    else
-      return false;
-  }
 
+    Status status;
+    int numOfRetries = 0;
+    do {
+      status = db.insert(table, dbkey, values);
+      if (status == Status.OK) {
+        break;
+      }
+      // Retry if configured. Without retrying, the load process will fail
+      // even if one single insertion fails. User can optionally configure
+      // an insertion retry limit (default is 0) to enable retry.
+      if (++numOfRetries <= insertionRetryLimit) {
+        System.err.println("Retrying insertion, retry count: " + numOfRetries);
+        try {
+          // Sleep for a random number between [0.8, 1.2)*insertionRetryInterval.
+          int sleepTime = (int) (1000 * insertionRetryInterval * (0.8 + 0.4 * Math.random()));
+          Thread.sleep(sleepTime);
+        } catch (InterruptedException e) {
+          break;
+        }
+
+      } else {
+        System.err.println("Error inserting, not retrying any more. number of attempts: " + numOfRetries +
+            "Insertion Retry Limit: " + insertionRetryLimit);
+        break;
+
+      }
+    } while (true);
+
+    return (status == Status.OK);
+  }
 
   /**
    * Do one transaction operation. Because it will be called concurrently from multiple client

workloads/workload_template

@@ -156,3 +156,15 @@ timeseries.granularity=1000
 #    property.
 # reportlatencyforeacherror=false
 # latencytrackederrors="<comma separated strings of error codes>"
+
+# Insertion error retry
+#
+# By default, YCSB does not retry any operations. However, during the
+# load process, if any insertion fails, the entire load process is terminated.
+# If a user desires to have more robust behavior during this phase, they can
+# enable retry for insertion by setting the following property to a positive
+# number.
+# insertionretrylimit = 0
+#
+# the following number controls the interval between retries (in seconds):
+# insertionretryinterval = 3