Merge pull request #581 from kruthar/orientdb-remote

[orientdb] Added remote connection capabilties

core/src/main/java/com/yahoo/ycsb/Client.java

 /**
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ * Copyright (c) 2010 - 2016 Yahoo! Inc. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License. You
@@ -469,6 +469,11 @@ public class Client
    */
   public static final String MAX_EXECUTION_TIME = "maxexecutiontime";
 
+  /**
+   * Whether or not this is the transaction phase (run) or not (load).
+   */
+  public static final String DO_TRANSACTIONS_PROPERTY = "dotransactions";
+
 
   public static void usageMessage()
   {
@@ -734,6 +739,8 @@ public class Client
       System.exit(0);
     }
 
+    props.setProperty(DO_TRANSACTIONS_PROPERTY, String.valueOf(dotransactions));
+
     long maxExecutionTime = Integer.parseInt(props.getProperty(MAX_EXECUTION_TIME, "0"));
 
     //get number of threads, target and db

orientdb/README.md

@@ -42,16 +42,20 @@ See the next section for the list of configuration parameters for OrientDB.
 ## OrientDB Configuration Parameters
 
 * ```orientdb.url``` - (required) The address to your database.
-    * Supported storage types: memory, plocal
+    * Supported storage types: memory, plocal, remote
     * EX. ```plocal:/path/to/database```
 * ```orientdb.user``` - The user to connect to the database with.
     * Default: ```admin```
 * ```orientdb.password``` - The password to connect to the database with.
     * Default: ```admin```
-* ```orientdb.newdb``` - Create the database if it does not exists, or overwrite the database if it does exist.
+* ```orientdb.newdb``` - Overwrite the database if it already exists.
+    * Only effects the ```load``` phase.
     * Default: ```false```
+* ```orientdb.remote.storagetype``` - Storage type of the database on remote server
+    * This is only required if using a ```remote:``` connection url
 
 ## Known Issues
 
 * There is a performance issue around the scan operation. This binding uses OIndex.iterateEntriesMajor() which will return unnecessarily large iterators. This has a performance impact as the recordcount goes up. There are ideas in the works to fix it, track it here: [#568](https://github.com/brianfrankcooper/YCSB/issues/568).
 * The OIndexCursor used to run the scan operation currently seems to be broken. Because of this, if the startkey and recordcount combination on a particular operation were to cause the iterator to go to the end, a NullPointerException is thrown. With sufficiently high record counts, this does not happen very often, but it could cause false negatives. Track that issue here: https://github.com/orientechnologies/orientdb/issues/5541.
+* Iterator methods needed to perform scans are Unsupported in the OrientDB API for remote database connections and so will return NOT_IMPLEMENTED status if attempted.

orientdb/pom.xml

@@ -44,7 +44,7 @@ LICENSE file.
 		</dependency>
 		<dependency>
 			<groupId>com.orientechnologies</groupId>
-			<artifactId>orientdb-core</artifactId>
+			<artifactId>orientdb-client</artifactId>
 			<version>${orientdb.version}</version>
 		</dependency>
     <dependency>

orientdb/src/main/java/com/yahoo/ycsb/db/OrientDBClient.java

@@ -17,11 +17,14 @@
 
 package com.yahoo.ycsb.db;
 
-import com.orientechnologies.orient.core.config.OGlobalConfiguration;
+import com.orientechnologies.common.exception.OException;
+import com.orientechnologies.orient.client.remote.OEngineRemote;
+import com.orientechnologies.orient.client.remote.OServerAdmin;
 import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal;
 import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
 import com.orientechnologies.orient.core.db.record.OIdentifiable;
 import com.orientechnologies.orient.core.dictionary.ODictionary;
+import com.orientechnologies.orient.core.exception.ODatabaseException;
 import com.orientechnologies.orient.core.index.OIndexCursor;
 import com.orientechnologies.orient.core.intent.OIntentMassiveInsert;
 import com.orientechnologies.orient.core.record.ORecord;
@@ -32,6 +35,7 @@ import com.yahoo.ycsb.DBException;
 import com.yahoo.ycsb.Status;
 import com.yahoo.ycsb.StringByteIterator;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map.Entry;
 import java.util.Properties;
@@ -46,6 +50,7 @@ public class OrientDBClient extends DB {
   private static final String             CLASS = "usertable";
   protected ODatabaseDocumentTx             db;
   private ODictionary<ORecord> dictionary;
+  private boolean isRemote = false;
 
   private static final String URL_PROPERTY = "orientdb.url";
 
@@ -58,6 +63,13 @@ public class OrientDBClient extends DB {
   private static final String NEWDB_PROPERTY = "orientdb.newdb";
   private static final String NEWDB_PROPERTY_DEFAULT = "false";
 
+  private static final String STORAGE_TYPE_PROPERTY = "orientdb.remote.storagetype";
+
+  private static final String DO_TRANSACTIONS_PROPERTY = "dotransactions";
+  private static final String DO_TRANSACTIONS_PROPERTY_DEFAULT = "true";
+
+  private static final String ORIENTDB_DOCUMENT_TYPE = "document";
+
   /**
    * Initialize any state for this DB. Called once per DB instance; there is one DB instance per client thread.
    */
@@ -68,42 +80,67 @@ public class OrientDBClient extends DB {
     String user = props.getProperty(USER_PROPERTY, USER_PROPERTY_DEFAULT);
     String password = props.getProperty(PASSWORD_PROPERTY, PASSWORD_PROPERTY_DEFAULT);
     Boolean newdb = Boolean.parseBoolean(props.getProperty(NEWDB_PROPERTY, NEWDB_PROPERTY_DEFAULT));
+    String remoteStorageType = props.getProperty(STORAGE_TYPE_PROPERTY);
+    Boolean dotransactions = Boolean.parseBoolean(props.getProperty(DO_TRANSACTIONS_PROPERTY, DO_TRANSACTIONS_PROPERTY_DEFAULT));
 
     if (url == null) {
       throw new DBException(String.format("Required property \"%s\" missing for OrientDBClient", URL_PROPERTY));
     }
 
-    try {
-      System.out.println("OrientDB loading database url = " + url);
-
-      OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false);
-      db = new ODatabaseDocumentTx(url);
-      if (db.exists()) {
-        db.open(user, password);
-        if (newdb) {
-          System.out.println("OrientDB drop and recreate fresh db");
-          db.drop();
-          db.create();
-        }
-      } else {
-        System.out.println("OrientDB database not found, create fresh db");
-        db.create();
-      }
+    System.err.println("OrientDB loading database url = " + url);
 
-      System.out.println("OrientDB connection created with " + url);
-
-      dictionary = db.getMetadata().getIndexManager().getDictionary();
-      if (!db.getMetadata().getSchema().existsClass(CLASS))
-        db.getMetadata().getSchema().createClass(CLASS);
+    // If using a remote database, use the OServerAdmin interface to connect
+    if (url.startsWith(OEngineRemote.NAME)) {
+      isRemote = true;
+      if (remoteStorageType == null) {
+        throw new DBException("When connecting to a remote OrientDB instance, specify a database storage type (plocal or memory) with " + STORAGE_TYPE_PROPERTY);
+      }
 
-      db.declareIntent(new OIntentMassiveInsert());
+      try {
+        OServerAdmin server = new OServerAdmin(url).connect(user, password);
+
+        if (server.existsDatabase()) {
+          if (newdb && !dotransactions) {
+            System.err.println("OrientDB dropping and recreating fresh db on remote server.");
+            server.dropDatabase(remoteStorageType);
+            server.createDatabase(server.getURL(), ORIENTDB_DOCUMENT_TYPE, remoteStorageType);
+          }
+        } else {
+          System.err.println("OrientDB database not found, creating fresh db");
+          server.createDatabase(server.getURL(), ORIENTDB_DOCUMENT_TYPE, remoteStorageType);
+        }
 
-    } catch (Exception e1) {
-      System.err.println("Could not initialize OrientDB connection pool for Loader: " + e1.toString());
-      e1.printStackTrace();
-      return;
+        server.close();
+        db = new ODatabaseDocumentTx(url).open(user, password);
+      } catch (IOException | OException e) {
+        throw new DBException(String.format("Error interfacing with %s", url), e);
+      }
+    } else {
+      try {
+        db = new ODatabaseDocumentTx(url);
+        if (db.exists()) {
+          db.open(user, password);
+          if (newdb && !dotransactions) {
+            System.err.println("OrientDB dropping and recreating fresh db.");
+            db.drop();
+            db.create();
+          }
+        } else {
+          System.err.println("OrientDB database not found, creating fresh db");
+          db.create();
+        }
+      } catch (ODatabaseException e) {
+        throw new DBException(String.format("Error interfacing with %s", url), e);
+      }
     }
 
+    System.err.println("OrientDB connection created with " + url);
+    dictionary = db.getMetadata().getIndexManager().getDictionary();
+    if (!db.getMetadata().getSchema().existsClass(CLASS))
+      db.getMetadata().getSchema().createClass(CLASS);
+
+    // TODO: This is a transparent optimization that should be openned up to the user.
+    db.declareIntent(new OIntentMassiveInsert());
   }
 
   @Override
@@ -225,6 +262,11 @@ public class OrientDBClient extends DB {
    * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
    */
   public Status scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
+    if (isRemote) {
+      // Iterator methods needed for scanning are Unsupported for remote database connections.
+      return Status.NOT_IMPLEMENTED;
+    }
+
     try {
       int entrycount = 0;
       final OIndexCursor entries = dictionary.getIndex().iterateEntriesMajor(startkey, true, true);