From bf4c4e579cec49c03cc2ebf507d8876dc16e6bd4 Mon Sep 17 00:00:00 2001
From: bigbes <bigbes@gmail.com>
Date: Thu, 11 Jun 2015 16:29:37 +0300
Subject: [PATCH] Add Tarantool database
http://tarantool.org and https://github.com/tarantool/tarantool
In "mail.ru" we wrote and widely use Tarantool key-value database.
It's key properties include:
* Defferent index types with iterators:
- HASH (the fastest)
- TREE (range and ordered retreival)
- BITSET (bit mask search)
- RTREE (geo search)
* multipart keys for HASH and TREE indexes
* Data persistence with by Write Ahead Log (WAL) and snapshots.
* asynchronous master-master replication, hot standby.
* coroutines and async. IO are used to implement high-performance lock-free access to data.
- socket-io/file-io with yeilds from lua
* stored procedures in Lua (Using LuaJIT)
* supports plugins written on C/C++ (Have two basic plugins for working with MySQL and PostgreSQL)
* Authentication and access control
Move 'distribution target' to the end (it's needed for .jar to be in the .tar.gz)
---
bin/ycsb | 3 +-
pom.xml | 4 +-
tarantool/README.md | 62 +++++++
tarantool/config/tarantool-hash.lua | 12 ++
tarantool/config/tarantool-tree.lua | 12 ++
tarantool/pom.xml | 59 ++++++
.../com/yahoo/ycsb/db/TarantoolClient.java | 168 ++++++++++++++++++
7 files changed, 318 insertions(+), 2 deletions(-)
create mode 100644 tarantool/README.md
create mode 100644 tarantool/config/tarantool-hash.lua
create mode 100644 tarantool/config/tarantool-tree.lua
create mode 100644 tarantool/pom.xml
create mode 100644 tarantool/src/main/java/com/yahoo/ycsb/db/TarantoolClient.java
diff --git a/bin/ycsb b/bin/ycsb
index 5494c767..9c15be80 100755
--- a/bin/ycsb
+++ b/bin/ycsb
@@ -33,6 +33,7 @@ DATABASES = {
"cassandra-8" : "com.yahoo.ycsb.db.CassandraClient8",
"cassandra-10" : "com.yahoo.ycsb.db.CassandraClient10",
"cassandra-cql": "com.yahoo.ycsb.db.CassandraCQLClient",
+ "couchbase" : "com.yahoo.ycsb.db.CouchbaseClient",
"dynamodb" : "com.yahoo.ycsb.db.DynamoDBClient",
"elasticsearch": "com.yahoo.ycsb.db.ElasticSearchClient",
"gemfire" : "com.yahoo.ycsb.db.GemFireClient",
@@ -47,8 +48,8 @@ DATABASES = {
"nosqldb" : "com.yahoo.ycsb.db.NoSqlDbClient",
"orientdb" : "com.yahoo.ycsb.db.OrientDBClient",
"redis" : "com.yahoo.ycsb.db.RedisClient",
+ "tarantool" : "com.yahoo.ycsb.db.TarantoolClient"
"voldemort" : "com.yahoo.ycsb.db.VoldemortClient",
- "couchbase" : "com.yahoo.ycsb.db.CouchbaseClient"
}
OPTIONS = {
diff --git a/pom.xml b/pom.xml
index c0bf48cf..930f0672 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,6 +67,7 @@
<thrift.version>0.8.0</thrift.version>
<hypertable.version>0.9.5.6</hypertable.version>
<couchbase.version>1.1.8</couchbase.version>
+ <tarantool.version>1.6.1-SNAPSHOT</tarantool.version>
</properties>
<modules>
@@ -85,10 +86,11 @@
<module>orientdb</module>
<module>redis</module>
<module>voldemort</module>
- <module>distribution</module>
<!--<module>mapkeeper</module>-->
<!--module>nosqldb</module-->
<module>couchbase</module>
+ <module>tarantool</module>
+ <module>distribution</module>
</modules>
<build>
diff --git a/tarantool/README.md b/tarantool/README.md
new file mode 100644
index 00000000..419c792a
--- /dev/null
+++ b/tarantool/README.md
@@ -0,0 +1,62 @@
+# Tarantool
+
+## Introduction
+
+Tarantool is a NoSQL In-Memory database.
+It's distributed under BSD licence and is hosted on [github][tarantool-github].
+
+Tarantool features:
+
+* Defferent index types with iterators:
+ - HASH (the fastest)
+ - TREE (range and ordered retreival)
+ - BITSET (bit mask search)
+ - RTREE (geo search)
+* multipart keys for HASH and TREE indexes
+* Data persistence with by Write Ahead Log (WAL) and snapshots.
+* asynchronous master-master replication, hot standby.
+* coroutines and async. IO are used to implement high-performance lock-free access to data.
+ - socket-io/file-io with yeilds from lua
+* stored procedures in Lua (Using LuaJIT)
+* supports plugins written on C/C++ (Have two basic plugins for working with MySQL and PostgreSQL)
+* Authentication and access control
+
+## Quick start
+
+This section descrives how to run YCSB against a local Tarantool instance
+
+### 1. Start Tarantool
+
+First, clone Tarantool from it's own git repo and build it (described in our [README.md][tarantool-readme]):
+
+ cp YCSB/tarantool/config/tarantool-tree.lua <vardir>/tarantool.lua
+ cp TNT/src/box/tarantool_box <vardir>
+ cd <vardir>
+ ./tarantool_box tarantool.lua
+
+OR you can simply download ans install a binary package for your GNU/Linux or BSD distro from http://tarantool.org/download.html
+
+### 2. Run YCSB
+
+Now you are ready to run! First, load the data:
+
+ ./bin/ycsb load tarantool -s -P workloads/workloada
+
+Then, run the workload:
+
+ ./bin/ycsb run tarantool -s -P workloads/workloada
+
+See the next section for the list of configuration parameters for Tarantool.
+
+## Tarantool Configuration Parameters
+
+#### 'tarantool.host' (default : 'localhost')
+Which host YCSB must use for connection with Tarantool
+#### 'tarantool.port' (default : 3301)
+Which port YCSB must use for connection with Tarantool
+#### 'tarantool.space' (default : 1024)
+ (possible values: 0 .. 255)
+Which space YCSB must use for benchmark Tarantool
+
+[tarantool-github]: https://github.com/tarantool/tarantool/
+[tarantool-readme]: https://github.com/tarantool/tarantool/blob/master/README.md
diff --git a/tarantool/config/tarantool-hash.lua b/tarantool/config/tarantool-hash.lua
new file mode 100644
index 00000000..256d93c6
--- /dev/null
+++ b/tarantool/config/tarantool-hash.lua
@@ -0,0 +1,12 @@
+box.cfg {
+ listen=3303,
+ logger="tarantool.log",
+ log_level=5,
+ logger_nonblock=true,
+ wal_mode="none",
+ pid_file="tarantool.pid"
+}
+
+box.schema.space.create("ycsb", {id = 1024})
+box.space.ycsb:create_index('primary', {type = 'hash', parts = {1, 'STR'}})
+box.schema.user.grant('guest', 'read,write,execute', 'universe')
diff --git a/tarantool/config/tarantool-tree.lua b/tarantool/config/tarantool-tree.lua
new file mode 100644
index 00000000..1e2a90e3
--- /dev/null
+++ b/tarantool/config/tarantool-tree.lua
@@ -0,0 +1,12 @@
+box.cfg {
+ listen=3303,
+ logger="tarantool.log",
+ log_level=5,
+ logger_nonblock=true,
+ wal_mode="none",
+ pid_file="tarantool.pid"
+}
+
+box.schema.space.create("ycsb", {id = 1024})
+box.space.ycsb:create_index('primary', {type = 'tree', parts = {1, 'STR'}})
+box.schema.user.grant('guest', 'read,write,execute', 'universe')
diff --git a/tarantool/pom.xml b/tarantool/pom.xml
new file mode 100644
index 00000000..90514f3e
--- /dev/null
+++ b/tarantool/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>root</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>tarantool-binding</artifactId>
+ <name>Tarantool DB Binding</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.tarantool</groupId>
+ <artifactId>connector</artifactId>
+ <version>${tarantool.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <repositories>
+ <repository>
+ <id>dgreenru-repo</id>
+ <name>dgreenru repository</name>
+ <url>http://dgreenru.github.com/repo/</url>
+ </repository>
+ </repositories>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/tarantool/src/main/java/com/yahoo/ycsb/db/TarantoolClient.java b/tarantool/src/main/java/com/yahoo/ycsb/db/TarantoolClient.java
new file mode 100644
index 00000000..0bf54d88
--- /dev/null
+++ b/tarantool/src/main/java/com/yahoo/ycsb/db/TarantoolClient.java
@@ -0,0 +1,168 @@
+package com.yahoo.ycsb.db;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.Arrays;
+import java.util.Vector;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.HashMap;
+import java.util.Properties;
+
+import java.io.IOException;
+
+import org.tarantool.TarantoolConnection16;
+import org.tarantool.TarantoolConnection16Impl;
+import org.tarantool.TarantoolException;
+import org.tarantool.CommunicationException;
+
+import com.yahoo.ycsb.DB;
+import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.StringByteIterator;
+
+public class TarantoolClient extends DB {
+
+ public static final String HOST_PROPERTY = "tarantool.host";
+ public static final String PORT_PROPERTY = "tarantool.port";
+ public static final String SPACE_PROPERTY = "tarantool.space";
+
+ public static final String DEFAULT_HOST = "localhost";
+ public static final int DEFAULT_PORT = 3301;
+ public static final int DEFAULT_SPACE = 1024;
+
+ private static final Logger log = Logger.getLogger(TarantoolClient.class.getName());
+ private TarantoolConnection16Impl connection;
+ private int spaceNo;
+
+ public void init() throws DBException {
+ Properties props = getProperties();
+
+ int port = DEFAULT_PORT;
+ String portString = props.getProperty(PORT_PROPERTY);
+ if (portString != null) {
+ port = Integer.parseInt(portString);
+ }
+
+ String host = props.getProperty(HOST_PROPERTY);
+ if (host == null) {
+ host = DEFAULT_HOST;
+ }
+
+ spaceNo = DEFAULT_SPACE;
+ String spaceString = props.getProperty(SPACE_PROPERTY);
+ if (spaceString != null) {
+ spaceNo = Integer.parseInt(spaceString);
+ }
+
+ try {
+ this.connection = new TarantoolConnection16Impl(host, port);
+ }
+ catch (Exception exc) {
+ System.err.println("Can't init Tarantool connection:" + exc.toString());
+ exc.printStackTrace();
+ return;
+ }
+ }
+
+ public void cleanup() throws DBException{
+ this.connection.close();
+ }
+
+ @Override
+ public int insert(String table, String key, HashMap<String, ByteIterator> values) {
+ int j = 0;
+ String[] tuple = new String[1 + 2 * values.size()];
+ tuple[0] = key;
+ for (Map.Entry<String, ByteIterator> i: values.entrySet()) {
+ tuple[j + 1] = i.getKey();
+ tuple[j + 2] = i.getValue().toString();
+ j += 2;
+ }
+ try {
+ this.connection.insert(this.spaceNo, tuple);
+ } catch (TarantoolException e) {
+ e.printStackTrace();
+ return 1;
+ }
+ return 0;
+ }
+
+ private HashMap<String, ByteIterator> tuple_convert_filter (List<String> input,
+ Set<String> fields) {
+ HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
+ if (input == null)
+ return result;
+ for (int i = 1; i < input.toArray().length; i += 2)
+ if (fields == null || fields.contains(input.get(i)))
+ result.put(input.get(i), new StringByteIterator(input.get(i+1)));
+ return result;
+ }
+
+ @Override
+ public int read(String table, String key, Set<String> fields,
+ HashMap<String, ByteIterator> result) {
+ try {
+ List<String> response;
+ response = this.connection.select(this.spaceNo, 0, Arrays.asList(key), 0, 1, 0);
+ result = tuple_convert_filter(response, fields);
+ return 0;
+ } catch (TarantoolException e) {
+ e.printStackTrace();
+ return 1;
+ } catch (IndexOutOfBoundsException e) {
+ return 1;
+ }
+ }
+
+ @Override
+ public int scan(String table, String startkey,
+ int recordcount, Set<String> fields,
+ Vector<HashMap<String, ByteIterator>> result) {
+ List<String> response;
+ try {
+ response = this.connection.select(this.spaceNo, 0, Arrays.asList(startkey), 0, recordcount, 6);
+ } catch (TarantoolException e) {
+ e.printStackTrace();
+ return 1;
+ }
+ HashMap<String, ByteIterator> temp = tuple_convert_filter(response, fields);
+ if (!temp.isEmpty())
+ result.add((HashMap<String, ByteIterator>) temp.clone());
+ return 0;
+ }
+
+ @Override
+ public int delete(String table, String key) {
+ try {
+ this.connection.delete(this.spaceNo, Arrays.asList(key));
+ } catch (TarantoolException e) {
+ e.printStackTrace();
+ return 1;
+ } catch (IndexOutOfBoundsException e) {
+ return 1;
+ }
+ return 0;
+ }
+ @Override
+ public int update(String table, String key,
+ HashMap<String, ByteIterator> values) {
+ int j = 0;
+ String[] tuple = new String[1 + 2 * values.size()];
+ tuple[0] = key;
+ for (Map.Entry<String, ByteIterator> i: values.entrySet()) {
+ tuple[j + 1] = i.getKey();
+ tuple[j + 2] = i.getValue().toString();
+ j += 2;
+ }
+ try {
+ this.connection.replace(this.spaceNo, tuple);
+ } catch (TarantoolException e) {
+ e.printStackTrace();
+ return 1;
+ }
+ return 0;
+
+ }
+}
--
GitLab