From 2a580440b350ce36b09ec4f91591eca16a570a75 Mon Sep 17 00:00:00 2001 From: "Robert J. Moore" <Robert.J.Moore@allanbank.com> Date: Sat, 6 Jun 2015 00:23:38 -0400 Subject: [PATCH] Fix #277 - Switch from updateOne to replaceOne to match changed semantics in the MongoDB 3.0 driver. Created tests for the basic DB operations. Updated MongoDB Inc. Driver version to 3.0.2. --- mongodb/pom.xml | 171 +++++----- .../java/com/yahoo/ycsb/db/MongoDbClient.java | 10 +- .../yahoo/ycsb/db/AbstractDBTestCases.java | 299 ++++++++++++++++++ .../yahoo/ycsb/db/AsyncMongoDbClientTest.java | 77 +++++ .../com/yahoo/ycsb/db/MongoDbClientTest.java | 77 +++++ pom.xml | 2 +- 6 files changed, 549 insertions(+), 87 deletions(-) create mode 100644 mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java create mode 100644 mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java create mode 100644 mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java diff --git a/mongodb/pom.xml b/mongodb/pom.xml index 0a81b248..3038e17d 100644 --- a/mongodb/pom.xml +++ b/mongodb/pom.xml @@ -1,87 +1,94 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.ycsb</groupId> - <artifactId>root</artifactId> - <version>0.2.0-SNAPSHOT</version> - </parent> +<project + xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>root</artifactId> + <version>0.2.0-SNAPSHOT</version> + </parent> - <artifactId>mongodb-binding</artifactId> - <name>MongoDB Binding</name> - <packaging>jar</packaging> + <artifactId>mongodb-binding</artifactId> + <name>MongoDB Binding</name> + <packaging>jar</packaging> - <dependencies> - <dependency> - <groupId>org.mongodb</groupId> - <artifactId>mongo-java-driver</artifactId> - <version>${mongodb.version}</version> - </dependency> - <dependency> - <groupId>com.allanbank</groupId> - <artifactId>mongodb-async-driver</artifactId> - <version>${mongodb.async.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.ycsb</groupId> - <artifactId>core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - <version>1.1.2</version> - </dependency> + <dependencies> + <dependency> + <groupId>org.mongodb</groupId> + <artifactId>mongo-java-driver</artifactId> + <version>${mongodb.version}</version> + </dependency> + <dependency> + <groupId>com.allanbank</groupId> + <artifactId>mongodb-async-driver</artifactId> + <version>${mongodb.async.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.ycsb</groupId> + <artifactId>core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <version>1.1.2</version> + </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>4.12</version> - <scope>test</scope> - </dependency> - </dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>de.flapdoodle.embed</groupId> + <artifactId>de.flapdoodle.embed.mongo</artifactId> + <version>1.47.3</version> + <scope>test</scope> + </dependency> + </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <version>${maven.assembly.version}</version> - <configuration> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - <appendAssemblyId>false</appendAssemblyId> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> - - <repositories> - <repository> - <releases> - <enabled>true</enabled> - <updatePolicy>always</updatePolicy> - <checksumPolicy>warn</checksumPolicy> - </releases> - <snapshots> - <enabled>false</enabled> - <updatePolicy>never</updatePolicy> - <checksumPolicy>fail</checksumPolicy> - </snapshots> - <id>allanbank</id> - <name>Allanbank Releases</name> - <url>http://www.allanbank.com/repo/</url> - <layout>default</layout> - </repository> - </repositories> -</project> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>${maven.assembly.version}</version> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <appendAssemblyId>false</appendAssemblyId> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + <repositories> + <repository> + <releases> + <enabled>true</enabled> + <updatePolicy>always</updatePolicy> + <checksumPolicy>warn</checksumPolicy> + </releases> + <snapshots> + <enabled>false</enabled> + <updatePolicy>never</updatePolicy> + <checksumPolicy>fail</checksumPolicy> + </snapshots> + <id>allanbank</id> + <name>Allanbank Releases</name> + <url>http://www.allanbank.com/repo/</url> + <layout>default</layout> + </repository> + </repositories> + </project> diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java index 511efda2..5163f05c 100644 --- a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java +++ b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java @@ -20,6 +20,7 @@ import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; import org.bson.Document; +import org.bson.types.Binary; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; @@ -251,9 +252,10 @@ public class MongoDbClient extends DB { // Do a single upsert. if (batchSize <= 1) { UpdateResult result = collection.withWriteConcern(writeConcern) - .updateOne(criteria, toInsert, UPSERT); + .replaceOne(criteria, toInsert, UPSERT); if (result.getMatchedCount() > 0 - || result.getModifiedCount() > 0) { + || result.getModifiedCount() > 0 + || result.getUpsertedId() != null) { return 0; } System.err.println("Nothing inserted for key " + key); @@ -465,9 +467,9 @@ public class MongoDbClient extends DB { */ protected void fillMap(HashMap<String, ByteIterator> resultMap, Document obj) { for (Map.Entry<String, Object> entry : obj.entrySet()) { - if (entry.getValue() instanceof byte[]) { + if (entry.getValue() instanceof Binary) { resultMap.put(entry.getKey(), new ByteArrayByteIterator( - (byte[]) entry.getValue())); + ((Binary) entry.getValue()).getData())); } } } diff --git a/mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java b/mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java new file mode 100644 index 00000000..94d082b5 --- /dev/null +++ b/mongodb/src/test/java/com/yahoo/ycsb/db/AbstractDBTestCases.java @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package com.yahoo.ycsb.db; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNoException; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Set; +import java.util.Vector; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.yahoo.ycsb.ByteArrayByteIterator; +import com.yahoo.ycsb.ByteIterator; +import com.yahoo.ycsb.DB; + +import de.flapdoodle.embed.mongo.Command; +import de.flapdoodle.embed.mongo.MongodExecutable; +import de.flapdoodle.embed.mongo.MongodProcess; +import de.flapdoodle.embed.mongo.MongodStarter; +import de.flapdoodle.embed.mongo.config.ArtifactStoreBuilder; +import de.flapdoodle.embed.mongo.config.IMongodConfig; +import de.flapdoodle.embed.mongo.config.MongodConfigBuilder; +import de.flapdoodle.embed.mongo.config.Net; +import de.flapdoodle.embed.mongo.config.RuntimeConfigBuilder; +import de.flapdoodle.embed.mongo.distribution.Version; +import de.flapdoodle.embed.process.io.directories.FixedPath; +import de.flapdoodle.embed.process.runtime.Network; + +/** + * MongoDbClientTest provides runs the basic DB test cases. + */ +@SuppressWarnings("boxing") +public abstract class AbstractDBTestCases { + + /** The handle to the running server. */ + private static MongodExecutable ourMongodExecutable = null; + + /** The running Mongodb process. */ + private static MongodProcess ourMongod = null; + + /** The directory to download the MongoDB executables to. */ + private static final File TMP_DIR = new File("target/mongodb"); + + /** + * Start a test mongd instance. + */ + @BeforeClass + public static void setUpBeforeClass() { + TMP_DIR.mkdirs(); + + MongodStarter starter = MongodStarter + .getInstance(new RuntimeConfigBuilder() + .defaults(Command.MongoD) + .artifactStore( + new ArtifactStoreBuilder() + .defaults(Command.MongoD) + .useCache(false) + .tempDir( + new FixedPath(TMP_DIR + .getAbsolutePath()))) + .build()); + int port = 27017; + + try { + IMongodConfig mongodConfig = new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(port, Network.localhostIsIPv6())).build(); + + ourMongodExecutable = starter.prepare(mongodConfig); + ourMongod = ourMongodExecutable.start(); + } + catch (IOException error) { + assumeNoException(error); + } + } + + /** + * Stops the test server. + */ + @AfterClass + public static void tearDownAfterClass() { + if (ourMongod != null) { + ourMongod.stop(); + ourMongod = null; + } + if (ourMongodExecutable != null) { + ourMongodExecutable.stop(); + ourMongodExecutable = null; + } + } + + /** + * Test method for {@link MongoDbClient#insert}, {@link MongoDbClient#read}, + * {@link com.yahoo.ycsb.db.MongoDbClient#delete}. + */ + @Test + public void testInsertReadDelete() { + final DB client = getDB(); + + final String table = "test"; + final String id = "delete"; + + HashMap<String, ByteIterator> inserted = new HashMap<String, ByteIterator>(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); + int result = client.insert(table, id, inserted); + assertThat("Insert did not return success (0).", result, is(0)); + + HashMap<String, ByteIterator> read = new HashMap<String, ByteIterator>(); + Set<String> keys = Collections.singleton("a"); + result = client.read(table, id, keys, read); + assertThat("Read did not return success (0).", result, is(0)); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); + assertFalse(iter.hasNext()); + } + + result = client.delete(table, id); + assertThat("Delete did not return success (0).", result, is(0)); + + read.clear(); + result = client.read(table, id, null, read); + assertThat("Read, after delete, did not return not found (1).", result, + is(1)); + assertThat("Found the deleted fields.", read.size(), is(0)); + + result = client.delete(table, id); + assertThat("Delete did not return not found (1).", result, is(1)); + } + + /** + * Test method for {@link MongoDbClient#scan} . + */ + @Test + public void testScan() { + final DB client = getDB(); + + final String table = "test"; + + // Insert a bunch of documents. + for (int i = 0; i < 100; ++i) { + HashMap<String, ByteIterator> inserted = new HashMap<String, ByteIterator>(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { + (byte) (i & 0xFF), (byte) (i >> 8 & 0xFF), + (byte) (i >> 16 & 0xFF), (byte) (i >> 24 & 0xFF) })); + int result = client.insert(table, padded(i), inserted); + assertThat("Insert did not return success (0).", result, is(0)); + } + + Set<String> keys = Collections.singleton("a"); + Vector<HashMap<String, ByteIterator>> results = new Vector<HashMap<String, ByteIterator>>(); + int result = client.scan(table, "00050", 5, null, results); + assertThat("Read did not return success (0).", result, is(0)); + assertThat(results.size(), is(5)); + for (int i = 0; i < 5; ++i) { + HashMap<String, ByteIterator> read = results.get(i); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 8 & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 16 & 0xFF)))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), + is(Byte.valueOf((byte) ((i + 50) >> 24 & 0xFF)))); + assertFalse(iter.hasNext()); + } + } + } + + /** + * Creates a zero padded integer. + * + * @param i + * The integer to padd. + * @return The padded integer. + */ + private String padded(int i) { + String result = String.valueOf(i); + while (result.length() < 5) { + result = "0" + result; + } + return result; + } + + /** + * Gets the test DB. + * + * @return The test DB. + */ + protected abstract DB getDB(); + + /** + * Test method for + * {@link com.yahoo.ycsb.db.MongoDbClient#update(java.lang.String, java.lang.String, java.util.HashMap)} + * . + */ + @Test + public void testInsertReadUpdate() { + DB client = getDB(); + + final String table = "test"; + final String id = "update"; + + HashMap<String, ByteIterator> inserted = new HashMap<String, ByteIterator>(); + inserted.put("a", new ByteArrayByteIterator(new byte[] { 1, 2, 3, 4 })); + int result = client.insert(table, id, inserted); + assertThat("Insert did not return success (0).", result, is(0)); + + HashMap<String, ByteIterator> read = new HashMap<String, ByteIterator>(); + Set<String> keys = Collections.singleton("a"); + result = client.read(table, id, keys, read); + assertThat("Read did not return success (0).", result, is(0)); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 1))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 2))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 3))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 4))); + assertFalse(iter.hasNext()); + } + + HashMap<String, ByteIterator> updated = new HashMap<String, ByteIterator>(); + updated.put("a", new ByteArrayByteIterator(new byte[] { 5, 6, 7, 8 })); + result = client.update(table, id, updated); + assertThat("Update did not return success (0).", result, is(0)); + + read.clear(); + result = client.read(table, id, null, read); + assertThat("Read, after update, did not return success (0).", result, + is(0)); + for (String key : keys) { + ByteIterator iter = read.get(key); + + assertThat("Did not read the inserted field: " + key, iter, + notNullValue()); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 5))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 6))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 7))); + assertTrue(iter.hasNext()); + assertThat(iter.nextByte(), is(Byte.valueOf((byte) 8))); + assertFalse(iter.hasNext()); + } + } + +} \ No newline at end of file diff --git a/mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java b/mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java new file mode 100644 index 00000000..f5ee7b65 --- /dev/null +++ b/mongodb/src/test/java/com/yahoo/ycsb/db/AsyncMongoDbClientTest.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package com.yahoo.ycsb.db; + +import static org.junit.Assume.assumeNoException; + +import java.util.Properties; + +import org.junit.After; +import org.junit.Before; + +import com.yahoo.ycsb.DB; + +/** + * AsyncMongoDbClientTest provides runs the basic workload operations. + */ +public class AsyncMongoDbClientTest extends AbstractDBTestCases { + + /** The client to use. */ + private AsyncMongoDbClient myClient = null; + + /** + * Start a test client. + */ + @Before + public void setUp() { + myClient = new AsyncMongoDbClient(); + myClient.setProperties(new Properties()); + try { + myClient.init(); + } + catch (Exception error) { + assumeNoException(error); + } + } + + /** + * Stops the test client. + */ + @After + public void tearDown() { + try { + myClient.cleanup(); + } + catch (Exception error) { + // Ignore. + } + finally { + myClient = null; + } + } + + /** + * {@inheritDoc} + * <p> + * Overriden to return the {@link AsyncMongoDbClient}. + * </p> + */ + @Override + protected DB getDB() { + return myClient; + } +} diff --git a/mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java b/mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java new file mode 100644 index 00000000..515fb802 --- /dev/null +++ b/mongodb/src/test/java/com/yahoo/ycsb/db/MongoDbClientTest.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2014, Yahoo!, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package com.yahoo.ycsb.db; + +import static org.junit.Assume.assumeNoException; + +import java.util.Properties; + +import org.junit.After; +import org.junit.Before; + +import com.yahoo.ycsb.DB; + +/** + * MongoDbClientTest provides runs the basic workload operations. + */ +public class MongoDbClientTest extends AbstractDBTestCases { + + /** The client to use. */ + private MongoDbClient myClient = null; + + /** + * Start a test client. + */ + @Before + public void setUp() { + myClient = new MongoDbClient(); + myClient.setProperties(new Properties()); + try { + myClient.init(); + } + catch (Exception error) { + assumeNoException(error); + } + } + + /** + * Stops the test client. + */ + @After + public void tearDown() { + try { + myClient.cleanup(); + } + catch (Exception error) { + // Ignore. + } + finally { + myClient = null; + } + } + + /** + * {@inheritDoc} + * <p> + * Overriden to return the {@link MongoDbClient}. + * </p> + */ + @Override + protected DB getDB() { + return myClient; + } +} diff --git a/pom.xml b/pom.xml index 9d430b7a..1b9f12cc 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ <infinispan.version>7.1.0.CR1</infinispan.version> <openjpa.jdbc.version>2.1.1</openjpa.jdbc.version> <!--<mapkeeper.version>1.0</mapkeeper.version>--> - <mongodb.version>3.0.1</mongodb.version> + <mongodb.version>3.0.2</mongodb.version> <mongodb.async.version>2.0.1</mongodb.async.version> <orientdb.version>1.0.1</orientdb.version> <redis.version>2.0.0</redis.version> -- GitLab