Merge pull request #583 from risdenk/solr-test

[solr] Added tests for Solr binding

solr/pom.xml

@@ -37,14 +37,24 @@ LICENSE file.
 			<artifactId>core</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
-		
 		</dependency>
 		<dependency>
 			<groupId>org.apache.solr</groupId>
 			<artifactId>solr-solrj</artifactId>
 			<version>${solr.version}</version>
 		</dependency>
-		
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>1.7.10</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.solr</groupId>
+      <artifactId>solr-test-framework</artifactId>
+      <version>${solr.version}</version>
+      <scope>test</scope>
+    </dependency>
 	</dependencies>
 	<build>
 		<plugins>

solr/src/main/resources/log4j.properties

+# Root logger option
+log4j.rootLogger=INFO, stderr
+
+# Direct log messages to stderr
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

solr/src/test/java/com/yahoo/ycsb/db/SolrClientBaseTest.java

+/**
+ * Copyright (c) 2016 YCSB contributors. All rights reserved.
+ * <p/>
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.db;
+
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.DB;
+import com.yahoo.ycsb.Status;
+import com.yahoo.ycsb.StringByteIterator;
+import org.apache.solr.client.solrj.embedded.JettyConfig;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.common.util.NamedList;
+import org.junit.*;
+
+import java.io.File;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public abstract class SolrClientBaseTest {
+
+  protected static MiniSolrCloudCluster miniSolrCloudCluster;
+  private DB instance;
+  private final static HashMap<String, ByteIterator> MOCK_DATA;
+  protected final static String MOCK_TABLE = "ycsb";
+  private final static String MOCK_KEY0 = "0";
+  private final static String MOCK_KEY1 = "1";
+  private final static int NUM_RECORDS = 10;
+
+  static {
+    MOCK_DATA = new HashMap<>(NUM_RECORDS);
+    for (int i = 0; i < NUM_RECORDS; i++) {
+      MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
+    }
+  }
+
+  @BeforeClass
+  public static void onlyOnce() throws Exception {
+    Path miniSolrCloudClusterTempDirectory = Files.createTempDirectory("miniSolrCloudCluster");
+    miniSolrCloudClusterTempDirectory.toFile().deleteOnExit();
+    miniSolrCloudCluster = new MiniSolrCloudCluster(1, miniSolrCloudClusterTempDirectory, JettyConfig.builder().build());
+
+    // Upload Solr configuration
+    URL configDir = SolrClientBaseTest.class.getClassLoader().getResource("solr_config");
+    assertNotNull(configDir);
+    miniSolrCloudCluster.uploadConfigDir(new File(configDir.toURI()), MOCK_TABLE);
+  }
+
+  @AfterClass
+  public static void destroy() throws Exception {
+    if(miniSolrCloudCluster != null) {
+      miniSolrCloudCluster.shutdown();
+    }
+  }
+
+  @Before
+  public void setup() throws Exception {
+    NamedList<Object> namedList = miniSolrCloudCluster.createCollection(MOCK_TABLE, 1, 1, MOCK_TABLE, null);
+    assertEquals(namedList.indexOf("success", 0), 1);
+    Thread.sleep(1000);
+
+    instance = getDB();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if(miniSolrCloudCluster != null) {
+      NamedList<Object> namedList = miniSolrCloudCluster.deleteCollection(MOCK_TABLE);
+      assertEquals(namedList.indexOf("success", 0), 1);
+      Thread.sleep(1000);
+    }
+  }
+
+  @Test
+  public void testInsert() throws Exception {
+    Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA);
+    assertEquals(Status.OK, result);
+  }
+
+  @Test
+  public void testDelete() throws Exception {
+    Status result = instance.delete(MOCK_TABLE, MOCK_KEY1);
+    assertEquals(Status.OK, result);
+  }
+
+  @Test
+  public void testRead() throws Exception {
+    Set<String> fields = MOCK_DATA.keySet();
+    HashMap<String, ByteIterator> resultParam = new HashMap<>(NUM_RECORDS);
+    Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam);
+    assertEquals(Status.OK, result);
+  }
+
+  @Test
+  public void testUpdate() throws Exception {
+    HashMap<String, ByteIterator> newValues = new HashMap<>(NUM_RECORDS);
+
+    for (int i = 0; i < NUM_RECORDS; i++) {
+      newValues.put("field" + i, new StringByteIterator("newvalue" + i));
+    }
+
+    Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues);
+    assertEquals(Status.OK, result);
+
+    //validate that the values changed
+    HashMap<String, ByteIterator> resultParam = new HashMap<>(NUM_RECORDS);
+    instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);
+
+    for (int i = 0; i < NUM_RECORDS; i++) {
+      assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
+    }
+  }
+
+  @Test
+  public void testScan() throws Exception {
+    Set<String> fields = MOCK_DATA.keySet();
+    Vector<HashMap<String, ByteIterator>> resultParam = new Vector<>(NUM_RECORDS);
+    Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, NUM_RECORDS, fields, resultParam);
+    assertEquals(Status.OK, result);
+  }
+
+  /**
+   * Gets the test DB.
+   *
+   * @return The test DB.
+   */
+  protected DB getDB() {
+    return getDB(new Properties());
+  }
+
+  /**
+   * Gets the test DB.
+   *
+   * @param props
+   *    Properties to pass to the client.
+   * @return The test DB.
+   */
+  protected abstract DB getDB(Properties props);
+}

solr/src/test/java/com/yahoo/ycsb/db/SolrClientCloudTest.java

+/**
+ * Copyright (c) 2016 YCSB contributors. All rights reserved.
+ * <p/>
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb.db;
+
+import com.yahoo.ycsb.DB;
+import org.junit.After;
+
+import java.util.Properties;
+
+import static org.junit.Assume.assumeNoException;
+
+public class SolrClientCloudTest extends SolrClientBaseTest {
+
+  private SolrClient instance;
+
+  @After
+  public void tearDown() throws Exception {
+    try {
+      if(instance != null) {
+        instance.cleanup();
+      }
+    } finally {
+      super.tearDown();
+    }
+  }
+
+  @Override
+  protected DB getDB(Properties props) {
+    instance = new SolrClient();
+
+    props.setProperty("solr.cloud", "true");
+    props.setProperty("solr.zookeeper.hosts", miniSolrCloudCluster.getSolrClient().getZkHost());
+
+    instance.setProperties(props);
+    try {
+      instance.init();
+    } catch (Exception error) {
+      assumeNoException(error);
+    }
+    return instance;
+  }
+}

solr/src/test/java/com/yahoo/ycsb/db/SolrClientTest.java

+/**
+ * Copyright (c) 2016 YCSB contributors. All rights reserved.
+ * <p/>
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb.db;
+
+import com.yahoo.ycsb.DB;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.junit.After;
+
+import java.net.URL;
+import java.util.Properties;
+import java.util.Set;
+
+import static org.junit.Assume.assumeNoException;
+
+public class SolrClientTest extends SolrClientBaseTest {
+
+  private SolrClient instance;
+
+  @After
+  public void tearDown() throws Exception {
+    try {
+      if(instance != null) {
+        instance.cleanup();
+      }
+    } finally {
+      super.tearDown();
+    }
+  }
+
+  @Override
+  protected DB getDB(Properties props) {
+    instance = new SolrClient();
+
+    // Use the first Solr server in the cluster.
+    // Doesn't matter if there are more since requests will be forwarded properly by Solr.
+    JettySolrRunner jettySolrRunner = miniSolrCloudCluster.getJettySolrRunners().get(0);
+    String solrBaseUrl = String.format("http://localhost:%s%s", jettySolrRunner.getLocalPort(),
+      jettySolrRunner.getBaseUrl());
+
+    props.setProperty("solr.base.url", solrBaseUrl);
+    instance.setProperties(props);
+
+    try {
+      instance.init();
+    } catch (Exception error) {
+      assumeNoException(error);
+    }
+    return instance;
+  }
+}

solr/src/test/resources/solr_config/schema.xml

+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Copied from Apache Solr 5.4.0 solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml
+ Modified to only required types and fields for YCSB testing.
+-->
+<schema name="test" version="1.6">
+  <types>
+    <fieldType name="int" docValues="true" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+
+    <fieldtype name="text" class="solr.TextField">
+      <analyzer>
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.StandardFilterFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.StopFilterFactory"/>
+        <filter class="solr.PorterStemFilterFactory"/>
+      </analyzer>
+    </fieldtype>
+  </types>
+
+  <fields>
+    <field name="id" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
+    <field name="text" type="text" indexed="true" stored="false"/>
+
+    <field name="_version_" type="long" indexed="true" stored="true"/>
+
+    <field name="field0" type="text" indexed="true" stored="true"/>
+    <field name="field1" type="text" indexed="true" stored="true"/>
+    <field name="field2" type="text" indexed="true" stored="true"/>
+    <field name="field3" type="text" indexed="true" stored="true"/>
+    <field name="field4" type="text" indexed="true" stored="true"/>
+    <field name="field5" type="text" indexed="true" stored="true"/>
+    <field name="field6" type="text" indexed="true" stored="true"/>
+    <field name="field7" type="text" indexed="true" stored="true"/>
+    <field name="field8" type="text" indexed="true" stored="true"/>
+    <field name="field9" type="text" indexed="true" stored="true"/>
+  </fields>
+
+  <defaultSearchField>text</defaultSearchField>
+  <uniqueKey>id</uniqueKey>
+</schema>

solr/src/test/resources/solr_config/solrconfig.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Copied from Apache Solr 5.4.0 solr/src/test/resources/solr_config/solrconfig.xml
+-->
+
+<!--
+ This is a stripped down config file used for a simple example...  
+ It is *not* a good example to work from. 
+-->
+<config>
+  <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+  <indexConfig>
+    <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+  </indexConfig>
+   <dataDir>${solr.data.dir:}</dataDir>
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
+
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <updateLog>
+      <str name="dir">${solr.data.dir:}</str>
+    </updateLog>
+  </updateHandler>
+
+  <requestDispatcher handleSelect="true" >
+    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
+  </requestDispatcher>
+
+  <requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />
+
+  <requestHandler name="/admin/ping" class="solr.PingRequestHandler">
+    <lst name="invariants">
+      <str name="q">*:*</str>
+    </lst>
+    <lst name="defaults">
+       <str name="echoParams">all</str>
+    </lst>
+    <str name="healthcheckFile">server-enabled.txt</str>
+  </requestHandler>
+
+  <!-- config for the admin interface --> 
+  <admin>
+    <defaultQuery>solr</defaultQuery>
+  </admin>
+
+</config>
+