core/org.eclipse.smila.solr.test/code/src/org/eclipse/smila/solr/SolrIndexPipelet_Index_Test.java

/*******************************************************************************
 * Copyright (c) 2008 empolis GmbH and brox IT Solutions GmbH. All rights reserved. This program and the accompanying
 * materials are made available under the terms of the Eclipse Public License v1.0 which accompanies this distribution,
 * and is available at http://www.eclipse.org/legal/epl-v10.html
 * 
 * Contributors: Thomas Menzel (brox IT Solution GmbH) - initial creator
 *******************************************************************************/
package org.eclipse.smila.solr;

import static org.apache.commons.lang.StringUtils.join;

import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.eclipse.smila.blackboard.BlackboardAccessException;
import org.eclipse.smila.blackboard.impl.BlackboardFactoryImpl;
import org.eclipse.smila.datamodel.AnyMap;
import org.eclipse.smila.datamodel.AnySeq;
import org.eclipse.smila.datamodel.DataFactoryCreator;
import org.eclipse.smila.datamodel.Record;
import org.eclipse.smila.datamodel.xml.XmlSerializationUtils;
import org.eclipse.smila.solr.index.SolrIndexPipelet;
import org.eclipse.smila.solr.search.SolrSearchPipelet;
import org.eclipse.smila.solr.util.SolrQueryUtils;

/**
 * this class uses the embedded solr to do its tests and hence needs the .solr config folder. since we want to test the
 * {@link SolrIndexPipelet} indexing is done thru this class while the check is done with solr classes directly and not
 * the {@link SolrSearchPipelet} to not a) duplicate/mingle the test units and also to have this as a
 * reference/regression against solr.
 * 
 * @author tmenzel
 */
public class SolrIndexPipelet_Index_Test extends SolrPipeletTestBase {

  /** value of the data to be indexed in that field. */
  private String _fieldValue;

  /**
   * name of the solr doc field containing the path. It is intentionally diff from REC_FIELD to show that the mappings
   * works.
   */
  private static final String SOLR_DOC_FIELD = "Path";

  /** name of the rec field to be put into SOLR_DOC_FIELD */
  private static final String REC_FIELD = "Folder";

  /** The _pipelet. */
  SolrIndexPipelet _pipelet = new SolrIndexPipelet();

  /**
   */
  @Override
  protected void setUp() throws Exception {
    super.setUp();
    _fieldValue = "c:\\data\\folder with spaces/mixed-delim$folder/and:other^sepcial&chars";
    // <rec:Val key="ExecutionMode">ADD</rec:Val>
    // <rec:Val key="CoreName">DefaultCore</rec:Val>
    // <rec:Seq key="CoreFields">
    // <rec:Map>
    // <rec:Val key="FieldName">Folder</rec:Val>
    // <rec:Val key="RecSourceName">Path</rec:Val>
    // <rec:Val key="RecSourceType">ATTRIBUTE</rec:Val>
    // </rec:Map>
    _config.put(SolrConstants.EXECUTION_MODE, SolrConstants.ExecutionMode.ADD.name());
    final AnySeq configCoreFields = _config.getSeq(SolrConstants.CORE_FIELDS, true);
    final AnyMap fieldConfig = DataFactoryCreator.createDefaultFactory().createAnyMap();
    configCoreFields.add(fieldConfig);
    fieldConfig.put(SolrConstants.CORE_FIELD_NAME, SOLR_DOC_FIELD);
    fieldConfig.put(SolrConstants.SOURCE_NAME, REC_FIELD);
    fieldConfig.put(SolrConstants.SOURCE_TYPE, SolrConstants.AttributeOrAttachment.ATTRIBUTE.name());

    _blackboard = new BlackboardFactoryImpl().createTransientBlackboard();
    _record = createRecord();

    _solrServer = Activator.getInstance().getSolrManager().getSolrServer(SolrConstants.DEFAULT_CORE);
    // clear the index in advance before the (next) test
  }

  /**
   * creates and adds the record to the BB of this test, but not to solr.
   */
  private Record createRecord(final String... idModifiers) throws BlackboardAccessException {
    final Record record = DataFactoryCreator.createDefaultFactory().createRecord();

    final String id = "test:\" " + getName() + "-" + join(idModifiers);
    record.setId(id);
    record.getMetadata().put(REC_FIELD, _fieldValue);
    _blackboard.setRecord(record);
    return record;

  }

  /**
   * tests that a document is actually added
   */
  public void test_AddAndSearch() throws Exception {
    indexAndCommit();

    final String queryString = SolrQueryUtils.toConstQueryOnField("_recordid", _record.getId());
    _log.debug(queryString);

    final QueryResponse response = search(queryString);

    assertFalse(SolrQueryUtils.responseStatusIsError(response));
    final SolrDocument solrDocument = response.getResults().get(0);
    assertEquals(_fieldValue, solrDocument.getFieldValue(SOLR_DOC_FIELD));
  }

  /**
   * tests that umlauts are working properly.
   */
  public void test_AddAndSearch_Umlauts() throws Exception {

    // aendern
    _fieldValue = "\u00e4ndern";
    _record.getMetadata().put(REC_FIELD, _fieldValue);
    indexAndCommit();

    final String queryString = SolrQueryUtils.toConstQueryOnField(SOLR_DOC_FIELD, _fieldValue);
    _log.debug(queryString);

    final QueryResponse response = search(queryString);

    assertFalse(SolrQueryUtils.responseStatusIsError(response));
    final SolrDocument solrDocument = response.getResults().get(0);
    assertEquals(_fieldValue, solrDocument.getFieldValue(SOLR_DOC_FIELD));
  }

  private QueryResponse search(final String queryString) throws SolrServerException {
    final SolrQuery solrQuery = new SolrQuery();
    solrQuery.setQuery(queryString);
    solrQuery.addField(SolrConstants.CORE_FIELD_SCORE);
    solrQuery.addField(SolrConstants.CORE_FIELD_ID);
    solrQuery.addField(SOLR_DOC_FIELD);
    _log.debug(solrQuery);
    final QueryResponse response = _solrServer.query(solrQuery);
    return response;
  }

  private void indexAndCommit() throws Exception {
    indexAndCommit(_record.getId());
  }

  private void indexAndCommit(final String id) throws Exception {
    _pipelet.configure(_config);
    _pipelet.process(_blackboard, new String[] { id });
    _solrServer.commit(true, true);
  }

  /**
   * indexes 2 indentical records (except id) but boosts the 2nd. the result should contain the 2nd @ pos 1 and its
   * score should be greater!
   */
  public void test_DocumentBoosting() throws Exception {
    final String id0 = _record.getId();
    indexAndCommit();
    _log.debug(XmlSerializationUtils.serialize2string(_record));

    _record = createRecord("1");
    final String id1 = _record.getId();
    _record.getMetadata().put(SolrConstants.DOC_BOOST, 2.0);
    indexAndCommit();

    final QueryResponse search = search(SolrQueryUtils.toConstQueryOnField(SOLR_DOC_FIELD, _fieldValue));

    final SolrDocumentList results = search.getResults();
    assertEquals(2, results.getNumFound());

    // since id1 is boosted it should be 1st
    assertEquals(id1, results.get(0).getFieldValue(SolrConstants.CORE_FIELD_ID));
    assertEquals(id0, results.get(1).getFieldValue(SolrConstants.CORE_FIELD_ID));

    // score: id0 < id1
    final float score0 = (Float) results.get(0).getFieldValue(SolrConstants.CORE_FIELD_SCORE);
    final float score1 = (Float) results.get(1).getFieldValue(SolrConstants.CORE_FIELD_SCORE);
    _log.debug(score0);
    _log.debug(score1);
    assertTrue(score0 > score1);
    assertEquals(score0, score1 * 2); // this might fail in certain circumstances but is the stricted test

  }

  /**
   * indexes 2 indentical records (except id) but boosts the 2nd. the result should contain the 2nd @ pos 1 and its
   * score should be greater!
   */
  public void test_SearchForIdwithSpecialChars() throws Exception {
    assertSearchForIdWithSpecialChars("g");
    assertSearchForIdWithSpecialChars("g@brox.de");
    assertSearchForIdWithSpecialChars("E:\\test-data\\Testdaten_TelekomTraining\\tt_ag\\Seminarunterlagen\\TP\\Informationstechnologie\\Offenes_Programm\\K000655_NET@Routing+Switching\\durchführungsrelevante_Dateien\\Dateisammlung\\Routing&Switching\\Weber Rolf\\Routing und Switching_Labguide_V2.7.doc");

  }

  private void assertSearchForIdWithSpecialChars(final String id) throws Exception, SolrServerException {
    clearIndex();
    final Record copy = _blackboard.getDataFactory().cloneRecord(_record, id);
    _blackboard.setRecord(copy);
    indexAndCommit(id);

    final QueryResponse search = search(SolrQueryUtils.toConstQueryOnField(SolrConstants.CORE_FIELD_ID, id));

    final SolrDocumentList results = search.getResults();
    assertEquals(1, results.getNumFound());
  }
}