core/org.eclipse.smila.solr/code/src/org/eclipse/smila/solr/search/ResponseParser.java

/**
 *
 */
package org.eclipse.smila.solr.search;

import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.Group;
import org.apache.solr.client.solrj.response.GroupCommand;
import org.apache.solr.client.solrj.response.GroupResponse;
import org.apache.solr.client.solrj.response.IntervalFacet;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.client.solrj.response.SolrResponseBase;
import org.apache.solr.client.solrj.response.SpellCheckResponse;
import org.apache.solr.client.solrj.response.SpellCheckResponse.Collation;
import org.apache.solr.client.solrj.response.SpellCheckResponse.Correction;
import org.apache.solr.client.solrj.response.SpellCheckResponse.Suggestion;
import org.apache.solr.client.solrj.response.TermsResponse;
import org.apache.solr.client.solrj.response.TermsResponse.Term;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.util.NamedList;
import org.eclipse.smila.datamodel.Any;
import org.eclipse.smila.datamodel.Any.ValueType;
import org.eclipse.smila.datamodel.AnyMap;
import org.eclipse.smila.datamodel.AnySeq;
import org.eclipse.smila.datamodel.DataFactory;
import org.eclipse.smila.datamodel.Record;
import org.eclipse.smila.datamodel.Value;
import org.eclipse.smila.datamodel.util.AnyUtil;
import org.eclipse.smila.search.api.helper.ResultBuilder;
import org.eclipse.smila.solr.SolrConfig;
import org.eclipse.smila.solr.SolrConstants;
import org.eclipse.smila.solr.SolrUtils;
import org.eclipse.smila.solr.administration.FieldInfoCache;
import org.eclipse.smila.solr.params.SolrParams;

/**
 * @author pwissel
 *
 */
public class ResponseParser extends ResultBuilder {

  private final Log _log = LogFactory.getLog(getClass());

  private final DataFactory _factory = DataFactory.DEFAULT;

  private final SolrConfig _config;

  private final String _index;

  public ResponseParser(final Record record, final SolrConfig config, String index) {
    super(record);
    _config = config;
    _index = index;
  }

  public Record toRecord(final SolrResponse response) {
    if (response instanceof SolrResponseBase) {
      parseResponseBase((SolrResponseBase) response);
    }
    if (response instanceof QueryResponse) {
      parseQueryResponse((QueryResponse) response);
    }
    return this.getResult();
  }

  public Any toAny(final SolrResponse response) {
    return toRecord(response).getMetadata();
  }

  private AnyMap getResponseMap() {
    return _result.getMetadata().getMap(SolrParams.SOLR_PARAMETER_ATTRIBUTE, true).getMap(ResponseAccessor.RESPONSE,
      true);
  }

  @SuppressWarnings("unchecked")
  void parseResponseBase(final SolrResponseBase response) {
    final NamedList<Object> header = response.getResponseHeader();
    if (header != null) {
      final AnyMap responseHeader = getResponseMap().getMap(SolrConstants.RESPONSE_HEADER, true);
      SolrUtils.parseNamedList(header, responseHeader);
    }
  }

  void parseQueryResponse(final QueryResponse response) {
    parseResults(response);
    parseFacets(response);
    parseGrouping(response);
    parseTerms(response);
    parseSpellcheck(response);
    parseMoreLikeThis(response);
    parseCursorMark(response);
    parseDebug(response);
    parseStats(response);
    parseSuggest(response);
  }

  void parseResults(final QueryResponse response) {
    final SolrDocumentList documents = response.getResults();
    if (documents == null) {
      return;
    }
    // add general result fields
    final long numFound = documents.getNumFound();
    setCount(numFound);
    final Float maxScore = documents.getMaxScore();
    if (maxScore != null) {
      getResponseMap().put(SolrConstants.MAX_SCORE, maxScore);
    }
    // add document result fields
    for (final SolrDocument document : documents) {
      final AnyMap item = createResultItem(document);
      getResultRecords().add(item);
      // add highlighting
      parseHighlighting(response, item);
    }
  }

  private AnyMap createResultItem(final SolrDocument document) {
    final AnyMap item = getResultRecords().getFactory().createAnyMap();
    // add result fields to item
    for (final Entry<String, Object> entry : document.entrySet()) {
      final String key = entry.getKey();
      final Object value = entry.getValue();
      if (value == null) {
        continue;
      }
      if (value instanceof Collection) {
        addMultiKeyValuePairToItem(item, key, value);
      } else {
        addKeyValuePairToItem(item, key, value);
      }
    }
    return item;
  }

  private void addKeyValuePairToItem(final AnyMap item, final String key, final Object value) {
    final DataFactory factory = item.getFactory();
    final Value autoConvertedValue = factory.autoConvertValue(value);
    item.put(key, autoConvertedValue);
  }

  private void addMultiKeyValuePairToItem(final AnyMap item, final String key, final Object value) {
    final DataFactory factory = item.getFactory();
    final Collection<?> multiValues = (Collection<?>) value;
    for (final Object multiValue : multiValues) {
      final Value autoConvertedValue = factory.autoConvertValue(multiValue);
      item.add(key, autoConvertedValue);
    }
  }

  private void parseHighlighting(final QueryResponse response, final AnyMap item) {
    final Map<String, Map<String, List<String>>> highlighting = response.getHighlighting();
    if (highlighting == null) {
      return;
    }
    final String id = item.getStringValue(_config.getIdField(_index));
    if (id != null) {
      for (final Entry<String, List<String>> entry : highlighting.get(id).entrySet()) {
        final String attribute = entry.getKey();
        final List<String> text = entry.getValue();
        if (text.size() > 1) {
          addHighlightText(item, attribute, text);
        } else {
          addHighlightText(item, attribute, text.get(0));
        }
      }
    }
  }

  @SuppressWarnings("rawtypes")
  void parseFacets(final QueryResponse response) {
    final boolean fetchFacetFieldType = _config.isFetchFacetFieldType();
    // facet fields
    final List<FacetField> fieldFacets = response.getFacetFields();
    if (fieldFacets != null) {
      addFacetFields(fieldFacets, fetchFacetFieldType);
    }
    final List<RangeFacet> rangeFacets = response.getFacetRanges();
    if (rangeFacets != null) {
      addFacetRanges(rangeFacets);
    }
    final Map<String, Integer> queryFacets = response.getFacetQuery();
    if (queryFacets != null) {
      addFacetQueries(queryFacets);
    }
    final NamedList<List<PivotField>> pivotFacets = response.getFacetPivot();
    if (pivotFacets != null) {
      addFacetPivots(pivotFacets);
    }
    final List<IntervalFacet> intervalFacets = response.getIntervalFacets();
    if (intervalFacets != null) {
      addFacetIntervals(intervalFacets);
    }
  }

  private void addFacetFields(final List<FacetField> fieldFacets, final boolean fetchFacetFieldType) {
    for (FacetField facetField : fieldFacets) {
      // create facet
      final String fieldName = facetField.getName();
      final AnySeq facet = addFacet(fieldName);
      // fetch field type
      final ValueType valueType = getFacetFieldType(fieldName);
      // add facet values
      for (final org.apache.solr.client.solrj.response.FacetField.Count countObj : facetField.getValues()) {
        final String value = countObj.getName();
        final long count = countObj.getCount();
        addTypedFacetValue(facet, value, valueType, count);
      }
    }
  }

  @SuppressWarnings("rawtypes")
  private void addFacetRanges(final List<RangeFacet> rangeFacets) {
    for (final RangeFacet rangeFacet : rangeFacets) {
      // create facet
      final String fieldName = rangeFacet.getName();
      final AnySeq facet = addFacet(fieldName);
      // fetch field type
      final ValueType valueType = getFacetFieldType(fieldName);
      // add facet values
      for (final Object obj : rangeFacet.getCounts()) {
        final org.apache.solr.client.solrj.response.RangeFacet.Count countObj =
          (org.apache.solr.client.solrj.response.RangeFacet.Count) obj;
        final String value = countObj.getValue();
        final long count = countObj.getCount();
        addTypedFacetValue(facet, value, valueType, count);
      }
    }
  }

  private void addFacetQueries(final Map<String, Integer> queryFacets) {
    final AnySeq facet = addFacet(SolrConstants.QUERIES);
    for (final String query : queryFacets.keySet()) {
      final String value = query;
      final long count = new Long(queryFacets.get(query));
      addFacetValue(facet, value, count);
    }
  }

  private void addFacetPivots(final NamedList<List<PivotField>> pivotFacets) {
    final AnyMap pivot = getResponseMap().getMap(SolrConstants.FACET_PIVOT, true);
    SolrUtils.parseNamedList(pivotFacets, pivot);
  }

  private void addFacetIntervals(final List<IntervalFacet> intervalFacets) {
    for (final IntervalFacet intervalFacet : intervalFacets) {
      // create facet
      final String fieldName = intervalFacet.getField();
      final AnySeq facet = addFacet(fieldName);
      // add facet values
      for (final org.apache.solr.client.solrj.response.IntervalFacet.Count countObj : intervalFacet
        .getIntervals()) {
        final String value = countObj.getKey();
        final long count = countObj.getCount();
        addFacetValue(facet, value, count);
      }
    }
  }

  private ValueType getFacetFieldType(final String fieldName) {
    if (_config.isFetchFacetFieldType()) {
      try {
        return FieldInfoCache.getFieldInfo(_index, fieldName).getTypeAsValueType();
      } catch (InterruptedException | IOException exception) {
        if (_log.isWarnEnabled()) {
          final String message = String.format("Error getting value type for fieldName: %s.", fieldName);
          _log.warn(message, exception);
        }
      }
    }
    return null;
  }

  private void addTypedFacetValue(final AnySeq facet, final String value, final ValueType valueType,
    final long count) {
    if (valueType != null) {
      addFacetValue(facet, value, valueType, count);
    } else {
      final Value val = _factory.autoConvertValue(value);
      addFacetValue(facet, val, count);
    }
  }

  void parseGrouping(final QueryResponse response) {
    final GroupResponse groupResponse = response.getGroupResponse();
    if (groupResponse == null) {
      return;
    }
    for (final GroupCommand command : groupResponse.getValues()) {
      final String name = command.getName();
      final int matches = command.getMatches();
      final AnySeq group = addGroup(name, new Long(matches));
      // Add ngroups if not null
      final Integer ngroups = command.getNGroups();
      if (ngroups != null) {
        getGroups().put(SolrConstants.NGROUPS, ngroups);
      }
      final DataFactory factory = group.getFactory();
      for (Group solrGroup : command.getValues()) {
        // get group value and check for null
        String groupValue = solrGroup.getGroupValue();
        if (groupValue == null) {
          // skip group value if 'processNullGroupValue ' is not configured
          if (!_config.isProcessGroupValueNull()) {
            continue;
          }
          // otherwise get 'nullGroupValue' or use default <null>
          groupValue = _config.getGroupValueNull();
        }
        final Value value = factory.autoConvertValue(groupValue);
        final SolrDocumentList documents = solrGroup.getResult();
        final Long count = documents.getNumFound();
        final AnySeq results = factory.createAnySeq();
        for (final SolrDocument document : documents) {
          final AnyMap result = createResultItem(document);
          parseHighlighting(response, result);
          results.add(result);
        }
        addGroupResults(group, value, count, results);
      }
    }
  }

  void parseTerms(final QueryResponse response) {
    final TermsResponse termsResponse = response.getTermsResponse();
    if (termsResponse == null) {
      return;
    }
    // _solr/response/terms
    final AnyMap terms = getResponseMap().getMap(SolrConstants.TERMS, true);
    final Map<String, List<Term>> termMap = termsResponse.getTermMap();
    for (final Entry<String, List<Term>> entry : termMap.entrySet()) {
      final AnyMap field = terms.getMap(entry.getKey(), true);
      for (final Term term : entry.getValue()) {
        field.put(term.getTerm(), term.getFrequency());
      }
    }
  }

  void parseSpellcheck(final QueryResponse response) {
    final SpellCheckResponse spellcheckResponse = response.getSpellCheckResponse();
    if (spellcheckResponse == null) {
      return;
    }
    // _solr/response/spellcheck
    final AnyMap spellcheck = getResponseMap().getMap(SolrConstants.SPELLCHECK, true);
    // suggestions
    final Map<String, Suggestion> suggestionMap = spellcheckResponse.getSuggestionMap();
    if (!MapUtils.isEmpty(suggestionMap)) {
      final AnyMap suggestions = spellcheck.getMap(SolrConstants.SUGGESTIONS, true);
      for (final Entry<String, Suggestion> entry : suggestionMap.entrySet()) {
        final String name = entry.getKey();
        final Suggestion suggestion = entry.getValue();
        if (suggestion != null) {
          final AnyMap map = suggestions.getMap(name, true);
          map.put(SolrConstants.NUM_FOUND, suggestion.getNumFound());
          map.put(SolrConstants.START_OFFSET, suggestion.getStartOffset());
          map.put(SolrConstants.END_OFFSET, suggestion.getEndOffset());
          map.put(SolrConstants.ORIG_FREQ, suggestion.getOriginalFrequency());
          final List<String> alternatives = suggestion.getAlternatives();
          final List<Integer> alternativeFrequencies = suggestion.getAlternativeFrequencies();
          if (alternativeFrequencies == null) {
            map.put(SolrConstants.SUGGESTION, AnyUtil.objectToAny(alternatives));
          } else {
            final AnySeq extendedResults = map.getSeq(SolrConstants.SUGGESTION, true);
            final DataFactory factory = extendedResults.getFactory();
            int index = 0;
            for (final String word : alternatives) {
              final AnyMap extended = factory.createAnyMap();
              extended.put(SolrConstants.WORD, word);
              extended.put(SolrConstants.FREQ, alternativeFrequencies.get(index));
              extendedResults.add(extended);
              index++;
            }
          }
        }
      }
    }
    // extended collation
    final List<Collation> collationList = spellcheckResponse.getCollatedResults();
    if (!CollectionUtils.isEmpty(collationList)) {
      final AnySeq collations = spellcheck.getSeq(SolrConstants.COLLATIONS, true);
      final DataFactory factory = collations.getFactory();
      for (final Collation collationObj : collationList) {
        if (collationObj != null) {
          final AnyMap collation = factory.createAnyMap();
          collation.put(SolrConstants.COLLATION_QUERY, collationObj.getCollationQueryString());
          collation.put(SolrConstants.HITS, collationObj.getNumberOfHits());
          if (!CollectionUtils.isEmpty(collationObj.getMisspellingsAndCorrections())) {
            final AnyMap misspellingsAndCorrection =
              collation.getMap(SolrConstants.MISSPELLINGS_AND_CORRECTIONS, true);
            for (final Correction correction : collationObj.getMisspellingsAndCorrections()) {
              misspellingsAndCorrection.put(correction.getOriginal(), correction.getCorrection());
            }
          }
          collations.add(collation);
        }
      }
    }
  }

  @SuppressWarnings("unchecked")
  void parseMoreLikeThis(final QueryResponse response) {
    final Object moreLikeThisObj = response.getResponse().get(SolrConstants.MORE_LIKE_THIS);
    if (moreLikeThisObj != null) {
      final AnyMap moreLikeThis = getResponseMap().getMap(SolrConstants.MORE_LIKE_THIS, true);
      final NamedList<SolrDocumentList> relatedDocuments = (NamedList<SolrDocumentList>) moreLikeThisObj;
      final Iterator<Entry<String, SolrDocumentList>> iterator = relatedDocuments.iterator();
      while (iterator.hasNext()) {
        final Entry<String, SolrDocumentList> entry = iterator.next();
        final String name = entry.getKey();
        final AnyMap result = moreLikeThis.getMap(name, true);
        final SolrDocumentList documents = entry.getValue();
        result.put(SolrConstants.NUM_FOUND, documents.getNumFound());
        result.put(SolrConstants.START, documents.getStart());
        final Float maxScore = documents.getMaxScore();
        if (maxScore != null) {
          result.put(SolrConstants.MAX_SCORE, maxScore);
        }
        for (final SolrDocument document : documents) {
          final AnyMap item = createResultItem(document);
          result.add(SolrConstants.RELATED, item);
        }
      }
    }
  }

  void parseCursorMark(final QueryResponse response) {
    final String nextCursorMark = response.getNextCursorMark();
    if (!StringUtils.isEmpty(nextCursorMark)) {
      getResponseMap().put(SolrConstants.NEXT_CURSOR_MARK, nextCursorMark);
    }
  }

  void parseDebug(final QueryResponse response) {
    final Map<String, Object> debugMap = response.getDebugMap();
    if (debugMap != null) {
      final AnyMap debug = getResponseMap().getMap(SolrConstants.DEBUG, true);
      for (final Entry<String, Object> entry : debugMap.entrySet()) {
        final String name = entry.getKey();
        final Object object = entry.getValue();
        if (object instanceof NamedList<?>) {
          SolrUtils.parseNamedList((NamedList<?>) object, debug.getMap(name, true));
        } else {
          final Value value = debug.getFactory().autoConvertValue(object);
          debug.put(name, value);
        }
      }
    }
  }

  void parseStats(final QueryResponse response) {
    final Map<String, FieldStatsInfo> fieldStatsInfoMap = response.getFieldStatsInfo();
    if (fieldStatsInfoMap != null) {
      final AnyMap stats = getResponseMap().getMap(SolrConstants.STATS, true);
      final DataFactory factory = stats.getFactory();
      for (final Entry<String, FieldStatsInfo> statsEntry : fieldStatsInfoMap.entrySet()) {
        final AnyMap field = stats.getMap(statsEntry.getKey(), true);
        final FieldStatsInfo fieldStatsInfo = statsEntry.getValue();
        addStats(field, fieldStatsInfo, factory);
        // facets
        final Map<String, List<FieldStatsInfo>> facetInfo = fieldStatsInfo.getFacets();
        if (facetInfo != null) {
          final AnyMap facets = field.getMap(SolrConstants.FACETS, true);
          for (final Entry<String, List<FieldStatsInfo>> facetsEntry : facetInfo.entrySet()) {
            final AnyMap facet = facets.getMap(facetsEntry.getKey(), true);
            for (final FieldStatsInfo facetStatsInfo : facetsEntry.getValue()) {
              addStats(facet, facetStatsInfo, factory);
            }
          }
        }
      }
    }
  }

  private void addStats(final AnyMap stats, final FieldStatsInfo fieldStatsInfo, final DataFactory factory) {
    stats.put(SolrConstants.MIN, factory.autoConvertValue(fieldStatsInfo.getMin()));
    stats.put(SolrConstants.MAX, factory.autoConvertValue(fieldStatsInfo.getMax()));
    stats.put(SolrConstants.COUNT, fieldStatsInfo.getCount());
    stats.put(SolrConstants.MISSING, fieldStatsInfo.getMissing());
    stats.put(SolrConstants.SUM, factory.autoConvertValue(fieldStatsInfo.getSum()));
    stats.put(SolrConstants.MEAN, factory.autoConvertValue(fieldStatsInfo.getMean()));
    stats.put(SolrConstants.STDDEV, fieldStatsInfo.getStddev());
  }

  @SuppressWarnings("unchecked")
  void parseSuggest(final QueryResponse response) {
    final Object obj = response.getResponse().get(SolrConstants.SUGGEST);
    if (obj != null) {
      final AnyMap suggest = getResponseMap().getMap(SolrConstants.SUGGEST, true);
      if (obj instanceof Map<?, ?>) {
        final Map<Object, Object> dictionaries = (Map<Object, Object>) obj;
        for (final Entry<Object, Object> dictionary : dictionaries.entrySet()) {
          final String name = (String) dictionary.getKey();
          final Object list = dictionary.getValue();
          if (list instanceof NamedList<?>) {
            final AnyMap target = suggest.getMap(name, true);
            SolrUtils.parseNamedList((NamedList<?>) list, target);
          }
        }
      }
    }
  }

}