tools/org.eclipse.persistence.tools.gen.nosql/src/org/eclipse/persistence/tools/gen/nosql/mongo/MongoEntityGenerator.java

/*******************************************************************************
 * Copyright (c) 2012 Oracle. All rights reserved.
 * This program and the accompanying materials are made available under the
 * terms of the Eclipse Public License v1.0 and Eclipse Distribution License v. 1.0
 * which accompanies this distribution.
 * The Eclipse Public License is available at http://www.eclipse.org/legal/epl-v10.html
 * and the Eclipse Distribution License is available at
 * http://www.eclipse.org/org/documents/edl-v10.php.
 *
 * Contributors:
 *     Oracle - initial API and implementation
 *
 ******************************************************************************/
package org.eclipse.persistence.tools.gen.nosql.mongo;

import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
import com.mongodb.MongoException;
import com.mongodb.ServerAddress;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.Vector;
import org.apache.velocity.VelocityContext;
import org.apache.velocity.app.VelocityEngine;
import org.eclipse.persistence.tools.gen.nosql.mongo.meta.CollectionDescriptor;
import org.eclipse.persistence.tools.gen.nosql.mongo.meta.ColumnDescriptor;
import org.eclipse.persistence.tools.gen.nosql.mongo.meta.LeafColumnDescriptor;
import org.eclipse.persistence.tools.gen.nosql.mongo.meta.NestedColumnDescriptor;
import org.eclipse.persistence.tools.mapping.orm.AccessType;
import org.eclipse.persistence.tools.mapping.orm.DataFormatType;
import org.eclipse.persistence.tools.mapping.orm.ExternalColumnMapping;
import org.eclipse.persistence.tools.mapping.orm.ExternalElementCollectionMapping;
import org.eclipse.persistence.tools.mapping.orm.ExternalEmbeddable;
import org.eclipse.persistence.tools.mapping.orm.ExternalEmbeddedMapping;
import org.eclipse.persistence.tools.mapping.orm.ExternalEntity;
import org.eclipse.persistence.tools.mapping.orm.ExternalNamedQuery;
import org.eclipse.persistence.tools.mapping.orm.ExternalNoSql;
import org.eclipse.persistence.tools.mapping.orm.ExternalORMConfiguration;
import org.eclipse.persistence.tools.mapping.orm.ORMDocumentType;
import org.eclipse.persistence.tools.mapping.orm.dom.ORMRepository;
import org.eclipse.persistence.tools.utility.NameTools;
import org.eclipse.persistence.tools.utility.StringTools;
import org.eclipse.persistence.tools.utility.StringUtil;
import org.eclipse.persistence.tools.utility.collection.ListTools;

/**
 * This class is an entry point for dynamic entity xml and source generation for
 * MongoDB. It also provides API for discovering metadata (table names, etc) and generating
 * EclipseLink-JPA NoSql, mapping files.
 * <p>
 * Provisional API: This interface is part of an interim API that is still under development and
 * expected to change significantly before reaching stability. It is available at this early stage
 * to solicit feedback from pioneering adopters on the understanding that any code that uses this
 * API will almost certainly be broken (repeatedly) as the API evolves.<p>
 *
 * @author John Bracken
 * @version 2.6
 */
@SuppressWarnings("nls")
public class MongoEntityGenerator {

	/** Mongo database connection to use in generation. */
	private Mongo connection;

	/** Mongo database instance to use in generation. */
	private DB database;

	/** Number of rows in each collection to sample during generation. */
	private int rowSampleSize;

	/**
	 * Constructs a new {@link MongoEntityGenerator} based on the provided
	 * db connection information.
	 *
	 * @param host the MongoDB network host.
	 * @param port the port of the Mongo database.
	 * @param dbName the name of the Mongo database.
	 * @param rowSampleSize number of rows in each collection to sample to determine the
	 * 		  table structure.  Since not all rows may have all possible values explicitly
	 * 		  defined, it's important to use a sufficient sample size.
	 *
	 * @throws MongoException
	 * @throws UnknownHostException
	 */
	public MongoEntityGenerator(String host,
	                            int port,
	                            String dbName,
	                            int rowSampleSize) throws MongoException, UnknownHostException {

		super();
		this.rowSampleSize = rowSampleSize;
		this.connection = new Mongo(new ServerAddress(host, port));
		this.database = this.connection.getDB(dbName);
	}

	/**
	 * Builds a list of {@link CollectionDescriptor}s that describe the collection and associated
	 * column metadata of the Mongo collections named in <code>collectionNames</code>.
	 *
	 * @param collectionNames names of the Mongo collections to build descriptors for.
	 *
	 * @return {@link List} of {@link CollectionDescriptor}s.
	 */
	private List<CollectionDescriptor> buildCollectionDescriptors(Collection<String> collectionNames) {
		List<CollectionDescriptor> collectionDescriptors = new LinkedList<CollectionDescriptor>();

		for (String collectionName : collectionNames) {
			CollectionDescriptor collectionDescriptor = new CollectionDescriptor(collectionName);
			collectionDescriptors.add(collectionDescriptor);
			updateColumnDescriptors(collectionDescriptor);
		}

		return collectionDescriptors;
	}

	/**
	 * Updates the {@link CollectionDescriptor}s associated with the provided
	 * {@link CollectionDescriptor}.
	 *
	 * @param collectionDescriptor the {@link CollectionDescriptor} to update.
	 */
	private void updateColumnDescriptors(CollectionDescriptor collectionDescriptor) {
		// Read the collection from the database
		DBCollection collection = this.database.getCollection(collectionDescriptor.getName());

		// Read a sampling of the rows to determine the super set of column keys
		// that are in the collection.
		DBCursor cursor = collection.find().limit(this.rowSampleSize);
		for ( ;cursor.hasNext();) {
			DBObject row = cursor.next();
			updateCollectionDescriptor(collectionDescriptor, row);
		}

	}

	/**
	 * Generates a set of EclipseLink, java entity files mapped to the underlying Mongo
	 * database for the given <code>collectionNames</code>.
	 *
	 * @param collectionNames names of the Mongo collections to generate entities for.
	 * @param packageName java package to use for the generated code.
	 * @param baseDirectory the base directory to use for the generated code.
	 * @param type the access type to use (method versus field annotations).
	 * @param characterEncoding the type of character encoding to use (e.g. "UTF-8").
	 *
	 * @throws Exception
	 */
	public void generateSource(Collection<String> collectionNames, String packageName,
													  File baseDirectory, AccessType type,
													  String characterEncoding) throws Exception {
		// Create metadata descriptors for the specified collections.
		List<CollectionDescriptor> collectionDescriptors = buildCollectionDescriptors(collectionNames);
		ExternalORMConfiguration config = generate(collectionDescriptors, packageName);

		// Ensure that the provided gen directory and package folders exists
		File packageDirectory = new File(baseDirectory, packageName.replace('.', '/') + "/");
		packageDirectory.mkdirs();

		// Build up the velocity generator
		Properties vep = new Properties();
		vep.setProperty("resource.loader", "class");
		vep.setProperty("class.resource.loader.class",
					    "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader");
		VelocityEngine ve = new VelocityEngine();
	    ve.init(vep);

	    // Generate source for entities
	    for (ExternalEntity entity : config.entities()) {
	    	generateSource(entity, type, "entity.java.vm", packageName, packageDirectory, ve, characterEncoding);
	    }

	    // Generate source for embeddables
	    for (ExternalEmbeddable entity : config.embeddables()) {
	    	generateSource(entity, type, "embeddable.java.vm", packageName, packageDirectory, ve, characterEncoding);
	    }
	}

	/**
	 *  Generates the entity source file for the provided mapping metadata.
	 *
	 * @param entity the metadata representation of the entity.
	 * @param accessType the {@link AccessType} to use (method or field) for annotations.
	 * @param templateName the source gen template to use (embeddable, entity, etc).
	 * @param packageName the package name to use.
	 * @param packageDirectory the directory to generate the source file in.
	 * @param ve the velocity engine to use.
	 * @param characterEncoding the type of character encoding to use.
	 *
	 * @throws Exception
	 */
	private void generateSource(ExternalEmbeddable entity, AccessType accessType,
														 String templateName, String packageName,
														 File packageDirectory, VelocityEngine ve,
														 String characterEncoding) throws Exception {
		VelocityContext context = new VelocityContext();
		context.put("entity", entity);
		context.put("mappings", ListTools.list(entity.mappings()));
		context.put("packageName", packageName);
		context.put("accessType", accessType);
		StringWriter w = new StringWriter();
		ve.mergeTemplate(templateName, context, w);

		String fileContent = w.toString();

		File javaFile = new File(packageDirectory, entity.getClassShortName() + ".java");

		byte[] content = fileContent.getBytes(characterEncoding);
		javaFile.createNewFile();
		FileOutputStream writer = new FileOutputStream(javaFile);
		writer.write(content);
		writer.flush();
		writer.close();
	}

	/**
	 * Generates and adds to the specified entity a mapping for the leaf mapping described by the
	 * {@link CollectionDescriptor}.
	 *
	 * @param exEntity entity to add the mapping to.
	 * @param column the column to generate the mapping from.
	 * @param columnName the name of the column.
	 * @param javaColumnName the java identifier name to use for the column.
	 */
	private void generateLeafMapping(ExternalEmbeddable exEntity, LeafColumnDescriptor column,
															  String columnName, String javaColumnName) {
		ExternalColumnMapping mapping;
		// If this is a list-type column, then a collection based mapping is required.
		if (column.isList()) {
			mapping = exEntity.addElementCollectionMapping(columnName);
			((ExternalElementCollectionMapping)mapping).
				setTargetClassName(column.getColumnType().getName());
			mapping.setAttributeType(Vector.class.getName());
		} else {
			// special case where _id is reserved as a pk name in mongo.
			// this is always intended to be an ID mapping.
			if (columnName.equals("_id")) {
				mapping = exEntity.addIdMapping(javaColumnName);
			} else {
				mapping = exEntity.addBasicMapping(javaColumnName);
			}
			mapping.setAttributeType(column.getColumnType().getName());
		}
		mapping.setNoSqlField(columnName);
	}

	/**
	 * Generates mappings for the provided mapped class ({@link ExternalEmbeddableEntity}) based on
	 * the columns enumerated in the provided {@link CollectionDescriptor}.
	 *
	 * @param exEntity the entity to generate mappings on.
	 * @param collection the Mongo DB collection descriptor to derive rows from.
	 * @param config the orm configuration.
	 * @param allEntityNames a {@link Set} of all of the entity names already in use.
	 */
	private void generateMappings(ExternalEmbeddable exEntity, CollectionDescriptor collection, String packageName,
														   ExternalORMConfiguration config, Set<String> allEntityNames) {
		for (ColumnDescriptor column : collection.columns()) {
			String columnName = column.getColumnName();
			String javaColumnName = NameTools.javaNameFromDatabaseName(columnName);
			if (column instanceof LeafColumnDescriptor) {
				generateLeafMapping(exEntity, (LeafColumnDescriptor)column,
									columnName, javaColumnName);
			} else if (column instanceof NestedColumnDescriptor) {
				generateNestedMapping(exEntity, config, packageName, allEntityNames,
									  (NestedColumnDescriptor)column, columnName, javaColumnName);
			}
		}
	}

	/**
	 * Generates a mapping on the provided mapped class that represents the nested mapping. This will
	 * also involved generating an {@link Embeddable} to represent the nested value.
	 *
	 * @param exEntity the entity to generate the mapping on.
	 * @param config the {@link ExternalORMConfiguration} that the entity is associated with.
	 * @param allEntityNames the existing used namespace for all mapped classes.
	 * @param column the nested column to generate the mapping for.
	 * @param columnName the name of the column.
	 * @param javaColumnName the java identifier name for the column.
	 */
	private void generateNestedMapping(ExternalEmbeddable exEntity,ExternalORMConfiguration config, String packageName,
									   Set<String> allEntityNames, NestedColumnDescriptor column, String columnName, String javaColumnName) {
		// Create the embeddable for the nested value
		String embeddableName = uniqueJavaClassName(columnName, packageName, allEntityNames);
		ExternalEmbeddable embeddable = config.addEmbeddable(embeddableName);
		embeddable.addNoSql().setDataFormat(DataFormatType.MAPPED);
		embeddable.setAccessType(AccessType.VIRTUAL);
		// generate mappings for the embeddable
		generateMappings(embeddable, column.getColumnDescriptor(), packageName, config, allEntityNames);

		// if a collection, generate an Element Collection
		if (column.isList()) {
			ExternalElementCollectionMapping mapping = exEntity.addElementCollectionMapping(javaColumnName);
			mapping.setNoSqlField(columnName);
			mapping.setAttributeType(Vector.class.getName());
			mapping.setTargetClassName(embeddableName);
		}
		// otherwise, just an embedded mapping.
		else {
			ExternalEmbeddedMapping mapping = exEntity.addEmbeddedMapping(javaColumnName);
			mapping.setNoSqlField(columnName);
			mapping.setAttributeType(embeddableName);
		}
	}

	/**
	 * Generates and adds a root level named query for each entity
	 * enumerate in the {@link ExternalORMConfiguration}.
	 *
	 * @param config the configuration to generate and add queries from.
	 */
	private void generateQueries(ExternalORMConfiguration config) {
		for (ExternalEntity entity : config.entities()) {
			String entityName = entity.getClassShortName();
			ExternalNamedQuery query = config.addNamedQuery(entityName + ".findAll");
			char identifier = Character.toLowerCase(entityName.charAt(0));
			query.setQuery("select " + identifier + " from " + entityName + " "  + identifier);

		}
	}

	/**
	 * Generates a {@link String} representation of an eclipselink-orm.xml descriptor based
	 * on the provided MongoDB {@link CollectionDescriptor} definitions.
	 *
	 * @param collectionDescriptors the MongoDB collections to generate entities from.
	 * @param packageName the package name to qualify all generated entities for.
	 *
	 * @return a {@link String} representation of the eclipselink-orm.xml.
	 */
	private ExternalORMConfiguration generate(List<CollectionDescriptor> collectionDescriptors, String packageName) {
		try {
			// Create a new orm.xml metadata model.
			ExternalORMConfiguration config = new ORMRepository().buildORMConfiguration(null, ORMDocumentType.ECLIPELINK_2_6);
			// track all entity names being used.
			Set<String> allEntityNames = new HashSet<String>();
			// Iterate over all collection descriptors and create an entity and associated mapping for them.
			for (CollectionDescriptor collection : collectionDescriptors) {
				// add an entity with a unique name
				String collectionName = collection.getName();
				String entityName = uniqueJavaClassName(collectionName, packageName, allEntityNames);
				ExternalEntity exEntity = config.addEntity(entityName);
				// access type is virtual
				exEntity.setAccessType(AccessType.VIRTUAL);
				// Configure NoSql settings
				ExternalNoSql noSqlDesc = exEntity.addNoSql();
				noSqlDesc.setDataFormat(DataFormatType.MAPPED);
				noSqlDesc.setDataType(collectionName);
				// add mappings
				generateMappings(exEntity, collection, packageName, config, allEntityNames);
			}

			// Generate a default, read all query for all entities
			generateQueries(config);

			return config;
		}
		catch (IOException e) {
			// Never happens
			return null;
		}
	}

	/**
	 * Creates an eclipselink-orm.xml descriptor mapping the provided <code>collectionNames</code>
	 * as EclipseLink dynamic entities.
	 *
	 * @param collectionNames the names of the Mongo collections to generate for.
	 * @param packageName the qualifying package name to use for the generated entities.
	 * @return a {@link String} representation of the eclipselink-orm.xml.
	 */
	public String generateXML(Collection<String> collectionNames, String packageName) {
		// Create metadata descriptors for the specified collections.
		List<CollectionDescriptor> collectionDescriptors = buildCollectionDescriptors(collectionNames);

		// Generate an eclpselink-orm.xml from the collection descriptors
		return generate(collectionDescriptors, packageName).getXML();
	}

	/**
	 * Updates the give {@link CollectionDescriptor} with the leaf column information defined
	 * by row value.
	 *
	 * @param collectionDescriptor the {@link CollectionDescriptor} to update.
	 * @param columnName the name of the column.
	 * @param value the row value of the column.
	 */
	private void handleLeafColumn(CollectionDescriptor collectionDescriptor,
													   String columnName, Object value) {
		LeafColumnDescriptor columnDescriptor = (LeafColumnDescriptor)collectionDescriptor.getColumn(columnName);
		if (columnDescriptor == null) {
			columnDescriptor = collectionDescriptor.addLeafColumn(columnName);
		}
		Class<?> valueClass = value.getClass();
		// Special case for an id-type column. If one is not explicitly defined,
		// Mongo auto-generates one and uses the noted class for the type. This
		// should be considered a String in java.
		valueClass = valueClass.getName().equals("org.bson.types.ObjectId") ? String.class : valueClass;
		// if the column type isn't consistent, just use Object as the type.
		if (columnDescriptor.getColumnType() == null) {
			columnDescriptor.setColumnType(valueClass);
		} else if (columnDescriptor.getColumnType() != valueClass) {
			columnDescriptor.setColumnType(Object.class);
		}
	}

	/**
	 * Updates the given {@link CollectionDescriptor} with the a column representing
	 * the list-type value.
	 *
	 * @param collectionDescriptor the {@link CollectionDescriptor} to update.
	 * @param columnName the name of the column to update.
	 * @param value the row value, or in this case the list value row.
	 */
	private void handleListColumn(CollectionDescriptor collectionDescriptor, String columnName,
																		    	Object value) {
		ColumnDescriptor columnDescriptor = collectionDescriptor.getColumn(columnName);
		BasicDBList listValue = (BasicDBList)value;
		if (listValue.size() > 0) {
			Iterator<?> listValues = listValue.listIterator();
			Object valueFromList = listValues.next();
			// Handle nested list
			if (valueFromList instanceof BasicDBObject) {
				NestedColumnDescriptor nestedColumnDesc;
				if (columnDescriptor == null) {
					nestedColumnDesc  = collectionDescriptor.addNestedColumn(columnName);
				} else {
					nestedColumnDesc = (NestedColumnDescriptor)columnDescriptor;
				}
				nestedColumnDesc.setList(true);
				updateCollectionDescriptor(nestedColumnDesc.getColumnDescriptor(), (DBObject)valueFromList);
				// Iterate over subsequent nested values to ensure the superset of all columns keys
				// are included
				for (;listValues.hasNext();) {
					valueFromList = listValues.next();
					updateCollectionDescriptor(nestedColumnDesc.getColumnDescriptor(), (DBObject)valueFromList);
				}

			}
			// Handle leaf list
			else {
				LeafColumnDescriptor leafColumnDescriptor;
				if (columnDescriptor == null) {
					leafColumnDescriptor  = collectionDescriptor.addLeafColumn(columnName);
				} else {
					leafColumnDescriptor = (LeafColumnDescriptor)columnDescriptor;
				}

				leafColumnDescriptor.setList(true);
				leafColumnDescriptor.setColumnType(valueFromList.getClass());
				// Iterate over subsequent elements. If the element type isn't the same as the last, default to Object
				// and break.
				for (;listValues.hasNext();) {
					valueFromList = listValues.next();
					if (leafColumnDescriptor.getColumnType() != valueFromList.getClass()) {
						leafColumnDescriptor.setColumnType(Object.class);
						break;
					}
				}
			}
		}
	}

	/**
	 * Updates the given {@link CollectionDescriptor} with the nested column information
	 * derived from the provided row value.
	 *
	 * @param collectionDescriptor the {@link CollectionDescriptor} to update.
	 * @param columnName name of the column.
	 * @param value row value of the column, or in this case the nest row.
	 */
	private void handleNestedColumn(CollectionDescriptor collectionDescriptor,
								   					   String columnName, Object value) {
		NestedColumnDescriptor columnDescriptor =
				(NestedColumnDescriptor)collectionDescriptor.getColumn(columnName);

		if (columnDescriptor == null) {
			columnDescriptor = collectionDescriptor.addNestedColumn(columnName);
		}

		updateCollectionDescriptor(columnDescriptor.getColumnDescriptor(), (DBObject)value);
	}

	/**
	 * Returns the names of the Mongo collections on the specified
	 * database.
	 *
	 * @return the name of the Mongo collections.
	 */
	public Set<String> listCollectionNames() {
		Set<String> collectionNames = this.database.getCollectionNames();
		// Remove the internal system.indexes table name
		collectionNames.remove("system.indexes");

		return collectionNames;
	}

	/**
	 * Generates a unique, fully qualified and singularised entity name for the specified
	 * Mongo collection name.
	 *
	 * @param collectionName the name of the Mongo collection.
	 * @param packageName the package name to qualify the entity name with.
	 * @param allEntityNames all of the existing entity names in use.
	 *
	 * @return the unique, qualifed and singularised entity name.
	 */
	private String uniqueJavaClassName(String collectionName, String packageName, Set<String> allEntityNames) {
		String entityName = StringUtil.singularise(NameTools.javaNameFromDatabaseName(collectionName, true));
		if (!StringTools.isBlank(packageName)) {
			entityName = packageName + "." + entityName;
		}
		entityName = NameTools.uniqueName(entityName, allEntityNames);
		allEntityNames.add(entityName);

		return entityName;
	}

	/**
	 * Updates the given {@link CollectionDescriptor} with columns implied by the provided row
	 * or {@link DBObject}.
	 *
	 * @param collectionDescriptor the {@link CollectionDescriptor} to update.
	 * @param dbObject the row to infer columns from.
	 */
	private void updateCollectionDescriptor(CollectionDescriptor collectionDescriptor, DBObject dbObject) {
		// iterate over all of the available names in the row.
		for (String columnName : dbObject.keySet()) {
			Object value = dbObject.get(columnName);
			// if a column does not yet exist in our descriptor, add one.
			if (value != null) {
				// List type
				if (value instanceof BasicDBList) {
					handleListColumn(collectionDescriptor, columnName, value);
				}
				// Single Nested
				else if (value instanceof BasicDBObject) {
					handleNestedColumn(collectionDescriptor, columnName, value);
				}
				// Leaf Value
				else {
					handleLeafColumn(collectionDescriptor, columnName, value);
				}
			}
		}
	}
}