core/plugins/org.eclipse.hawk.localfolder/src/org/eclipse/hawk/localfolder/LocalFolder.java - gerrit/hawk/hawk - Git at Google

 /*******************************************************************************
  * Copyright (c) 2011-2020 The University of York, Aston University.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Public License 2.0 which is available at
  * http://www.eclipse.org/legal/epl-2.0.
  *
  * This Source Code may also be made available under the following Secondary
  * Licenses when the conditions for such availability set forth in the Eclipse
  * Public License, v. 2.0 are satisfied: GNU General Public License, version 3.
  *
  * SPDX-License-Identifier: EPL-2.0 OR GPL-3.0
  *
  * Contributors:
  *     Konstantinos Barmpis - initial API and implementation
  *     Antonio Garcia-Dominguez - use Java 7 Path instead of File+string processing,
  *       use MapDB, refactor into shared part with LocalFile, add file filtering
  ******************************************************************************/
 package org.eclipse.hawk.localfolder;

 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLDecoder;
 import java.nio.file.FileVisitResult;
 import java.nio.file.FileVisitor;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;

 import org.eclipse.hawk.core.IModelIndexer;
 import org.eclipse.hawk.core.VcsChangeType;
 import org.eclipse.hawk.core.VcsCommit;
 import org.eclipse.hawk.core.VcsCommitItem;
 import org.eclipse.hawk.core.VcsRepositoryDelta;
 import org.mapdb.DB;
 import org.mapdb.DBMaker;

 /**
  * VCS manager that watches over the contents of a directory, including its
  * subdirectories. Revisions are based on "last modified" timestamps, which
  * have millisecond resolution in Linux from Java 9 onwards.
  */
 public class LocalFolder extends FileBasedLocation {

 	private long maxLastModified = 0;

 	private final class LastModifiedFileVisitor implements FileVisitor<Path> {

 		private boolean alter;

 		public LastModifiedFileVisitor(boolean alter) {
 			this.alter = alter;
 		}

 		@Override
 		public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
 			final File f = dir.toFile();
 			final long currentlatest = getRevisionFromFileMetadata(f);
 			final Long lastRev = recordedModifiedDates.get(dir.toString());
 			if (lastRev == null || lastRev < currentlatest) {
 				if (alter) {
 					recordedModifiedDates.put(dir.toString(), currentlatest);
 				}
 				maxLastModified = Math.max(f.lastModified(), maxLastModified);
 			}
 			return FileVisitResult.CONTINUE;
 		}

 		@Override
 		public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 			if (isFileInteresting(dir.toFile())) {
 				return FileVisitResult.CONTINUE;
 			} else {
 				return FileVisitResult.SKIP_SUBTREE;
 			}
 		}

 		@Override
 		public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 			final File f = file.toFile();
 			if (isFileInteresting(f)) {
 				final long currentlatest = getRevisionFromFileMetadata(f);
 				final Long lastRev = recordedModifiedDates.get(file.toString());
 				if (lastRev == null || lastRev < currentlatest) {
 					if (alter) {
 						recordedModifiedDates.put(file.toString(), currentlatest);
 					}
 					maxLastModified = Math.max(f.lastModified(), maxLastModified);
 				}
 			}
 			return FileVisitResult.CONTINUE;
 		}

 		@Override
 		public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
 			return FileVisitResult.CONTINUE;
 		}
 	}

 	private Path rootLocation;
 	private long lastIndexedRevision = 0;
 	private Set<File> previousFiles;
 	private Map<String, Long> recordedModifiedDates;
 	private Function<File, Boolean> fileFilter;

 	/**
 	 * MapDB database: using file-backed Java collections allows us to save memory
 	 * when handling folders with a large number of files.
 	 */
 	private DB db;

 	@Override
 	public void init(String vcsloc, IModelIndexer indexer) throws Exception {
 		final File fMapDB = File.createTempFile("localfolder", "mapdb");
 		db = DBMaker.newFileDB(fMapDB).deleteFilesAfterClose().closeOnJvmShutdown().make();
 		previousFiles = db.createHashSet("previousFiles").make();
 		recordedModifiedDates = db.createHashMap("recordedModifiedDates").make();

 		console = indexer.getConsole();

 		// Accept both regular paths and file:// URIs
 		Path path;
 		try {
 			path = Paths.get(new URI(vcsloc));
 		} catch (URISyntaxException | IllegalArgumentException ex) {
 			path = Paths.get(vcsloc);
 		}

 		final File canonicalFile = path.toFile().getCanonicalFile();
 		rootLocation = canonicalFile.toPath();
 		String repositoryURI = rootLocation.toUri().toString();

 		// If the file doesn't exist, it might be because this is a local folder in
 		// a remote server - try to preserve the provided vcsloc as is. Otherwise,
 		// if the server and the client use different operating systems we could end
 		// up with an unusable URL in the server.
 		if (canonicalFile.exists()) {
 			repositoryURL = repositoryURI;
 		} else {
 			repositoryURL = vcsloc;
 		}

 		// dont decode it to ensure consistency with other managers
 		// URLDecoder.decode(repositoryURI.replace("+", "%2B"), "UTF-8");
 	}

 	protected String getCurrentRevision(boolean alter) {
 		try {
 			final LastModifiedFileVisitor visitor = new LastModifiedFileVisitor(alter);
 			Files.walkFileTree(rootLocation, visitor);
 			long ret = maxLastModified;
 			if (alter) {
 				lastIndexedRevision = ret;
 			}
 			return ret + "";
 		} catch (IOException ex) {
 			console.printerrln(ex);
 			return FIRST_REV;
 		}
 	}

 	@Override
 	public File importFile(String revision, String p, File temp) {
 		try {
 			final String path = URLDecoder.decode(p.replace("+", "%2B"), "UTF-8");
 			final Path resolvedPath = rootLocation.resolve(path.startsWith("/") ? path.replaceFirst("/", "") : path);
 			return resolvedPath.toFile();
 		} catch (Exception e) {
 			console.printerrln(e);
 			return null;
 		}
 	}

 	@Override
 	public boolean isActive() {
 		return rootLocation == null ? false : rootLocation.toFile().exists();
 	}

 	@Override
 	public void shutdown() {
 		rootLocation = null;
 		if (!db.isClosed()) {
 			db.close();
 		}
 	}

 	@Override
 	public String getHumanReadableName() {
 		return "Local Folder Monitor";
 	}

 	@Override
 	public VcsRepositoryDelta getDelta(String startRevision, String endRevision) throws Exception {
 		List<VcsCommit> commits = new ArrayList<>();
 		VcsRepositoryDelta delta = new VcsRepositoryDelta(commits);
 		delta.setManager(this);

 		// Update maxLastModified (there may be deletes, updating the folder timestamp)
 		getCurrentRevision();

 		Set<File> files = new HashSet<>();
 		addAllFiles(rootLocation.toFile(), files);
 		previousFiles.removeAll(files);

 		for (File f : previousFiles) {
 			VcsCommit commit = new VcsCommit();
 			commit.setAuthor("i am a local folder driver - no authors recorded");
 			commit.setJavaDate(new Date());
 			commit.setMessage("i am a local folder driver - no messages recorded");
 			commit.setRevision(maxLastModified + "");
 			commits.add(commit);

 			VcsCommitItem c = new VcsCommitItem();
 			c.setChangeType(VcsChangeType.DELETED);
 			c.setCommit(commit);

 			Path path = f.toPath();

 			// Don't decode it to ensure consistency with other managers
 			String relativepath = makeRelative(repositoryURL,
 					f.toPath().toUri().toString()
 			);

 			c.setPath(relativepath.startsWith("/") ? relativepath : "/" + relativepath);
 			commit.getItems().add(c);
 			recordedModifiedDates.remove(path.toString());
 		}

 		previousFiles.clear();

 		if (files != null && files.size() > 0) {
 			// long newLastModifiedRepository = lastModifiedRepository;
 			for (File f : files) {
 				previousFiles.add(f);
 				Path filePath = f.toPath();
 				final long latestRev = getRevisionFromFileMetadata(f);
 				final Long lastDate = recordedModifiedDates.get(filePath.toString());
 				if (lastDate != null && lastDate == latestRev) {
 					if ((lastIndexedRevision + "").equals(startRevision))
 						continue;
 				}
 				recordedModifiedDates.put(filePath.toString(), latestRev);

 				VcsCommit commit = new VcsCommit();
 				commit.setAuthor("i am a local folder driver - no authors recorded");
 				commit.setJavaDate(new Date());
 				commit.setMessage("i am a local folder driver - no messages recorded");
 				commit.setRevision(maxLastModified + "");
 				commits.add(commit);

 				VcsCommitItem c = new VcsCommitItem();
 				c.setChangeType(VcsChangeType.UPDATED);
 				c.setCommit(commit);

 				String relativepath = makeRelative(repositoryURL, f.toPath().toUri().toString());
 				c.setPath(relativepath.startsWith("/") ? relativepath : ("/" + relativepath));
 				commit.getItems().add(c);
 			}
 		}

 		// Update the latest revision seen
 		lastIndexedRevision = maxLastModified;

 		return delta;
 	}

 	public Function<File, Boolean> getFileFilter() {
 		return fileFilter;
 	}

 	public void setFileFilter(Function<File, Boolean> fileFilter) {
 		this.fileFilter = fileFilter;
 	}

 	protected void addAllFiles(File dir, Set<File> ret) {
 		File[] files = dir.listFiles();
 		if (files == null) {
 			// couldn't list files in that directory
 			console.printerrln("Could not list the entries of " + dir);
 			return;
 		}
 		for (File file : files) {
 			if (isFileInteresting(file)) {
 				if (!file.isDirectory()) {
 					ret.add(file);
 				} else {
 					addAllFiles(file, ret);
 				}
 			}
 		}
 	}

 	private long getRevisionFromFileMetadata(final File f) {
 		return f.lastModified();
 	}

 	private boolean isFileInteresting(File f) {
 		return fileFilter == null || fileFilter.apply(f);
 	}

 	@Override
 	public String getDefaultLocation() {
 		return "file://path/to/folder";
 	}
 }
	/*******************************************************************************
	* Copyright (c) 2011-2020 The University of York, Aston University.
	*
	* This program and the accompanying materials are made available under the
	* terms of the Eclipse Public License 2.0 which is available at
	* http://www.eclipse.org/legal/epl-2.0.
	*
	* This Source Code may also be made available under the following Secondary
	* Licenses when the conditions for such availability set forth in the Eclipse
	* Public License, v. 2.0 are satisfied: GNU General Public License, version 3.
	*
	* SPDX-License-Identifier: EPL-2.0 OR GPL-3.0
	*
	* Contributors:
	* Konstantinos Barmpis - initial API and implementation
	* Antonio Garcia-Dominguez - use Java 7 Path instead of File+string processing,
	* use MapDB, refactor into shared part with LocalFile, add file filtering
	******************************************************************************/
	package org.eclipse.hawk.localfolder;

	import java.io.File;
	import java.io.IOException;
	import java.net.URI;
	import java.net.URISyntaxException;
	import java.net.URLDecoder;
	import java.nio.file.FileVisitResult;
	import java.nio.file.FileVisitor;
	import java.nio.file.Files;
	import java.nio.file.Path;
	import java.nio.file.Paths;
	import java.nio.file.attribute.BasicFileAttributes;
	import java.util.ArrayList;
	import java.util.Date;
	import java.util.HashSet;
	import java.util.List;
	import java.util.Map;
	import java.util.Set;
	import java.util.function.Function;

	import org.eclipse.hawk.core.IModelIndexer;
	import org.eclipse.hawk.core.VcsChangeType;
	import org.eclipse.hawk.core.VcsCommit;
	import org.eclipse.hawk.core.VcsCommitItem;
	import org.eclipse.hawk.core.VcsRepositoryDelta;
	import org.mapdb.DB;
	import org.mapdb.DBMaker;

	/**
	* VCS manager that watches over the contents of a directory, including its
	* subdirectories. Revisions are based on "last modified" timestamps, which
	* have millisecond resolution in Linux from Java 9 onwards.
	*/
	public class LocalFolder extends FileBasedLocation {

	private long maxLastModified = 0;

	private final class LastModifiedFileVisitor implements FileVisitor<Path> {

	private boolean alter;

	public LastModifiedFileVisitor(boolean alter) {
	this.alter = alter;
	}

	@Override
	public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
	final File f = dir.toFile();
	final long currentlatest = getRevisionFromFileMetadata(f);
	final Long lastRev = recordedModifiedDates.get(dir.toString());
	if (lastRev == null \|\| lastRev < currentlatest) {
	if (alter) {
	recordedModifiedDates.put(dir.toString(), currentlatest);
	}
	maxLastModified = Math.max(f.lastModified(), maxLastModified);
	}
	return FileVisitResult.CONTINUE;
	}

	@Override
	public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
	if (isFileInteresting(dir.toFile())) {
	return FileVisitResult.CONTINUE;
	} else {
	return FileVisitResult.SKIP_SUBTREE;
	}
	}

	@Override
	public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
	final File f = file.toFile();
	if (isFileInteresting(f)) {
	final long currentlatest = getRevisionFromFileMetadata(f);
	final Long lastRev = recordedModifiedDates.get(file.toString());
	if (lastRev == null \|\| lastRev < currentlatest) {
	if (alter) {
	recordedModifiedDates.put(file.toString(), currentlatest);
	}
	maxLastModified = Math.max(f.lastModified(), maxLastModified);
	}
	}
	return FileVisitResult.CONTINUE;
	}

	@Override
	public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
	return FileVisitResult.CONTINUE;
	}
	}

	private Path rootLocation;
	private long lastIndexedRevision = 0;
	private Set<File> previousFiles;
	private Map<String, Long> recordedModifiedDates;
	private Function<File, Boolean> fileFilter;

	/**
	* MapDB database: using file-backed Java collections allows us to save memory
	* when handling folders with a large number of files.
	*/
	private DB db;

	@Override
	public void init(String vcsloc, IModelIndexer indexer) throws Exception {
	final File fMapDB = File.createTempFile("localfolder", "mapdb");
	db = DBMaker.newFileDB(fMapDB).deleteFilesAfterClose().closeOnJvmShutdown().make();
	previousFiles = db.createHashSet("previousFiles").make();
	recordedModifiedDates = db.createHashMap("recordedModifiedDates").make();

	console = indexer.getConsole();

	// Accept both regular paths and file:// URIs
	Path path;
	try {
	path = Paths.get(new URI(vcsloc));
	} catch (URISyntaxException \| IllegalArgumentException ex) {
	path = Paths.get(vcsloc);
	}

	final File canonicalFile = path.toFile().getCanonicalFile();
	rootLocation = canonicalFile.toPath();
	String repositoryURI = rootLocation.toUri().toString();

	// If the file doesn't exist, it might be because this is a local folder in
	// a remote server - try to preserve the provided vcsloc as is. Otherwise,
	// if the server and the client use different operating systems we could end
	// up with an unusable URL in the server.
	if (canonicalFile.exists()) {
	repositoryURL = repositoryURI;
	} else {
	repositoryURL = vcsloc;
	}

	// dont decode it to ensure consistency with other managers
	// URLDecoder.decode(repositoryURI.replace("+", "%2B"), "UTF-8");
	}

	protected String getCurrentRevision(boolean alter) {
	try {
	final LastModifiedFileVisitor visitor = new LastModifiedFileVisitor(alter);
	Files.walkFileTree(rootLocation, visitor);
	long ret = maxLastModified;
	if (alter) {
	lastIndexedRevision = ret;
	}
	return ret + "";
	} catch (IOException ex) {
	console.printerrln(ex);
	return FIRST_REV;
	}
	}

	@Override
	public File importFile(String revision, String p, File temp) {
	try {
	final String path = URLDecoder.decode(p.replace("+", "%2B"), "UTF-8");
	final Path resolvedPath = rootLocation.resolve(path.startsWith("/") ? path.replaceFirst("/", "") : path);
	return resolvedPath.toFile();
	} catch (Exception e) {
	console.printerrln(e);
	return null;
	}
	}

	@Override
	public boolean isActive() {
	return rootLocation == null ? false : rootLocation.toFile().exists();
	}

	@Override
	public void shutdown() {
	rootLocation = null;
	if (!db.isClosed()) {
	db.close();
	}
	}

	@Override
	public String getHumanReadableName() {
	return "Local Folder Monitor";
	}

	@Override
	public VcsRepositoryDelta getDelta(String startRevision, String endRevision) throws Exception {
	List<VcsCommit> commits = new ArrayList<>();
	VcsRepositoryDelta delta = new VcsRepositoryDelta(commits);
	delta.setManager(this);

	// Update maxLastModified (there may be deletes, updating the folder timestamp)
	getCurrentRevision();

	Set<File> files = new HashSet<>();
	addAllFiles(rootLocation.toFile(), files);
	previousFiles.removeAll(files);

	for (File f : previousFiles) {
	VcsCommit commit = new VcsCommit();
	commit.setAuthor("i am a local folder driver - no authors recorded");
	commit.setJavaDate(new Date());
	commit.setMessage("i am a local folder driver - no messages recorded");
	commit.setRevision(maxLastModified + "");
	commits.add(commit);

	VcsCommitItem c = new VcsCommitItem();
	c.setChangeType(VcsChangeType.DELETED);
	c.setCommit(commit);

	Path path = f.toPath();

	// Don't decode it to ensure consistency with other managers
	String relativepath = makeRelative(repositoryURL,
	f.toPath().toUri().toString()
	);

	c.setPath(relativepath.startsWith("/") ? relativepath : "/" + relativepath);
	commit.getItems().add(c);
	recordedModifiedDates.remove(path.toString());
	}

	previousFiles.clear();

	if (files != null && files.size() > 0) {
	// long newLastModifiedRepository = lastModifiedRepository;
	for (File f : files) {
	previousFiles.add(f);
	Path filePath = f.toPath();
	final long latestRev = getRevisionFromFileMetadata(f);
	final Long lastDate = recordedModifiedDates.get(filePath.toString());
	if (lastDate != null && lastDate == latestRev) {
	if ((lastIndexedRevision + "").equals(startRevision))
	continue;
	}
	recordedModifiedDates.put(filePath.toString(), latestRev);

	VcsCommit commit = new VcsCommit();
	commit.setAuthor("i am a local folder driver - no authors recorded");
	commit.setJavaDate(new Date());
	commit.setMessage("i am a local folder driver - no messages recorded");
	commit.setRevision(maxLastModified + "");
	commits.add(commit);

	VcsCommitItem c = new VcsCommitItem();
	c.setChangeType(VcsChangeType.UPDATED);
	c.setCommit(commit);

	String relativepath = makeRelative(repositoryURL, f.toPath().toUri().toString());
	c.setPath(relativepath.startsWith("/") ? relativepath : ("/" + relativepath));
	commit.getItems().add(c);
	}
	}

	// Update the latest revision seen
	lastIndexedRevision = maxLastModified;

	return delta;
	}

	public Function<File, Boolean> getFileFilter() {
	return fileFilter;
	}

	public void setFileFilter(Function<File, Boolean> fileFilter) {
	this.fileFilter = fileFilter;
	}

	protected void addAllFiles(File dir, Set<File> ret) {
	File[] files = dir.listFiles();
	if (files == null) {
	// couldn't list files in that directory
	console.printerrln("Could not list the entries of " + dir);
	return;
	}
	for (File file : files) {
	if (isFileInteresting(file)) {
	if (!file.isDirectory()) {
	ret.add(file);
	} else {
	addAllFiles(file, ret);
	}
	}
	}
	}

	private long getRevisionFromFileMetadata(final File f) {
	return f.lastModified();
	}

	private boolean isFileInteresting(File f) {
	return fileFilter == null \|\| fileFilter.apply(f);
	}

	@Override
	public String getDefaultLocation() {
	return "file://path/to/folder";
	}
	}