Updates to CDA generation
diff --git a/cda/plugins/org.eclipse.mdht.cda.xml.ui/src/org/eclipse/mdht/cda/xml/ui/handlers/GenerateCDADataHandler.java b/cda/plugins/org.eclipse.mdht.cda.xml.ui/src/org/eclipse/mdht/cda/xml/ui/handlers/GenerateCDADataHandler.java
index 288d478..bb572f3 100644
--- a/cda/plugins/org.eclipse.mdht.cda.xml.ui/src/org/eclipse/mdht/cda/xml/ui/handlers/GenerateCDADataHandler.java
+++ b/cda/plugins/org.eclipse.mdht.cda.xml.ui/src/org/eclipse/mdht/cda/xml/ui/handlers/GenerateCDADataHandler.java
@@ -12,6 +12,7 @@
*******************************************************************************/
package org.eclipse.mdht.cda.xml.ui.handlers;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -22,6 +23,7 @@
import java.lang.management.MemoryUsage;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
@@ -888,6 +890,13 @@
}
}
+ public static String formatSize(long v) {
+ if (v < 1024)
+ return v + " B";
+ int z = (63 - Long.numberOfLeadingZeros(v)) / 10;
+ return String.format("%.1f %sB", (double) v / (1L << (z * 10)), " KMGTPE".charAt(z));
+ }
+
void processFolder(IFolder folder, IProgressMonitor monitor, String splitOption, HashSet<EClass> sectionFilter,
HashMap<EClass, HashSet<EClass>> theSectionCache) throws Exception {
@@ -962,6 +971,13 @@
Collections.sort(documents, c);
+ String fileLocation2 = folder.getParent().getLocation().toOSString() + System.getProperty("file.separator") +
+ CDAValueUtil.DATE_FORMAT3.format(new Date()) + "_" + folder.getName().toUpperCase() + "performance.log";
+
+ Path path = Paths.get(fileLocation2);
+
+ BufferedWriter performance = Files.newBufferedWriter(path);
+
int totalFiles = folder.members().length;
for (IFile file : documents) {
stopwatch.reset();
@@ -982,17 +998,20 @@
if (estimatedTimeLeft > 60) {
monitor.setTaskName(
- "Generate Spreadsheet, Estimated Time to finish : " + ((int) estimatedTimeLeft / 60) +
- " Minutes ");
+ "Processing " + file.getName() + ", " + StringUtils.leftPad(String.valueOf(filectr++), 5) +
+ " of " + StringUtils.leftPad(String.valueOf(totalFiles), 5) +
+ ", Estimated Time to finish : " + ((int) estimatedTimeLeft / 60) + " Minutes ");
} else {
monitor.setTaskName(
- "Generate Spreadsheet, Estimated Time to finish : " + ((int) estimatedTimeLeft) + " Seconds ");
+ "Processing " + file.getName() + ", " + StringUtils.leftPad(String.valueOf(filectr++), 5) +
+ " of " + StringUtils.leftPad(String.valueOf(totalFiles), 5) +
+ ", Estimated Time to finish : " + ((int) estimatedTimeLeft) + " Seconds ");
}
- monitor.subTask(
- "Processing File " + StringUtils.leftPad(String.valueOf(filectr++), 5) + " of " +
- StringUtils.leftPad(String.valueOf(totalFiles), 5) + " Average Time per File " +
- (currentProcessingTime / filectr) / 1000.0 + " Seconds ");
+ // monitor.subTask(
+ // "Processing File " + StringUtils.leftPad(String.valueOf(filectr++), 5) + " of " +
+ // StringUtils.leftPad(String.valueOf(totalFiles), 5) + " General Performance" +
+ // fileSize / (currentProcessingTime / 1000.0) + " Bytes per Second ");
try {
console.println(getMemoryUssage());
@@ -1093,6 +1112,17 @@
// }
}
+
+ long t = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+ String s = file.getName() + " : " + formatSize(fileSize) + " took " + t / 1000.0 +
+ " Second(s) at " +
+ String.format("%.2f MB/S", (fileSize / (1024 * 1024)) / (t / 1000.0));
+ monitor.subTask(s);
+
+ performance.write(s);
+ performance.newLine();
+ performance.flush();
+
console.println("End Processing " + stopwatch.elapsed(TimeUnit.MILLISECONDS));
} else {
diff --git a/cda/plugins/org.eclipse.mdht.cda.xml.ui/src/org/eclipse/mdht/cda/xml/ui/handlers/StripCDANarrativeHandler.java b/cda/plugins/org.eclipse.mdht.cda.xml.ui/src/org/eclipse/mdht/cda/xml/ui/handlers/StripCDANarrativeHandler.java
index 65fd36c..0545a5a 100644
--- a/cda/plugins/org.eclipse.mdht.cda.xml.ui/src/org/eclipse/mdht/cda/xml/ui/handlers/StripCDANarrativeHandler.java
+++ b/cda/plugins/org.eclipse.mdht.cda.xml.ui/src/org/eclipse/mdht/cda/xml/ui/handlers/StripCDANarrativeHandler.java
@@ -39,9 +39,7 @@
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.emf.common.util.URI;
-import org.eclipse.emf.ecore.EClass;
import org.eclipse.emf.ecore.resource.Resource.IOWrappedException;
-import org.eclipse.emf.ecore.resource.ResourceSet;
import org.eclipse.jface.dialogs.ProgressMonitorDialog;
import org.eclipse.jface.dialogs.TitleAreaDialog;
import org.eclipse.jface.operation.IRunnableWithProgress;
@@ -52,6 +50,7 @@
import org.eclipse.mdht.uml.cda.Section;
import org.eclipse.mdht.uml.cda.StrucDocText;
import org.eclipse.mdht.uml.cda.util.CDAUtil;
+import org.eclipse.mdht.uml.cda.util.CDAUtil.ValidationHandler;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.ScrolledComposite;
import org.eclipse.swt.dnd.Clipboard;
@@ -179,17 +178,15 @@
}
- public void stripNarratives(ResourceSet rs, final IFile file) throws Exception {
+ public void stripNarratives(final IFile file) throws Exception {
URI cdaURI = URI.createFileURI(file.getLocation().toOSString());
ClinicalDocument clinicalDocument = null;
try (InputStream is = Files.newInputStream(Paths.get(cdaURI.toFileString()))) {
- //
- ;
- // clinicalDocument = CDAUtil.load(is, ((ValidationHandler) null));
- clinicalDocument = CDAUtil.load(rs, CDAUtil.generateURI(), is, null);
+
+ clinicalDocument = CDAUtil.load(is, ((ValidationHandler) null));
is.close();
} catch (IOWrappedException iowe) {
System.out.println("error processing " + cdaURI.toFileString());
@@ -224,16 +221,14 @@
}
- // clinicalDocument.eResource().unload();
-
- rs.getResources().remove(clinicalDocument.eResource());
+ clinicalDocument.eResource().unload();
}
ArrayList<IFile> documents = new ArrayList<IFile>();
protected static DecimalFormat format2Places = new DecimalFormat(".##");
- protected void processFolder(ResourceSet rs, IFolder folder, IProgressMonitor monitor) throws CoreException {
+ protected void processFolder(IFolder folder, IProgressMonitor monitor) throws CoreException {
int filectr = 1;
long currentProcessingTime = 1;
@@ -251,7 +246,7 @@
}
if (resource instanceof IFolder && !resource.getName().equals("NarrativeRemoved")) {
- processFolder(rs, (IFolder) resource, monitor);
+ processFolder((IFolder) resource, monitor);
}
if (resource instanceof IFile) {
@@ -291,7 +286,6 @@
currentProcessingTime = 0;
long estimatedTimeLeft = 0;
-
for (IFile document : documents) {
monitor.worked(1);
@@ -318,14 +312,11 @@
try {
stopwatch.reset();
stopwatch.start();
- stripNarratives(rs, document);
+ stripNarratives(document);
stopwatch.stop();
currentProcessingTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
- long denominator = stopwatch.elapsed(TimeUnit.MILLISECONDS);
- long ratePerSecond = fileSize / (denominator > 0
- ? denominator
- : 1);
+ long ratePerSecond = fileSize / stopwatch.elapsed(TimeUnit.MILLISECONDS);
totalBytes -= fileSize;
@@ -352,8 +343,6 @@
try {
- ResourceSet rs = CDAUtil.createResourceSet((EClass) null);
-
ProgressMonitorDialog pd = new ProgressMonitorDialog(
PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell());
@@ -376,10 +365,10 @@
if (o instanceof IFolder) {
IFolder folder = (IFolder) o;
monitor.beginTask("DeIdentify CDA Documents", folder.members().length);
- processFolder(rs, folder, monitor);
+ processFolder(folder, monitor);
}
if (o instanceof IFile) {
- stripNarratives(rs, (IFile) o);
+ stripNarratives((IFile) o);
}
}
} catch (PartInitException e) {
