benchmark/src/example/PerformanceTest.java - gerrit/eclipselink/examples/performance - Git at Google

 /*******************************************************************************
  * Copyright (c) 1998, 2012 Oracle and/or its affiliates. All rights reserved.
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Public License v1.0 and Eclipse Distribution License v. 1.0
  * which accompanies this distribution.
  * The Eclipse Public License is available at http://www.eclipse.org/legal/epl-v10.html
  * and the Eclipse Distribution License is available at
  * http://www.eclipse.org/org/documents/edl-v10.php.
  *
  * Contributors:
  *      Oracle - initial impl
  ******************************************************************************/
 package example;

 import java.util.ArrayList;
 import java.util.List;

 /**
  * Generic performance test.
  * This allows executing a timed run and measures and logs the performance.
  * Logs the executions per run (minute), logs average and % stdev.
  */
 public class PerformanceTest {

     public static int REPEATS = 5;
     public static long RUN_TIME = 10 * 10000L * 1000000L; //10 seconds (in nanos).
     public static int THREADS = 1;

     static WorkerThread[] threads;

     /**
      * Defines the work thread.
      * A worker thread calls the run method in a loop,
      * until it is suspended.
      */
     protected static class WorkerThread extends Thread {
         volatile boolean isSuspended = true;
         volatile boolean isDead = false;
         Runnable runnable;
         long min;
         long max;
         long total;
         int count;

         public void stopExecution() {
             isDead = true;
         }

         /**
          * After the next completion of the run method, suspend execution.
          * The thread is still alive, but waiting to be signaled to resumed.
          */
         public void suspendExecution() {
             isSuspended = true;
         }

         /**
          * Wait for the thread to suspend.
          * Synchronize will wait for wait to release synchronization.
          */
         public synchronized void joinExecution() {
             if (!isSuspended) {
                 throw new RuntimeException("Must suspend first");
             }
         }

         /**
          * Continue running the run method.
          */
         public synchronized void resumeExecution() {
             isSuspended = false;
             min = Long.MAX_VALUE;
             max = 0;
             total = 0;
             count = 0;
             try {
                 notify();
             } catch (Exception exception) {
                 throw new RuntimeException(exception.getMessage());
             }
         }

         /**
          * Run the test run method in a loop until killed.
          */
         public synchronized void run() {
             try {
                 while (!isDead) {
                     // Allows the thread to suspend itself when the current test is done.
                     if (isSuspended) {
                         wait();
                     }
                     long start = System.nanoTime();
                     this.runnable.run();
                     long end = System.nanoTime();
                     long time = end - start;
                     total = total + time;
                     if (time > max) {
                         max = time;
                     }
                     if (time < min) {
                         min = time;
                     }
                     count++;
                 }
             } catch (Exception exception) {
                 exception.printStackTrace();
                 throw new RuntimeException(exception.getMessage());
             }
         }
     }

     /**
      * Measure the performance of the run.
      * Repeat the run REPEATS (5) times,
      * and measure the number of execution in RUN_TIME (60s).
      */
     public PerformanceResult executeRun(String name, Runnable runnable) {
         if (THREADS > 1) {
             return executeMultiThreadedRun(name, runnable);
         }
         System.out.println("Starting run: " + name);

         List<Integer> results = new ArrayList<Integer>();
         long max = 0;
         long min = Long.MAX_VALUE;
         // Repeat the test and baseline for the number of repeats.
         for (int index = 0; index < REPEATS; index++) {
             int executions = 0;
             System.gc();
             try {
                 Thread.sleep(1000);
             } catch (InterruptedException ignore) {}
             long startTime = System.nanoTime();
             long endTime = startTime;
             long lastEndTime = endTime;
             // Count how many times the test can be invoked in the run time.
             // This allows for the test run time to be easily changed.
             while ((startTime + (RUN_TIME)) >= endTime) {
                 runnable.run();
                 executions++;
                 endTime = System.nanoTime();
                 long time = endTime - lastEndTime;
                 lastEndTime = endTime;
                 if (time > max) {
                     max = time;
                 }
                 if (time < min) {
                     min = time;
                 }
             }
             results.add(executions);
             System.out.println("Done run: " +  index + " for: " + name + " - " + executions);
             reset();
         }

         System.out.println("Completed run: " + name);

         PerformanceResult result = new PerformanceResult();
         result.testName = name;
         result.runRepeats = REPEATS;
         result.runTime = RUN_TIME;
         result.average = averageResults(results);
         result.max = maxResults(results);
         result.min = minResults(results);
         result.standardDeviation = standardDeviationResults(results);
         result.maxOperationTime = max;
         result.minOperationTime = min;
         result.averageOperationTime = (long) (RUN_TIME / result.average);

         System.out.println("");
         System.out.println(result);
         System.out.println("");
         System.out.println("");

         return result;
     }

     /**
      * Allow the test to reset after a run.
      */
     public void reset() {
         return;
     }

     /**
      * Measure the performance of the run.
      * Repeat the run REPEATS (5) times,
      * and measure the number of execution in RUN_TIME (60s).
      */
     public static PerformanceResult executeMultiThreadedRun(String name, Runnable runnable) {
         System.out.println("Starting run: " + name);

         threads = new WorkerThread[THREADS];
         for (int index = 0; index < THREADS; index++) {
             threads[index] = new WorkerThread();
         }
         for (int index = 0; index < THREADS; index++) {
             threads[index].runnable = runnable;
         }
         for (int index = 0; index < THREADS; index++) {
             threads[index].start();
         }

         long min = Long.MAX_VALUE;
         long max = 0;
         long totalTime = 0;
         long totalExecutions = 0;
         List<Integer> results = new ArrayList<Integer>();
         // Repeat the test and baseline for the number of repeats.
         for (int repeat = 0; repeat < REPEATS; repeat++) {
             System.gc();
             try {
                 Thread.sleep(1000);
             } catch (InterruptedException ignore) {}
             long startTime = System.nanoTime();
             long endTime = startTime;
             for (int index = 0; index < THREADS; index++) {
                 threads[index].resumeExecution();
             }
             while ((startTime + RUN_TIME) >= endTime) {
                 try {
                     Thread.sleep(1);
                 } catch (Exception ignore) {}
                 endTime = System.nanoTime();
             }
             for (int index = 0; index < THREADS; index++) {
                 threads[index].suspendExecution();
             }
             Thread.yield();
             for (int index = 0; index < THREADS; index++) {
                 threads[index].joinExecution();
             }
             int executions = 0;
             for (int index = 0; index < THREADS; index++) {
                 WorkerThread thread = threads[index];
                 executions = executions + thread.count;
                 totalTime = totalTime + thread.total;
                 if (thread.max > max) {
                     max = thread.max;
                 }
                 if (thread.min < min) {
                     min = thread.min;
                 }
             }
             totalExecutions = totalExecutions + executions;
             results.add(executions);
             System.out.println("Done run: " +  repeat + " for: " + name);

         }

         for (int index = 0; index < THREADS; index++) {
             threads[index].stopExecution();
         }
         Thread.yield();
         threads = null;

         System.out.println("Completed run: " + name);

         PerformanceResult result = new PerformanceResult();
         result.testName = name;
         result.threads = THREADS;
         result.runRepeats = REPEATS;
         result.runTime = RUN_TIME;
         result.average = averageResults(results);
         result.max = maxResults(results);
         result.min = minResults(results);
         result.standardDeviation = standardDeviationResults(results);
         result.maxOperationTime = max;
         result.minOperationTime = min;
         result.averageOperationTime = (totalTime / totalExecutions);

         System.out.println("");
         System.out.println(result);
         System.out.println("");
         System.out.println("");

         return result;
     }

     /**
      * Compute the max of the results.
      */
     public static int maxResults(List<Integer> times) {
         int testMax = 0;
         for (int index = 0; index < times.size(); index++) {
             int time = (int)times.get(index);
             if (time > testMax) {
                 testMax = time;
             }
         }
         return testMax;
     }

     /**
      * Compute the min of the results.
      */
     public static int minResults(List<Integer> times) {
         int testMin = 0;
         for (int index = 0; index < times.size(); index++) {
             int time = (int)times.get(index);
             if ((testMin == 0) || (time < testMin)) {
                 testMin = time;
             }
         }
         return testMin;
     }

     /**
      * Filter max and min from results.
      */
     public static List<Integer> filterMaxMinResults(List<Integer> times) {
         List filteredTimes = new ArrayList(times);
         if (filteredTimes.size() > 3) {
             filteredTimes.remove((Integer)maxResults(times));
             filteredTimes.remove((Integer)minResults(times));
         }
         return filteredTimes;
     }

     /**
      * Compute the average of the results rejecting the min and max.
      */
     public static double averageResults(List<Integer> allTimes) {
         // Compute the average reject the min and max to improve consistency.
         List<Integer> times = filterMaxMinResults(allTimes);
         double testAverage = 0;
         for (int index = 0; index < times.size(); index++) {
             int time = (int)times.get(index);
             testAverage = testAverage + time;
         }
         testAverage = testAverage / times.size();
         return testAverage;
     }

     /**
      * Compute the standard deviation of the results rejecting the min and max.
      */
     public static double standardDeviationResults(List<Integer> allTimes) {
         // Compute the average reject the min and max to improve consistency.
         double testAverage = averageResults(allTimes);

         // Compute the standard deviation reject the min and max to improve consistency.
         List<Integer> times = filterMaxMinResults(allTimes);
         double testStandardDeviation = 0;
         for (int index = 0; index < times.size(); index++) {
             int time = (int)times.get(index);
             testStandardDeviation = testStandardDeviation + Math.pow(time - testAverage, 2);
         }
         testStandardDeviation = testStandardDeviation / times.size();
         testStandardDeviation = Math.sqrt(testStandardDeviation);
         // As percent of average
         testStandardDeviation = (testStandardDeviation / testAverage) * 100;
         return testStandardDeviation;
     }
 }
	/*******************************************************************************
	* Copyright (c) 1998, 2012 Oracle and/or its affiliates. All rights reserved.
	* This program and the accompanying materials are made available under the
	* terms of the Eclipse Public License v1.0 and Eclipse Distribution License v. 1.0
	* which accompanies this distribution.
	* The Eclipse Public License is available at http://www.eclipse.org/legal/epl-v10.html
	* and the Eclipse Distribution License is available at
	* http://www.eclipse.org/org/documents/edl-v10.php.
	*
	* Contributors:
	* Oracle - initial impl
	******************************************************************************/
	package example;

	import java.util.ArrayList;
	import java.util.List;

	/**
	* Generic performance test.
	* This allows executing a timed run and measures and logs the performance.
	* Logs the executions per run (minute), logs average and % stdev.
	*/
	public class PerformanceTest {

	public static int REPEATS = 5;
	public static long RUN_TIME = 10 * 10000L * 1000000L; //10 seconds (in nanos).
	public static int THREADS = 1;

	static WorkerThread[] threads;

	/**
	* Defines the work thread.
	* A worker thread calls the run method in a loop,
	* until it is suspended.
	*/
	protected static class WorkerThread extends Thread {
	volatile boolean isSuspended = true;
	volatile boolean isDead = false;
	Runnable runnable;
	long min;
	long max;
	long total;
	int count;

	public void stopExecution() {
	isDead = true;
	}

	/**
	* After the next completion of the run method, suspend execution.
	* The thread is still alive, but waiting to be signaled to resumed.
	*/
	public void suspendExecution() {
	isSuspended = true;
	}

	/**
	* Wait for the thread to suspend.
	* Synchronize will wait for wait to release synchronization.
	*/
	public synchronized void joinExecution() {
	if (!isSuspended) {
	throw new RuntimeException("Must suspend first");
	}
	}

	/**
	* Continue running the run method.
	*/
	public synchronized void resumeExecution() {
	isSuspended = false;
	min = Long.MAX_VALUE;
	max = 0;
	total = 0;
	count = 0;
	try {
	notify();
	} catch (Exception exception) {
	throw new RuntimeException(exception.getMessage());
	}
	}

	/**
	* Run the test run method in a loop until killed.
	*/
	public synchronized void run() {
	try {
	while (!isDead) {
	// Allows the thread to suspend itself when the current test is done.
	if (isSuspended) {
	wait();
	}
	long start = System.nanoTime();
	this.runnable.run();
	long end = System.nanoTime();
	long time = end - start;
	total = total + time;
	if (time > max) {
	max = time;
	}
	if (time < min) {
	min = time;
	}
	count++;
	}
	} catch (Exception exception) {
	exception.printStackTrace();
	throw new RuntimeException(exception.getMessage());
	}
	}
	}

	/**
	* Measure the performance of the run.
	* Repeat the run REPEATS (5) times,
	* and measure the number of execution in RUN_TIME (60s).
	*/
	public PerformanceResult executeRun(String name, Runnable runnable) {
	if (THREADS > 1) {
	return executeMultiThreadedRun(name, runnable);
	}
	System.out.println("Starting run: " + name);

	List<Integer> results = new ArrayList<Integer>();
	long max = 0;
	long min = Long.MAX_VALUE;
	// Repeat the test and baseline for the number of repeats.
	for (int index = 0; index < REPEATS; index++) {
	int executions = 0;
	System.gc();
	try {
	Thread.sleep(1000);
	} catch (InterruptedException ignore) {}
	long startTime = System.nanoTime();
	long endTime = startTime;
	long lastEndTime = endTime;
	// Count how many times the test can be invoked in the run time.
	// This allows for the test run time to be easily changed.
	while ((startTime + (RUN_TIME)) >= endTime) {
	runnable.run();
	executions++;
	endTime = System.nanoTime();
	long time = endTime - lastEndTime;
	lastEndTime = endTime;
	if (time > max) {
	max = time;
	}
	if (time < min) {
	min = time;
	}
	}
	results.add(executions);
	System.out.println("Done run: " + index + " for: " + name + " - " + executions);
	reset();
	}

	System.out.println("Completed run: " + name);

	PerformanceResult result = new PerformanceResult();
	result.testName = name;
	result.runRepeats = REPEATS;
	result.runTime = RUN_TIME;
	result.average = averageResults(results);
	result.max = maxResults(results);
	result.min = minResults(results);
	result.standardDeviation = standardDeviationResults(results);
	result.maxOperationTime = max;
	result.minOperationTime = min;
	result.averageOperationTime = (long) (RUN_TIME / result.average);

	System.out.println("");
	System.out.println(result);
	System.out.println("");
	System.out.println("");

	return result;
	}

	/**
	* Allow the test to reset after a run.
	*/
	public void reset() {
	return;
	}

	/**
	* Measure the performance of the run.
	* Repeat the run REPEATS (5) times,
	* and measure the number of execution in RUN_TIME (60s).
	*/
	public static PerformanceResult executeMultiThreadedRun(String name, Runnable runnable) {
	System.out.println("Starting run: " + name);

	threads = new WorkerThread[THREADS];
	for (int index = 0; index < THREADS; index++) {
	threads[index] = new WorkerThread();
	}
	for (int index = 0; index < THREADS; index++) {
	threads[index].runnable = runnable;
	}
	for (int index = 0; index < THREADS; index++) {
	threads[index].start();
	}

	long min = Long.MAX_VALUE;
	long max = 0;
	long totalTime = 0;
	long totalExecutions = 0;
	List<Integer> results = new ArrayList<Integer>();
	// Repeat the test and baseline for the number of repeats.
	for (int repeat = 0; repeat < REPEATS; repeat++) {
	System.gc();
	try {
	Thread.sleep(1000);
	} catch (InterruptedException ignore) {}
	long startTime = System.nanoTime();
	long endTime = startTime;
	for (int index = 0; index < THREADS; index++) {
	threads[index].resumeExecution();
	}
	while ((startTime + RUN_TIME) >= endTime) {
	try {
	Thread.sleep(1);
	} catch (Exception ignore) {}
	endTime = System.nanoTime();
	}
	for (int index = 0; index < THREADS; index++) {
	threads[index].suspendExecution();
	}
	Thread.yield();
	for (int index = 0; index < THREADS; index++) {
	threads[index].joinExecution();
	}
	int executions = 0;
	for (int index = 0; index < THREADS; index++) {
	WorkerThread thread = threads[index];
	executions = executions + thread.count;
	totalTime = totalTime + thread.total;
	if (thread.max > max) {
	max = thread.max;
	}
	if (thread.min < min) {
	min = thread.min;
	}
	}
	totalExecutions = totalExecutions + executions;
	results.add(executions);
	System.out.println("Done run: " + repeat + " for: " + name);

	}

	for (int index = 0; index < THREADS; index++) {
	threads[index].stopExecution();
	}
	Thread.yield();
	threads = null;

	System.out.println("Completed run: " + name);

	PerformanceResult result = new PerformanceResult();
	result.testName = name;
	result.threads = THREADS;
	result.runRepeats = REPEATS;
	result.runTime = RUN_TIME;
	result.average = averageResults(results);
	result.max = maxResults(results);
	result.min = minResults(results);
	result.standardDeviation = standardDeviationResults(results);
	result.maxOperationTime = max;
	result.minOperationTime = min;
	result.averageOperationTime = (totalTime / totalExecutions);

	System.out.println("");
	System.out.println(result);
	System.out.println("");
	System.out.println("");

	return result;
	}

	/**
	* Compute the max of the results.
	*/
	public static int maxResults(List<Integer> times) {
	int testMax = 0;
	for (int index = 0; index < times.size(); index++) {
	int time = (int)times.get(index);
	if (time > testMax) {
	testMax = time;
	}
	}
	return testMax;
	}

	/**
	* Compute the min of the results.
	*/
	public static int minResults(List<Integer> times) {
	int testMin = 0;
	for (int index = 0; index < times.size(); index++) {
	int time = (int)times.get(index);
	if ((testMin == 0) \|\| (time < testMin)) {
	testMin = time;
	}
	}
	return testMin;
	}

	/**
	* Filter max and min from results.
	*/
	public static List<Integer> filterMaxMinResults(List<Integer> times) {
	List filteredTimes = new ArrayList(times);
	if (filteredTimes.size() > 3) {
	filteredTimes.remove((Integer)maxResults(times));
	filteredTimes.remove((Integer)minResults(times));
	}
	return filteredTimes;
	}

	/**
	* Compute the average of the results rejecting the min and max.
	*/
	public static double averageResults(List<Integer> allTimes) {
	// Compute the average reject the min and max to improve consistency.
	List<Integer> times = filterMaxMinResults(allTimes);
	double testAverage = 0;
	for (int index = 0; index < times.size(); index++) {
	int time = (int)times.get(index);
	testAverage = testAverage + time;
	}
	testAverage = testAverage / times.size();
	return testAverage;
	}

	/**
	* Compute the standard deviation of the results rejecting the min and max.
	*/
	public static double standardDeviationResults(List<Integer> allTimes) {
	// Compute the average reject the min and max to improve consistency.
	double testAverage = averageResults(allTimes);

	// Compute the standard deviation reject the min and max to improve consistency.
	List<Integer> times = filterMaxMinResults(allTimes);
	double testStandardDeviation = 0;
	for (int index = 0; index < times.size(); index++) {
	int time = (int)times.get(index);
	testStandardDeviation = testStandardDeviation + Math.pow(time - testAverage, 2);
	}
	testStandardDeviation = testStandardDeviation / times.size();
	testStandardDeviation = Math.sqrt(testStandardDeviation);
	// As percent of average
	testStandardDeviation = (testStandardDeviation / testAverage) * 100;
	return testStandardDeviation;
	}
	}