org.eclipse.rephraserengine-support/src-cdtdb-4.0.3/org/eclipse/rephraserengine/internal/db/org/eclipse/cdt/internal/core/pdom/db/Database.java - ptp/org.eclipse.photran - Git at Google

 /*******************************************************************************
  * Copyright (c) 2005, 2008 QNX Software Systems and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
  * http://www.eclipse.org/legal/epl-v10.html
  *
  * Contributors:
  *     QNX - Initial API and implementation
  *     Symbian - Add some non-javadoc implementation notes
  *     Markus Schorn (Wind River Systems)
  *     IBM Corporation
  *******************************************************************************/
 package org.eclipse.rephraserengine.internal.db.org.eclipse.cdt.internal.core.pdom.db;

 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedByInterruptException;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
 import java.util.ArrayList;
 import java.util.Iterator;

 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.rephraserengine.internal.db.org.eclipse.cdt.core.CCorePlugin;


 /**
  * Database encapsulates access to a flat binary format file with a memory-manager-like API for
  * obtaining and releasing areas of storage (memory).
  *
  * @author Doug Schaefer
  */
 /*
  * The file encapsulated is divided into Chunks of size CHUNK_SIZE, and a table of contents
  * mapping chunk index to chunk address is maintained. Chunk structure exists only conceptually -
  * its not a structure that appears in the file.
  *
  * ===== The first chunk is used by Database itself for house-keeping purposes and has structure
  *
  * offset            content
  * 	                 _____________________________
  * 0                | version number
  * INT_SIZE         | pointer to head of linked list of blocks of size MIN_SIZE
  * ..               | ...
  * INT_SIZE * m (1) | pointer to head of linked list of blocks of size MIN_SIZE * m
  * DATA_AREA        | undefined (PDOM stores its own house-keeping data in this area)
  *
  * (1) where m <= (CHUNK_SIZE / MIN_SIZE)
  *
  * ===== block structure
  *
  * offset            content
  * 	                 _____________________________
  * 0                | size of block (negative indicates in use, positive unused)
  * PREV_OFFSET      | pointer to prev block (of same size)
  * NEXT_OFFSET      | pointer to next block (of same size)
  *
  */
 public class Database {

 	private final File fLocation;
 	private final boolean fReadOnly;
 	private RandomAccessFile fFile;
 	private boolean fExclusiveLock= false;	// necessary for any write operation
 	private boolean fLocked;				// necessary for any operation.
 	private boolean fIsMarkedIncomplete= false;

 	private int fVersion;
 	private final Chunk fHeaderChunk;
 	private Chunk[] fChunks;
 	private ChunkCache fCache;

 	private long malloced;
 	private long freed;
 	private long cacheHits;
 	private long cacheMisses;

 	// public for tests only, you shouldn't need these
 	public static final int VERSION_OFFSET = 0;
 	public static final int CHUNK_SIZE = 1024 * 4;
 	public static final int MIN_SIZE = 16;
 	public static final int INT_SIZE = 4;
 	public static final int CHAR_SIZE = 2;
 	public static final int PREV_OFFSET = INT_SIZE;
 	public static final int NEXT_OFFSET = INT_SIZE * 2;
 	public static final int DATA_AREA = CHUNK_SIZE / MIN_SIZE * INT_SIZE + INT_SIZE;

 	public static final int MAX_SIZE = CHUNK_SIZE - 4; // Room for overhead

 	/**
 	 * Construct a new Database object, creating a backing file if necessary.
 	 * @param location the local file path for the database
 	 * @param cache the cache to be used optimization
 	 * @param version the version number to store in the database (only applicable for new databases)
 	 * @param permanentReadOnly whether this Database object will ever need writing to
 	 * @throws CoreException
 	 */
 	public Database(File location, ChunkCache cache, int version, boolean openReadOnly) throws CoreException {
 		try {
 			fLocation = location;
 			fReadOnly= openReadOnly;
 			fCache= cache;
 			openFile();

 			int nChunksOnDisk = (int) (fFile.length() / CHUNK_SIZE);
 			fHeaderChunk= new Chunk(this, 0);
 			fHeaderChunk.fLocked= true;		// never makes it into the cache, needed to satisfy assertions
 			if (nChunksOnDisk <= 0) {
 				fVersion= version;
 				fChunks= new Chunk[1];
 			}
 			else {
 				fHeaderChunk.read();
 				fVersion= fHeaderChunk.getInt(VERSION_OFFSET);
 				fChunks = new Chunk[nChunksOnDisk];	// chunk[0] is unused.
 			}
 		} catch (IOException e) {
 			throw new CoreException(new DBStatus(e));
 		}
 	}

 	private void openFile() throws FileNotFoundException {
 		fFile = new RandomAccessFile(fLocation, fReadOnly ? "r" : "rw"); //$NON-NLS-1$ //$NON-NLS-2$
 	}

 	void read(ByteBuffer buf, int i) throws IOException {
 		int retries= 0;
 		do {
 			try {
 				fFile.getChannel().read(buf, i);
 				return;
 			}
 			catch (ClosedChannelException e) {
 				// bug 219834 file may have be closed by interrupting a thread during an I/O operation.
 				reopen(e, ++retries);
 			}
 		} while (true);
 	}

 	void write(ByteBuffer buf, int i) throws IOException {
 		int retries= 0;
 		do {
 			try {
 				fFile.getChannel().write(buf, i);
 				return;
 			}
 			catch (ClosedChannelException e) {
 				// bug 219834 file may have be closed by interrupting a thread during an I/O operation.
 				reopen(e, ++retries);
 			}
 		} while(true);
 	}

 	private void reopen(ClosedChannelException e, int attempt) throws ClosedChannelException, FileNotFoundException {
 		// only if the current thread was not interrupted we try to reopen the file.
 		if (e instanceof ClosedByInterruptException || attempt >= 20) {
 			throw e;
 		}
 		openFile();
 	}


 	public void transferTo(FileChannel target) throws IOException {
 		assert fLocked;
 		final FileChannel from= fFile.getChannel();
 		from.transferTo(0, from.size(), target);
 	}

 	public int getVersion() throws CoreException {
 		return fVersion;
 	}

 	public void setVersion(int version) throws CoreException {
 		assert fExclusiveLock;
 		fHeaderChunk.putInt(VERSION_OFFSET, version);
 		fVersion= version;
 	}

 	/**
 	 * Empty the contents of the Database, make it ready to start again
 	 * @throws CoreException
 	 */
 	public void clear(int version) throws CoreException {
 		assert fExclusiveLock;
 		removeChunksFromCache();

 		fVersion= version;
 		// clear the first chunk.
 		fHeaderChunk.clear(0, CHUNK_SIZE);
 		// chunks have been removed from the cache, so we may just reset the array of chunks.
 		fChunks = new Chunk[] {null};
 		try {
 			fHeaderChunk.flush();	// zero out header chunk
 			fFile.getChannel().truncate(CHUNK_SIZE);	// truncate database
 		}
 		catch (IOException e) {
 			CCorePlugin.log(e);
 		}
 		malloced = freed = 0;
 	}

 	private void removeChunksFromCache() {
 		synchronized (fCache) {
 			for (int i=1; i < fChunks.length; i++) {
 				Chunk chunk= fChunks[i];
 				if (chunk != null) {
 					fCache.remove(chunk);
 					fChunks[i]= null;
 				}
 			}
 		}
 	}


 	/**
 	 * Return the Chunk that contains the given offset.
 	 * @throws CoreException
 	 */
 	public Chunk getChunk(int offset) throws CoreException {
 		if (offset < CHUNK_SIZE) {
 			return fHeaderChunk;
 		}
 		synchronized(fCache) {
 			assert fLocked;
 			final int index = offset / CHUNK_SIZE;
 			Chunk chunk= fChunks[index];
 			if (chunk == null) {
 				cacheMisses++;
 				chunk = fChunks[index] = new Chunk(this, index);
 				chunk.read();
 			}
 			else {
 				cacheHits++;
 			}
 			fCache.add(chunk, fExclusiveLock);
 			return chunk;
 		}
 	}

 	/**
 	 * Allocate a block out of the database.
 	 *
 	 * @param size
 	 * @return
 	 */
 	public int malloc(int size) throws CoreException {
 		assert fExclusiveLock;
 		if (size > MAX_SIZE)
 			// Too Big
 			throw new CoreException(new Status(IStatus.ERROR, CCorePlugin.PLUGIN_ID, 0,
 					CCorePlugin.getResourceString("pdom.requestTooLarge"), new IllegalArgumentException())); //$NON-NLS-1$

 		// Which block size
 		int freeblock = 0;
 		int blocksize;
 		int matchsize = 0;
 		for (blocksize = MIN_SIZE; blocksize <= CHUNK_SIZE; blocksize += MIN_SIZE) {
 			if (blocksize - INT_SIZE >= size) {
 				if (matchsize == 0) // our real size
 					matchsize = blocksize;
 				freeblock = getFirstBlock(blocksize);
 				if (freeblock != 0)
 					break;
 			}
 		}

 		// get the block
 		Chunk chunk;
 		if (freeblock == 0) {
 			// allocate a new chunk
 			freeblock= createNewChunk();
 			blocksize = CHUNK_SIZE;
 			chunk = getChunk(freeblock);
 		} else {
 			chunk = getChunk(freeblock);
 			removeBlock(chunk, blocksize, freeblock);
 		}

 		if (blocksize != matchsize) {
 			// Add in the unused part of our block
 			addBlock(chunk, blocksize - matchsize, freeblock + matchsize);
 		}

 		// Make our size negative to show in use
 		chunk.putInt(freeblock, - matchsize);

 		// Clear out the block, lots of people are expecting this
 		chunk.clear(freeblock + 4, size);

 		malloced += matchsize;
 		return freeblock + 4;
 	}

 	private int createNewChunk() throws CoreException {
 		assert fExclusiveLock;
 		synchronized (fCache) {
 			final int oldLen= fChunks.length;
 			final Chunk chunk= new Chunk(this, oldLen);
 			chunk.fDirty= true;

 			Chunk[] newchunks = new Chunk[oldLen+1];
 			System.arraycopy(fChunks, 0, newchunks, 0, oldLen);
 			newchunks[oldLen]= chunk;
 			fChunks= newchunks;
 			fCache.add(chunk, true);
 			return oldLen * CHUNK_SIZE;
 		}
 	}

 	private int getFirstBlock(int blocksize) throws CoreException {
 		assert fLocked;
 		return fHeaderChunk.getInt((blocksize / MIN_SIZE) * INT_SIZE);
 	}

 	private void setFirstBlock(int blocksize, int block) throws CoreException {
 		assert fExclusiveLock;
 		fHeaderChunk.putInt((blocksize / MIN_SIZE) * INT_SIZE, block);
 	}

 	private void removeBlock(Chunk chunk, int blocksize, int block) throws CoreException {
 		assert fExclusiveLock;
 		int prevblock = chunk.getInt(block + PREV_OFFSET);
 		int nextblock = chunk.getInt(block + NEXT_OFFSET);
 		if (prevblock != 0)
 			putInt(prevblock + NEXT_OFFSET, nextblock);
 		else // we were the head
 			setFirstBlock(blocksize, nextblock);

 		if (nextblock != 0)
 			putInt(nextblock + PREV_OFFSET, prevblock);
 	}

 	private void addBlock(Chunk chunk, int blocksize, int block) throws CoreException {
 		assert fExclusiveLock;
 		// Mark our size
 		chunk.putInt(block, blocksize);

 		// Add us to the head of the list
 		int prevfirst = getFirstBlock(blocksize);
 		chunk.putInt(block + PREV_OFFSET, 0);
 		chunk.putInt(block + NEXT_OFFSET, prevfirst);
 		if (prevfirst != 0)
 			putInt(prevfirst + PREV_OFFSET, block);
 		setFirstBlock(blocksize, block);
 	}

 	/**
 	 * Free an allocate block.
 	 *
 	 * @param offset
 	 */
 	public void free(int offset) throws CoreException {
 		assert fExclusiveLock;
 		// TODO - look for opportunities to merge blocks
 		int block = offset - 4;
 		Chunk chunk = getChunk(block);
 		int blocksize = - chunk.getInt(block);
 		if (blocksize < 0)
 			// already freed
 			throw new CoreException(new Status(IStatus.ERROR, CCorePlugin.PLUGIN_ID, 0, "Already Freed", new Exception())); //$NON-NLS-1$
 		addBlock(chunk, blocksize, block);
 		freed += blocksize;
 	}

 	public void putByte(int offset, byte value) throws CoreException {
 		getChunk(offset).putByte(offset, value);
 	}

 	public byte getByte(int offset) throws CoreException {
 		return getChunk(offset).getByte(offset);
 	}

 	public void putInt(int offset, int value) throws CoreException {
 		getChunk(offset).putInt(offset, value);
 	}

 	public int getInt(int offset) throws CoreException {
 		return getChunk(offset).getInt(offset);
 	}

 	public void putShort(int offset, short value) throws CoreException {
 		getChunk(offset).putShort(offset, value);
 	}

 	public short getShort(int offset) throws CoreException {
 		return getChunk(offset).getShort(offset);
 	}

 	public void putLong(int offset, long value) throws CoreException {
 		getChunk(offset).putLong(offset, value);
 	}

 	public long getLong(int offset) throws CoreException {
 		return getChunk(offset).getLong(offset);
 	}

 	public void putChar(int offset, char value) throws CoreException {
 		getChunk(offset).putChar(offset, value);
 	}

 	public char getChar(int offset) throws CoreException {
 		return getChunk(offset).getChar(offset);
 	}

 	public IString newString(String string) throws CoreException {
 		if (string.length() > ShortString.MAX_LENGTH)
 			return new LongString(this, string);
 		else
 			return new ShortString(this, string);
 	}

 	public IString newString(char[] chars) throws CoreException {
 		if (chars.length > ShortString.MAX_LENGTH)
 			return new LongString(this, chars);
 		else
 			return new ShortString(this, chars);
 	}

 	public IString getString(int offset) throws CoreException {
 		int length = getInt(offset);
 		if (length > ShortString.MAX_LENGTH)
 			return new LongString(this, offset);
 		else
 			return new ShortString(this, offset);
 	}

 	/**
 	 * For debugging purposes, only.
 	 */
 	public void reportFreeBlocks() throws CoreException {
 		System.out.println("Allocated size: " + fChunks.length * CHUNK_SIZE); //$NON-NLS-1$
 		System.out.println("malloc'ed: " + malloced); //$NON-NLS-1$
 		System.out.println("free'd: " + freed); //$NON-NLS-1$
 		System.out.println("wasted: " + (fChunks.length * CHUNK_SIZE - (malloced - freed))); //$NON-NLS-1$
 		System.out.println("Free blocks"); //$NON-NLS-1$
 		for (int bs = MIN_SIZE; bs <= CHUNK_SIZE; bs += MIN_SIZE) {
 			int count = 0;
 			int block = getFirstBlock(bs);
 			while (block != 0) {
 				++count;
 				block = getInt(block + NEXT_OFFSET);
 			}
 			if (count != 0)
 				System.out.println("Block size: " + bs + "=" + count); //$NON-NLS-1$ //$NON-NLS-2$
 		}
 	}

 	/**
 	 * Closes the database.
 	 * <p>
 	 * The behavior of any further calls to the Database is undefined
 	 * @throws IOException
 	 * @throws CoreException
 	 */
 	public void close() throws CoreException {
 		assert fExclusiveLock;
 		flush();
 		removeChunksFromCache();

 		// chunks have been removed from the cache, so we are fine
 		fHeaderChunk.clear(0, CHUNK_SIZE);
 		fHeaderChunk.fDirty= false;
 		fChunks= new Chunk[] {null};
 		try {
 			fFile.close();
 		} catch (IOException e) {
 			throw new CoreException(new DBStatus(e));
 		}
 	}

 	/**
      * This method is public for testing purposes only.
      */
 	public File getLocation() {
 		return fLocation;
 	}

 	/**
 	 * Called from any thread via the cache, protected by {@link #fCache}.
 	 */
 	void releaseChunk(final Chunk chunk) {
 		if (!chunk.fLocked) {
 			fChunks[chunk.fSequenceNumber]= null;
 		}
 	}

 	/**
 	 * Returns the cache used for this database.
 	 * @since 4.0
 	 */
 	public ChunkCache getChunkCache() {
 		return fCache;
 	}

 	/**
 	 * Asserts that database is used by one thread exclusively. This is necessary when doing
 	 * write operations.
 	 */
 	public void setExclusiveLock() {
 		fExclusiveLock= true;
 		fLocked= true;
 	}

 	public void setLocked(boolean val) {
 		fLocked= val;
 	}

 	public void giveUpExclusiveLock(final boolean flush) throws CoreException {
 		if (fExclusiveLock) {
 			try {
 				ArrayList dirtyChunks= new ArrayList();
 				synchronized (fCache) {
 					for (int i= 1; i < fChunks.length; i++) {
 						Chunk chunk= fChunks[i];
 						if (chunk != null) {
 							if (chunk.fCacheIndex < 0) {
 								// locked chunk that has been removed from cache.
 								if (chunk.fDirty) {
 									dirtyChunks.add(chunk); // keep in fChunks until it is flushed.
 								}
 								else {
 									chunk.fLocked= false;
 									fChunks[i]= null;
 								}
 							}
 							else if (chunk.fLocked) {
 								// locked chunk, still in cache.
 								if (chunk.fDirty) {
 									if (flush) {
 										dirtyChunks.add(chunk);
 									}
 								}
 								else {
 									chunk.fLocked= false;
 								}
 							}
 							else {
 								assert !chunk.fDirty; // dirty chunks must be locked.
 							}
 						}
 					}
 				}
 				// also handles header chunk
 				flushAndUnlockChunks(dirtyChunks, flush);
 			}
 			finally {
 				fExclusiveLock= false;
 			}
 		}
 	}

 	public void flush() throws CoreException {
 		assert fLocked;
 		if (fExclusiveLock) {
 			try {
 				giveUpExclusiveLock(true);
 			}
 			finally {
 				setExclusiveLock();
 			}
 			return;
 		}

 		// be careful as other readers may access chunks concurrently
 		ArrayList dirtyChunks= new ArrayList();
 		synchronized (fCache) {
 			for (int i= 1; i < fChunks.length ; i++) {
 				Chunk chunk= fChunks[i];
 				if (chunk != null && chunk.fDirty) {
 					dirtyChunks.add(chunk);
 				}
 			}
 		}

 		// also handles header chunk
 		flushAndUnlockChunks(dirtyChunks, true);
 	}

 	private void flushAndUnlockChunks(final ArrayList dirtyChunks, boolean isComplete) throws CoreException {
 		assert !Thread.holdsLock(fCache);
 		synchronized(fHeaderChunk) {
 			if (!fHeaderChunk.fDirty) {
 				if (!(isComplete && fIsMarkedIncomplete)) {
 					return;
 				}
 			}
 			if (!dirtyChunks.isEmpty()) {
 				markFileIncomplete();
 				for (Iterator it = dirtyChunks.iterator(); it.hasNext();) {
 					Chunk chunk = (Chunk) it.next();
 					if (chunk.fDirty) {
 						chunk.flush();
 					}
 				}

 				// only after the chunks are flushed we may unlock and release them.
 				synchronized (fCache) {
 					for (Iterator it = dirtyChunks.iterator(); it.hasNext();) {
 						Chunk chunk = (Chunk) it.next();
 						chunk.fLocked= false;
 						if (chunk.fCacheIndex < 0) {
 							fChunks[chunk.fSequenceNumber]= null;
 						}
 					}
 				}
 			}

 			if (isComplete) {
 				if (fHeaderChunk.fDirty || fIsMarkedIncomplete) {
 					fHeaderChunk.putInt(VERSION_OFFSET, fVersion);
 					fHeaderChunk.flush();
 					fIsMarkedIncomplete= false;
 				}
 			}
 		}
 	}

 	private void markFileIncomplete() throws CoreException {
 		if (!fIsMarkedIncomplete) {
 			fIsMarkedIncomplete= true;
 			try {
 				final ByteBuffer buf= ByteBuffer.wrap(new byte[4]);
 				fFile.getChannel().write(buf, 0);
 			} catch (IOException e) {
 				throw new CoreException(new DBStatus(e));
 			}
 		}
 	}

 	public void resetCacheCounters() {
 		cacheHits= cacheMisses= 0;
 	}

 	public long getCacheHits() {
 		return cacheHits;
 	}

 	public long getCacheMisses() {
 		return cacheMisses;
 	}
 }
	/*******************************************************************************
	* Copyright (c) 2005, 2008 QNX Software Systems and others.
	* All rights reserved. This program and the accompanying materials
	* are made available under the terms of the Eclipse Public License v1.0
	* which accompanies this distribution, and is available at
	* http://www.eclipse.org/legal/epl-v10.html
	*
	* Contributors:
	* QNX - Initial API and implementation
	* Symbian - Add some non-javadoc implementation notes
	* Markus Schorn (Wind River Systems)
	* IBM Corporation
	*******************************************************************************/
	package org.eclipse.rephraserengine.internal.db.org.eclipse.cdt.internal.core.pdom.db;

	import java.io.File;
	import java.io.FileNotFoundException;
	import java.io.IOException;
	import java.io.RandomAccessFile;
	import java.nio.ByteBuffer;
	import java.nio.channels.ClosedByInterruptException;
	import java.nio.channels.ClosedChannelException;
	import java.nio.channels.FileChannel;
	import java.util.ArrayList;
	import java.util.Iterator;

	import org.eclipse.core.runtime.CoreException;
	import org.eclipse.core.runtime.IStatus;
	import org.eclipse.core.runtime.Status;
	import org.eclipse.rephraserengine.internal.db.org.eclipse.cdt.core.CCorePlugin;


	/**
	* Database encapsulates access to a flat binary format file with a memory-manager-like API for
	* obtaining and releasing areas of storage (memory).
	*
	* @author Doug Schaefer
	*/
	/*
	* The file encapsulated is divided into Chunks of size CHUNK_SIZE, and a table of contents
	* mapping chunk index to chunk address is maintained. Chunk structure exists only conceptually -
	* its not a structure that appears in the file.
	*
	* ===== The first chunk is used by Database itself for house-keeping purposes and has structure
	*
	* offset content
	* _____________________________
	* 0 \| version number
	* INT_SIZE \| pointer to head of linked list of blocks of size MIN_SIZE
	* .. \| ...
	* INT_SIZE * m (1) \| pointer to head of linked list of blocks of size MIN_SIZE * m
	* DATA_AREA \| undefined (PDOM stores its own house-keeping data in this area)
	*
	* (1) where m <= (CHUNK_SIZE / MIN_SIZE)
	*
	* ===== block structure
	*
	* offset content
	* _____________________________
	* 0 \| size of block (negative indicates in use, positive unused)
	* PREV_OFFSET \| pointer to prev block (of same size)
	* NEXT_OFFSET \| pointer to next block (of same size)
	*
	*/
	public class Database {

	private final File fLocation;
	private final boolean fReadOnly;
	private RandomAccessFile fFile;
	private boolean fExclusiveLock= false; // necessary for any write operation
	private boolean fLocked; // necessary for any operation.
	private boolean fIsMarkedIncomplete= false;

	private int fVersion;
	private final Chunk fHeaderChunk;
	private Chunk[] fChunks;
	private ChunkCache fCache;

	private long malloced;
	private long freed;
	private long cacheHits;
	private long cacheMisses;

	// public for tests only, you shouldn't need these
	public static final int VERSION_OFFSET = 0;
	public static final int CHUNK_SIZE = 1024 * 4;
	public static final int MIN_SIZE = 16;
	public static final int INT_SIZE = 4;
	public static final int CHAR_SIZE = 2;
	public static final int PREV_OFFSET = INT_SIZE;
	public static final int NEXT_OFFSET = INT_SIZE * 2;
	public static final int DATA_AREA = CHUNK_SIZE / MIN_SIZE * INT_SIZE + INT_SIZE;

	public static final int MAX_SIZE = CHUNK_SIZE - 4; // Room for overhead

	/**
	* Construct a new Database object, creating a backing file if necessary.
	* @param location the local file path for the database
	* @param cache the cache to be used optimization
	* @param version the version number to store in the database (only applicable for new databases)
	* @param permanentReadOnly whether this Database object will ever need writing to
	* @throws CoreException
	*/
	public Database(File location, ChunkCache cache, int version, boolean openReadOnly) throws CoreException {
	try {
	fLocation = location;
	fReadOnly= openReadOnly;
	fCache= cache;
	openFile();

	int nChunksOnDisk = (int) (fFile.length() / CHUNK_SIZE);
	fHeaderChunk= new Chunk(this, 0);
	fHeaderChunk.fLocked= true; // never makes it into the cache, needed to satisfy assertions
	if (nChunksOnDisk <= 0) {
	fVersion= version;
	fChunks= new Chunk[1];
	}
	else {
	fHeaderChunk.read();
	fVersion= fHeaderChunk.getInt(VERSION_OFFSET);
	fChunks = new Chunk[nChunksOnDisk]; // chunk[0] is unused.
	}
	} catch (IOException e) {
	throw new CoreException(new DBStatus(e));
	}
	}

	private void openFile() throws FileNotFoundException {
	fFile = new RandomAccessFile(fLocation, fReadOnly ? "r" : "rw"); //$NON-NLS-1$ //$NON-NLS-2$
	}

	void read(ByteBuffer buf, int i) throws IOException {
	int retries= 0;
	do {
	try {
	fFile.getChannel().read(buf, i);
	return;
	}
	catch (ClosedChannelException e) {
	// bug 219834 file may have be closed by interrupting a thread during an I/O operation.
	reopen(e, ++retries);
	}
	} while (true);
	}

	void write(ByteBuffer buf, int i) throws IOException {
	int retries= 0;
	do {
	try {
	fFile.getChannel().write(buf, i);
	return;
	}
	catch (ClosedChannelException e) {
	// bug 219834 file may have be closed by interrupting a thread during an I/O operation.
	reopen(e, ++retries);
	}
	} while(true);
	}

	private void reopen(ClosedChannelException e, int attempt) throws ClosedChannelException, FileNotFoundException {
	// only if the current thread was not interrupted we try to reopen the file.
	if (e instanceof ClosedByInterruptException \|\| attempt >= 20) {
	throw e;
	}
	openFile();
	}


	public void transferTo(FileChannel target) throws IOException {
	assert fLocked;
	final FileChannel from= fFile.getChannel();
	from.transferTo(0, from.size(), target);
	}

	public int getVersion() throws CoreException {
	return fVersion;
	}

	public void setVersion(int version) throws CoreException {
	assert fExclusiveLock;
	fHeaderChunk.putInt(VERSION_OFFSET, version);
	fVersion= version;
	}

	/**
	* Empty the contents of the Database, make it ready to start again
	* @throws CoreException
	*/
	public void clear(int version) throws CoreException {
	assert fExclusiveLock;
	removeChunksFromCache();

	fVersion= version;
	// clear the first chunk.
	fHeaderChunk.clear(0, CHUNK_SIZE);
	// chunks have been removed from the cache, so we may just reset the array of chunks.
	fChunks = new Chunk[] {null};
	try {
	fHeaderChunk.flush(); // zero out header chunk
	fFile.getChannel().truncate(CHUNK_SIZE); // truncate database
	}
	catch (IOException e) {
	CCorePlugin.log(e);
	}
	malloced = freed = 0;
	}

	private void removeChunksFromCache() {
	synchronized (fCache) {
	for (int i=1; i < fChunks.length; i++) {
	Chunk chunk= fChunks[i];
	if (chunk != null) {
	fCache.remove(chunk);
	fChunks[i]= null;
	}
	}
	}
	}


	/**
	* Return the Chunk that contains the given offset.
	* @throws CoreException
	*/
	public Chunk getChunk(int offset) throws CoreException {
	if (offset < CHUNK_SIZE) {
	return fHeaderChunk;
	}
	synchronized(fCache) {
	assert fLocked;
	final int index = offset / CHUNK_SIZE;
	Chunk chunk= fChunks[index];
	if (chunk == null) {
	cacheMisses++;
	chunk = fChunks[index] = new Chunk(this, index);
	chunk.read();
	}
	else {
	cacheHits++;
	}
	fCache.add(chunk, fExclusiveLock);
	return chunk;
	}
	}

	/**
	* Allocate a block out of the database.
	*
	* @param size
	* @return
	*/
	public int malloc(int size) throws CoreException {
	assert fExclusiveLock;
	if (size > MAX_SIZE)
	// Too Big
	throw new CoreException(new Status(IStatus.ERROR, CCorePlugin.PLUGIN_ID, 0,
	CCorePlugin.getResourceString("pdom.requestTooLarge"), new IllegalArgumentException())); //$NON-NLS-1$

	// Which block size
	int freeblock = 0;
	int blocksize;
	int matchsize = 0;
	for (blocksize = MIN_SIZE; blocksize <= CHUNK_SIZE; blocksize += MIN_SIZE) {
	if (blocksize - INT_SIZE >= size) {
	if (matchsize == 0) // our real size
	matchsize = blocksize;
	freeblock = getFirstBlock(blocksize);
	if (freeblock != 0)
	break;
	}
	}

	// get the block
	Chunk chunk;
	if (freeblock == 0) {
	// allocate a new chunk
	freeblock= createNewChunk();
	blocksize = CHUNK_SIZE;
	chunk = getChunk(freeblock);
	} else {
	chunk = getChunk(freeblock);
	removeBlock(chunk, blocksize, freeblock);
	}

	if (blocksize != matchsize) {
	// Add in the unused part of our block
	addBlock(chunk, blocksize - matchsize, freeblock + matchsize);
	}

	// Make our size negative to show in use
	chunk.putInt(freeblock, - matchsize);

	// Clear out the block, lots of people are expecting this
	chunk.clear(freeblock + 4, size);

	malloced += matchsize;
	return freeblock + 4;
	}

	private int createNewChunk() throws CoreException {
	assert fExclusiveLock;
	synchronized (fCache) {
	final int oldLen= fChunks.length;
	final Chunk chunk= new Chunk(this, oldLen);
	chunk.fDirty= true;

	Chunk[] newchunks = new Chunk[oldLen+1];
	System.arraycopy(fChunks, 0, newchunks, 0, oldLen);
	newchunks[oldLen]= chunk;
	fChunks= newchunks;
	fCache.add(chunk, true);
	return oldLen * CHUNK_SIZE;
	}
	}

	private int getFirstBlock(int blocksize) throws CoreException {
	assert fLocked;
	return fHeaderChunk.getInt((blocksize / MIN_SIZE) * INT_SIZE);
	}

	private void setFirstBlock(int blocksize, int block) throws CoreException {
	assert fExclusiveLock;
	fHeaderChunk.putInt((blocksize / MIN_SIZE) * INT_SIZE, block);
	}

	private void removeBlock(Chunk chunk, int blocksize, int block) throws CoreException {
	assert fExclusiveLock;
	int prevblock = chunk.getInt(block + PREV_OFFSET);
	int nextblock = chunk.getInt(block + NEXT_OFFSET);
	if (prevblock != 0)
	putInt(prevblock + NEXT_OFFSET, nextblock);
	else // we were the head
	setFirstBlock(blocksize, nextblock);

	if (nextblock != 0)
	putInt(nextblock + PREV_OFFSET, prevblock);
	}

	private void addBlock(Chunk chunk, int blocksize, int block) throws CoreException {
	assert fExclusiveLock;
	// Mark our size
	chunk.putInt(block, blocksize);

	// Add us to the head of the list
	int prevfirst = getFirstBlock(blocksize);
	chunk.putInt(block + PREV_OFFSET, 0);
	chunk.putInt(block + NEXT_OFFSET, prevfirst);
	if (prevfirst != 0)
	putInt(prevfirst + PREV_OFFSET, block);
	setFirstBlock(blocksize, block);
	}

	/**
	* Free an allocate block.
	*
	* @param offset
	*/
	public void free(int offset) throws CoreException {
	assert fExclusiveLock;
	// TODO - look for opportunities to merge blocks
	int block = offset - 4;
	Chunk chunk = getChunk(block);
	int blocksize = - chunk.getInt(block);
	if (blocksize < 0)
	// already freed
	throw new CoreException(new Status(IStatus.ERROR, CCorePlugin.PLUGIN_ID, 0, "Already Freed", new Exception())); //$NON-NLS-1$
	addBlock(chunk, blocksize, block);
	freed += blocksize;
	}

	public void putByte(int offset, byte value) throws CoreException {
	getChunk(offset).putByte(offset, value);
	}

	public byte getByte(int offset) throws CoreException {
	return getChunk(offset).getByte(offset);
	}

	public void putInt(int offset, int value) throws CoreException {
	getChunk(offset).putInt(offset, value);
	}

	public int getInt(int offset) throws CoreException {
	return getChunk(offset).getInt(offset);
	}

	public void putShort(int offset, short value) throws CoreException {
	getChunk(offset).putShort(offset, value);
	}

	public short getShort(int offset) throws CoreException {
	return getChunk(offset).getShort(offset);
	}

	public void putLong(int offset, long value) throws CoreException {
	getChunk(offset).putLong(offset, value);
	}

	public long getLong(int offset) throws CoreException {
	return getChunk(offset).getLong(offset);
	}

	public void putChar(int offset, char value) throws CoreException {
	getChunk(offset).putChar(offset, value);
	}

	public char getChar(int offset) throws CoreException {
	return getChunk(offset).getChar(offset);
	}

	public IString newString(String string) throws CoreException {
	if (string.length() > ShortString.MAX_LENGTH)
	return new LongString(this, string);
	else
	return new ShortString(this, string);
	}

	public IString newString(char[] chars) throws CoreException {
	if (chars.length > ShortString.MAX_LENGTH)
	return new LongString(this, chars);
	else
	return new ShortString(this, chars);
	}

	public IString getString(int offset) throws CoreException {
	int length = getInt(offset);
	if (length > ShortString.MAX_LENGTH)
	return new LongString(this, offset);
	else
	return new ShortString(this, offset);
	}

	/**
	* For debugging purposes, only.
	*/
	public void reportFreeBlocks() throws CoreException {
	System.out.println("Allocated size: " + fChunks.length * CHUNK_SIZE); //$NON-NLS-1$
	System.out.println("malloc'ed: " + malloced); //$NON-NLS-1$
	System.out.println("free'd: " + freed); //$NON-NLS-1$
	System.out.println("wasted: " + (fChunks.length * CHUNK_SIZE - (malloced - freed))); //$NON-NLS-1$
	System.out.println("Free blocks"); //$NON-NLS-1$
	for (int bs = MIN_SIZE; bs <= CHUNK_SIZE; bs += MIN_SIZE) {
	int count = 0;
	int block = getFirstBlock(bs);
	while (block != 0) {
	++count;
	block = getInt(block + NEXT_OFFSET);
	}
	if (count != 0)
	System.out.println("Block size: " + bs + "=" + count); //$NON-NLS-1$ //$NON-NLS-2$
	}
	}

	/**
	* Closes the database.
	* <p>
	* The behavior of any further calls to the Database is undefined
	* @throws IOException
	* @throws CoreException
	*/
	public void close() throws CoreException {
	assert fExclusiveLock;
	flush();
	removeChunksFromCache();

	// chunks have been removed from the cache, so we are fine
	fHeaderChunk.clear(0, CHUNK_SIZE);
	fHeaderChunk.fDirty= false;
	fChunks= new Chunk[] {null};
	try {
	fFile.close();
	} catch (IOException e) {
	throw new CoreException(new DBStatus(e));
	}
	}

	/**
	* This method is public for testing purposes only.
	*/
	public File getLocation() {
	return fLocation;
	}

	/**
	* Called from any thread via the cache, protected by {@link #fCache}.
	*/
	void releaseChunk(final Chunk chunk) {
	if (!chunk.fLocked) {
	fChunks[chunk.fSequenceNumber]= null;
	}
	}

	/**
	* Returns the cache used for this database.
	* @since 4.0
	*/
	public ChunkCache getChunkCache() {
	return fCache;
	}

	/**
	* Asserts that database is used by one thread exclusively. This is necessary when doing
	* write operations.
	*/
	public void setExclusiveLock() {
	fExclusiveLock= true;
	fLocked= true;
	}

	public void setLocked(boolean val) {
	fLocked= val;
	}

	public void giveUpExclusiveLock(final boolean flush) throws CoreException {
	if (fExclusiveLock) {
	try {
	ArrayList dirtyChunks= new ArrayList();
	synchronized (fCache) {
	for (int i= 1; i < fChunks.length; i++) {
	Chunk chunk= fChunks[i];
	if (chunk != null) {
	if (chunk.fCacheIndex < 0) {
	// locked chunk that has been removed from cache.
	if (chunk.fDirty) {
	dirtyChunks.add(chunk); // keep in fChunks until it is flushed.
	}
	else {
	chunk.fLocked= false;
	fChunks[i]= null;
	}
	}
	else if (chunk.fLocked) {
	// locked chunk, still in cache.
	if (chunk.fDirty) {
	if (flush) {
	dirtyChunks.add(chunk);
	}
	}
	else {
	chunk.fLocked= false;
	}
	}
	else {
	assert !chunk.fDirty; // dirty chunks must be locked.
	}
	}
	}
	}
	// also handles header chunk
	flushAndUnlockChunks(dirtyChunks, flush);
	}
	finally {
	fExclusiveLock= false;
	}
	}
	}

	public void flush() throws CoreException {
	assert fLocked;
	if (fExclusiveLock) {
	try {
	giveUpExclusiveLock(true);
	}
	finally {
	setExclusiveLock();
	}
	return;
	}

	// be careful as other readers may access chunks concurrently
	ArrayList dirtyChunks= new ArrayList();
	synchronized (fCache) {
	for (int i= 1; i < fChunks.length ; i++) {
	Chunk chunk= fChunks[i];
	if (chunk != null && chunk.fDirty) {
	dirtyChunks.add(chunk);
	}
	}
	}

	// also handles header chunk
	flushAndUnlockChunks(dirtyChunks, true);
	}

	private void flushAndUnlockChunks(final ArrayList dirtyChunks, boolean isComplete) throws CoreException {
	assert !Thread.holdsLock(fCache);
	synchronized(fHeaderChunk) {
	if (!fHeaderChunk.fDirty) {
	if (!(isComplete && fIsMarkedIncomplete)) {
	return;
	}
	}
	if (!dirtyChunks.isEmpty()) {
	markFileIncomplete();
	for (Iterator it = dirtyChunks.iterator(); it.hasNext();) {
	Chunk chunk = (Chunk) it.next();
	if (chunk.fDirty) {
	chunk.flush();
	}
	}

	// only after the chunks are flushed we may unlock and release them.
	synchronized (fCache) {
	for (Iterator it = dirtyChunks.iterator(); it.hasNext();) {
	Chunk chunk = (Chunk) it.next();
	chunk.fLocked= false;
	if (chunk.fCacheIndex < 0) {
	fChunks[chunk.fSequenceNumber]= null;
	}
	}
	}
	}

	if (isComplete) {
	if (fHeaderChunk.fDirty \|\| fIsMarkedIncomplete) {
	fHeaderChunk.putInt(VERSION_OFFSET, fVersion);
	fHeaderChunk.flush();
	fIsMarkedIncomplete= false;
	}
	}
	}
	}

	private void markFileIncomplete() throws CoreException {
	if (!fIsMarkedIncomplete) {
	fIsMarkedIncomplete= true;
	try {
	final ByteBuffer buf= ByteBuffer.wrap(new byte[4]);
	fFile.getChannel().write(buf, 0);
	} catch (IOException e) {
	throw new CoreException(new DBStatus(e));
	}
	}
	}

	public void resetCacheCounters() {
	cacheHits= cacheMisses= 0;
	}

	public long getCacheHits() {
	return cacheHits;
	}

	public long getCacheMisses() {
	return cacheMisses;
	}
	}