r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/dataeditor/AbstractRDataProvider.java

/*=============================================================================#
 # Copyright (c) 2010, 2019 Stephan Wahlbrink and others.
 # 
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
 # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
 # which is available at https://www.apache.org/licenses/LICENSE-2.0.
 # 
 # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
 # 
 # Contributors:
 #     Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation
 #=============================================================================*/

package org.eclipse.statet.internal.r.ui.dataeditor;

import static org.eclipse.statet.internal.r.ui.RUIPlugin.logError;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;

import com.ibm.icu.util.TimeZone;

import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.osgi.util.NLS;
import org.eclipse.swt.widgets.Display;

import org.eclipse.statet.jcommons.collections.CopyOnWriteIdentityListSet;
import org.eclipse.statet.jcommons.collections.ImCollections;
import org.eclipse.statet.jcommons.collections.ImList;
import org.eclipse.statet.jcommons.status.ProgressMonitor;
import org.eclipse.statet.jcommons.status.Status;
import org.eclipse.statet.jcommons.status.StatusException;
import org.eclipse.statet.jcommons.ts.core.SystemRunnable;
import org.eclipse.statet.jcommons.ts.core.Tool;
import org.eclipse.statet.jcommons.ts.core.ToolRunnable;
import org.eclipse.statet.jcommons.ts.core.ToolService;

import org.eclipse.statet.ecommons.runtime.core.util.StatusUtils;
import org.eclipse.statet.ecommons.ui.util.UIAccess;
import org.eclipse.statet.ecommons.waltable.coordinate.PositionId;
import org.eclipse.statet.ecommons.waltable.data.ControlData;
import org.eclipse.statet.ecommons.waltable.data.IDataProvider;
import org.eclipse.statet.ecommons.waltable.sort.ISortModel;
import org.eclipse.statet.ecommons.waltable.sort.SortDirection;

import org.eclipse.statet.r.core.model.RElementName;
import org.eclipse.statet.r.core.tool.TmpUtils;
import org.eclipse.statet.r.nico.ICombinedRDataAdapter;
import org.eclipse.statet.r.ui.dataeditor.IRDataTableInput;
import org.eclipse.statet.r.ui.dataeditor.IRDataTableVariable;
import org.eclipse.statet.r.ui.dataeditor.RDataTableColumn;
import org.eclipse.statet.rj.data.RCharacterStore;
import org.eclipse.statet.rj.data.RDataUtils;
import org.eclipse.statet.rj.data.RFactorStore;
import org.eclipse.statet.rj.data.RIntegerStore;
import org.eclipse.statet.rj.data.RLanguage;
import org.eclipse.statet.rj.data.RObject;
import org.eclipse.statet.rj.data.RObjectFactory;
import org.eclipse.statet.rj.data.RStore;
import org.eclipse.statet.rj.data.RVector;
import org.eclipse.statet.rj.data.UnexpectedRDataException;
import org.eclipse.statet.rj.data.impl.DefaultRObjectFactory;
import org.eclipse.statet.rj.data.impl.RFactorStructStore;
import org.eclipse.statet.rj.services.BasicFQRObjectRef;
import org.eclipse.statet.rj.services.FQRObjectRef;
import org.eclipse.statet.rj.services.FunctionCall;
import org.eclipse.statet.rj.services.RService;
import org.eclipse.statet.rj.services.util.dataaccess.AbstractRDataAdapter;
import org.eclipse.statet.rj.services.util.dataaccess.LazyRStore;
import org.eclipse.statet.rj.services.util.dataaccess.LazyRStore.Fragment;
import org.eclipse.statet.rj.services.util.dataaccess.RDataAssignment;
import org.eclipse.statet.rj.ts.core.RToolService;


public abstract class AbstractRDataProvider<T extends RObject> implements IDataProvider {
	
	
	public static final ControlData LOADING= new ControlData(ControlData.ASYNC, "loading...");
	public static final ControlData ERROR= new ControlData(ControlData.ERROR, "ERROR");
	public static final ControlData NA= new ControlData(ControlData.NA, "NA"); //$NON-NLS-1$
	public static final ControlData DUMMY= new ControlData(0, ""); //$NON-NLS-1$
	
	protected static final RElementName BASE_NAME= RElementName.create(RElementName.MAIN_DEFAULT, "x"); //$NON-NLS-1$
	
	
	static final void checkCancel(final Exception e) throws StatusException {
		if (e instanceof StatusException
				&& ((StatusException) e).getStatus().getSeverity() == Status.CANCEL) {
			throw (StatusException) e;
		}
	}
	
	
	public static final class SortColumn {
		
		
		private final long id;
		
		public final boolean decreasing;
		
		
		public SortColumn(final long columnId, final boolean decreasing) {
			this.id= columnId;
			this.decreasing= decreasing;
		}
		
		
		public long getId() {
			return this.id;
		}
		
		public long getIdx() {
			return (this.id & PositionId.NUM_MASK);
		}
		
		
		@Override
		public int hashCode() {
			final int h= (int) (this.id ^ (this.id >>> 32));
			return (this.decreasing) ? (-h) : h;
		}
		
		@Override
		public boolean equals(final Object obj) {
			if (!(obj instanceof SortColumn)) {
				return false;
			}
			final SortColumn other= (SortColumn) obj;
			return (this.id == other.id && this.decreasing == other.decreasing);
		}
		
	}
	
	public static interface IDataProviderListener {
		
		
		static final int ERROR_STRUCT_CHANGED= 1;
		
		
		void onInputInitialized(boolean structChanged);
		
		void onInputFailed(int error);
		
		void onRowCountChanged();
		
		void onRowsChanged(long begin, long end);
		
	}
	
	private static final long DEFAULT_WAIT= 25_000000;
	private static final long CANCEL_WAIT= 200_000000;
	private static final long BLOCKING_WAIT= 300_000000;
	
	private class MainLock extends Lock implements LazyRStore.Updater {
		
		
		private final List<Fragment> waitingFragments= new ArrayList<>();
		
		
		@Override
		public void scheduleUpdate(final LazyRStore store,
				final RDataAssignment assignment, final Fragment fragment,
				final int flags, final ProgressMonitor m) {
			long waitNanos;
			if (!this.scheduled) {
				this.scheduled= true;
				AbstractRDataProvider.this.schedule(AbstractRDataProvider.this.updateRunnable);
				
				waitNanos= ((flags & FORCE_SYNC) != 0) ? 0 : DEFAULT_WAIT;
			}
			else {
				waitNanos= ((flags & FORCE_SYNC) != 0) ? 0 : -1;
			}
			
			if (waitNanos >= 0 && fragment != null) {
				this.waitingFragments.add(fragment);
				this.worker.signalAll();
//				long blockingMonitor= (monitor instanceof IProgressMonitorWithBlocking) ? 0 : -1;
				try {
					if (waitNanos == 0) {
						do {
							// wait required to check cancel state
							waitNanos= this.requestor.awaitNanos(CANCEL_WAIT);
							
//							if (blockingMonitor >= 0 && blockingMonitor < BLOCKING_WAIT) {
//								blockingMonitor+= CANCEL_WAIT - waitNanos;
//								if (blockingMonitor >= BLOCKING_WAIT) {
//									((IProgressMonitorWithBlocking) monitor).setBlocked(
//											new Status(IStatus.INFO, RUI.BUNDLE_ID, "Waiting for R."));
//								}
//							}
						}
						while (this.waitingFragments.contains(fragment)
								&& this.state == 0
								&& (m == null || !m.isCanceled()) );
					}
					else {
						do {
							waitNanos= this.requestor.awaitNanos(waitNanos);
						}
						while (this.waitingFragments.contains(fragment)
								&& this.state == 0
								&& (m == null || !m.isCanceled())
								&& (waitNanos > 0) );
					}
				}
				catch (final InterruptedException e) {}
				finally {
					this.waitingFragments.remove(fragment);
					
//					if (blockingMonitor >= BLOCKING_WAIT) {
//						((IProgressMonitorWithBlocking) monitor).clearBlocked();
//					}
				}
			}
		}
		
		void notify(final Object obj) {
			if (this.waitingFragments.remove(obj)) {
				this.requestor.signalAll();
			}
		}
		
		@Override
		void clear() {
			this.waitingFragments.clear();
			super.clear();
		}
		
	}
	
	protected class ColumnDataProvider implements IDataProvider {
		
		
		public ColumnDataProvider() {
		}
		
		
		@Override
		public long getColumnCount() {
			return AbstractRDataProvider.this.getColumnCount();
		}
		
		@Override
		public long getRowCount() {
			return 1;
		}
		
		@Override
		public Object getDataValue(final long columnIndex, final long rowIndex,
				final int flags, final IProgressMonitor monitor) {
			try {
				final LazyRStore.Fragment<T> fragment= AbstractRDataProvider.this.fragmentsLock.getFragment(
						AbstractRDataProvider.this.dataStore, 0, columnIndex,
						flags, StatusUtils.convert(monitor) );
				if (fragment != null) {
					return getColumnName(fragment, columnIndex);
				}
				else {
					return handleMissingData(flags);
				}
			}
			catch (final LoadDataException e) {
				return handleLoadDataException(e, flags);
			}
		}
		
		@Override
		public void setDataValue(final long columnIndex, final long rowIndex, final Object newValue) {
			throw new UnsupportedOperationException();
		}
		
	}
	
	protected class RowDataProvider implements IDataProvider {
		
		
		private final LazyRStore<RVector<?>> rowNamesStore= new LazyRStore<>(AbstractRDataProvider.this.rowCount, 1,
				10, AbstractRDataProvider.this.fragmentsLock);
		
		
		public RowDataProvider() {
		}
		
		
		@Override
		public long getColumnCount() {
			return 1;
		}
		
		@Override
		public long getRowCount() {
			return AbstractRDataProvider.this.getRowCount();
		}
		
		@Override
		public Object getDataValue(final long columnIndex, final long rowIndex,
				final int flags, final IProgressMonitor monitor) {
			try {
				final LazyRStore.Fragment<RVector<?>> fragment= AbstractRDataProvider
						.this.fragmentsLock.getFragment(this.rowNamesStore, rowIndex, 0,
								flags, StatusUtils.convert(monitor) );
				if (fragment != null) {
					final RVector<?> vector= fragment.getRObject();
					if (vector != null) {
						RStore<?> names= vector.getNames();
						if (names == null) {
							names= vector.getData();
						}
						return names.get(rowIndex - fragment.getRowBeginIdx());
					}
					return Long.toString(rowIndex + 1);
				}
				else {
					return handleMissingData(flags);
				}
			}
			catch (final LoadDataException e) {
				return handleLoadDataException(e, flags);
			}
		}
		
		public long getRowIdx(final long rowIndex) {
			try {
				final LazyRStore.Fragment<RVector<?>> fragment= AbstractRDataProvider
						.this.fragmentsLock.getFragment(this.rowNamesStore, rowIndex, 0, 0, null);
				if (fragment != null) {
					final RVector<?> vector= fragment.getRObject();
					if (vector != null) {
						final RStore<?> idxs= vector.getData();
						return ((idxs.getStoreType() == RStore.INTEGER) ?
										(long) idxs.getInt(rowIndex - fragment.getRowBeginIdx()) :
										(long) idxs.getNum(rowIndex - fragment.getRowBeginIdx()) )
								- 1;
					}
					return rowIndex;
				}
				else {
					return -1;
				}
			}
			catch (final LoadDataException e) {
				return -2;
			}
		}
		
		@Override
		public void setDataValue(final long columnIndex, final long rowIndex, final Object newValue) {
			throw new UnsupportedOperationException();
		}
		
	}
	
	protected class SortModel implements ISortModel {
		
		
		@Override
		public List<Long> getSortedColumnIds() {
			final SortColumn sortColumn= getSortColumn();
			if (sortColumn != null) {
				return Collections.singletonList(sortColumn.id);
			}
			return Collections.<Long>emptyList();
		}
		
		@Override
		public void sort(final long columnId, final SortDirection sortDirection, final boolean accumulate) {
			SortColumn sortColumn;
			switch (sortDirection) {
			case ASC:
				sortColumn= new SortColumn(columnId, false);
				break;
			case DESC:
				sortColumn= new SortColumn(columnId, true);
				break;
			default:
				sortColumn= null;
				break;
			}
			setSortColumn(sortColumn);
		}
		
		@Override
		public int getSortOrder(final long columnId) {
			final SortColumn sortColumn= getSortColumn();
			if (sortColumn != null && sortColumn.id == columnId) {
				return 0;
			}
			return -1;
		}
		
		@Override
		public boolean isSorted(final long columnId) {
			final SortColumn sortColumn= getSortColumn();
			if (sortColumn != null && sortColumn.id == columnId) {
				return true;
			}
			return false;
		}
		
		@Override
		public SortDirection getSortDirection(final long columnId) {
			final SortColumn sortColumn= getSortColumn();
			if (sortColumn != null && sortColumn.id == columnId) {
				return (!sortColumn.decreasing) ? SortDirection.ASC : SortDirection.DESC;
			}
			return SortDirection.NONE;
		}
		
		@Override
		public void clear() {
			setSortColumn(null);
		}
		
	}
	
	
	private final ToolRunnable initRunnable= new SystemRunnable() {
		
		@Override
		public String getTypeId() {
			return "r/dataeditor/init";
		}
		
		@Override
		public String getLabel() {
			return "Prepare Data Viewer (" + AbstractRDataProvider.this.input.getName() + ")";
		}
		
		@Override
		public boolean canRunIn(final Tool tool) {
			return true; // TODO
		}
		
		@Override
		public boolean changed(final int event, final Tool tool) {
			if (event == MOVING_FROM) {
				return false;
			}
			return true;
		}
		
		@Override
		public void run(final ToolService service, final ProgressMonitor m) throws StatusException {
			runInit((RToolService) service, m);
		}
		
	};
	
	private final ToolRunnable updateRunnable= new SystemRunnable() {
		
		@Override
		public String getTypeId() {
			return "r/dataeditor/load"; //$NON-NLS-1$
		}
		
		@Override
		public String getLabel() {
			return "Load Data (" + AbstractRDataProvider.this.input.getName() + ")";
		}
		
		@Override
		public boolean canRunIn(final Tool tool) {
			return true; // TODO
		}
		
		@Override
		public boolean changed(final int event, final Tool tool) {
			switch (event) {
			case MOVING_FROM:
				return false;
			case REMOVING_FROM:
			case BEING_ABANDONED:
				AbstractRDataProvider.this.fragmentsLock.lock();
				try {
					AbstractRDataProvider.this.fragmentsLock.scheduled= false;
					AbstractRDataProvider.this.fragmentsLock.clear();
				}
				finally {
					AbstractRDataProvider.this.fragmentsLock.unlock();
				}
				break;
			default:
				break;
			}
			return true;
		}
		
		@Override
		public void run(final ToolService service, final ProgressMonitor m) throws StatusException {
			runUpdate((RToolService) service, m);
		}
		
	};
	
	private final ToolRunnable cleanRunnable= new SystemRunnable() {
		
		@Override
		public String getTypeId() {
			return "r/dataeditor/clean"; //$NON-NLS-1$
		}
		
		@Override
		public String getLabel() {
			return "Clean Cache (" + AbstractRDataProvider.this.input.getName() + ")";
		}
		
		@Override
		public boolean canRunIn(final Tool tool) {
			return true; // TODO
		}
		
		@Override
		public boolean changed(final int event, final Tool tool) {
			switch (event) {
			case MOVING_FROM:
			case REMOVING_FROM:
				return false;
			default:
				break;
			}
			return true;
		}
		
		@Override
		public void run(final ToolService service, final ProgressMonitor m) throws StatusException {
			runClean((RToolService) service, m);
		}
		
	};
	
	
	private final Display realm;
	
	private final IRDataTableInput input;
	
	private final long columnCount;
	private long fullRowCount;
	private long rowCount;
	
	private final CopyOnWriteIdentityListSet<IDataProviderListener> dataListeners= new CopyOnWriteIdentityListSet<>();
	
	private boolean initScheduled;
	private volatile boolean disposeScheduled;
	
	
	private RDataTableContentDescription description;
	
	private final IDataProvider columnDataProvider;
	private final IDataProvider rowDataProvider;
	
	private final IDataProvider columnLabelProvider;
	private final IDataProvider rowLabelProvider;
	
	private final MainLock fragmentsLock= new MainLock();
	
	protected final AbstractRDataAdapter<T, T> adapter;
	private final LazyRStore<T> dataStore;
	
	private final List<Object> activeOperations= new ArrayList<>();
	
	private boolean updateSorting;
	private boolean updateFiltering;
	
	private final StringBuilder rStringBuilder= new StringBuilder(128);
	private TmpUtils.Item rTmpItem; // only in R jobs
	private T rObjectStruct;
	
	private final ISortModel sortModel;
	private SortColumn sortColumn= null;
	private String rCacheSort; // only in R jobs
	
	private String filter;
	private String rCacheFilter;
	
	private boolean updateIdx; // only in R jobs
	private String rCacheIdx; // only in R jobs
	private String rCacheIdxR; // only in R jobs
	
	private final FindManager findManager;
	
	
	protected AbstractRDataProvider(final IRDataTableInput input,
			final AbstractRDataAdapter<T, T> adapter, final T initialRObject) {
		this.realm= UIAccess.getDisplay();
		this.input= input;
		
		this.adapter= adapter;
		this.fullRowCount= this.rowCount= this.adapter.getRowCount(initialRObject);
		this.columnCount= this.adapter.getColumnCount(initialRObject);
		
		final int dataMax;
		if (this.columnCount <= 25) {
			dataMax= 10;
		}
		else if (this.columnCount <= 50) {
			dataMax= 20;
		}
		else {
			dataMax= 25;
		}
		this.dataStore= new LazyRStore<>(this.rowCount, this.columnCount, dataMax, this.fragmentsLock);
		this.findManager= new FindManager(this);
		
		this.columnDataProvider= createColumnDataProvider();
		this.rowDataProvider= createRowDataProvider();
		this.columnLabelProvider= createColumnLabelProvider();
		this.rowLabelProvider= createRowLabelProvider();
		this.sortModel= createSortModel();
	}
	
	
	public final IRDataTableInput getInput() {
		return this.input;
	}
	
	protected final AbstractRDataAdapter<T, T> getAdapter() {
		return this.adapter;
	}
	
	public final T getRObject() {
		return this.rObjectStruct;
	}
	
	final int getLockState() {
		return this.fragmentsLock.state;
	}
	
	public final void beginOperation(final Object o) {
		this.fragmentsLock.lock();
		try {
			this.activeOperations.add(o);
		}
		finally {
			this.fragmentsLock.unlock();
		}
	}
	
	public final void endOperation(final Object o) {
		this.fragmentsLock.lock();
		try {
			if (this.activeOperations.remove(o)
					&& this.activeOperations.isEmpty() ) {
				this.fragmentsLock.notifyWorker();
			}
		}
		finally {
			this.fragmentsLock.unlock();
		}
	}
	
	final void schedule(final ToolRunnable runnable) {
		try {
			final Tool tool= this.input.getTool();
			final Status status= tool.getQueue().add(runnable);
			if (status.getSeverity() == Status.ERROR && !tool.isTerminated()) {
				throw new StatusException(status);
			}
		}
		catch (final StatusException e) {
			clear(Lock.ERROR_STATE);
			logError("An error occurred when scheduling job for data viewer.", e);
		}
	}
	
	private void runInit(final RToolService r, final ProgressMonitor m) throws StatusException {
		if (this.disposeScheduled) {
			synchronized (this.initRunnable) {
				this.initScheduled= false;
			}
			return;
		}
		
		try {
			if (this.rTmpItem == null) {
//				r.evalVoid("require(\"rj\", quietly= TRUE)", m);
				this.rTmpItem= TmpUtils.newItem("viewer", r, m);
			}
		}
		catch (final Exception e) {
			synchronized (this.initRunnable) {
				this.initScheduled= false;
			}
			checkCancel(e);
			clear(Lock.ERROR_STATE);
			logError("An error occurred when preparing tmp variables for data viewer.", e);
			
			this.realm.syncExec(new Runnable() {
				@Override
				public void run() {
					for (final IDataProviderListener listener : AbstractRDataProvider.this.dataListeners.toList()) {
						listener.onInputFailed(0);
					}
				}
			});
			return;
		}
		
		try {
			final RObject rObject= (r instanceof ICombinedRDataAdapter) ?
					((ICombinedRDataAdapter) r).evalCombinedStruct(this.input.getElementName(), 0, 1, m) :
					r.evalData(this.input.getFullName(), null, RObjectFactory.F_ONLY_STRUCT, 1, m);
			if (this.rObjectStruct == null) {
				this.rObjectStruct= this.adapter.validate(rObject);
			}
			else {
				this.rObjectStruct= this.adapter.validate(rObject, this.rObjectStruct, 0);
			}
		}
		catch (final Exception e) {
			synchronized (this.initRunnable) {
				this.initScheduled= false;
			}
			checkCancel(e);
			clear(Lock.RELOAD_STATE);
			logError("An error occurred when initializing structure data for data viewer.", e);
			
			this.realm.syncExec(new Runnable() {
				@Override
				public void run() {
					for (final IDataProviderListener listener : AbstractRDataProvider.this.dataListeners.toList()) {
						listener.onInputFailed(IDataProviderListener.ERROR_STRUCT_CHANGED);
					}
				}
			});
			return;
		}
		final RDataTableContentDescription description;
		try {
			description= loadDescription(this.input.getElementName(), this.rObjectStruct, r, m);
		}
		catch (final Exception e) {
			synchronized (this.initRunnable) {
				this.initScheduled= false;
			}
			checkCancel(e);
			clear(Lock.RELOAD_STATE);
			logError("An error occurred when initializing default formats for data viewer.", e);
			return;
		}
		
		this.realm.syncExec(new Runnable() {
			@Override
			public void run() {
				AbstractRDataProvider.this.description= description;
				final long rowCount= AbstractRDataProvider.this.adapter.getRowCount(AbstractRDataProvider.this.rObjectStruct);
				final boolean rowsChanged= (rowCount != getRowCount());
				clear(0, rowCount, rowCount, true, true, true);
				
				synchronized (AbstractRDataProvider.this.initRunnable) {
					AbstractRDataProvider.this.initScheduled= false;
				}
//				if (rowsChanged) {
//					for (final IDataProviderListener listener : dataListeners) {
//						listener.onRowCountChanged();
//					}
//				}
				for (final IDataProviderListener listener : AbstractRDataProvider.this.dataListeners.toList()) {
					listener.onInputInitialized(rowsChanged);
				}
			}
		});
	}
	
	final TmpUtils.Item getTmpItem() {
		return this.rTmpItem;
	}
	
	private void runUpdate(final RToolService r, final ProgressMonitor m) throws StatusException {
		int work= -1;
		while (true) {
			try {
				final boolean updateSorting;
				final boolean updateFiltering;
				
				this.fragmentsLock.lock();
				try {
					switch (work) {
					case -1: 
						this.fragmentsLock.scheduled= false;
						break;
					case 0:
						if (!this.activeOperations.isEmpty()) {
							try {
								this.fragmentsLock.worker.await();
							}
							catch (final InterruptedException e) {}
							break;
						}
						else {
							return;
						}
					default:
						break;
					}
					work= 0;
					
					updateSorting= this.updateSorting;
					updateFiltering= this.updateFiltering;
				}
				finally {
					this.fragmentsLock.unlock();
				}
				
				FQRObjectRef elementRef= null;
				
				if (updateSorting) {
					if (elementRef == null) {
						elementRef= checkElementRef(this.input.getElementRef(), r, m);
						this.adapter.check(elementRef, this.rObjectStruct, r, m);
					}
					updateSorting(r, m);
					work++;
				}
				if (updateFiltering) {
					if (elementRef == null) {
						elementRef= checkElementRef(this.input.getElementRef(), r, m);
						this.adapter.check(elementRef, this.rObjectStruct, r, m);
					}
					updateFiltering(r, m);
					work++;
				}
				if (this.updateIdx) {
					if (elementRef == null) {
						elementRef= checkElementRef(this.input.getElementRef(), r, m);
						this.adapter.check(elementRef, this.rObjectStruct, r, m);
					}
					updateIdx(r, m);
					work++;
				}
				
				if (work == 0
						&& this.rowDataProvider instanceof AbstractRDataProvider<?>.RowDataProvider) {
					final LazyRStore<RVector<?>> namesStore= ((RowDataProvider) this.rowDataProvider).rowNamesStore;
					while (true) {
						final Fragment<RVector<?>> fragment;
						this.fragmentsLock.lock();
						try {
							fragment= namesStore.getNextScheduledFragment();
						}
						finally {
							this.fragmentsLock.unlock();
						}
						if (fragment == null) {
							break;
						}
						
						if (elementRef == null) {
							elementRef= checkElementRef(this.input.getElementRef(), r, m);
							this.adapter.check(elementRef, this.rObjectStruct, r, m);
						}
						final RVector<?> fragmentObject= this.adapter.loadRowNames(elementRef,
								this.rObjectStruct, fragment, this.rCacheIdx, r, m);
						this.fragmentsLock.lock();
						try {
							namesStore.updateFragment(fragment, fragmentObject);
							
							this.fragmentsLock.notify(fragment);
						}
						finally {
							this.fragmentsLock.unlock();
						}
						notifyListener(fragment);
						work++;
					}
				}
				if (work == 0) {
					while (true) {
						final Fragment<T> fragment;
						this.fragmentsLock.lock();
						try {
							fragment= this.dataStore.getNextScheduledFragment();
						}
						finally {
							this.fragmentsLock.unlock();
						}
						if (fragment == null) {
							break;
						}
						
						if (elementRef == null) {
							elementRef= checkElementRef(this.input.getElementRef(), r, m);
							this.adapter.check(elementRef, this.rObjectStruct, r, m);
						}
						final T fragmentObject= this.adapter.loadData(elementRef,
								this.rObjectStruct, fragment, this.rCacheIdx, r, m);
						
						this.fragmentsLock.lock();
						try {
							this.dataStore.updateFragment(fragment, fragmentObject);
							
							this.fragmentsLock.notify(fragment);
						}
						finally {
							this.fragmentsLock.unlock();
						}
						notifyListener(fragment);
						work++;
					}
				}
			}
			catch (final Exception e) {
				checkCancel(e);
				clear(Lock.RELOAD_STATE);
				logError(NLS.bind("An error occurred when loading data of ''{0}'' for data viewer.",
								this.input.getFullName()),
						e );
				return;
			}
		}
	}
	
	private FQRObjectRef checkElementRef(final FQRObjectRef elementRef,
			final RService r, final ProgressMonitor m) throws StatusException, UnexpectedRDataException {
		RObject env= elementRef.getEnv();
		switch (env.getRObjectType()) {
		case RObject.TYPE_REFERENCE:
			return elementRef;
		case RObject.TYPE_LANGUAGE:
			env= RDataUtils.checkRReference(
					r.evalData(((RLanguage) env).getSource(),
							null, 0, RService.DEPTH_REFERENCE, m ),
					RObject.TYPE_ENVIRONMENT );
			return new BasicFQRObjectRef(elementRef.getRHandle(), env, elementRef.getName());
		default:
			throw new UnexpectedRDataException(
					"Unexpected R object type: " + RDataUtils.getObjectTypeName(env.getRObjectType()) );
		}
	}
	
	private void updateSorting(
			final RToolService r, final ProgressMonitor m) throws StatusException, UnexpectedRDataException {
		cleanSorting(m);
		
		final SortColumn sortColumn;
		this.fragmentsLock.lock();
		try {
			sortColumn= this.sortColumn;
			this.updateSorting= false;
		}
		finally {
			this.fragmentsLock.unlock();
		}
		
		if (sortColumn != null) {
			if (this.rCacheSort == null) {
				this.rCacheSort= this.rTmpItem.createSub("order"); //$NON-NLS-1$
			}
			final StringBuilder cmd= getRCmdStringBuilder();
			appendOrderCmd(cmd, sortColumn);
			this.rTmpItem.set(this.rCacheSort, cmd.toString(), m);
		}
	}
	
	private void updateFiltering(
			final RToolService r, final ProgressMonitor m) throws StatusException, UnexpectedRDataException {
		cleanFiltering(m);
		
		String filter;
		this.fragmentsLock.lock();
		try {
			filter= this.filter;
			this.updateFiltering= false;
		}
		finally {
			this.fragmentsLock.unlock();
		}
		
		final long filteredRowCount;
		if (filter == null) {
			filteredRowCount= getFullRowCount();
		}
		else {
			if (this.rCacheFilter == null) {
				this.rCacheFilter= this.rTmpItem.createSub("include"); //$NON-NLS-1$
			}
			this.rTmpItem.set(this.rCacheFilter, filter, m);
			{	final FunctionCall call= r.createFunctionCall(RJTmp.GET_FILTERED_COUNT);
				call.addChar(RJTmp.FILTER_PAR, this.rCacheFilter);
				filteredRowCount= RDataUtils.checkSingleIntValue(call.evalData(m));
			}
		}
		this.realm.syncExec(new Runnable() {
			@Override
			public void run() {
				clear(0, filteredRowCount, getFullRowCount(), false, false, false);
				for (final IDataProviderListener listener : AbstractRDataProvider.this.dataListeners.toList()) {
					listener.onRowCountChanged();
				}
			}
		});
	}
	
	private void updateIdx(
			final RToolService r, final ProgressMonitor m) throws StatusException, UnexpectedRDataException {
		cleanIdx(m);
		if (this.rCacheSort != null || this.rCacheFilter != null) {
			if (this.rCacheIdx == null) {
				this.rCacheIdx= this.rTmpItem.createSub("idx"); //$NON-NLS-1$
			}
			if (this.rCacheFilter == null) { // fRCacheSort != null
				final StringBuilder cmd= getRCmdStringBuilder();
				cmd.append(TmpUtils.ENV_FQ_NAME+'$').append(this.rCacheSort);
				this.rTmpItem.set(this.rCacheIdx, cmd.toString(), m);
			}
			else if (this.rCacheSort == null) { // fRCacheFilter != null
				final FunctionCall call= r.createFunctionCall(RJTmp.SET_WHICH_INDEX);
				call.addChar(RJTmp.NAME_PAR, this.rCacheIdx);
				call.addChar(RJTmp.FILTER_PAR, this.rCacheFilter);
				call.evalVoid(m);
			}
			else { // fRCacheSort != null && fRCacheFilter != null
				final FunctionCall call= r.createFunctionCall(RJTmp.SET_FILTERED_INDEX);
				call.addChar(RJTmp.NAME_PAR, this.rCacheIdx);
				call.addChar(RJTmp.FILTER_PAR, this.rCacheFilter);
				call.addChar(RJTmp.INDEX_PAR, this.rCacheSort);
				call.evalVoid(m);
			}
		}
		this.updateIdx= false;
	}
	
	String checkFilter() {
		return this.rCacheFilter;
	}
	
	String checkRevIndex(final RToolService r, final ProgressMonitor m) throws StatusException {
		if (this.rCacheIdx != null) {
			if (this.rCacheIdxR == null) {
				final String name= this.rTmpItem.createSub("idx.r"); //$NON-NLS-1$
				try {
					final FunctionCall call= r.createFunctionCall(RJTmp.SET_REVERSE_INDEX);
					call.addChar(RJTmp.NAME_PAR, name);
					call.addChar(RJTmp.INDEX_PAR, this.rCacheIdx);
					call.addNum(RJTmp.LEN_PAR, getFullRowCount());
					call.evalVoid(m);
					return this.rCacheIdxR= name;
				}
				finally {
					if (this.rCacheIdxR == null) {
						this.rTmpItem.remove(name, m);
					}
				}
			}
			return this.rCacheIdxR;
		}
		else {
			return null;
		}
	}
	
	protected abstract RDataTableContentDescription loadDescription(RElementName name,
			T struct,
			RToolService r, ProgressMonitor m) throws StatusException, UnexpectedRDataException;
	
	
	protected RDataTableColumn createColumn(final RStore store, final String expression,
			final RElementName elementName, final long columnIndex, final String columnName,
			final RToolService r, final ProgressMonitor m) throws StatusException, UnexpectedRDataException {
		
		final ImList<String> classNames;
		
		RObject rObject;
		{	final FunctionCall call= r.createFunctionCall("class"); //$NON-NLS-1$
			call.add(expression);
			rObject= call.evalData(m);
			final RVector<RCharacterStore> names= RDataUtils.checkRCharVector(rObject);
			classNames= ImCollections.newList(names.getData().toArray());
		}
		RDataTableColumn column;
		final RDataFormatter format= new RDataFormatter();
		switch (store.getStoreType()) {
		case RStore.LOGICAL:
			format.setAutoWidth(5);
			column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
					IRDataTableVariable.LOGI, store, classNames, format);
			break;
		case RStore.NUMERIC:
			if (checkDateFormat(expression, classNames, format, r, m)) {
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.DATE, store, classNames, format);
				break;
			}
			if (checkDateTimeFormat(expression, classNames, format, r, m)) {
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.DATETIME, store, classNames, format);
				break;
			}
			{	final FunctionCall call= r.createFunctionCall("rj:::.getFormatInfo"); //$NON-NLS-1$
				call.add("x", expression); //$NON-NLS-1$
				rObject= call.evalData(m);
			}
			{	final RIntegerStore formatInfo= RDataUtils.checkRIntVector(rObject).getData();
				RDataUtils.checkLengthGreaterOrEqual(formatInfo, 3);
				format.setAutoWidth(Math.max(formatInfo.getInt(0), 3));
				format.initNumFormat(formatInfo.getInt(1), formatInfo.getInt(2) > 0 ?
						formatInfo.getInt(2) + 1 : 0);
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.NUM, store, classNames, format);
				break;
			}
		case RStore.INTEGER:
			if (checkDateFormat(expression, classNames, format, r, m)) {
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.DATE, store, classNames, format);
				break;
			}
			if (checkDateTimeFormat(expression, classNames, format, r, m)) {
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.DATETIME, store, classNames, format);
				break;
			}
			{	final FunctionCall call= r.createFunctionCall("rj:::.getFormatInfo"); //$NON-NLS-1$
				call.add("x", expression); //$NON-NLS-1$
				rObject= call.evalData(m);
			}
			{	final RIntegerStore formatInfo= RDataUtils.checkRIntVector(rObject).getData();
				RDataUtils.checkLengthGreaterOrEqual(formatInfo, 1);
				format.setAutoWidth(Math.max(formatInfo.getInt(0), 3));
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.INT, store, classNames, format);
				break;
			}
		case RStore.CHARACTER:
			{	final FunctionCall call= r.createFunctionCall("rj:::.getFormatInfo"); //$NON-NLS-1$
				call.add("x", expression); //$NON-NLS-1$
				rObject= call.evalData(m);
			}
			{	final RIntegerStore formatInfo= RDataUtils.checkRIntVector(rObject).getData();
				RDataUtils.checkLengthGreaterOrEqual(formatInfo, 1);
				format.setAutoWidth(Math.max(formatInfo.getInt(0), 3));
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.CHAR, store, classNames, format);
				break;
			}
		case RStore.COMPLEX:
			{	final FunctionCall call= r.createFunctionCall("rj:::.getFormatInfo"); //$NON-NLS-1$
				call.add("x", expression); //$NON-NLS-1$
				rObject= call.evalData(m);
			}
			{	final RIntegerStore formatInfo= RDataUtils.checkRIntVector(rObject).getData();
				RDataUtils.checkLengthGreaterOrEqual(formatInfo, 3);
				format.setAutoWidth(Math.max(formatInfo.getInt(0), 3));
				format.initNumFormat(formatInfo.getInt(1), formatInfo.getInt(2) > 0 ?
						formatInfo.getInt(2) + 1 : 0);
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.CPLX, store, classNames, format);
				break;
			}
		case RStore.RAW:
			format.setAutoWidth(2);
			column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
					IRDataTableVariable.RAW, store, classNames, format);
			break;
		case RStore.FACTOR:
			{	final FunctionCall call= r.createFunctionCall("levels"); //$NON-NLS-1$
				call.add(expression);
				rObject= call.evalData(m);
			}
			{	format.setAutoWidth(3);
				final RCharacterStore levels= RDataUtils.checkRCharVector(rObject).getData();
				final int l= RDataUtils.checkIntLength(levels);
				for (int i= 0; i < l; i++) {
					if (!levels.isNA(i)) {
						final int length= levels.getChar(i).length();
						if (length > format.getAutoWidth()) {
							format.setAutoWidth(length);
						}
					}
				}
				format.initFactorLevels(levels);
				column= new RDataTableColumn(columnIndex, columnName, expression, elementName,
						IRDataTableVariable.FACTOR, RFactorStructStore.addLevels((RFactorStore) store, levels),
						classNames, format);
				break;
			}
		default:
			throw new UnexpectedRDataException("store type: " + store.getStoreType());
		}
		return column;
	}
	
	protected boolean checkDateFormat(final String expression, final List<String> classNames,
			final RDataFormatter formatter,
			final RToolService r, final ProgressMonitor m) throws StatusException, UnexpectedRDataException {
		if (classNames.contains("Date")) { //$NON-NLS-1$
			formatter.initDateFormat(RDataFormatter.MILLIS_PER_DAY);
			formatter.setAutoWidth(10);
			return true;
		}
		return false;
	}
	
	protected boolean checkDateTimeFormat(final String expression, final List<String> classNames,
			final RDataFormatter formatter,
			final RToolService r, final ProgressMonitor m) throws StatusException, UnexpectedRDataException {
		RObject rObject;
		if (classNames.contains("POSIXct")) { //$NON-NLS-1$
			formatter.initDateTimeFormat(RDataFormatter.MILLIS_PER_SECOND);
			formatter.setAutoWidth(27);
			
			{	final FunctionCall call= r.createFunctionCall("base::attr"); //$NON-NLS-1$
				call.add(expression);
				call.addChar("tzone"); //$NON-NLS-1$
				rObject= call.evalData(m);
			}
			if (rObject.getRObjectType() != RObject.TYPE_NULL) {
				formatter.setDateTimeZone(TimeZone.getTimeZone(RDataUtils.checkSingleCharValue(rObject)));
			}
			return true;
		}
		return false;
	}
	
	protected RDataTableColumn createNamesColumn(final String expression, final long count,
			final RToolService r, final ProgressMonitor m) throws StatusException, UnexpectedRDataException {
		final RObject names= r.evalData(expression, null, RObjectFactory.F_ONLY_STRUCT, 1, m);
		if (names != null && names.getRObjectType() == RObject.TYPE_VECTOR
				&& names.getLength() == count
				&& (names.getData().getStoreType() == RStore.CHARACTER
						|| names.getData().getStoreType() == RStore.INTEGER)) {
			return createColumn(names.getData(), expression, null, -1, null, r, m);
		}
		return createAutoNamesColumn(count);
	}
	
	private RDataTableColumn createAutoNamesColumn(final long count) {
		final RDataFormatter format= new RDataFormatter();
		format.setAutoWidth(Math.max(Long.toString(count).length(), 3));
		return new RDataTableColumn(-1, null, null, null,
				IRDataTableVariable.INT, DefaultRObjectFactory.INT_STRUCT_DUMMY,
				ImCollections.newList(RObject.CLASSNAME_INTEGER),
				format);
	}
	
	protected abstract void appendOrderCmd(StringBuilder cmd, SortColumn sortColumn);
	
	
	private void runClean(final RToolService r, final ProgressMonitor m) throws StatusException {
		clear(Lock.ERROR_STATE);
		cleanSorting(m);
		cleanFiltering(m);
		this.findManager.clean(m);
		this.rTmpItem.dispose(m);
		this.rTmpItem= null;
	}
	
	private void cleanSorting(final ProgressMonitor m) throws StatusException {
		cleanIdx(m);
		if (this.rCacheSort != null) {
			this.rTmpItem.remove(this.rCacheSort, m);
			this.rCacheSort= null;
		}
	}
	
	private void cleanFiltering(final ProgressMonitor m) throws StatusException {
		cleanIdx(m);
		if (this.rCacheFilter != null) {
			this.rTmpItem.remove(this.rCacheFilter, m);
			this.rCacheFilter= null;
		}
	}
	
	private void cleanIdx(final ProgressMonitor m) throws StatusException {
		this.updateIdx= true;
		if (this.rCacheIdx != null) {
			this.rTmpItem.remove(this.rCacheIdx, m);
			this.rCacheIdx= null;
		}
		if (this.rCacheIdxR != null) {
			this.rTmpItem.remove(this.rCacheIdxR,  m);
			this.rCacheIdxR= null;
		}
	}
	
	
	protected final StringBuilder getRCmdStringBuilder() {
		this.rStringBuilder.setLength(0);
		return this.rStringBuilder;
	}
	
	
	public boolean getAllColumnsEqual() {
		return false;
	}
	
	public RDataTableContentDescription getDescription() {
		return this.description;
	}
	
	@Override
	public long getColumnCount() {
		return this.columnCount;
	}
	
	public long getFullRowCount() {
		return this.fullRowCount;
	}
	
	@Override
	public long getRowCount() {
		return this.rowCount;
	}
	
	@Override
	public Object getDataValue(final long columnIndex, final long rowIndex, final int flags,
			final IProgressMonitor monitor) {
		try {
			final LazyRStore.Fragment<T> fragment= this.fragmentsLock.getFragment(
					this.dataStore, rowIndex, columnIndex,
					flags, StatusUtils.convert(monitor) );
			if (fragment != null) {
				return getDataValue(fragment, rowIndex, columnIndex);
			}
			else {
				return handleMissingData(flags);
			}
		}
		catch (final LoadDataException e) {
			return handleLoadDataException(e, flags);
		}
	}
	
	protected abstract Object getDataValue(LazyRStore.Fragment<T> fragment, long rowIdx, long columnIdx);
	
	@Override
	public void setDataValue(final long columnIndex, final long rowIndex, final Object newValue) {
		throw new UnsupportedOperationException();
	}
	
	protected abstract Object getColumnName(LazyRStore.Fragment<T> fragment, long columnIdx);
	
	public boolean hasRealColumns() {
		return true;
	}
	
	public boolean hasRealRows() {
		return true;
	}
	
	public IDataProvider getColumnDataProvider() {
		return this.columnDataProvider;
	}
	
	public IDataProvider getRowDataProvider() {
		return this.rowDataProvider;
	}
	
	public IDataProvider getColumnLabelProvider() {
		return this.columnLabelProvider;
	}
	
	public IDataProvider getRowLabelProvider() {
		return this.rowLabelProvider;
	}
	
	
	protected IDataProvider createColumnDataProvider() {
		return new ColumnDataProvider();
	}
	
	protected IDataProvider createRowDataProvider() {
		return new RowDataProvider();
	}
	
	protected IDataProvider createColumnLabelProvider() {
		return null;
	}
	
	protected IDataProvider createRowLabelProvider() {
		return null;
	}
	
	protected ISortModel createSortModel() {
		return new SortModel();
	}
	
	private Object handleMissingData(final int flags) {
		return ((flags & IDataProvider.FORCE_SYNC) != 0) ? ERROR : LOADING;
	}
	
	private Object handleLoadDataException(final LoadDataException e, final int flags) {
		if (!e.isRecoverable()) {
			return ERROR;
		}
		reset();
		return handleMissingData(flags);
	}
	
	
	public ISortModel getSortModel() {
		return this.sortModel;
	}
	
	public SortColumn getSortColumn() {
		return this.sortColumn;
	}
	
	private void setSortColumn(final SortColumn column) {
		this.fragmentsLock.lock();
		try {
			if (Objects.equals(this.sortColumn, column)) {
				return;
			}
			this.sortColumn= column;
			
			clear(-1, -1, -1, true, false, false);
			
			this.findManager.reset(false);
		}
		finally {
			this.fragmentsLock.unlock();
		}
	}
	
	public void setFilter(final String filter) {
		this.fragmentsLock.lock();
		try {
			if ((this.filter != null) ? this.filter.equals(filter) : null == filter) {
				return;
			}
			this.filter= filter;
			
			clear(-1, -1, -1, false, true, false);
			
			this.findManager.reset(true);
			
			this.fragmentsLock.scheduleUpdate(null, null, null, 0, null);
		}
		finally {
			this.fragmentsLock.unlock();
		}
	}
	
	public String getFilter() {
		return this.filter;
	}
	
	
	public void addFindListener(final IFindListener listener) {
		this.findManager.addFindListener(listener);
	}
	
	public void removeFindListener(final IFindListener listener) {
		this.findManager.removeFindListener(listener);
	}
	
	public void find(final FindTask task) {
		this.findManager.find(task);
	}
	
	
	public long[] toDataIdxs(final long columnIndex, final long rowIndex) {
		if (getFilter() != null || getSortColumn() != null) {
			if (this.rowDataProvider instanceof AbstractRDataProvider.RowDataProvider) {
				final long rowIdx= ((AbstractRDataProvider.RowDataProvider) this.rowDataProvider)
						.getRowIdx(rowIndex);
				return new long[] { columnIndex, rowIdx };
			}
			return new long[] { columnIndex, -2 };
		}
		return new long[] { columnIndex, rowIndex };
	}
	
	
	public void addDataChangedListener(final IDataProviderListener listener) {
		this.dataListeners.add(listener);
	}
	
	public void removeDataChangedListener(final IDataProviderListener listener) {
		this.dataListeners.remove(listener);
	}
	
	protected void notifyListener(final LazyRStore.Fragment<?> item) {
		try {
			for (final IDataProviderListener listener : this.dataListeners.toList()) {
				listener.onRowsChanged(item.getRowBeginIdx(), item.getRowEndIdx());
			}
		}
		catch (final Exception e) {
			logError("An error occurred when notifying about row updates.", e);
		}
	}
	
	
	public void reset() {
		clear(Lock.PAUSE_STATE);
		synchronized (this.initRunnable) {
			if (this.initScheduled) {
				return;
			}
			this.initScheduled= true;
		}
		try {
			final Status status= this.input.getTool().getQueue().add(this.initRunnable);
			if (status.getSeverity() >= Status.ERROR) {
				throw new StatusException(status);
			}
		}
		catch (final StatusException e) {
		}
	}
	
	private void clear(final int newState) {
		clear(newState, -1, -1, true, true, true);
	}
	
	private void clear(final int newState, final long filteredRowCount, final long fullRowCount,
			final boolean updateSorting, final boolean updateFiltering,
			final boolean clearFind) {
		this.fragmentsLock.lock();
		try {
			this.dataStore.clear(filteredRowCount);
			if (this.rowDataProvider instanceof AbstractRDataProvider<?>.RowDataProvider) {
				((RowDataProvider) this.rowDataProvider).rowNamesStore.clear(filteredRowCount);
			}
			this.updateSorting |= updateSorting;
			this.updateFiltering |= updateFiltering;
			
			if (newState >= 0 && this.fragmentsLock.state < Lock.ERROR_STATE) {
				this.fragmentsLock.state= newState;
			}
			this.fragmentsLock.clear();
			
			if (filteredRowCount >= 0) {
				this.rowCount= filteredRowCount;
				this.fullRowCount= fullRowCount;
			}
			
			if (clearFind) {
				this.findManager.clear(newState);
			}
		}
		finally {
			this.fragmentsLock.unlock();
		}
	}
	
	public void dispose() {
		this.disposeScheduled= true;
		schedule(this.cleanRunnable);
	}
	
}