Bug 574748: [R-Console] Add support for "formattings" in R Pager text
file input

Change-Id: I1575e3b2e52da295d0ad054f4fc54afdc792ca3e
diff --git a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/RPagerEditor.java b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/RPagerEditor.java
index 38947d1..bdd5c54 100644
--- a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/RPagerEditor.java
+++ b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/RPagerEditor.java
@@ -29,6 +29,7 @@
 import org.eclipse.statet.internal.r.ui.RUIPlugin;
 import org.eclipse.statet.internal.r.ui.pager.RPagerEditorInput.TextFile;
 import org.eclipse.statet.ltk.core.input.BasicSourceFragment;
+import org.eclipse.statet.ltk.ui.input.BasicSourceFragmentEditorInput;
 import org.eclipse.statet.r.ui.RUI;
 
 
@@ -84,8 +85,8 @@
 					final var sourceFragment= new BasicSourceFragment("RPagerFile#" + id++,
 							textFile.getName(), textFile.getName(),
 							new ImmutableDocument(textFile.getContent(), 0) );
-					final TextFilePage page= new TextFilePage();
-					addPage(page, new TextFileEditorInput(sourceFragment));
+					final TextFileEditorPage page= new TextFileEditorPage();
+					addPage(page, new BasicSourceFragmentEditorInput(sourceFragment));
 				}
 				catch (final Exception e) {
 					RUIPlugin.logError(String.format("An error occurred when creating R Pager editor page for '%1$s'.",
diff --git a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileDocumentProvider.java b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileDocumentProvider.java
new file mode 100644
index 0000000..8f981f5
--- /dev/null
+++ b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileDocumentProvider.java
@@ -0,0 +1,125 @@
+/*=============================================================================#
+ # Copyright (c) 2010, 2021 Stephan Wahlbrink and others.
+ # 
+ # This program and the accompanying materials are made available under the
+ # terms of the Eclipse Public License 2.0 which is available at
+ # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
+ # which is available at https://www.apache.org/licenses/LICENSE-2.0.
+ # 
+ # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+ # 
+ # Contributors:
+ #     Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation
+ #=============================================================================*/
+
+package org.eclipse.statet.internal.r.ui.pager;
+
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.jface.operation.IRunnableContext;
+import org.eclipse.jface.text.AbstractDocument;
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.ITypedRegion;
+import org.eclipse.jface.text.source.AnnotationModel;
+import org.eclipse.jface.text.source.IAnnotationModel;
+import org.eclipse.ui.texteditor.AbstractDocumentProvider;
+
+import org.eclipse.statet.jcommons.collections.ImCollections;
+import org.eclipse.statet.jcommons.collections.ImList;
+import org.eclipse.statet.jcommons.lang.NonNullByDefault;
+import org.eclipse.statet.jcommons.lang.Nullable;
+
+import org.eclipse.statet.ecommons.text.core.util.ImmutableDocument;
+
+import org.eclipse.statet.ltk.core.input.SourceFragment;
+import org.eclipse.statet.ltk.ui.input.SourceFragmentEditorInput;
+
+
+@NonNullByDefault
+public class TextFileDocumentProvider extends AbstractDocumentProvider {
+	
+	
+	public class TextFileElementInfo extends ElementInfo {
+		
+		private final ImList<ITypedRegion> formatRegions;
+		
+		public TextFileElementInfo(final IDocument document, final ImList<ITypedRegion> formatRegions,
+				final IAnnotationModel model) {
+			super(document, model);
+			this.formatRegions= formatRegions;
+		}
+		
+	}
+	
+	
+	final TextFileParser parser= new TextFileParser();
+	
+	
+	public TextFileDocumentProvider() {
+	}
+	
+	
+	@Override
+	protected @Nullable ElementInfo createElementInfo(final Object element) throws CoreException {
+		AbstractDocument document= null;
+		if (document == null) {
+			document= createDocument(element);
+		}
+		if (document != null) {
+			final ImList<ITypedRegion> formatRegions;
+			final String orgText= document.get();
+			synchronized (this.parser) {
+				this.parser.parse(orgText);
+				if (this.parser.getCleanText() != orgText) {
+					document= new ImmutableDocument(this.parser.getCleanText(), document.getModificationStamp());
+				}
+				formatRegions= this.parser.getFormatRegions();
+			}
+			
+			setupDocument(document);
+			final ElementInfo info= new TextFileElementInfo(document, formatRegions,
+					createAnnotationModel(element) );
+			return info;
+		}
+		return null;
+	}
+	
+	@Override
+	protected @Nullable AbstractDocument createDocument(final Object element) throws CoreException {
+		if (element instanceof SourceFragmentEditorInput) {
+			final SourceFragment fragment= ((SourceFragmentEditorInput)element).getSourceFragment();
+			return fragment.getDocument();
+		}
+		return null;
+	}
+	
+	protected void setupDocument(final AbstractDocument document) {
+		// we use the default partitioner
+	}
+	
+	@Override
+	protected IAnnotationModel createAnnotationModel(final Object element) throws CoreException {
+		return new AnnotationModel();
+	}
+	
+	@Override
+	protected @Nullable IRunnableContext getOperationRunner(final @Nullable IProgressMonitor monitor) {
+		return null;
+	}
+	
+	@Override
+	protected void doSaveDocument(final @Nullable IProgressMonitor monitor, final Object element,
+			final IDocument document, final boolean overwrite) throws CoreException {
+	}
+	
+	public ImList<ITypedRegion> getTextFormatRegions(final @Nullable Object element) {
+		if (element instanceof SourceFragmentEditorInput) {
+			final var elementInfo= getElementInfo(element);
+			if (elementInfo instanceof TextFileElementInfo) {
+				return ((TextFileElementInfo)elementInfo).formatRegions;
+			}
+		}
+		return ImCollections.emptyList();
+	}
+	
+}
diff --git a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileEditorPage.java b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileEditorPage.java
new file mode 100644
index 0000000..31913b2
--- /dev/null
+++ b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileEditorPage.java
@@ -0,0 +1,89 @@
+/*=============================================================================#
+ # Copyright (c) 2021 Stephan Wahlbrink and others.
+ # 
+ # This program and the accompanying materials are made available under the
+ # terms of the Eclipse Public License 2.0 which is available at
+ # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
+ # which is available at https://www.apache.org/licenses/LICENSE-2.0.
+ # 
+ # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+ # 
+ # Contributors:
+ #     Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation
+ #=============================================================================*/
+
+package org.eclipse.statet.internal.r.ui.pager;
+
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.jface.text.source.CompositeRuler;
+import org.eclipse.jface.text.source.IVerticalRuler;
+import org.eclipse.ui.IEditorInput;
+import org.eclipse.ui.texteditor.StatusTextEditor;
+
+import org.eclipse.statet.jcommons.lang.NonNullByDefault;
+import org.eclipse.statet.jcommons.lang.Nullable;
+
+
+@NonNullByDefault
+public class TextFileEditorPage extends StatusTextEditor {
+	
+	
+	private static final TextFileDocumentProvider DOCUMENT_PROVIDER= new TextFileDocumentProvider();
+	
+	
+	private final TextFileSourceViewerConfiguration sourceViewerConfiguration;
+	
+	
+	public TextFileEditorPage() {
+		setDocumentProvider(TextFileEditorPage.DOCUMENT_PROVIDER);
+		this.sourceViewerConfiguration= new TextFileSourceViewerConfiguration();
+		setSourceViewerConfiguration(this.sourceViewerConfiguration);
+		setRulerContextMenuId(""); //$NON-NLS-1$
+	}
+	
+	
+	@Override
+	@SuppressWarnings("null")
+	public TextFileDocumentProvider getDocumentProvider() {
+		return (TextFileDocumentProvider)super.getDocumentProvider();
+	}
+	
+	
+	@Override
+	protected void doSetInput(final @Nullable IEditorInput input) throws CoreException {
+		final var documentProvider= getDocumentProvider();
+		documentProvider.connect(input);
+		try {
+			this.sourceViewerConfiguration.setStyleRegions(
+					TextFileEditorPage.DOCUMENT_PROVIDER.getTextFormatRegions(input) );
+			super.doSetInput(input);
+		}
+		finally {
+			documentProvider.disconnect(input);
+		}
+	}
+	
+	
+	@Override
+	public boolean isDirty() {
+		return false;
+	}
+	
+	@Override
+	public boolean isSaveAsAllowed() {
+		return false;
+	}
+	
+	@Override
+	protected void performSaveAs(final @Nullable IProgressMonitor progressMonitor) {
+		// for save as: implement and change isSaveAsAllowed to true
+	}
+	
+	
+	@Override
+	protected IVerticalRuler createVerticalRuler() {
+		return new CompositeRuler();
+	}
+	
+}
diff --git a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFilePage.java b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFilePage.java
deleted file mode 100644
index 9095d66..0000000
--- a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFilePage.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*=============================================================================#
- # Copyright (c) 2021 Stephan Wahlbrink and others.
- # 
- # This program and the accompanying materials are made available under the
- # terms of the Eclipse Public License 2.0 which is available at
- # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
- # which is available at https://www.apache.org/licenses/LICENSE-2.0.
- # 
- # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
- # 
- # Contributors:
- #     Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation
- #=============================================================================*/
-
-package org.eclipse.statet.internal.r.ui.pager;
-
-import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.jface.text.source.CompositeRuler;
-import org.eclipse.jface.text.source.IVerticalRuler;
-import org.eclipse.ui.texteditor.IDocumentProvider;
-import org.eclipse.ui.texteditor.StatusTextEditor;
-
-import org.eclipse.statet.jcommons.lang.NonNullByDefault;
-import org.eclipse.statet.jcommons.lang.Nullable;
-
-import org.eclipse.statet.ltk.ui.sourceediting.FragmentDocumentProvider;
-
-
-@NonNullByDefault
-public class TextFilePage extends StatusTextEditor {
-	
-	
-	private final IDocumentProvider documentProvider= new FragmentDocumentProvider("", null);
-	
-	
-	public TextFilePage() {
-		setDocumentProvider(this.documentProvider);
-		setRulerContextMenuId(""); //$NON-NLS-1$
-	}
-	
-	
-	@Override
-	public boolean isDirty() {
-		return false;
-	}
-	
-	@Override
-	public boolean isSaveAsAllowed() {
-		return false;
-	}
-	
-	@Override
-	protected void performSaveAs(final @Nullable IProgressMonitor progressMonitor) {
-		// for save as: implement and change isSaveAsAllowed to true
-	}
-	
-	
-	@Override
-	protected IVerticalRuler createVerticalRuler() {
-		return new CompositeRuler();
-	}
-	
-}
diff --git a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileParser.java b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileParser.java
new file mode 100644
index 0000000..1b845f6
--- /dev/null
+++ b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileParser.java
@@ -0,0 +1,137 @@
+/*=============================================================================#
+ # Copyright (c) 2021 Stephan Wahlbrink and others.
+ # 
+ # This program and the accompanying materials are made available under the
+ # terms of the Eclipse Public License 2.0 which is available at
+ # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
+ # which is available at https://www.apache.org/licenses/LICENSE-2.0.
+ # 
+ # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+ # 
+ # Contributors:
+ #     Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation
+ #=============================================================================*/
+
+package org.eclipse.statet.internal.r.ui.pager;
+
+import java.util.ArrayList;
+
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.ITypedRegion;
+import org.eclipse.jface.text.TypedRegion;
+
+import org.eclipse.statet.jcommons.collections.ImCollections;
+import org.eclipse.statet.jcommons.collections.ImList;
+import org.eclipse.statet.jcommons.lang.NonNullByDefault;
+
+
+@NonNullByDefault
+public class TextFileParser {
+	
+	
+	public static final String DEFAULT_FORMAT_TYPE= IDocument.DEFAULT_CONTENT_TYPE;
+	public static final String UNDERLINE_FORMAT_TYPE= "Pager.Underline";
+	
+	
+	private boolean createDefaultRegions= false;
+	
+	private final StringBuilder cleanTextBuilder= new StringBuilder();
+	
+	private final ArrayList<ITypedRegion> formatRegions= new ArrayList<>();
+	private String cleanText= ""; //$NON-NLS-1$
+	
+	
+	public TextFileParser() {
+	}
+	
+	
+	public void setCreateDefaultRegions(final boolean enable) {
+		this.createDefaultRegions= enable;
+	}
+	
+	
+	public void parse(final String in) {
+		this.formatRegions.clear();
+		this.cleanTextBuilder.setLength(0);
+		
+		final int inLength= in.length();
+		int offsetDone= 0;
+		while (offsetDone < inLength) {
+			final int backOffset= in.indexOf(0x08, offsetDone);
+			if (backOffset >= 0) {
+				this.cleanTextBuilder.append(in, offsetDone, backOffset - 1);
+				if (backOffset + 1 == inLength || backOffset == offsetDone) {
+					offsetDone= backOffset + 1;
+					continue;
+				}
+				switch (in.charAt(backOffset - 1)) {
+				case '_':
+					checkDefaultFormat(this.cleanTextBuilder.length());
+					offsetDone= readBackspaceFormat(in, backOffset - 1, '_', UNDERLINE_FORMAT_TYPE);
+					continue;
+				default:
+					offsetDone= backOffset + 1;
+					continue;
+				}
+			}
+			else {
+				if (offsetDone > 0) {
+					this.cleanTextBuilder.append(in, offsetDone, inLength);
+					offsetDone= inLength;
+				}
+				break;
+			}
+		}
+		
+		this.cleanText= (offsetDone > 0) ? this.cleanTextBuilder.toString() : in;
+		checkDefaultFormat(this.cleanText.length());
+	}
+	
+	private void checkDefaultFormat(final int offset) {
+		if (!this.createDefaultRegions) {
+			return;
+		}
+		final int lastEndOffset;
+		if (this.formatRegions.isEmpty()) {
+			lastEndOffset= 0;
+		}
+		else {
+			final ITypedRegion lastFormat= this.formatRegions.get(this.formatRegions.size() - 1);
+			lastEndOffset= lastFormat.getOffset() + lastFormat.getLength();
+		}
+		if (offset > lastEndOffset) {
+			this.formatRegions.add(new TypedRegion(
+					lastEndOffset, offset - lastEndOffset,
+					DEFAULT_FORMAT_TYPE ));
+		}
+	}
+	
+	private int readBackspaceFormat(final String in, int offset, final char formatChar,
+			final String formatType) {
+		final int inLength= in.length();
+		final int cleanStartOffset= this.cleanTextBuilder.length();
+		
+		do {
+			this.cleanTextBuilder.append(in.charAt(offset + 2));
+			offset+= 3;
+		}
+		while (offset + 2 < inLength
+				&& in.charAt(offset) == formatChar
+				&& in.charAt(offset + 1) == 0x08);
+		
+		this.formatRegions.add(new TypedRegion(
+				cleanStartOffset, this.cleanTextBuilder.length() - cleanStartOffset,
+				formatType ));
+		return offset;
+	}
+	
+	
+	public String getCleanText() {
+		return this.cleanText;
+	}
+	
+	public ImList<ITypedRegion> getFormatRegions() {
+		return ImCollections.toList(this.formatRegions);
+	}
+	
+}
diff --git a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileSourceViewerConfiguration.java b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileSourceViewerConfiguration.java
new file mode 100644
index 0000000..05b2aa6
--- /dev/null
+++ b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileSourceViewerConfiguration.java
@@ -0,0 +1,79 @@
+/*=============================================================================#
+ # Copyright (c) 2021 Stephan Wahlbrink and others.
+ # 
+ # This program and the accompanying materials are made available under the
+ # terms of the Eclipse Public License 2.0 which is available at
+ # https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
+ # which is available at https://www.apache.org/licenses/LICENSE-2.0.
+ # 
+ # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+ # 
+ # Contributors:
+ #     Stephan Wahlbrink <sw@wahlbrink.eu> - initial API and implementation
+ #=============================================================================*/
+
+package org.eclipse.statet.internal.r.ui.pager;
+
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.ITypedRegion;
+import org.eclipse.jface.text.presentation.IPresentationReconciler;
+import org.eclipse.jface.text.presentation.PresentationReconciler;
+import org.eclipse.jface.text.rules.DefaultDamagerRepairer;
+import org.eclipse.jface.text.source.ISourceViewer;
+import org.eclipse.jface.text.source.SourceViewerConfiguration;
+
+import org.eclipse.statet.jcommons.collections.ImList;
+import org.eclipse.statet.jcommons.lang.NonNullByDefault;
+import org.eclipse.statet.jcommons.lang.Nullable;
+
+import org.eclipse.statet.ecommons.text.ui.presentation.FixTokenScanner;
+import org.eclipse.statet.ecommons.text.ui.presentation.TextStyleManager;
+
+
+@NonNullByDefault
+public class TextFileSourceViewerConfiguration extends SourceViewerConfiguration {
+	
+	
+	private static final TextStyleManager TEXT_STYLE_MANAGER= new TextFilesTextStyleManager();
+	
+	
+	private final TextStyleManager textStyles;
+	
+	private @Nullable IPresentationReconciler presentationReconciler;
+	private final FixTokenScanner scanner;
+	
+	
+	public TextFileSourceViewerConfiguration() {
+		this.textStyles= TEXT_STYLE_MANAGER;
+		
+		this.scanner= new FixTokenScanner(this.textStyles, TextFileParser.DEFAULT_FORMAT_TYPE);
+	}
+	
+	
+	public void setStyleRegions(final ImList<ITypedRegion> regions) {
+		this.scanner.setStyleRegions(regions);
+	}
+	
+	
+	@Override
+	public IPresentationReconciler getPresentationReconciler(final ISourceViewer sourceViewer) {
+		var presentationReconciler= this.presentationReconciler;
+		if (presentationReconciler == null) {
+			presentationReconciler= createPresentationReconciler();
+			this.presentationReconciler= presentationReconciler;
+		}
+		return presentationReconciler;
+	}
+	
+	protected IPresentationReconciler createPresentationReconciler() {
+		final PresentationReconciler reconciler= new PresentationReconciler();
+		reconciler.setDocumentPartitioning(getConfiguredDocumentPartitioning(null));
+		
+		final DefaultDamagerRepairer dr= new DefaultDamagerRepairer(this.scanner);
+		reconciler.setDamager(dr, IDocument.DEFAULT_CONTENT_TYPE);
+		reconciler.setRepairer(dr, IDocument.DEFAULT_CONTENT_TYPE);
+		
+		return reconciler;
+	}
+	
+}
diff --git a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileEditorInput.java b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFilesTextStyleManager.java
similarity index 61%
rename from r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileEditorInput.java
rename to r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFilesTextStyleManager.java
index af10973..1d3dbc8 100644
--- a/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFileEditorInput.java
+++ b/r/org.eclipse.statet.r.ui/src/org/eclipse/statet/internal/r/ui/pager/TextFilesTextStyleManager.java
@@ -14,19 +14,29 @@
 
 package org.eclipse.statet.internal.r.ui.pager;
 
+import org.eclipse.jface.text.TextAttribute;
+import org.eclipse.swt.SWT;
+
 import org.eclipse.statet.jcommons.lang.NonNullByDefault;
 
-import org.eclipse.statet.ltk.core.input.SourceFragment;
-import org.eclipse.statet.ltk.ui.input.BasicSourceFragmentEditorInput;
+import org.eclipse.statet.ecommons.text.ui.presentation.BasicTextStyleManager;
 
 
 @NonNullByDefault
-public class TextFileEditorInput extends BasicSourceFragmentEditorInput<SourceFragment> {
+public class TextFilesTextStyleManager extends BasicTextStyleManager<TextAttribute> {
 	
 	
-	public TextFileEditorInput(final SourceFragment fragment) {
-		super(fragment);
+	public TextFilesTextStyleManager() {
 	}
 	
 	
+	@Override
+	protected TextAttribute createTextAttributes(final String key) {
+		int style= 0;
+		if (key == TextFileParser.UNDERLINE_FORMAT_TYPE) {
+			style= SWT.BOLD;
+		}
+		return new TextAttribute(null, null, style);
+	}
+	
 }