Refactor LineList class to AbstractLineList to implement it and do tests

easily for synch between IDocument and LineList. See
#5

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/AbstractLineList.java

@@ -0,0 +1,84 @@
+/**
+ *  Copyright (c) 2015-2017 Angelo ZERR.
+ *  All rights reserved. This program and the accompanying materials
+ *  are made available under the terms of the Eclipse Public License v1.0
+ *  which accompanies this distribution, and is available at
+ *  http://www.eclipse.org/legal/epl-v10.html
+ *
+ *  Contributors:
+ *  Angelo Zerr <[email protected]> - initial API and implementation
+ */
+package org.eclipse.tm4e.core.model;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * Abstract class for Model lines used by the TextMate model. Implementation
+ * class must :
+ * 
+ * <ul>
+ * <li>synchronizes lines with the lines of the editor content when it changed.</li>
+ * <li>call {@link AbstractLineList#invalidateLine(int)} with the first changed line.</li>
+ * </ul>
+ *
+ */
+public abstract class AbstractLineList implements IModelLines {
+
+	private final List<ModelLine> list = Collections.synchronizedList(new ArrayList<>());
+
+	private TMModel model;
+
+	public AbstractLineList() {
+	}
+
+	void setModel(TMModel model) {
+		this.model = model;
+	}
+
+	@Override
+	public void addLine(int line) {
+		try {
+			this.list.add(line, new ModelLine());
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+	@Override
+	public void removeLine(int line) {
+		this.list.remove(line);
+	}
+
+	@Override
+	public void updateLine(int line) {
+		try {
+			// this.list.get(line).text = this.lineToTextResolver.apply(line);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+	@Override
+	public ModelLine get(int index) {
+		return this.list.get(index);
+	}
+
+	@Override
+	public int getSize() {
+		return this.list.size();
+	}
+
+	@Override
+	public void forEach(Consumer<ModelLine> consumer) {
+		this.list.forEach(consumer);
+	}
+
+	protected void invalidateLine(int lineIndex) {
+		if (model != null) {
+			model.invalidateLine(lineIndex);
+		}
+	}
+}

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/IModelLines.java

@@ -10,6 +10,8 @@
  */
 package org.eclipse.tm4e.core.model;
 
+import java.util.function.Consumer;
+
 /**
  * Mode lines API which must be initalize with a document and changed of
  * document.
@@ -45,4 +47,17 @@ public interface IModelLines {
 	 */
 	int getSize();
 
+	ModelLine get(int index);
+
+	void forEach(Consumer<ModelLine> consumer);
+
+	int getNumberOfLines();
+
+	String getLineText(int line) throws Exception;
+
+	int getLineLength(int line) throws Exception;
+
+	void dispose();
+
+
 }

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/ModelLine.java

@@ -12,7 +12,7 @@
 
 import java.util.List;
 
-class ModelLine {
+public class ModelLine {
 
 	//String text;
 	boolean isInvalid;

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/AbstractTMModel.java → org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMModel.java

@@ -18,10 +18,10 @@
 import org.eclipse.tm4e.core.grammar.IGrammar;
 
 /**
- * Abstract class for TextMate model.
+ * TextMate model class.
  *
  */
-public abstract class AbstractTMModel implements ITMModel {
+public class TMModel implements ITMModel {
 
 	/**
 	 * The TextMate grammar to use to parse for each lines of the document the
@@ -35,34 +35,27 @@ public abstract class AbstractTMModel implements ITMModel {
 	/** The background thread. */
 	private TokenizerThread fThread;
 
-	private final LineList lines;
+	private final IModelLines lines;
 	private PriorityBlockingQueue<Integer> invalidLines = new PriorityBlockingQueue<>();
 
-	public AbstractTMModel() {
+	public TMModel(IModelLines lines) {
 		this.listeners = new ArrayList<>();
-		this.lines = new LineList(/*lineNumber -> {
-			try {
-				// No need to store text in the line list (helpful just for debugging).
-				return null; //getLineText(lineNumber);
-			} catch (Exception ex) {
-				ex.printStackTrace();
-				return "";
-			}
-		}*/);
+		this.lines = lines;
+		((AbstractLineList)lines).setModel(this);
 		invalidLines.add(0);
 	}
 
 	/**
-	 * The {@link TokenizerThread} takes as input an {@link AbstractTMModel} and continuously
-	 * runs tokenizing in background on the lines found in {@link AbstractTMModel#lines}.
-	 * The {@link AbstractTMModel#lines} are expected to be accessed through {@link AbstractTMModel#getLines()}
+	 * The {@link TokenizerThread} takes as input an {@link TMModel} and continuously
+	 * runs tokenizing in background on the lines found in {@link TMModel#lines}.
+	 * The {@link TMModel#lines} are expected to be accessed through {@link TMModel#getLines()}
 	 * and manipulated by the UI part to inform of needs to (re)tokenize area, then the {@link TokenizerThread}
 	 * processes them and emits events through the model. UI elements are supposed to subscribe and react to the events with
-	 * {@link AbstractTMModel#addModelTokensChangedListener(IModelTokensChangedListener)}.
+	 * {@link TMModel#addModelTokensChangedListener(IModelTokensChangedListener)}.
 	 *
 	 */
 	static class TokenizerThread extends Thread {
-		private AbstractTMModel model;
+		private TMModel model;
 		private TMState _lastState;
 		private Tokenizer tokenizer;
 
@@ -73,7 +66,7 @@ static class TokenizerThread extends Thread {
 		 * @param name
 		 *            the thread's name
 		 */
-		public TokenizerThread(String name, AbstractTMModel model) {
+		public TokenizerThread(String name, TMModel model) {
 			super(name);
 			this.model = model;
 			this.tokenizer = new Tokenizer(model.getGrammar());
@@ -87,7 +80,7 @@ public void run() {
 				return;
 
 			// initialize
-			int linesLength = model.getNumberOfLines();
+			int linesLength = model.lines.getNumberOfLines();
 			for (int line = 0; line < linesLength; line++) {
 				try {
 					model.lines.addLine(line);
@@ -147,7 +140,7 @@ private void _revalidateTokensNow(Integer startIndex, Integer toLineIndexOrNull)
 
 					// Compute how many characters will be tokenized for this line
 					try {
-						currentCharsToTokenize = model.getLineLength(lineIndex);
+						currentCharsToTokenize = model.lines.getLineLength(lineIndex);
 					} catch (Exception e) {
 						// TODO Auto-generated catch block
 						e.printStackTrace();
@@ -194,7 +187,7 @@ private int _updateTokensInRange(ModelTokensChangedEventBuilder eventBuilder, in
 				String text = null;
 				ModelLine modeLine = model.lines.get(lineIndex);
 				try {
-					text = model.getLineText(lineIndex);
+					text = model.lines.getLineText(lineIndex);
 					// Tokenize only the first X characters
 					r = tokenizer.tokenize(text, modeLine.getState(), 0, stopLineTokenizationAfter);
 				} catch (Throwable e) {
@@ -270,7 +263,6 @@ private LineTokens nullTokenize(String buffer, TMState state) {
 			int deltaOffset = 0;
 			List<TMToken> tokens = new ArrayList<>();
 			tokens.add(new TMToken(deltaOffset, ""));
-
 			return new LineTokens(tokens, deltaOffset + buffer.length(), state);
 		}
 
@@ -311,6 +303,7 @@ public void removeModelTokensChangedListener(IModelTokensChangedListener listene
 	@Override
 	public void dispose() {
 		stop();
+		getLines().dispose();
 	}
 
 	/**
@@ -350,7 +343,7 @@ public boolean isLineInvalid(int lineNumber) {
 		return lines.get(lineNumber).isInvalid;
 	}
 
-	protected void invalidateLine(int lineIndex) {
+	void invalidateLine(int lineIndex) {
 		this.lines.get(lineIndex).isInvalid = true;
 		//Integer currentInvalidIndex = this.invalidLines.peek();
 		this.invalidLines.add(lineIndex);
@@ -366,9 +359,4 @@ public IModelLines getLines() {
 		return this.lines;
 	}
 
-	protected abstract int getNumberOfLines();
-
-	protected abstract String getLineText(int line) throws Exception;
-
-	protected abstract int getLineLength(int line) throws Exception;
 }

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/model/TMModel.java → org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/model/DocumentLineList.java

@@ -14,17 +14,22 @@
 import org.eclipse.jface.text.DocumentEvent;
 import org.eclipse.jface.text.IDocument;
 import org.eclipse.jface.text.IDocumentListener;
-import org.eclipse.tm4e.core.model.AbstractTMModel;
+import org.eclipse.tm4e.core.model.AbstractLineList;
 
 /**
- * TextMate model implementation linked to an {@link IDocument}.
+ * TextMate {@link AbstractLineList} implementation with Eclipse
+ * {@link IDocument}.
+ * 
+ * Goal of this class is to synchronize Eclipse {@link DocumentEvent} with
+ * TextMate model lines.
+ *
  */
-public class TMModel extends AbstractTMModel {
+public class DocumentLineList extends AbstractLineList {
 
 	private final IDocument document;
-	private final InternalListener listener;
+	private InternalListener listener;
 
-	public TMModel(IDocument document) {
+	public DocumentLineList(IDocument document) {
 		this.document = document;
 		this.listener = new InternalListener();
 		document.addDocumentListener(listener);
@@ -48,7 +53,7 @@ private void removeLine(DocumentEvent event) throws BadLocationException {
 			int startLine = DocumentHelper.getStartLine(event);
 			int endLine = DocumentHelper.getEndLine(event, true);
 			for (int i = endLine; i > startLine; i--) {
-				getLines().removeLine(i);
+				DocumentLineList.this.removeLine(i);
 			}
 		}
 
@@ -58,19 +63,19 @@ public void documentChanged(DocumentEvent event) {
 			try {
 				int startLine = DocumentHelper.getStartLine(event);
 				if (!DocumentHelper.isRemove(event)) {
-					if (getLines().getSize() != DocumentHelper.getNumberOfLines(document)) {
+					if (DocumentLineList.this.getSize() != DocumentHelper.getNumberOfLines(document)) {
 						int endLine = DocumentHelper.getEndLine(event, false);
 						// Insert new lines
 						for (int i = startLine; i < endLine; i++) {
-							getLines().addLine(i + 1);
+							DocumentLineList.this.addLine(i + 1);
 						}
 					} else {
 						// Update line
-						getLines().updateLine(startLine);
+						DocumentLineList.this.updateLine(startLine);
 					}
 				} else {
 					// Update line
-					getLines().updateLine(startLine);
+					DocumentLineList.this.updateLine(startLine);
 				}
 				invalidateLine(startLine);
 			} catch (BadLocationException e) {
@@ -80,28 +85,22 @@ public void documentChanged(DocumentEvent event) {
 	}
 
 	@Override
-	protected int getNumberOfLines() {
+	public int getNumberOfLines() {
 		return DocumentHelper.getNumberOfLines(document);
 	}
 
 	@Override
-	protected String getLineText(int line) throws Exception {
+	public String getLineText(int line) throws Exception {
 		return DocumentHelper.getLineText(document, line, false);
 	}
 
 	@Override
-	protected int getLineLength(int line) throws Exception {
+	public int getLineLength(int line) throws Exception {
 		return DocumentHelper.getLineLength(document, line);
 	}
 
 	@Override
 	public void dispose() {
-		super.dispose();
 		document.removeDocumentListener(listener);
 	}
-
-	public IDocument getDocument() {
-		return document;
-	}
-
 }