Ticket #29984: patch-mecab.diff

app/guts/pipes/DocumentPipe.java

@@ -16 +16 @@
 import cc.mallet.types.Instance;
 import cc.mallet.util.CharSequenceLexer;
 
+import dualist.ja.SimpleMecabPipe;
+
 public class DocumentPipe extends Pipe {
 
     private Pipe myPipe = new SerialPipes(new Pipe[] {
@@ -27 +29 @@
             new CharSequenceReplace(Pattern.compile("&(.*?);"), ""),
             new CharSequenceReplace(Pattern.compile("[0-9]+"), "00"),
             new CharSequenceLowercase(),
+            (System.getProperty("dualist.lang") != null &&
+             System.getProperty("dualist.lang").equals("ja")) ?
+            new SimpleMecabPipe() :
             new CharSequence2TokenSequence(CharSequenceLexer.LEX_WORD_CLASSES),
             new TokenSequenceRemoveStopwords(),
             new TokenSequence2FeatureSequence(),