Ticket #19284: smlv35_encode_jp.patch

Classes/SMLTextPerformer.h

@@ -34 +34 @@
 
 - (NSString *)removeAllLineEndingsInString:(NSString *)string;
 
+- (BOOL)isEUC:(unsigned char*)charData nLen:(unsigned int)nLen;
 
+- (BOOL)isSJIS:(unsigned char*)charData nLen:(unsigned int)nLen;
 
+- (unsigned int)is2022JP:(unsigned char*)charData nLen:(unsigned int)nLen;
+
+- (unsigned int)isUTF8:(const unsigned char *)charData nLen:(unsigned int)nLen;
+
+- (BOOL)isUTF16:(unsigned char*)charData nLen:(unsigned int)nLen;
+
 @end

Classes/SMLTextPerformer.m

@@ -81 +81 @@
 
 - (NSStringEncoding)guessEncodingFromData:(NSData *)textData
 {
-        NSString *string = [[NSString alloc] initWithData:textData encoding:NSISOLatin1StringEncoding];
+//      NSString *string = [[NSString alloc] initWithData:textData encoding:NSISOLatin1StringEncoding];
         NSStringEncoding encoding = 0;
         BOOL foundExplicitEncoding = NO;
-        
+/*      
         if ([string length] > 9) { // If it's shorter than this you can't check for encoding string
                 NSScanner *scannerHTML = [[NSScanner alloc] initWithString:string];
                 NSInteger beginning;
@@ -114 +114 @@
                         }
                 }
         }
-        
+*/      
         // If the scanner hasn't found an explicitly defined encoding, check for either EFBBBF, FEFF or FFFE and, if found, set the encoding to UTF-8 or UTF-16
         if (!foundExplicitEncoding && [textData length] > 2) {
                 NSString *lookForEncodingInBytesString = [NSString stringWithString:[textData description]];
                 if ([[lookForEncodingInBytesString substringWithRange:NSMakeRange(1,6)] isEqualToString:@"efbbbf"]) encoding = NSUTF8StringEncoding;
                 else if ([[lookForEncodingInBytesString substringWithRange:NSMakeRange(1,4)] isEqualToString:@"feff"] || [[lookForEncodingInBytesString substringWithRange:NSMakeRange(1,4)] isEqualToString:@"fffe"]) encoding = NSUnicodeStringEncoding;
         }
-
+  
+  unsigned int pos,utf8,jp2022,euc,sjis,utf16,r;
+  pos = utf8 = euc = sjis = jp2022 = r = 0;
+  
+  if (encoding == 0 && [textData length] > 2)
+                {
+                        unsigned int nLen = [textData length];
+                        unsigned char* charData = (unsigned char*)[textData bytes];
+                        while(pos < nLen)
+                                {
+                                        if ((r=[self isUTF8:charData+pos nLen:nLen-pos])>0)
+                                                {
+                                                        utf8++;
+                                                        pos+=r;
+                                                }
+                                        else if ([self isEUC:charData+pos nLen:nLen-pos])
+                                                {
+                                                        euc++;
+                                                        pos+=2;
+                                                }
+                else if ([self isSJIS:charData+pos nLen:nLen-pos])
+                                                {
+                                sjis++;
+                                                        pos+=2;
+                        }
+                else if ((r=[self is2022JP:charData+pos nLen:nLen-pos])>0)
+                                                {
+                                jp2022++;
+                                                        pos+=r;
+                        }
+                                        else
+                                                {
+                                pos++;
+                                                }
+        }
+                }
+  
+        if (utf8 > 0 && utf8 > euc && utf8 > sjis && utf8 > jp2022)
+                {
+                        encoding = NSUTF8StringEncoding;
+                }
+        else if (euc > 0 && euc > utf8 && euc > sjis && euc > jp2022)
+                {
+                        encoding = NSJapaneseEUCStringEncoding;
+                }
+        else if (sjis > 0 && sjis > utf8 && sjis > euc && sjis > jp2022)
+                {
+                        encoding = NSShiftJISStringEncoding;
+                }
+        else if (jp2022 > 0 && jp2022 > utf8 && jp2022 > euc && jp2022 > sjis)
+                {
+                        encoding = NSISO2022JPStringEncoding;
+                }
+  
+  NSString* try_str = [[[NSString alloc] initWithData:textData encoding:encoding] autorelease];
+        if (try_str != nil) return encoding;
+  
+  NSStringEncoding encodings[] =
+  {
+    -2147483647,// MacOS Japanese
+    NSUnicodeStringEncoding,
+    0
+  };
+  
+  // try to encoding
+  int i = 0;
+  while(encodings[i] != 0)
+                {
+        try_str = [[[NSString alloc] initWithData:textData encoding:encodings[i]] autorelease];
+        if (try_str != nil)
+                                {
+                encoding = encodings[i];
+                break;
+                }
+                        i++;
+                }
         return encoding;
 }
 
-
 - (NSString *)replaceAllNewLineCharactersWithSymbolInString:(NSString *)string
 {
         // To remove all newline characters in textString and replace it with a symbol, use NSMakeRange every time as the length changes
@@ -149 +223 @@
         return returnString;
 }
 
+-(BOOL)isUTF16:(unsigned char*)charData nLen:(unsigned int)nLen
+{// NOT USED
+        if (nLen < 2) return NO;
+  // surrogate 
+  if ((*charData >= 0xd8 && *charData <= 0xdb) || (*charData >= 0xdc && *charData <= 0xdf))
+    {
+      return YES;
+    }
+  if ((*charData <= 0xd7) || (*charData >= 0xe0))
+    {
+      return YES;
+    }
+  return NO;
+}
+
+-(unsigned int)is2022JP:(unsigned char*)charData nLen:(unsigned int)nLen
+{
+        if (nLen < 3) return 0;
+  if (*charData == 0x1b && *(charData+1) == 0x24 && (*(charData+2) == 0x40 || *(charData+2) == 0x42))
+    {// ESC$@(78JIS) || ESC$B(83JIS) 
+      return 3;
+    }
+  return 0;
+}
+
+-(unsigned int)isUTF8:(const unsigned char *)charData nLen:(unsigned int)nLen
+{
+  if( (nLen >= 3) &&
+     (*(charData+0) & (unsigned char)0xf0)==(unsigned char)0xe0 &&
+     (*(charData+1) & (unsigned char)0xc0)==(unsigned char)0x80 &&
+     (*(charData+2) & (unsigned char)0xc0)==(unsigned char)0x80)
+    {
+      return 3;
+    }
+  if( (nLen >= 2) &&
+     (*(charData+0) & (unsigned char)0xe0)==(unsigned char)0xc0 &&
+     (*(charData+1) & (unsigned char)0xc0)==(unsigned char)0x80)
+    {
+      return 2;
+    }
+  return 0;
+}
+
+-(BOOL)isSJIS:(unsigned char*)charData nLen:(unsigned int)nLen
+{
+  if (nLen < 2) return NO;
+  if ((*charData >= 0x81 && *charData <= 0x9f) || (*charData >= 0xe0 && *charData <= 0xfc))
+    {
+      if ((*(charData+1) >= 0x40 && *(charData+1)< 0xfc) && (*(charData+1) != 0x7f))
+        {
+          return YES;
+        }
+    }
+  return NO;
+}
+
+-(BOOL)isEUC:(unsigned char*)charData nLen:(unsigned int)nLen
+{
+  if (nLen < 2) return NO;
+  if ((*charData >= 0x8e && *charData <= 0xfe) && (*charData != 0x8f && *charData != 0xa0))
+    {
+      if (*(charData+1) >= 0xa0 && *(charData+1) <= 0xfe)
+        {
+          return YES;
+        }
+    }
+  return NO;
+}
+
 @end

Classes/SMLOpenSavePerformer.m

@@ -196 +196 @@
                 encoding = chosenEncoding;
         } else if ([[SMLDefaults valueForKey:@"EncodingsMatrix"] integerValue] == 0) {
                 NSError *error = nil;
-                string = [NSString stringWithContentsOfFile:path usedEncoding:&encoding error:&error];
-                if (error != nil || string == nil) { // It hasn't found an encoding so return the default
-                        if (textData == nil) {
-                                textData = [[NSData alloc] initWithContentsOfFile:path];
-                        }
-                        encoding = [SMLText guessEncodingFromData:textData];
-                        if (encoding == 0 || encoding == -1) { // Something has gone wrong or it hasn't found an encoding, so use default
-                                encoding = [[SMLDefaults valueForKey:@"EncodingsPopUp"] integerValue];
-                        }
-                }
+    if (textData == nil)
+      {
+        textData = [[NSData alloc] initWithContentsOfFile:path];
+      }
+    encoding = [SMLText guessEncodingFromData:textData];
+    if (encoding == 0 || encoding == -1) 
+      { // Something has gone wrong or it hasn't found an encoding, so use default
+        encoding = [[SMLDefaults valueForKey:@"EncodingsPopUp"] integerValue];
+      }
+    else
+      {
+        string = [NSString stringWithContentsOfFile:path encoding:encoding error:&error];
+      }
+    if (error != nil || string == nil)
+      {
+        string = [NSString stringWithContentsOfFile:path usedEncoding:&encoding error:&error];
+      }
+//              string = [NSString stringWithContentsOfFile:path usedEncoding:&encoding error:&error];
+//              if (error != nil || string == nil) { // It hasn't found an encoding so return the default
+//                      if (textData == nil) {
+//                              textData = [[NSData alloc] initWithContentsOfFile:path];
+//                      }
+//                      encoding = [SMLText guessEncodingFromData:textData];
+//                      if (encoding == 0 || encoding == -1) { // Something has gone wrong or it hasn't found an encoding, so use default
+//                              encoding = [[SMLDefaults valueForKey:@"EncodingsPopUp"] integerValue];
+//                      }
+//              }
                 
         } else {
                 encoding = [[SMLDefaults valueForKey:@"EncodingsPopUp"] integerValue];