From 001771d5c4c92adf071b1881e98ba76cff722379 Mon Sep 17 00:00:00 2001 From: Robert Virkus Date: Sat, 4 Jul 2020 17:42:51 +0200 Subject: [PATCH] Decode/encode mailbox names. Close #69. --- CHANGELOG.md | 5 + README.md | 2 +- lib/codecs/mail_codec.dart | 3 + lib/codecs/modified_utf7_codec.dart | 257 ++++++++++++++++++++++ lib/imap/imap_client.dart | 44 +++- lib/imap/mailbox.dart | 23 +- lib/mail/mail_client.dart | 12 +- lib/src/util/ascii_runes.dart | 3 + pubspec.yaml | 4 +- test/codecs/modified_utf7_codec_test.dart | 67 ++++++ test/imap/imap_client_test.dart | 4 +- 11 files changed, 405 insertions(+), 19 deletions(-) create mode 100644 lib/codecs/modified_utf7_codec.dart create mode 100644 test/codecs/modified_utf7_codec_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index f000c37e..e5684dbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.0.29 +- Add `discconect()` method to high level `MailClient` API +- Encode and decode mailbox names using Modified UTF7 encoding +- Add [IMAP support for UTF-8](https://tools.ietf.org/html/rfc6855) + ## 0.0.28 - High level `MailClient` API supports IMAP IDLE, POP and SMTP. diff --git a/README.md b/README.md index e64cdd8a..33c8a92d 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ The following IMAP extensions are supported: * ✅ [CONDSTORE](https://tools.ietf.org/html/rfc7162) * ✅ [QRESYNC](https://tools.ietf.org/html/rfc7162) * ✅ [ENABLE](https://tools.ietf.org/html/rfc5161) - +* ✅ [IMAP Support for UTF-8](https://tools.ietf.org/html/rfc6855) ### Supported encodings Character encodings: diff --git a/lib/codecs/mail_codec.dart b/lib/codecs/mail_codec.dart index 91ac16bc..95a16848 100644 --- a/lib/codecs/mail_codec.dart +++ b/lib/codecs/mail_codec.dart @@ -16,6 +16,9 @@ abstract class MailCodec { static const String _encodingEndSequence = '?='; static final RegExp _encodingExpression = RegExp( r'\=\?.+?\?.+?\?.+?\?\='); // the question marks after plus make this regular expression non-greedy + static const Encoding encodingUtf8 = utf8; + static const Encoding encodingLatin1 = latin1; + static const Encoding encodingAscii = ascii; static final Map _codecsByName = { 'utf-8': utf8, 'utf8': utf8, diff --git a/lib/codecs/modified_utf7_codec.dart b/lib/codecs/modified_utf7_codec.dart new file mode 100644 index 00000000..8b510f18 --- /dev/null +++ b/lib/codecs/modified_utf7_codec.dart @@ -0,0 +1,257 @@ +import 'dart:convert'; + +/// Provides Modified UTF7 encoder and decoder. +/// Compare https://tools.ietf.org/html/rfc3501#section-5.1.3 and https://tools.ietf.org/html/rfc2152 for details. +/// Inspired by https://github.com/jstedfast/MailKit/blob/master/MailKit/Net/Imap/ImapEncoding.cs +class ModifiedUtf7Codec { + const ModifiedUtf7Codec(); + + static const String _utf7Alphabet = + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,'; + + static const List _utf7Rank = [ + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 62, + 63, + 255, + 255, + 255, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 255, + 255, + 255, + 255, + 255, + 255, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 255, + 255, + 255, + 255, + 255, + ]; + + void _utf7ShiftOut(StringBuffer output, int u, int bits) { + if (bits > 0) { + final x = (u << (6 - bits)) & 0x3f; + output.write(_utf7Alphabet[x]); + } + + output.write('-'); + } + + /// Encodes the specified text in Modified UTF7 format. + /// [text] specifies the text to be encoded. + String encodeText(String text) { + final encoded = StringBuffer(); + var shifted = false; + var bits = 0, u = 0; + + for (var index = 0; index < text.length; index++) { + var character = text[index]; + var codeUnit = character.codeUnitAt(0); + if (codeUnit >= 0x20 && codeUnit < 0x7f) { + // characters with octet values 0x20-0x25 and 0x27-0x7e + // represent themselves while 0x26 ("&") is represented + // by the two-octet sequence "&-" + + if (shifted) { + _utf7ShiftOut(encoded, u, bits); + shifted = false; + bits = 0; + } + + if (codeUnit == 0x26) { + encoded.write('&-'); + } else { + encoded.write(character); + } + } else { + // base64 encode + if (!shifted) { + encoded.write('&'); + shifted = true; + } + + u = (u << 16) | (codeUnit & 0xffff); + bits += 16; + + while (bits >= 6) { + final x = (u >> (bits - 6)) & 0x3f; + encoded.write(_utf7Alphabet[x]); + bits -= 6; + } + } + } + + if (shifted) { + _utf7ShiftOut(encoded, u, bits); + } + + return encoded.toString(); + } + + /// Decodes the specified [text] + /// + /// [codec] the optional character encoding (charset, defaults to utf-8) + String decodeText(String text, [Encoding codec = utf8]) { + var decoded = StringBuffer(); + var shifted = false; + var bits = 0, v = 0; + var index = 0; + String c; + + while (index < text.length) { + c = text[index++]; + + if (shifted) { + final codeUnit = c.codeUnitAt(0); + if (c == '-') { + // shifted back out of modified UTF-7 + shifted = false; + bits = v = 0; + } else if (codeUnit > 127) { + // invalid UTF-7 + return text; + } else { + final rank = _utf7Rank[codeUnit]; + + if (rank == 0xff) { + // invalid UTF-7 + return text; + } + + v = (v << 6) | rank; + bits += 6; + + if (bits >= 16) { + var u = ((v >> (bits - 16)) & 0xffff); + decoded.write(String.fromCharCode(u)); + bits -= 16; + } + } + } else if (c == '&' && index < text.length) { + if (text[index] == '-') { + decoded.write('&'); + index++; + } else { + // shifted into modified UTF-7 + shifted = true; + } + } else { + decoded.write(c); + } + } + + return decoded.toString(); + } +} diff --git a/lib/imap/imap_client.dart b/lib/imap/imap_client.dart index 4dcf76ae..2ecb644c 100644 --- a/lib/imap/imap_client.dart +++ b/lib/imap/imap_client.dart @@ -38,6 +38,20 @@ class ImapServerInfo { String capabilitiesText; List capabilities; List enabledCapabilities = []; + + /// Checks if the capability with the specified [capabilityName] is supported. + bool supports(String capabilityName) { + return (capabilities?.firstWhere((c) => c.name == capabilityName, + orElse: () => null) != + null); + } + + /// Checks if the capability with the specified [capabilityName] has been enabled. + bool isEnabled(String capabilityName) { + return (enabledCapabilities?.firstWhere((c) => c.name == capabilityName, + orElse: () => null) != + null); + } } enum StoreAction { add, remove, replace } @@ -591,11 +605,17 @@ class ImapClient { path, (recursive ? '*' : '%')); // list all folders in that path } + String _encodeMailboxPath(String path) { + return (serverInfo.isEnabled('UTF8=ACCEPT')) ? path : Mailbox.encode(path); + } + /// lists all mailboxes in the path [referenceName] that match the given [mailboxName] that can contain wildcards. /// /// The LIST command will set the [serverInfo.pathSeparator] as a side-effect Future>> listMailboxesByReferenceAndName( String referenceName, String mailboxName) { + referenceName = _encodeMailboxPath(referenceName); + mailboxName = _encodeMailboxPath(mailboxName); var cmd = Command('LIST $referenceName $mailboxName'); var parser = ListParser(serverInfo); return sendCommand>(cmd, parser); @@ -608,7 +628,7 @@ class ImapClient { /// The LIST command will set the [serverInfo.pathSeparator] as a side-effect Future>> listSubscribedMailboxes( {String path = '""', bool recursive = false}) { - //Command cmd = Command("LIST \"INBOX/\" %"); + path = _encodeMailboxPath(path); var cmd = Command('LSUB $path ' + (recursive ? '*' : '%')); // list all folders in that path var parser = ListParser(serverInfo, isLsubParser: true); @@ -687,7 +707,8 @@ class ImapClient { /// implementation for both SELECT as well as EXAMINE Future> _selectOrExamine(String command, Mailbox box, {bool enableCondStore = false, QResyncParameters qresync}) { - var buffer = StringBuffer()..write(command)..write(' ')..write(box.path); + var path = _encodeMailboxPath(box.path); + var buffer = StringBuffer()..write(command)..write(' ')..write(path); if (enableCondStore || qresync != null) { buffer.write(' ('); if (enableCondStore) { @@ -1036,6 +1057,7 @@ class ImapClient { /// /// Spefify the name with [path] Future> createMailbox(String path) async { + path = _encodeMailboxPath(path); var cmd = Command('CREATE $path'); var response = await sendCommand(cmd, null); if (response.isOkStatus) { @@ -1054,8 +1076,7 @@ class ImapClient { /// /// [box] the mailbox to be deleted Future> deleteMailbox(Mailbox box) { - var cmd = Command('DELETE ${box.path}'); - return sendCommand(cmd, null); + return _sendMailboxCommand('DELETE', box); } /// Renames the specified mailbox @@ -1063,7 +1084,10 @@ class ImapClient { /// [box] the mailbox that should be renamed /// [newName] the desired future name of the mailbox Future> renameMailbox(Mailbox box, String newName) async { - var cmd = Command('RENAME ${box.path} $newName'); + var path = _encodeMailboxPath(box.path); + newName = _encodeMailboxPath(newName); + + var cmd = Command('RENAME ${path} $newName'); var response = await sendCommand(cmd, null); if (response.isOkStatus) { if (box.name == 'INBOX') { @@ -1085,15 +1109,19 @@ class ImapClient { /// The mailbox is listed in future LSUB commands, compare [listSubscribedMailboxes]. /// [box] the mailbox that is subscribed Future> subscribeMailbox(Mailbox box) { - var cmd = Command('SUBSCRIBE ${box.path}'); - return sendCommand(cmd, null); + return _sendMailboxCommand('SUBSCRIBE', box); } /// Unsubscribes the specified mailbox. /// /// [box] the mailbox that is unsubscribed Future> unsubscribeMailbox(Mailbox box) { - var cmd = Command('UNSUBSCRIBE ${box.path}'); + return _sendMailboxCommand('UNSUBSCRIBE', box); + } + + Future> _sendMailboxCommand(String command, Mailbox box) { + var path = _encodeMailboxPath(box.path); + var cmd = Command('$command $path'); return sendCommand(cmd, null); } diff --git a/lib/imap/mailbox.dart b/lib/imap/mailbox.dart index 38e52a44..414d763b 100644 --- a/lib/imap/mailbox.dart +++ b/lib/imap/mailbox.dart @@ -1,3 +1,5 @@ +import 'package:enough_mail/codecs/modified_utf7_codec.dart'; + /// Contains common flags for mailboxes enum MailboxFlag { marked, @@ -22,8 +24,16 @@ enum MailboxFlag { /// Stores meta data about a folder aka Mailbox class Mailbox { - String name; - String path; + static const ModifiedUtf7Codec _modifiedUtf7Codec = ModifiedUtf7Codec(); + String get encodedName => _modifiedUtf7Codec.encodeText(_name); + String _name; + String get name => _name; + set name(String value) => _name = _modifiedUtf7Codec.decodeText(value); + + String get encodedPath => _modifiedUtf7Codec.encodeText(_path); + String _path; + String get path => _path; + set path(String value) => _path = _modifiedUtf7Codec.decodeText(value); bool isMarked = false; bool hasChildren = false; bool isSelected = false; @@ -58,7 +68,9 @@ class Mailbox { isInbox || isDrafts || isSent || isJunk || isTrash || isArchive; Mailbox(); - Mailbox.setup(this.name, this.path, this.flags) { + Mailbox.setup(String name, String path, this.flags) { + this.name = name; + this.path = path; isMarked = hasFlag(MailboxFlag.marked); hasChildren = hasFlag(MailboxFlag.hasChildren); isSelected = hasFlag(MailboxFlag.select); @@ -105,4 +117,9 @@ class Mailbox { } return buffer.toString(); } + + /// Helper method to encode the specified [path] in Modified UTF7 encoding. + static String encode(String path) { + return _modifiedUtf7Codec.encodeText(path); + } } diff --git a/lib/mail/mail_client.dart b/lib/mail/mail_client.dart index 11478fd0..d172ae69 100644 --- a/lib/mail/mail_client.dart +++ b/lib/mail/mail_client.dart @@ -383,10 +383,18 @@ class _IncomingImapClient extends _IncomingMailClient { if (response.isOkStatus) { //TODO compare with previous capabilities and possibly fire events for new or removed server capabilities _config.serverCapabilities = _imapClient.serverInfo.capabilities; + var enableCaps = []; if (_config.supports('QRESYNC')) { - var enabledResponse = await _imapClient.enable(['QRESYNC']); - _isQResyncEnabled = enabledResponse.isOkStatus; + enableCaps.add('QRESYNC'); } + if (_config.supports('UTF8=ACCEPT') || _config.supports('UTF8=ONLY')) { + enableCaps.add('UTF8=ACCEPT'); + } + if (enableCaps.isNotEmpty) { + await _imapClient.enable(enableCaps); + _isQResyncEnabled = _imapClient.serverInfo.isEnabled('QRESYNC'); + } + _supportsIdle = _config.supports('IDLE'); } return response; diff --git a/lib/src/util/ascii_runes.dart b/lib/src/util/ascii_runes.dart index a02a742c..cf1ba59a 100644 --- a/lib/src/util/ascii_runes.dart +++ b/lib/src/util/ascii_runes.dart @@ -2,13 +2,16 @@ class AsciiRunes { static const int runeTab = 9; static const int runeSpace = 32; static const int runeDoubleQuote = 34; + static const int runeAmpersand = 38; static const int runeSingleQuote = 39; + static const int runeMinus = 45; static const int runeDot = 46; static const int rune0 = 48; static const int rune9 = 57; static const int runeComma = 44; static const int runeSemicolon = 59; static const int runeSmallerThan = 60; + static const int runeEquals = 61; static const int runeGreaterThan = 62; static const int runeAt = 64; static const int runeAUpperCase = 65; diff --git a/pubspec.yaml b/pubspec.yaml index c5ba7e88..8a3f270e 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: enough_mail -description: IMAP, POP3 and SMTP clients in pure Dart. Choose between a low level and a high level API for mailing. -version: 0.0.28 +description: IMAP, POP3 and SMTP email clients in pure Dart. Choose between a low level and a high level API for mailing. +version: 0.0.29 homepage: https://github.com/Enough-Software/enough_mail environment: diff --git a/test/codecs/modified_utf7_codec_test.dart b/test/codecs/modified_utf7_codec_test.dart new file mode 100644 index 00000000..388aa75d --- /dev/null +++ b/test/codecs/modified_utf7_codec_test.dart @@ -0,0 +1,67 @@ +import 'dart:convert'; + +import 'package:test/test.dart'; +import 'package:enough_mail/codecs/mail_codec.dart'; +import 'package:enough_mail/codecs/modified_utf7_codec.dart'; + +void main() { + final codec = ModifiedUtf7Codec(); + final encoding = utf8; + + group('Modified UTF7 decoding', () { + test('Simple case 1', () { + var input = '&Jjo-!'; + expect(codec.decodeText(input, encoding), '☺!'); + }); + test('Simple case 2', () { + var input = 'Hello, &ThZ1TA-'; + expect(codec.decodeText(input, encoding), 'Hello, 世界'); + }); + + test('Encoded Ampersand', () { + var input = 'hello&-goodbye'; + expect(codec.decodeText(input, encoding), 'hello&goodbye'); + }); + + test('English, Japanese, and Chinese', () { + var input = '~peter/mail/&ZeVnLIqe-/&U,BTFw-'; + expect(codec.decodeText(input, encoding), '~peter/mail/日本語/台北'); + }); + }); + + group('Modified UTF7 encoding', () { + test('Simple case 1', () { + var input = '☺!'; + expect(codec.encodeText(input), '&Jjo-!'); + }); + test('Simple case 2', () { + var input = 'Hello, 世界'; + expect(codec.encodeText(input), 'Hello, &ThZ1TA-'); + }); + + test('Encoded Ampersand', () { + var input = 'hello&goodbye'; + expect(codec.encodeText(input), 'hello&-goodbye'); + }); + + test('English, Japanese, and Chinese', () { + var input = '~peter/mail/日本語/台北'; + expect(codec.encodeText(input), '~peter/mail/&ZeVnLIqe-/&U,BTFw-'); + }); + + test('quotes', () { + var input = '""'; + expect(codec.encodeText(input), '""'); + }); + + test('* wildcard', () { + var input = '*'; + expect(codec.encodeText(input), '*'); + }); + + test('% wildcard', () { + var input = '%'; + expect(codec.encodeText(input), '%'); + }); + }); +} diff --git a/test/imap/imap_client_test.dart b/test/imap/imap_client_test.dart index 58f3dd51..aecfd007 100644 --- a/test/imap/imap_client_test.dart +++ b/test/imap/imap_client_test.dart @@ -1236,7 +1236,7 @@ void main() { test('ImapClient idle', () async { _log(''); expungedMessages.clear(); - var idleResponseFuture = client.idleStart(); + await client.idleStart(); if (mockServer != null) { mockInbox.messagesExists += 4; @@ -1245,8 +1245,6 @@ void main() { } await Future.delayed(Duration(milliseconds: 200)); await client.idleDone(); - var idleResponse = await idleResponseFuture; - expect(idleResponse.status, ResponseStatus.OK); if (mockServer != null) { expect(expungedMessages.length, 2); expect(expungedMessages[0], 2);