save dev

got totally drowned in zend/mail, zend/mime, and eventum bugs

zendframework/zend-mime#26
zendframework/zend-mail#146

db/migrations/20170518174755_eventum_fix_sup_fields.php

@@ -0,0 +1,108 @@
+<?php
+
+/*
+ * This file is part of the Eventum (Issue Tracking System) package.
+ *
+ * @copyright (c) Eventum Team
+ * @license GNU General Public License, version 2 or later (GPL-2+)
+ *
+ * For the full copyright and license information,
+ * please see the COPYING and AUTHORS files
+ * that were distributed with this source code.
+ */
+
+use Eventum\Db\AbstractMigration;
+use Eventum\Mail\MailMessage;
+use Zend\Mail\Headers;
+
+class EventumFixSupFields extends AbstractMigration
+{
+    /** @var string */
+    private $email_table;
+    /** @var string */
+    private $body_table;
+
+    /**
+     * This would be in init() but it's too early to use adapter.
+     *
+     * @see https://github.com/robmorgan/phinx/issues/1095
+     */
+    public function initTables()
+    {
+        $this->email_table = $this->quoteColumnName('support_email');
+        $this->body_table = $this->quoteColumnName('support_email_body');
+    }
+
+    public function up()
+    {
+        $this->initTables();
+        $this->fixTruncatedAddressFields('sup_cc', 'Cc');
+        $this->fixTruncatedAddressFields('sup_to', 'To');
+        throw new RuntimeException();
+    }
+
+    public function fixTruncatedAddressFields($field, $header)
+    {
+        $st = $this->getTruncatedRecords($field);
+        foreach ($st as $row) {
+            $sup_id = $row['sup_id'];
+            file_put_contents("/tmp/{$row['sup_id']}", $row['body']);
+            $mail = MailMessage::createFromString($row['body']);
+            $value = $this->unfold($mail->{$header});
+            $this->updateFieldValue($sup_id, $field, $value);
+        }
+    }
+
+    /**
+     * @see \Zend\Mail\Header\AbstractAddressList::fromString
+     * @param $fieldValue
+     * @return mixed
+     */
+    private function unfold($fieldValue)
+    {
+        $fieldValue = str_replace(Headers::FOLDING, ' ', $fieldValue);
+
+        return $fieldValue;
+    }
+
+    private function updateFieldValue($sup_id, $field, $value)
+    {
+        $field = $this->quoteTableName($field);
+        $value = $this->quoteValue($value);
+
+        $stmt
+            = "
+            UPDATE {$this->email_table}
+            SET $field=$value
+            WHERE sup_id=$sup_id AND 1=0
+        ";
+        $this->query($stmt);
+    }
+
+    /**
+     * Get sup_id where $field length appears to be truncated
+     * Thus length is exactly 255 characters.
+     * This will return only records where support_email_body is not truncated.
+     *
+     * @see https://github.com/eventum/eventum/issues/266
+     *
+     * @param string $field
+     * @param int $length
+     * @return Traversable|array
+     */
+    private function getTruncatedRecords($field, $length = 255)
+    {
+        $field = $this->quoteTableName($field);
+
+        $stmt
+            = "
+            SELECT sup_id, $field field, seb_full_email body
+            FROM {$this->email_table} e, {$this->body_table} b
+            WHERE sup_id=seb_sup_id
+            AND LENGTH($field) = $length
+            aND sup_id=10357
+        ";
+
+        return $this->query($stmt);
+    }
+}

src/Mail/Helper/SplitHeaderBody.php

@@ -0,0 +1,87 @@
+<?php
+
+/*
+ * This file is part of the Eventum (Issue Tracking System) package.
+ *
+ * @copyright (c) Eventum Team
+ * @license GNU General Public License, version 2 or later (GPL-2+)
+ *
+ * For the full copyright and license information,
+ * please see the COPYING and AUTHORS files
+ * that were distributed with this source code.
+ */
+
+namespace Eventum\Mail\Helper;
+
+use Mail_Helper;
+
+class SplitHeaderBody
+{
+    /**
+     * Split $raw email to headers array and content part.
+     *
+     * @param string $raw
+     * @param array &$headers
+     * @param string &$content
+     */
+    public static function splitMessage($raw, &$headers, &$content)
+    {
+        // this method could be really easy
+        // if we wouldn't need to handle broken input.
+
+        // Handle messages broken by Mail_Helper::rewriteThreadingHeaders()
+        // which by inserted References: to \r\n separated headers
+        // used \n separator.
+
+        // split by standard headers separator
+        list($headers, $content) = explode("\r\n\r\n", $raw, 2);
+
+        // split by \r\n, but \r may be optional
+        $headers = preg_split("/\r?\n/", $headers);
+        // strip any leftover \r
+        $headers = array_map('trim', $headers);
+
+        // now headers is in array form, hard to break that again.
+    }
+
+    private function jurajaokse()
+    {
+        // some old emails that were \r separated
+        // eventum filled as \r\r\nheader\nheader\r\n
+        //    Mail_Helper::rewriteThreadingHeaders()
+        // corrupted them
+
+        // split headers/body by \r\n and join headers back by \n
+        // this ensures Headers::fromString doesn't assume email headers are \n\n separated
+        // splitMessage)_
+        list($headers, $content) = explode("\r\n\r\n", $raw, 2);
+
+        $headers = preg_split("/\r?\n/", $headers);
+        // strip any leftover \r
+        $headers = array_map('trim', $headers);
+//        echo json_encode($headersArray);die;
+//        $headers = join("\n", $headersArray);
+//        $raw = $headers."\n\n".$content;
+
+        /*        // try with default \n
+                // then retry with \r\n
+                try {
+                    Mime\Decode::splitMessage($raw, $headers, $content, "\n");
+                } catch (\Zend\Mail\Exception\RuntimeException $e) {
+                    Mime\Decode::splitMessage($raw, $headers, $content, "\r\n");
+                }*/
+
+        /*
+                // if $raw is "\r" only separated, replace it with "\n"
+                if (substr_count($headers, "\n") == 0) {
+                    $parts = explode("\r", $raw);
+                    $raw = join("\n", $parts);
+                }*/
+        return new CreateFromRaw();
+
+//        $message = new self(['root' => true, 'raw' => $raw]);
+        $message = new self(['root' => true, 'headers' => $headers, 'content' => $content]);
+
+        return $message;
+    }
+}

src/Mail/MailMessage.php

@@ -16,6 +16,7 @@
 use DomainException;
 use Eventum\Mail\Helper\MimePart;
 use Eventum\Mail\Helper\SanitizeHeaders;
+use Eventum\Mail\Helper\SplitHeaderBody;
 use InvalidArgumentException;
 use LogicException;
 use Mime_Helper;
@@ -87,7 +88,42 @@ public static function createNew()
      */
     public static function createFromString($raw)
     {
-        $message = new self(['root' => true, 'raw' => $raw]);
+        // some old emails that were \r separated
+        // eventum filled as \r\r\nheader\nheader\r\n
+        //    Mail_Helper::rewriteThreadingHeaders()
+        // corrupted them
+
+        // split headers/body by \r\n and join headers back by \n
+        // this ensures Headers::fromString doesn't assume email headers are \n\n separated
+        // splitMessage)_
+//        list($headers, $content) = explode("\r\n\r\n", $raw, 2);
+
+//        $headers = preg_split("/\r?\n/", $headers);
+        // strip any leftover \r
+//        $headers = array_map('trim', $headers);
+//        echo json_encode($headersArray);die;
+//        $headers = join("\n", $headersArray);
+//        $raw = $headers."\n\n".$content;
+
+        /*        // try with default \n
+                // then retry with \r\n
+                try {
+                    Mime\Decode::splitMessage($raw, $headers, $content, "\n");
+                } catch (\Zend\Mail\Exception\RuntimeException $e) {
+                    Mime\Decode::splitMessage($raw, $headers, $content, "\r\n");
+                }*/
+
+        /*
+                // if $raw is "\r" only separated, replace it with "\n"
+                if (substr_count($headers, "\n") == 0) {
+                    $parts = explode("\r", $raw);
+                    $raw = join("\n", $parts);
+                }*/
+//        return new CreateFromRaw();
+        SplitHeaderBody::splitMessage($raw, $headers, $content);
+
+//        $message = new self(['root' => true, 'raw' => $raw]);
+        $message = new self(['root' => true, 'headers' => $headers, 'content' => $content]);
 
         return $message;
     }
@@ -653,8 +689,8 @@ public function removeFromAddressList($header, $address)
     public function isSeen()
     {
         return $this->hasFlag(ZendMailStorage::FLAG_SEEN)
-        || $this->hasFlag(ZendMailStorage::FLAG_DELETED)
-        || $this->hasFlag(ZendMailStorage::FLAG_ANSWERED);
+            || $this->hasFlag(ZendMailStorage::FLAG_DELETED)
+            || $this->hasFlag(ZendMailStorage::FLAG_ANSWERED);
     }
 
     /**

tests/Mail/MailParseTest.php

@@ -13,8 +13,13 @@
 
 namespace Eventum\Test\Mail;
 
+use Eventum\Mail\MailMessage;
 use Eventum\Test\TestCase;
+use Mail_Helper;
 use Mime_Helper;
+use Zend\Mail\Header\GenericHeader;
+use Zend\Mail\Header\To;
+use Zend\Mail\Headers;
 
 /**
  * @group mail
@@ -42,4 +47,41 @@ public function testBug684922()
         $message_body = $structure->body;
         $this->assertEquals('', $message_body);
     }
+
+    /**
+     * Test reading email whose headers are \r\n separated
+     * but body is \n separated. AND it contains \n\n in the email body
+     */
+    public function testReadHeadersSeparator()
+    {
+        /*        $content = $this->readDataFile('10357_fixed.txt');
+
+        $m = MailMessage::createFromString($content);
+        $headers = $m->getHeadersArray();
+        list($text_headers, $body) = Mail_Helper::rewriteThreadingHeaders(1, $content, $headers);
+
+        file_put_contents('/tmp/broken2.txt', $text_headers);
+*/
+
+        $content = $this->readDataFile('10357_original.txt');
+        $mail = MailMessage::createFromString($content);
+
+//        $content = $this->readDataFile('10357.txt');
+//        $mail = MailMessage::createFromString($content);
+    }
+
+    public function testToHeaderNameWithComma()
+    {
+        $to = To::fromString('To: "=?iso-8859-1?Q?Wetlesen=2C_Asbj=F8rn?=" <[email protected]>');
+        echo $to->toString();
+        $to = To::fromString($to->toString());
+        echo $to->toString();
+    }
+
+    public function testDogFood()
+    {
+        $headers = new Headers();
+        $headers->addHeader(GenericHeader::fromString('To: "=?iso-8859-1?Q?Wetlesen=2C_Asbj=F8rn?=" <[email protected]>'));
+        $headers->get('To');
+    }
 }