ZMS-47: Fix utf-8 header round-trip in Headers.add()

Author: NickOvt

lib/headers.js

@@ -42,7 +42,7 @@ class Headers {
             this._parseHeaders();
         }
         key = this._normalizeHeader(key);
-        let lines = this.lines.filter(line => line.key === key).map(line => line.line);
+        let lines = this.lines.filter(line => line.key === key).map(line => this._decodeHeaderValue(line.line));
 
         return lines;
     }
@@ -62,7 +62,7 @@ class Headers {
         if (!header) {
             return '';
         }
-        return ((this.libmime.decodeHeader(header.line) || {}).value || '').toString().trim();
+        return ((this.libmime.decodeHeader(this._decodeHeaderValue(header.line)) || {}).value || '').toString().trim();
     }
 
     getList() {
@@ -174,17 +174,27 @@ class Headers {
 
         lineEnd = lineEnd || '\r\n';
 
-        let headers = this.lines.map(line => line.line.replace(/\r?\n/g, lineEnd)).join(lineEnd) + `${lineEnd}${lineEnd}`;
+        let headers = this.lines
+            .map(line => this._buildHeaderLine(line.line.replace(/\r?\n/g, lineEnd)))
+            .reduce((joined, line, idx) => {
+                if (idx) {
+                    joined.push(Buffer.from(lineEnd, 'binary'));
+                }
+                joined.push(line);
+                return joined;
+            }, []);
+
+        headers.push(Buffer.from(lineEnd + lineEnd, 'binary'));
 
         if (this.mbox) {
-            headers = this.mbox + lineEnd + headers;
+            headers.unshift(Buffer.from(this.mbox + lineEnd, 'binary'));
         }
 
         if (this.http) {
-            headers = this.http + lineEnd + headers;
+            headers.unshift(Buffer.from(this.http + lineEnd, 'binary'));
         }
 
-        return Buffer.from(headers, 'binary');
+        return Buffer.concat(headers);
     }
 
     _normalizeHeader(key) {
@@ -232,6 +242,20 @@ class Headers {
         this.lines = lines;
         this.parsed = true;
     }
+
+    _buildHeaderLine(line) {
+        let value = this._decodeHeaderValue(line);
+        return Buffer.from(value, value === line ? 'binary' : 'utf8');
+    }
+
+    _decodeHeaderValue(str) {
+        if (!str) {
+            return str;
+        }
+
+        let utf8 = Buffer.from(str, 'binary').toString('utf8');
+        return utf8.includes('\uFFFD') ? str : utf8;
+    }
 }
 
 // expose to the world

test/headers-test.js

@@ -130,3 +130,47 @@ module.exports['Replace header at relative key index'] = test => {
     test.equal(headers.build().toString(), generatedHeaderStr);
     test.done();
 };
+
+module.exports['Preserve SMTPUTF8 mailbox local part'] = test => {
+    let value = 'öäöä@wildduck.xn--4caaa.test';
+    let headers = new Headers();
+    headers.add('From', value);
+
+    test.equal(headers.get('from')[0], `From: ${value}`);
+    test.equal(headers.getDecoded('from')[0].value, value);
+    test.equal(headers.build().toString(), `From: ${value}\r\n\r\n`);
+    test.done();
+};
+
+module.exports['Preserve SMTPUTF8 mailbox with UTF-8 domain'] = test => {
+    let value = 'öäöä@wildduck.äää.test';
+    let headers = new Headers();
+    headers.add('From', value);
+
+    test.equal(headers.get('from')[0], `From: ${value}`);
+    test.equal(headers.getDecoded('from')[0].value, value);
+    test.equal(headers.build().toString(), `From: ${value}\r\n\r\n`);
+    test.done();
+};
+
+module.exports['Preserve UTF-8 display name and mailbox'] = test => {
+    let value = '"Jöhn Dœ" <öäöä@wildduck.xn--4caaa.test>';
+    let headers = new Headers();
+    headers.add('From', value);
+
+    test.equal(headers.get('from')[0], `From: ${value}`);
+    test.equal(headers.getDecoded('from')[0].value, value);
+    test.equal(headers.build().toString(), `From: ${value}\r\n\r\n`);
+    test.done();
+};
+
+module.exports['Preserve ASCII address round-trip'] = test => {
+    let value = 'ascii@example.com';
+    let headers = new Headers();
+    headers.add('From', value);
+
+    test.equal(headers.get('from')[0], `From: ${value}`);
+    test.equal(headers.getDecoded('from')[0].value, value);
+    test.equal(headers.build().toString(), `From: ${value}\r\n\r\n`);
+    test.done();
+};