improve commonmark compliance

Gruntfile.js

@@ -2,6 +2,7 @@
  * Created by Tivie on 12-11-2014.
  */
 
+const commonmark = require('commonmark-spec');
 module.exports = function (grunt) {
 
   if (grunt.option('q') || grunt.option('quiet')) {
@@ -230,6 +231,13 @@ module.exports = function (grunt) {
     grunt.task.run(['lint', 'concat:test', 'mochaTest:single', 'clean']);
   });
 
+  grunt.registerTask('extract-commonmark-tests', function () {
+    'use strict';
+    let commonmark = require('commonmark-spec');
+    let testsuite = JSON.stringify(commonmark.tests, null, 2);
+    grunt.file.write('test/functional/makehtml/cases/commonmark.testsuite.json', testsuite)
+  });
+
   /**
    * Tasks
    */
@@ -237,19 +245,14 @@ module.exports = function (grunt) {
   grunt.registerTask('test-functional', ['concat:test', 'mochaTest:functional', 'clean']);
   grunt.registerTask('test-unit', ['concat:test', 'mochaTest:unit', 'clean']);
   grunt.registerTask('test-cli', ['clean', 'lint', 'concat:test', 'mochaTest:cli', 'clean']);
-  grunt.registerTask('test-commonmark', ['clean', 'lint', 'concat:test', 'mochaTest:commonmark', 'clean']);
+  grunt.registerTask('test-commonmark', ['clean', 'lint', 'concat:test', 'extract-commonmark-tests', 'mochaTest:commonmark', 'clean']);
 
   grunt.registerTask('performance', ['concat:test', 'performancejs', 'clean']);
   grunt.registerTask('build', ['test', 'concat:dist', 'concat:cli', 'uglify:dist', 'uglify:cli', 'endline']);
   grunt.registerTask('build-without-test', ['concat:dist', 'uglify', 'endline']);
   grunt.registerTask('prep-release', ['build', 'performance', 'generate-changelog']);
 
-  grunt.registerTask('extract-commonmark-tests', function () {
-    'use strict';
-    let commonmark = require('commonmark-spec');
 
-    grunt.file.write('test/functional/makehtml/cases/commonmark.testsuite.json', JSON.stringify(commonmark.tests, null, 2))
-  });
 
   // Default task(s).
   grunt.registerTask('default', ['test']);

docs/spec-compliance.md

@@ -0,0 +1,35 @@
+# Spec Compliance
+
+## Commonmark
+
+Compliance percentage:
+
+### How to enable commonmark flavor
+
+Enable Commonmark flavor
+
+```
+let converter = new showdown.Converter();
+converter.setFlavor('commonmark');
+```
+
+### Known differences:
+
+#### ATX Headings
+
+ - Showdown doesn't support empty headings
+
+    Input:
+    ```md
+    # 
+    ```
+   
+    Showdown Output:
+    ```html
+    <p>#</p>
+    ```
+   
+    Commonmark output:
+    ```
+    <h1></h1>
+    ```

package-lock.json

@@ -29,6 +29,7 @@
         "grunt-endline": "^0.7.0",
         "grunt-eslint": "^24.0.0",
         "grunt-mocha-test": "^0.13.3",
+        "html-prettify": "^1.0.6",
         "karma": "^6.3.17",
         "karma-browserstack-launcher": "^1.6.0",
         "karma-chai": "^0.1.0",
@@ -3271,6 +3272,12 @@
         "node": ">=12"
       }
     },
+    "node_modules/html-prettify": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/html-prettify/-/html-prettify-1.0.6.tgz",
+      "integrity": "sha512-a2e5NX3pjP1io0Up0d3JOr+tMwKy8IsT4JaMMLznXzuuqPlthnvNdKd3lesQpu37/XWiA28FvDYQU0w/RlAymA==",
+      "dev": true
+    },
     "node_modules/htmlparser2": {
       "version": "3.8.3",
       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",
@@ -9249,6 +9256,12 @@
         "whatwg-encoding": "^2.0.0"
       }
     },
+    "html-prettify": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/html-prettify/-/html-prettify-1.0.6.tgz",
+      "integrity": "sha512-a2e5NX3pjP1io0Up0d3JOr+tMwKy8IsT4JaMMLznXzuuqPlthnvNdKd3lesQpu37/XWiA28FvDYQU0w/RlAymA==",
+      "dev": true
+    },
     "htmlparser2": {
       "version": "3.8.3",
       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",

package.json

@@ -60,6 +60,7 @@
     "grunt-endline": "^0.7.0",
     "grunt-eslint": "^24.0.0",
     "grunt-mocha-test": "^0.13.3",
+    "html-prettify": "^1.0.6",
     "karma": "^6.3.17",
     "karma-browserstack-launcher": "^1.6.0",
     "karma-chai": "^0.1.0",

src/converter.js

@@ -303,7 +303,8 @@ showdown.Converter = function (converterOptions) {
     text = '\n\n' + text + '\n\n';
 
     // detab
-    text = showdown.subParser('makehtml.detab')(text, options, globals);
+    //text = showdown.subParser('makehtml.detab')(text, options, globals);
+    text = showdown.helper.normalizeLeadingTabs(text);
 
     /**
      * Strip any lines consisting only of spaces and tabs.

src/helpers.js

@@ -705,6 +705,14 @@ showdown.helper._populateAttributes = function (attributes) {
   return text;
 };
 
+showdown.helper.normalizeLeadingTabs = function (text) {
+  // 1. (1 to 3 spaces followed by a tab at the start of the line) becomes (1 tab)
+  text = text.replace(/^ {1,3}\t/gm, '\t');
+
+  // 2.
+  return text;
+};
+
 /**
  * Remove one level of line-leading tabs or spaces
  * @param {string} text

src/showdown.js

@@ -30,21 +30,9 @@ var showdown = {},
         noHeaderId:                           true,
         ghCodeBlocks:                         false
       },
-      ghost: {
-        omitExtraWLInCodeBlocks:              true,
-        parseImgDimensions:                   true,
-        simplifiedAutoLink:                   true,
-        literalMidWordUnderscores:            true,
-        strikethrough:                        true,
-        tables:                               true,
-        tablesHeaderId:                       true,
-        ghCodeBlocks:                         true,
-        tasklists:                            true,
-        smoothLivePreview:                    true,
-        simpleLineBreaks:                     true,
-        requireSpaceBeforeHeadingText:        true,
-        ghMentions:                           false,
-        encodeEmails:                         true
+      commonmark: {
+        noHeaderId:                           true,
+        requireSpaceBeforeHeadingText:        true
       },
       vanilla: getDefaultOpts(true),
       allOn: allOptionsOn()

src/subParsers/makehtml/codeBlock.js

@@ -23,7 +23,7 @@ showdown.subParser('makehtml.codeBlock', function (text, options, globals) {
   // sentinel workarounds for lack of \A and \Z, safari\khtml bug
   text += '¨0';
 
-  let pattern = /(?:\n\n|^)((?:(?:[ ]{4}|\t).*\n+)+)(\n*[ ]{0,3}[^ \t\n]|(?=¨0))/g;
+  let pattern = /(?:\n\n|^)((?:(?: {4}|\t).*\n+)+)(\n* {0,3}[^ \t\n]|(?=¨0))/g;
   text = text.replace(pattern, function (wholeMatch, m1, m2) {
     let codeblock = m1,
         nextChar = m2,
@@ -55,7 +55,7 @@ showdown.subParser('makehtml.codeBlock', function (text, options, globals) {
       codeblock = captureStartEvent.matches.codeblock;
       codeblock = showdown.helper.outdent(codeblock);
       codeblock = showdown.subParser('makehtml.encodeCode')(codeblock, options, globals);
-      codeblock = showdown.subParser('makehtml.detab')(codeblock, options, globals);
+      //codeblock = showdown.subParser('makehtml.detab')(codeblock, options, globals);
       codeblock = codeblock.replace(/^\n+/g, ''); // trim leading newlines
       codeblock = codeblock.replace(/\n+$/g, ''); // trim trailing newlines
       attributes = captureStartEvent.attributes;

src/subParsers/makehtml/detab.js

@@ -12,6 +12,7 @@
 showdown.subParser('makehtml.detab', function (text, options, globals) {
   'use strict';
 
+  return text;
   let startEvent = new showdown.Event('makehtml.detab.onStart', text);
   startEvent
     .setOutput(text)

src/subParsers/makehtml/githubCodeBlock.js

@@ -68,7 +68,7 @@ showdown.subParser('makehtml.githubCodeBlock', function (text, options, globals)
       let lang = infostring.trim().split(' ')[0];
       codeblock = captureStartEvent.matches.codeblock;
       codeblock = showdown.subParser('makehtml.encodeCode')(codeblock, options, globals);
-      codeblock = showdown.subParser('makehtml.detab')(codeblock, options, globals);
+      //codeblock = showdown.subParser('makehtml.detab')(codeblock, options, globals);
       codeblock = codeblock
         .replace(/^\n+/g, '')  // trim leading newlines
         .replace(/\n+$/g, ''); // trim trailing whitespace

src/subParsers/makehtml/heading.js

@@ -73,16 +73,18 @@ showdown.subParser('makehtml.heading', function (text, options, globals) {
   startEvent = globals.converter.dispatch(startEvent);
   text = startEvent.output;
 
-  let setextRegexH1 = (options.smoothLivePreview) ? /^(.+)[ \t]*\n={2,}[ \t]*\n+/gm : /^(.+)[ \t]*\n=+[ \t]*\n+/gm,
-      setextRegexH2 = (options.smoothLivePreview) ? /^(.+)[ \t]*\n-{2,}[ \t]*\n+/gm : /^(.+)[ \t]*\n-+[ \t]*\n+/gm,
-      atxRegex      = (options.requireSpaceBeforeHeadingText) ? /^(#{1,6})[ \t]+(.+?)[ \t]*#*\n+/gm : /^(#{1,6})[ \t]*(.+?)[ \t]*#*\n+/gm;
+  let setextRegexH1 = (options.smoothLivePreview) ? /^(.+[ \t]*\n)(.+[ \t]*\n)?(.+[ \t]*\n)?={2,}[ \t]*\n+/gm : /^( {0,3}[^ \t\n].+[ \t]*\n)(.+[ \t]*\n)?(.+[ \t]*\n)? {0,3}=+[ \t]*$/gm,
+      setextRegexH2 = (options.smoothLivePreview) ? /^(.+[ \t]*\n)(.+[ \t]*\n)?(.+[ \t]*\n)?-{2,}[ \t]*\n+/gm : /^( {0,3}[^ \t\n].+[ \t]*\n)(.+[ \t]*\n)?(.+[ \t]*\n)? {0,3}-+[ \t]*$/gm,
+      atxRegex      = (options.requireSpaceBeforeHeadingText) ? /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/gm : /^ {0,3}(#{1,6})[ \t]*(.+?)[ \t]*#*[ \t]*$/gm;
 
-  text = text.replace(setextRegexH1, function (wholeMatch, headingText) {
+  text = text.replace(setextRegexH1, function (wholeMatch, line1, line2, line3) {
+    let headingText = line1.trim() + ((line2) ? '\n' + line2.trim() : '') + ((line3) ? '\n' + line3.trim() : '');
     let id = (options.noHeaderId) ? null : showdown.subParser('makehtml.heading.id')(headingText, options, globals);
     return parseHeader(setextRegexH1, wholeMatch, headingText, options.headerLevelStart, id);
   });
 
-  text = text.replace(setextRegexH2, function (wholeMatch, headingText) {
+  text = text.replace(setextRegexH2, function (wholeMatch, line1, line2, line3) {
+    let headingText = line1.trim() + ((line2) ? '\n' + line2.trim() : '') + ((line3) ? '\n' + line3.trim() : '');
     let id = (options.noHeaderId) ? null : showdown.subParser('makehtml.heading.id')(headingText, options, globals);
     return parseHeader(setextRegexH2, wholeMatch, headingText, options.headerLevelStart + 1, id);
   });
@@ -154,7 +156,7 @@ showdown.subParser('makehtml.heading.id', function (m, options, globals) {
       .toLowerCase();
   } else {
     title = title
-      .replace(/[^\w]/g, '')
+      .replace(/\W/g, '')
       .toLowerCase();
   }
 

src/subParsers/makehtml/paragraphs.js

@@ -15,7 +15,7 @@ showdown.subParser('makehtml.paragraphs', function (text, options, globals) {
   for (var i = 0; i < end; i++) {
     var str = grafs[i];
     // if this is an HTML marker, copy it
-    if (str.search(/¨(K|G)(\d+)\1/g) >= 0) {
+    if (str.search(/¨([KG])(\d+)\1/g) >= 0) {
       grafsOut.push(str);
 
     // test for presence of characters to prevent empty lines being parsed

test/functional/makehtml/cases/features/underline/fulltext.html

@@ -104,11 +104,11 @@ ___triple underscores___
 <h6 id="heading6markupheading6">Heading 6 markup  <code>###### Heading 6</code></h6>
 <h6 id="-5"> </h6>
 <p>You can also create Setext-style headings which have two levels.</p>
-<h1 id="level1markupuseanequalsignequalsign">Level 1 markup use an equal sign = (equal sign) </h1>
+<h1 id="level1markupuseanequalsignequalsign">Level 1 markup use an equal sign = (equal sign)</h1>
 <pre><code> Level 1 markup use an equal sign = (equal sign)        
  ==============================
 </code></pre>
-<h2 id="level2markupusesdashes">Level 2 markup uses - (dashes) </h2>
+<h2 id="level2markupusesdashes">Level 2 markup uses - (dashes)</h2>
 <pre><code>Level 2 markup uses - (dashes) 
 -------------
 </code></pre>

test/functional/makehtml/cases/karlcow/header-level1-hash-sign-trailing-1-space.html

@@ -1 +1 @@
-<p># This is an H1</p>
+<h1>This is an H1</h1>

test/functional/makehtml/cases/karlcow/paragraph-trailing-tab.html

@@ -1 +1 @@
-<p>This is a paragraph with 1 trailing tab.    </p>
+<p>This is a paragraph with 1 trailing tab.	</p>

test/functional/makehtml/extra.testsuite.commonmark.js

@@ -3,19 +3,35 @@
  */
 
 // jshint ignore: start
-var bootstrap = require('./makehtml.bootstrap.js'),
-    converter = new bootstrap.showdown.Converter(),
+let bootstrap = require('./makehtml.bootstrap.js'),
+    converter = new bootstrap.showdown.Converter({
+      noHeaderId: true,
+      requireSpaceBeforeHeadingText: true
+    }),
     assertion = bootstrap.assertion,
     testsuite = bootstrap.getJsonTestSuite('test/functional/makehtml/cases/commonmark.testsuite.json');
 
 describe('makeHtml() commonmark testsuite', function () {
   'use strict';
 
-  for (var section in testsuite) {
+  for (let section in testsuite) {
     if (testsuite.hasOwnProperty(section)) {
       describe(section, function () {
-        for (var i = 0; i < testsuite[section].length; ++i) {
-          it(testsuite[section][i].name, assertion(testsuite[section][i], converter));
+        for (let i = 0; i < testsuite[section].length; ++i) {
+          let name = testsuite[section][i].name;
+          switch (name) {
+            case 'ATX headings_79': // empty headings don't make sense
+            case 'Setext headings_92': // lazy continuation is needed for compatibility
+            case 'Setext headings_93': // lazy continuation is needed for compatibility
+            case 'Setext headings_94': // lazy continuation is needed for compatibility
+              continue;
+          }
+
+
+          if (testsuite[section][i].name === 'ATX headings_79') {
+            continue;
+          }
+          it(name, assertion(testsuite[section][i], converter, true));
         }
       });
     }

test/functional/makehtml/makehtml.bootstrap.js

@@ -8,6 +8,7 @@
 
   require('source-map-support').install();
   require('chai').should();
+  const htmlPrettify = require('html-prettify');
   let fs = require('fs');
 
   function getTestSuite (dir) {
@@ -52,7 +53,13 @@
       let section = jsonArray[i].section;
       let name = jsonArray[i].section + '_' + (jsonArray[i].example || jsonArray[i].number);
       let md = jsonArray[i].markdown;
+      // transformations
+      md = md.replace(/→/g, '\t'); // replace → with tabs
+
       let html = jsonArray[i].html;
+      // transformations
+      html = html.replace(/→/g, '\t'); // replace → with tabs
+
       if (!tcObj.hasOwnProperty(section)) {
         tcObj[section] = [];
       }
@@ -67,10 +74,15 @@
   }
 
 
-  function assertion (testCase, converter) {
+  function assertion (testCase, converter, prettify) {
+    prettify = prettify || false;
     return function () {
       testCase.actual = converter.makeHtml(testCase.input);
-      testCase = normalize(testCase);
+      // transformations for readability
+      //testCase.expected = testCase.expected.replace(/\t/g, '→');
+      //testCase.actual = testCase.actual.replace(/\t/g, '→');
+
+      testCase = normalize(testCase, prettify);
 
       // Compare
       testCase.actual.should.equal(testCase.expected, testCase.file);
@@ -78,7 +90,7 @@
   }
 
   //Normalize input/output
-  function normalize (testCase) {
+  function normalize (testCase, prettify) {
 
     // Normalize line returns
     testCase.expected = testCase.expected.replace(/(\r\n)|\n|\r/g, '\n');
@@ -96,9 +108,12 @@
     testCase.expected = testCase.expected.trim();
     testCase.actual = testCase.actual.trim();
 
-    //Beautify
-    //testCase.expected = beautify(testCase.expected, beauOptions);
-    //testCase.actual = beautify(testCase.actual, beauOptions);
+    //prettify
+    if (prettify) {
+      testCase.expected = htmlPrettify(testCase.expected);
+      testCase.actual = htmlPrettify(testCase.actual);
+    }
+
 
     // Normalize line returns
     testCase.expected = testCase.expected.replace(/(\r\n)|\n|\r/g, '\n');