use npm@next (temporarily); use html-frontmatter; use npm HTML docs i…

…nstead of markdown docs.

README.md

@@ -47,6 +47,9 @@ The [content.json](/content.json) file is served publicly at `/content.json`
 with CORS support, allowing browsers on other domains to fetch all the npm
 documentation and accompanying metadata with a single HTTP call.
 
+The `content` property in each page object is Base64 encoded so it doesn't
+break JSON.
+
 ## Deployment
 
 ```sh

bin/build.sh

@@ -2,6 +2,9 @@
 
 set -x
 
-cp -r node_modules/npm/doc/ content/
+cp -r node_modules/npm/html/partial/doc/api content/
+cp -r node_modules/npm/html/partial/doc/cli content/
+cp -r node_modules/npm/html/partial/doc/files content/
+cp -r node_modules/npm/html/partial/doc/misc content/
 ./bin/tree.js
 ./bin/nav.js

bin/tree.js

@@ -3,57 +3,48 @@
 // Walk the content directory looking for markdown files
 // and build an object of all the pages and sections (directories)
 
-var path = require("path")
-var fs = require("fs")
-var marked = require("marked")
-var fmt = require("util").format
-var _ = require("lodash")
-var compact = _.compact
-var uniq = _.uniq
-var pluck = _.pluck
-var front = new RegExp("<!--\n([^]*)\n-->")
+var path        = require("path")
+var fs          = require("fs")
+var marked      = require("marked")
+var fmt         = require("util").format
+var _           = require("lodash")
+var compact     = _.compact
+var uniq        = _.uniq
+var pluck       = _.pluck
+var merge       = _.merge
+var fm          = require("html-frontmatter")
+var contentFile = path.resolve(__dirname, "../content.json")
 var content = {
   sections: [],
   pages: []
 }
 
-var contentFile = path.resolve(__dirname, "../content.json")
-
 // Walk the content directory tree
 var emitter = require("walkdir")(path.resolve(__dirname,  "../content/"))
 
 emitter.on("file", function(filename,stat){
 
-  // Skip non-markdown files
-  if (!filename.match(/\.md$/)) return
+  // We only want .html and .md files
+  if (!filename.match(/\.(md|html)$/)) return
 
   var page = {}
   page.content = fs.readFileSync(filename).toString()
 
   // Look for an HTML comment in the file, then parse each
   // colon-delimited line within as metadata
-  if (page.content.match(front)) {
-    var frontmatter = front.exec(page.content)[1]
-    frontmatter = frontmatter
-      .replace(/\n  /, " ") // treat two-space indentation as a newline
-      .replace(/\s{2,}/g, " ") // remove excess spaces
-
-    frontmatter.split("\n").forEach(function(line) {
-      var parts = line.trim().split(": ")
-      page[parts[0]] = parts[1]
-    })
-  }
+  merge(page, fm(page.content))
 
-  page.contentHTML = marked(page.content)
+  // Convert markdown to HTML
+  if (filename.match(/\.md$/))
+    page.content = marked(page.content)
 
   // Base64 encode content so it can be saved as JSON
   page.content = new Buffer(page.content).toString('base64')
-  page.contentHTML = new Buffer(page.contentHTML).toString('base64')
 
   // Clean up the filename
   filename = filename
     .replace(/.*\/content\//, "") // remove basepath
-    .replace(/\.md$/, "")        // remove extension
+    .replace(/\.md$/, "")         // remove extension
 
   // Use filename as title if not specified in frontmatter
   // (and remove superfluous npm- prefix)

index.js

@@ -12,12 +12,11 @@ app.use(express.static(__dirname + "/public"))
 app.use(harp.mount(__dirname + "/public"))
 hbs.registerPartials(__dirname + "/views/partials")
 
-// All the content is in content.json
-// Convert Base64-encoded content back to regular strings
+// Fetch all the content, and convert Base64-encoded
+// content back to regular strings
 var content = require(path.resolve(__dirname, "content.json"))
 content.pages = content.pages.map(function(page) {
   page.content = (new Buffer(page.content, "base64")).toString()
-  page.contentHTML = (new Buffer(page.contentHTML, "base64")).toString()
   return page
 })
 

package.json

@@ -28,10 +28,11 @@
     "express": "^4.9.5",
     "harp": "^0.13.0",
     "hbs": "^2.7.0",
+    "html-frontmatter": "^1.0.0",
     "js-beautify": "^1.5.2",
     "lodash": "^2.4.1",
     "marked": "^0.3.2",
-    "npm": "^2.0.2",
+    "npm": "next",
     "walkdir": "0.0.7"
   }
 }

views/layout.hbs

@@ -16,7 +16,7 @@
     </section>
   </header>
   <nav>{{> nav}}</nav>
-  <section>{{{contentHTML}}}</section>
+  <section>{{{content}}}</section>
 
   <script>
     (function(i, s, o, g, r, a, m) {