diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 9580d13ca60..cc2f84305fe 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -172,6 +172,35 @@ Checklist: - [ ] I made sure all breaking changes are noted - [x] I'm not going to say "nifty" anymore +### Updating Documentation + +All of the documentation is powered by JSDoc, we parse it into JSON and feed it to an [application](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/site) built in Angular. Hopefully our CI builds and updates the documentation for each successful merge to master, but if for whatever reason a manual update is required please refer to the following steps. + +We can build the documentation using the `npm run prepare-ghpages` command. This command optionally accepts two parameters. + +* module - The name of the module to build (e.g. `google-cloud`). +* version - The target version of the module. (e.g. `0.43.0`) Defaults to master. + +If both parameters are omitted, we will build the master docs for all modules. + +```sh +$ npm run prepare-ghpages google-cloud 0.43.0 +``` + +Now it's time to push the docs! If you wish to preview locally you can optionally run an http server in the `gh-pages` folder. + +```sh +$ cd gh-pages +$ http-server . # Run the server to look for any visual errors +$ git push origin gh-pages +$ cd .. +``` +Finally the last thing to do is cleanup the submodule we created to copy over the JSON. + +```sh +$ npm run remove-ghpages +``` + [elsewhere]: ../README.md#elsewhere [gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/ [indvcla]: https://developers.google.com/open-source/cla/individual diff --git a/.gitignore b/.gitignore index 73c5f4abc8c..34790c257eb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,5 @@ packages/*/CONTRIBUTORS packages/*/COPYING **/node_modules **/coverage/* -docs/json/**/*.json +docs/json *.log diff --git a/.jshintrc b/.jshintrc index 5d3398ff037..f293ceee02c 100644 --- a/.jshintrc +++ b/.jshintrc @@ -29,6 +29,7 @@ "echo": true, "exit": true, "cd": true, - "mkdir": true + "mkdir": true, + "test": true } } diff --git a/.travis.yml b/.travis.yml index 01da1a71be8..993fba7c23b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,26 @@ before_script: fi script: "node ./scripts/build.js" after_success: - - if [ $IS_PUSH_TO_MASTER ]; then - node ./scripts/build-docs.js; + # We check to see if the job number ends with 1 (e.g. 234234.1) + # letting us know it is the first pass for this build + - if [ $IS_PUSH_TO_MASTER && "${TRAVIS_JOB_NUMBER}" == *1 ]; then + rm -rf docs/json; + npm run docs; + git submodule add -q -b gh-pages https://${GH_OAUTH_TOKEN}@github.com/${GH_OWNER}/${GH_PROJECT_NAME} ghpages; + cp -rf docs/json/* ghpages/json; + cp docs/manifest.json ghpages; + cd ghpages; + set +e; + git add .; + set -e; + if [[ -z "$(git status --porcelain)" ]]; then + echo "Nothing to commit. Exiting without pushing changes."; + exit 0; + fi; + git config user.name "travis-ci"; + git config user.email "travis@travis-ci.org"; + git commit -m "Update docs for ${TRAVIS_TAG:-master} [ci skip]"; + git status; + git push -q https://${GH_OAUTH_TOKEN}@github.com/${GH_OWNER}/${GH_PROJECT_NAME} HEAD:gh-pages; fi sudo: false diff --git a/docs/json/bigquery/master/.gitkeep b/docs/json/bigquery/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/bigtable/master/.gitkeep b/docs/json/bigtable/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/compute/master/.gitkeep b/docs/json/compute/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/datastore/master/.gitkeep b/docs/json/datastore/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/dns/master/.gitkeep b/docs/json/dns/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/google-cloud/master/.gitkeep b/docs/json/google-cloud/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/language/master/.gitkeep b/docs/json/language/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/logging/master/.gitkeep b/docs/json/logging/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/monitoring/master/.gitkeep b/docs/json/monitoring/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/prediction/master/.gitkeep b/docs/json/prediction/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/pubsub/master/.gitkeep b/docs/json/pubsub/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/resource/master/.gitkeep b/docs/json/resource/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/speech/master/.gitkeep b/docs/json/speech/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/storage/master/.gitkeep b/docs/json/storage/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/translate/master/.gitkeep b/docs/json/translate/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/json/vision/master/.gitkeep b/docs/json/vision/master/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/manifest.json b/docs/manifest.json index a747cdf1ef8..4b4e7ac17ea 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -13,6 +13,7 @@ "name": "google-cloud", "defaultService": "google-cloud", "versions": [ + "0.43.0", "0.42.2", "0.41.2", "0.41.0", @@ -132,6 +133,7 @@ "name": "@google-cloud/language", "defaultService": "language", "versions": [ + "0.5.0", "0.4.0", "0.3.0", "0.2.0", @@ -145,6 +147,7 @@ "name": "@google-cloud/logging", "defaultService": "logging", "versions": [ + "0.5.0", "0.4.0", "0.3.0", "0.2.0", @@ -152,6 +155,15 @@ "master" ] }, + { + "id": "monitoring", + "name": "@google-cloud/monitoring", + "defaultService": "monitoring", + "versions": [ + "0.1.1", + "master" + ] + }, { "id": "prediction", "name": "@google-cloud/prediction", @@ -169,6 +181,7 @@ "name": "@google-cloud/pubsub", "defaultService": "pubsub", "versions": [ + "0.5.0", "0.4.0", "0.3.0", "0.2.0", @@ -192,6 +205,7 @@ "name": "@google-cloud/speech", "defaultService": "speech", "versions": [ + "0.4.0", "0.3.0", "0.2.0", "master" @@ -202,6 +216,7 @@ "name": "@google-cloud/storage", "defaultService": "storage", "versions": [ + "0.4.0", "0.3.0", "0.2.0", "0.1.1", @@ -225,6 +240,7 @@ "name": "@google-cloud/vision", "defaultService": "vision", "versions": [ + "0.5.0", "0.4.0", "0.3.0", "0.2.0", @@ -237,4 +253,4 @@ "title": "npm", "href": "https://www.npmjs.com/package/gcloud" } -} +} \ No newline at end of file diff --git a/package.json b/package.json index 56e432dd8ee..d439491ac3d 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,9 @@ "postinstall": "node ./scripts/install.js", "link-common": "node ./scripts/link-common.js", "update-deps": "node ./scripts/update-deps.js", - "docs": "node ./scripts/docs/packages.js", - "bundle": "node ./scripts/docs/bundle.js", + "docs": "node ./scripts/docs", + "prepare-ghpages": "node ./scripts/docs/prepare.js", + "remove-ghpages": "node ./scripts/docs/remove.js", "lint": "jshint scripts/ packages/ system-test/ test/ && jscs packages/ system-test/ test/", "test": "npm run docs && npm run snippet-test && npm run unit-test", "unit-test": "mocha --timeout 5000 --bail packages/*/test/*.js", diff --git a/scripts/build-docs.js b/scripts/build-docs.js deleted file mode 100644 index cc723d59b11..00000000000 --- a/scripts/build-docs.js +++ /dev/null @@ -1,154 +0,0 @@ -/*! - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*! - * Developer Documentation - * - * This script should be ran anytime after a successful build occurs on Travis. - * This script is responsible for building the JSON docs after a push to master. - * Here's a breakdown of what happens (and when) - * - * - First we check to see if this the first job to be ran on Travis and that - * it's a push to master (not a PR). If these conditions are not met, we - * exit early. - * - * - Then we'll submodule in the gh-pages branch and generate the latest - * master docs for ALL modules. - * - * - If this is CI job is the result of a release, we'll add the release version - * to the manifest and generate the JSON docs for said release. - * - * - After all this, if any doc changes have been made, we'll commit them to - * gh-pages. - */ - -'use strict'; - -require('shelljs/global'); - -var fs = require('fs'); -var globby = require('globby'); -var path = require('path'); - -var helpers = require('./helpers'); - -var ci = helpers.ci; -var git = helpers.git; -var Module = helpers.Module; -var run = helpers.run; - -if (!ci.isFirstPass()) { - echo('Skipping documentation updates.'); - exit(); -} - -// set user to travis -git.setUser('travis-ci', 'travis@travis-ci.org'); - -// Create a submodule for the gh-pages branch -var ghpages = git.submodule('gh-pages', 'ghpages'); -var GHPAGES_JSON = path.join(ghpages.cwd, 'json'); - -// generate latest json docs -Module.buildDocs(); - -// copy docs to gh-pages -cp('-R', 'docs/json/*', GHPAGES_JSON); -cp('docs/home.html', GHPAGES_JSON); -cp('docs/manifest.json', ghpages.cwd); - -globby - .sync(path.join(GHPAGES_JSON, '*/master')) - .forEach(function(dir) { - cp('docs/*.md', dir); - }); - -// create the latest bundled docs -run('npm run bundle'); - -var release = ci.getRelease(); - -// check to see if we have a release and it's not common.. -if (release && release.name !== 'common') { - // we need to add the new release to official docs.. - var master = git.submodule('master'); - - var manifestPath = path.resolve(master.cwd, 'docs/manifest.json'); - var manifest = require(manifestPath); - - // update manifest to include latest version of whatever module got a release - for (var i = 0, docs; i < manifest.modules.length; i++) { - docs = manifest.modules[i]; - - if (docs.id === release.name) { - if (docs.versions.indexOf(release.version) === -1) { - docs.versions.unshift(release.version); - } - - break; - } - } - - // save the latest manifest in master - manifest = JSON.stringify(manifest, null, 2) + '\n'; - fs.writeFileSync(manifestPath, manifest); - - // add updated manfiest to gh-pages - cp(manifestPath, ghpages.cwd); - - // commit it if the version is actually new (how else did we get here?!) - if (master.hasUpdates()) { - master.add('docs/manifest.json'); - master.commit('Update docs/manifest.json for ' + ci.getTagName()); - master.push('master'); - } else { - echo('manifest already includes the new version. Skipping commit.'); - } - - // store the output folder in a variable - var RELEASE_DIR = path.resolve(GHPAGES_JSON, release.name, release.version); - - // if the output folder doesn't exist, then we create it! - try { - fs.accessSync(RELEASE_DIR, fs.F_OK); - } catch (e) { - mkdir('-p', RELEASE_DIR); - } - - var JSON_DIR = path.join( - release.name === Module.UMBRELLA ? ghpages.cwd : 'docs', - 'json', - release.name, - 'master/*' - ); - - cp('-rf', JSON_DIR, RELEASE_DIR); - - var tocPath = path.join(RELEASE_DIR, 'toc.json'); - var toc = require(tocPath); - - toc.tagName = ci.getTagName(); - toc = JSON.stringify(toc); - fs.writeFileSync(tocPath, toc); -} - -if (ghpages.hasUpdates()) { - ghpages.add('manifest.json', 'json'); - ghpages.commit('Update docs for ' + (ci.getTagName() || 'master')); - ghpages.push('HEAD:gh-pages'); -} else { - echo('Nothing to commit. Exiting without pushing changes.'); -} diff --git a/scripts/docs/builder.js b/scripts/docs/builder.js new file mode 100644 index 00000000000..da0ec3bfb3c --- /dev/null +++ b/scripts/docs/builder.js @@ -0,0 +1,459 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +require('shelljs/global'); + +var flatten = require('lodash.flatten'); +var fs = require('fs'); +var globby = require('globby'); +var path = require('path'); +var semver = require('semver'); + +var config = require('./config'); +var helpers = require('../helpers'); +var parser = require('./parser'); + +var git = helpers.git; + +var DOCS_ROOT = './docs/json'; +var MARKDOWN = path.resolve(DOCS_ROOT, '../*.md'); +var PACKAGES_ROOT = './packages'; +var UMBRELLA_PACKAGE = config.UMBRELLA_PACKAGE; + +var manifestPath = path.resolve(DOCS_ROOT, '../manifest.json'); +var manifest = require(manifestPath); + +/** + * @class Builder + * + * This is a custom builder used to generate documentation json! + * + * @param {string} name - Name of the module to build docs for. + * @param {string=} version - Target version of the docs. + * @param {string=} cwd - The current working directory to build from. + * + * @example + * var builder = new Builder('vision', '0.5.0'); + */ +function Builder(name, version, cwd) { + this.name = name; + this.version = version || config.DEFAULT_VERSION; + this.dir = path.join(cwd || '', DOCS_ROOT, name, this.version); + this.isUmbrella = name === UMBRELLA_PACKAGE; + this.isMaster = this.version === config.DEFAULT_VERSION; +} + +/** + * This simply generates documentation for the specified module. + * All files will end up in /docs/json/{builder.name}/{builder.version} + * + * If there are any malformed docs/etc., an error will be thrown. + * + * @throws {error} + * + * @example + * builder.build(); + */ +Builder.prototype.build = function() { + var self = this; + + mkdir('-p', this.dir); + cp(MARKDOWN, this.dir); + + var docs = globby.sync(path.join(this.name, 'src/*.js'), { + cwd: PACKAGES_ROOT, + ignore: config.IGNORE + }).map(function(file) { + var json = self.parseFile(file); + var outputFile = file.replace(/^.+src\//, '') + 'on'; + + json.path = path.basename(outputFile); + self.write(outputFile, json); + + return json; + }); + + var types = parser.createTypesDictionary(docs); + var toc = parser.createToc(types); + + toc.tagName = this.getTagName(); + this.write(config.TYPES_DICT, types); + this.write(config.TOC, toc); + + if (this.isUmbrella) { + new Bundler(this).bundle(); + } +}; + +/** + * Retrieve the tag name for the current release. Will throw an error if + * version is not set. + * + * @throws {error} + * @return {string} tagName + * + * @example + * var tagName = builder.getTagName(); // bigtable-0.2.0 + */ +Builder.prototype.getTagName = function() { + if (this.isMaster) { + return this.version; + } + + if (this.isUmbrella) { + return 'v' + this.version; + } + + return [this.name, this.version].join('-'); +}; + +/** + * Retrieves the "types" list for the the current module. + * + * @return {object[]} types + * + * @example + * var types = builder.getTypes(); + */ +Builder.prototype.getTypes = function() { + return require(path.resolve(this.dir, config.TYPES_DICT)); +}; + +/** + * Attempts to parse the specified file. Will throw an error if parsing fails. + * + * @throw {error} + * @param {string} file - The file to parse. + * @return {object} json - The parsed file JSON. + * + * @example + * var json = builder.parseFile('bigtable/src/index.js'); + */ +Builder.prototype.parseFile = function(file) { + var filePath = path.join(PACKAGES_ROOT, file); + var contents = fs.readFileSync(filePath, 'utf8'); + + try { + return parser.parseFile(filePath, contents); + } catch (e) { + throw new Error( + 'Unable to generate docs for file: ' + file + '. Reason: ' + e.message + ); + } +}; + +/** + * Updates the manifest.json file to include the module and/or tagged version. + */ +Builder.prototype.updateManifest = function() { + var config = findWhere(manifest.modules, { id: this.name }); + + if (!config) { + config = { + id: this.name, + name: '@google-cloud/' + this.name, + defaultService: this.name, + versions: ['master'] + }; + + manifest.modules.push(config); + manifest.modules.sort(sortByModId); + } + + if (config.versions.indexOf(this.version) === -1) { + config.versions.unshift(this.version); + } + + fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); +}; + +/** + * Convenience method to make writing JSON files a little less tedious. + * + * @param {string} file - The file name. + * @param {object} json - The JSON object. + */ +Builder.prototype.write = function(file, json) { + fs.writeFileSync(path.join(this.dir, file), JSON.stringify(json)); +}; + +/** + * @class Bundler + * + * This class is used to generate/update the umbrella documentation. + * + * @param {builder|string} builder - Either a Builder instance or version of + * the umbrella docs. + * + * @example + * var builder = new Builder('google-cloud', '0.44.0'); + * var bundler = new Bundler(builder); + * + * // or the shorthand version.. + * var bundler = new Bundler('0.44.0'); + */ +function Bundler(builder) { + this.builder = builder; + + if (typeof this.builder === 'string') { + this.builder = new Builder(UMBRELLA_PACKAGE, builder); + } +} + +/** + * @static + * + * This iterates over all known umbrella package versions and checks to see + * if the supplied builder is the max satisfying semver version. If it is, then + * it will update the JSON files for that particular builder. + * + * @param {builder} builder - The Builder instance. + * + * @example + * // pretend this was a patch release, it would automatically be installed + * // to umbrella packages that list 0.4.0 as the semver version. + * var builder = new Builder('bigtable', '0.4.1'); + * + * Bundler.updateDep(builder); + */ +Bundler.updateDep = function(builder) { + if (builder.isMaster) { + throw new Error('Must supply valid version to update bundles with.'); + } + + var bundleConfig = findWhere(manifest.modules, { + id: UMBRELLA_PACKAGE + }); + + var bundleVersions = bundleConfig.versions; + var config = findWhere(manifest.modules, { id: builder.name }); + var versions = config.versions.filter(semver.valid); + var bundler, dep; + + for (var i = 0; i < bundleVersions.length - 1; i++) { + bundler = new Bundler(bundleVersions[i]); + git.checkout(bundler.builder.getTagName()); + dep = findWhere(bundler.getDeps(), { name: builder.name }); + git.checkout('-'); + + if (semver.maxSatisfying(versions, dep.version) !== builder.version) { + break; + } + + bundler.add(builder); + } +}; + +/** + * Adds the supplied builder to the bundled docs. + * + * @param {builder} builder - The builder instance. + * + * @example + * var builder = new Builder('bigtable', '0.5.0'); + * var bundler = new Bundler('0.45.0'); + * + * bundler.add(builder); + * // The 0.45.0 bundle now contains the bigtable-0.5.0 docs + */ +Bundler.prototype.add = function(builder) { + var self = this; + var outputFolder = path.join(this.builder.dir, builder.name); + + mkdir('-p', outputFolder); + + globby.sync(path.resolve(builder.dir, '*.json'), { + ignore: [config.TYPES_DICT, config.TOC].map(function(file) { + return path.resolve(builder.dir, file); + }) + }).forEach(function(file) { + var json = require(file); + var service = json.parent || json.id; + var outputFile = path.join(builder.name, path.basename(file)); + + json.overview = parser.createOverview(service, true); + self.builder.write(outputFile, json); + }); +}; + +/** + * Generates all JSON files for the umbrella version. + * + * @example + * var bundler = new Bundler('0.45.0'); + * + * bundler.bundle(); + */ +Bundler.prototype.bundle = function() { + var self = this; + + var baseTypes = this.builder.getTypes(); + var deps = this.getDeps().map(function(dep) { + var config = findWhere(manifest.modules, { id: dep.name }); + var versions = config.versions.filter(semver.valid); + + return { + name: dep.name, + version: semver.maxSatisfying(versions, dep.version) + }; + }); + + var submodule = git.submodule('master', 'bundler'); + + var depTypes = deps.map(function(dep) { + var builder = new Builder(dep.name, dep.version, submodule.cwd); + + submodule.checkout(builder.getTagName()); + builder.build(); + self.add(builder); + + return builder.getTypes().map(function(type) { + type.contents = path.join(dep.name, type.contents); + return type; + }); + }); + + var types = baseTypes.concat(flatten(depTypes)); + var toc = parser.createToc(types, true); + + toc.tagName = this.builder.getTagName(); + this.builder.write(config.TYPES_DICT, types); + this.builder.write(config.TOC, toc); + + git.deinit(submodule); +}; + +/** + * Gets the umbrella package @google-cloud specific dependencies for the + * current state of the package. + * + * @return {object[]} + * + * @example + * var bundler = new Bundler('0.44.0'); + * var deps = bundler.getDeps(); + * // deps = [ + * // { + * // name: 'bigtable', + * // version: '^0.4.0' + * // } + * // ] + */ +Bundler.prototype.getDeps = function() { + var jsonPath = path.resolve(PACKAGES_ROOT, UMBRELLA_PACKAGE, 'package.json'); + var contents = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); + var dependencies = contents.dependencies; + + return Object.keys(dependencies) + .filter(function(dep) { + return /^\@google\-cloud/.test(dep); + }) + .map(function(dep) { + return { + name: dep.replace('@google-cloud/', ''), + version: dependencies[dep] + }; + }); +}; + +/** + * Searches a collection to find a model containing the specified props. + * + * @param {object[]} collection - The collection to search. + * @param {object} props - The properties to match against. + * @return {object|null} + */ +function findWhere(collection, props) { + var i, key, obj; + + collLoop: for (i = 0; i < collection.length; i++) { + obj = collection[i]; + + for (key in props) { + if (obj[key] !== props[key]) { + continue collLoop; + } + } + + return obj; + } + + return null; +} + +/** + * Used to sort manifest modules by id. + * + * @param {object} a - module A. + * @param {object} b - module B. + * @return {number} + */ +function sortByModId(a, b) { + if (a.id === UMBRELLA_PACKAGE) { + return -1; + } + + if (b.id === UMBRELLA_PACKAGE) { + return 1; + } + + return +(a.id > b.id) || +(a.id === b.id) - 1; +} + +/** + * Builds docs for the specified module. + * + * @param {string} name - The name of the module to build docs for. + * @param {string=} version - The version of the module. + * + * @example + * builder.build('bigtable', '0.2.0'); + */ +function build(name, version) { + var builder = new Builder(name, version); + + git.checkout(builder.getTagName()); + builder.build(); + git.checkout('-'); + builder.updateManifest(); + + if (!builder.isUmbrella && !builder.isMaster) { + Bundler.updateDep(builder); + } +} + +module.exports.build = build; + +/** + * Builds docs for all modules. + * + * @example + * builder.buildAll(); + */ +function buildAll() { + var modules = globby.sync('*', { + cwd: PACKAGES_ROOT, + ignore: config.IGNORE + }); + + modules.forEach(function(name) { + build(name, config.DEFAULT_VERSION); + }); +} + +module.exports.buildAll = buildAll; diff --git a/scripts/docs/bundle.js b/scripts/docs/bundle.js deleted file mode 100644 index fcdba004163..00000000000 --- a/scripts/docs/bundle.js +++ /dev/null @@ -1,223 +0,0 @@ -/** - * Copyright 2014 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -var path = require('path'); -var globby = require('globby'); -var semver = require('semver'); -var mkdirp = require('mkdirp'); -var fs = require('fs'); -var async = require('async'); -var flatten = require('lodash.flatten'); -var parser = require('./parser'); -var config = require('./config'); - -// since we version the JSON in ghpages and we don't create tags for -// individual modules, we'll look for the appropriate semver version in -// the ghpages submodule -var JSON_DIR = './ghpages/json'; - -var pkgJson = require('../../packages/google-cloud/package.json'); -var dependencies = pkgJson.dependencies; - -var deps = Object.keys(dependencies).reduce(function(deps, dep) { - if (/^\@google\-cloud/.test(dep)) { - deps.push({ - packageName: dep.replace('@google-cloud/', ''), - semver: dependencies[dep] - }); - } - - return deps; -}, []); - -function getPackageVersion(dep, callback) { - var globOptions = { - cwd: path.join(JSON_DIR, dep.packageName) - }; - - globby('*', globOptions) - .then(function(versions) { - var validVersions = versions.filter(function(version) { - return semver.valid(version); - }); - - var version = semver.maxSatisfying(validVersions, dep.semver); - - if (!version) { - throw new Error( - 'Unable to find suitable version for package: ' + dep.packageName); - } - - callback(null, version); - }) - .then(null, callback); -} - -function getDocsFiles(packageName, version, callback) { - var globOptions = { - cwd: path.join(JSON_DIR, packageName), - ignore: [ - path.join(version, config.TYPES_DICT), - path.join(version, config.TOC) - ] - }; - - globby(path.join(version, '*.json'), globOptions) - .then(callback.bind(null, null), callback); -} - -function copyFile(dep, file, callback) { - var outputDir = path.join( - JSON_DIR, - config.UMBRELLA_PACKAGE, - config.DEFAULT_VERSION, - dep.packageName - ); - - mkdirp(outputDir, function(err) { - if (err) { - callback(err); - return; - } - - var inputFile = path.join( - JSON_DIR, - dep.packageName, - file - ); - - var outputFile = path.join( - outputDir, - path.basename(file) - ); - - fs.readFile(inputFile, 'utf8', function(err, contents) { - if (err) { - callback(err); - return; - } - - var fileJson = JSON.parse(contents); - var service = fileJson.parent || fileJson.id; - - fileJson.overview = parser.createOverview(service, true); - fs.writeFile(outputFile, JSON.stringify(fileJson), callback); - }); - }); -} - -function getTypes(dep, callback) { - var inputFile = path.join( - JSON_DIR, - dep.packageName, - dep.version, - config.TYPES_DICT - ); - - fs.readFile(inputFile, 'utf8', function(err, contents) { - var types = JSON.parse(contents); - - types.forEach(function(type) { - type.contents = path.join(dep.packageName, type.contents); - }); - - callback(null, types); - }); -} - -function createUmbrellaTypes(types, callback) { - var inputFile = path.join( - './docs/json', - config.UMBRELLA_PACKAGE, - config.DEFAULT_VERSION, - config.TYPES_DICT - ); - - fs.readFile(inputFile, 'utf8', function(err, contents) { - if (err) { - callback(err); - return; - } - - var umbrellaTypes = JSON.parse(contents); - var allTypes = umbrellaTypes.concat(types); - - var outputFile = path.join( - JSON_DIR, - config.UMBRELLA_PACKAGE, - config.DEFAULT_VERSION, - config.TYPES_DICT - ); - - fs.writeFile(outputFile, JSON.stringify(allTypes), function(err) { - callback(err, allTypes); - }); - }); -} - -function createUmbrellaToc(types, callback) { - var outputFile = path.join( - JSON_DIR, - config.UMBRELLA_PACKAGE, - config.DEFAULT_VERSION, - config.TOC - ); - - var toc = parser.createToc(types, true); - - fs.writeFile(outputFile, JSON.stringify(toc), callback); -} - -async.map(deps, function(dep, callback) { - async.waterfall([ - function(callback) { - getPackageVersion(dep, callback); - }, - function(version, callback) { - dep.version = version; - getDocsFiles(dep.packageName, version, callback); - }, - function(files, callback) { - async.each(files, copyFile.bind(null, dep), callback); - } - ], function(err) { - if (err) { - callback(err); - return; - } - - getTypes(dep, callback); - }); -}, function(err, types) { - if (err) { - throw err; - } - - async.waterfall([ - function(callback) { - createUmbrellaTypes(flatten(types), callback); - }, - function(types, callback) { - createUmbrellaToc(types, callback); - } - ], function(err) { - if (err) { - throw err; - } - }); -}); diff --git a/scripts/docs/config.js b/scripts/docs/config.js index bea5ca240ec..495b1c3e4ba 100644 --- a/scripts/docs/config.js +++ b/scripts/docs/config.js @@ -29,7 +29,7 @@ module.exports = { 'storage/src/acl.js', 'bigtable/src/mutation.js' ], - OVERVIEW_TEMPLATE: 'overview.template.html', + OVERVIEW_TEMPLATE: 'templates/overview.html', OVERVIEW: { 'google-cloud': { title: 'Google Cloud', diff --git a/scripts/docs/index.js b/scripts/docs/index.js new file mode 100644 index 00000000000..364ec99370b --- /dev/null +++ b/scripts/docs/index.js @@ -0,0 +1,33 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var builder = require('./builder'); +var helpers = require('../helpers'); + +var args = process.argv.splice(1); +var target = helpers.ci.getRelease() || { + name: args[1], + version: args[2] +}; + +if (target.name) { + builder.build(target.name, target.version); + return; +} + +builder.buildAll(); diff --git a/scripts/docs/packages.js b/scripts/docs/packages.js deleted file mode 100644 index 939766078f9..00000000000 --- a/scripts/docs/packages.js +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2014 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -var async = require('async'); -var globby = require('globby'); -var path = require('path'); -var fs = require('fs'); - -var parser = require('./parser'); -var config = require('./config'); - - -var JSON_FOLDER = './docs/json'; - -var globOptions = { - cwd: './packages', - ignore: config.IGNORE -}; - -function getPackages(callback) { - globby('*', globOptions) - .then(callback.bind(null, null), callback); -} - -function getPackageFiles(packageName, callback) { - globby(path.join(packageName, 'src/*.js'), globOptions) - .then(callback.bind(null, null), callback); -} - -function createJSON(packageName, file, callback) { - var inputFile = path.join(globOptions.cwd, file); - var outputFile = path.join( - JSON_FOLDER, - packageName, - config.DEFAULT_VERSION, - file.replace(/^.+src\//, '') + 'on' - ); - - fs.readFile(inputFile, 'utf8', function(err, contents) { - if (err) { - callback(err); - return; - } - - var json = parser.parseFile(inputFile, contents); - - fs.writeFile(outputFile, JSON.stringify(json), function(err) { - json.path = path.basename(outputFile); - callback(err, json); - }); - }); -} - -function createTypesDictionary(packageName, json, callback) { - var outputFile = path.join( - JSON_FOLDER, - packageName, - config.DEFAULT_VERSION, - config.TYPES_DICT - ); - - var types = parser.createTypesDictionary(json); - - fs.writeFile(outputFile, JSON.stringify(types), function(err) { - callback(err, types); - }); -} - -function createTableOfContents(packageName, types, callback) { - var outputFile = path.join( - JSON_FOLDER, - packageName, - config.DEFAULT_VERSION, - config.TOC - ); - - var toc = parser.createToc(types); - - fs.writeFile(outputFile, JSON.stringify(toc), callback); -} - -getPackages(function(err, packages) { - if (err) { - throw err; - } - - async.each(packages, function(packageName, pcallback) { - getPackageFiles(packageName, function(err, files) { - if (err) { - throw err; - } - - async.map(files, function(file, fcallback) { - createJSON(packageName, file, fcallback); - }, function(err, json) { - if (err) { - throw err; - } - - async.waterfall([ - function(callback) { - createTypesDictionary(packageName, json, callback); - }, - function(types, callback) { - createTableOfContents(packageName, types, callback); - } - ], pcallback); - }); - }); - }); -}); diff --git a/scripts/docs/parser.js b/scripts/docs/parser.js index 6e1164b12ce..33bc88d2753 100644 --- a/scripts/docs/parser.js +++ b/scripts/docs/parser.js @@ -25,7 +25,7 @@ var prop = require('propprop'); var extend = require('extend'); var template = require('lodash.template'); var config = require('./config'); -var baseToc = require('./base-toc.json'); +var baseToc = require('./templates/toc.json'); var templateFile = fs.readFileSync( path.join(__dirname, config.OVERVIEW_TEMPLATE)); @@ -138,7 +138,7 @@ function getParent(id) { } function getChildren(id) { - var childrenGlob = './packages/' + id + '/*'; + var childrenGlob = './packages/' + id + '/src/*'; return globby .sync(childrenGlob, { ignore: config.IGNORE }) @@ -376,6 +376,11 @@ function createToc(types, collapse) { function createOverview(moduleName, umbrellaMode) { var modConfig = config.OVERVIEW[moduleName]; + + if (!modConfig) { + throw new Error('Missing doc configs for "' + moduleName + '" package.'); + } + var pkgJson = path.join( __dirname, '../../packages', diff --git a/scripts/docs/prepare.js b/scripts/docs/prepare.js new file mode 100644 index 00000000000..e3b4d67a6cd --- /dev/null +++ b/scripts/docs/prepare.js @@ -0,0 +1,58 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +require('shelljs/global'); + +var multiline = require('multiline'); + +var helpers = require('../helpers'); +var git = helpers.git; +var run = helpers.run; + +var args = process.argv.splice(1); +var moduleName = args[1] || ''; +var version = args[2] || ''; + +var ghpages = git.submodule('gh-pages'); + +rm('-rf', 'docs/json'); +run(['npm run docs', moduleName, version]); +cp('-rf', 'docs/json/*', 'gh-pages/json'); +cp('docs/manifest.json', 'gh-pages'); + +ghpages.add('manifest.json', 'json'); +ghpages.commit('Update docs for ' + getTagName(moduleName, version)); + +console.log(multiline(function() {/* + +Now you just need to push to gh-pages: cd gh-pages && git push origin gh-pages + +To clean up the gh-pages submodule: npm run remove-ghpages +*/})); + +function getTagName(moduleName, version) { + if (!version) { + return 'master'; + } + + if (moduleName === 'google-cloud') { + return 'v' + version; + } + + return [moduleName, version].join('-'); +} diff --git a/scripts/docs/remove.js b/scripts/docs/remove.js new file mode 100644 index 00000000000..cc9745147fa --- /dev/null +++ b/scripts/docs/remove.js @@ -0,0 +1,32 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +require('shelljs/global'); + +var git = require('../helpers').git; + +var SUBMODULE_NAME = 'gh-pages'; + +// submodule data might have gotten wiped out when switching branches +if (test('-e', '.gitmodules')) { + git.deinit({ alias: SUBMODULE_NAME }); + git.remove('-rf', '.gitmodules'); +} + +git.remove('-rf', 'gh-pages'); +rm('-rf', '.git/modules/gh-pages'); diff --git a/scripts/docs/overview.template.html b/scripts/docs/templates/overview.html similarity index 100% rename from scripts/docs/overview.template.html rename to scripts/docs/templates/overview.html diff --git a/scripts/docs/base-toc.json b/scripts/docs/templates/toc.json similarity index 100% rename from scripts/docs/base-toc.json rename to scripts/docs/templates/toc.json diff --git a/scripts/helpers.js b/scripts/helpers.js index cc7f29e2c85..78dd1284e2a 100644 --- a/scripts/helpers.js +++ b/scripts/helpers.js @@ -255,7 +255,7 @@ function run(command, options) { console.log(command); - var response = exec(command, options); + var response = exec(command.trim(), options); if (response.code) { exit(response.code); @@ -275,8 +275,18 @@ function Git(cwd) { this.cwd = cwd || ROOT_DIR; } -// We'll use this for cloning/submoduling/pushing purposes on CI -Git.REPO = 'https://${GH_OAUTH_TOKEN}@github.com/${GH_OWNER}/${GH_PROJECT_NAME}'; +Git.REPO = 'git@github.com:GoogleCloudPlatform/google-cloud-node.git'; + +/** + * Checks out a branch. + * + * @param {string} branch - The branch to check out. + */ +Git.prototype.checkout = function(branch) { + run(['git checkout', branch], { + cwd: this.cwd + }); +}; /** * Creates a submodule in the root directory in quiet mode. @@ -292,7 +302,12 @@ Git.prototype.submodule = function(branch, alias) { cwd: this.cwd }); - return new Git(path.join(this.cwd, alias)); + var git = new Git(path.join(this.cwd, alias)); + + git.branch = branch; + git.alias = alias; + + return git; }; /** @@ -338,6 +353,21 @@ Git.prototype.add = function() { }); }; +/** + * Removes files via git + * + * @param {string=} options - Command line options like -rf + * @param {...string} file - File to remove. + */ +Git.prototype.remove = function() { + var files = [].slice.call(arguments); + var command = ['git rm'].concat(files); + + run(command, { + cwd: this.cwd + }); +}; + /** * Commits to git via commit message. * @@ -364,6 +394,22 @@ Git.prototype.push = function(branch) { }); }; +/** + * Deinits a submodule. + * + * @param {git} submodule - The submodule instance. + */ +Git.prototype.deinit = function(submodule) { + var options = { + cwd: this.cwd + }; + + run(['git submodule deinit -f', submodule.alias], options); + run(['git rm -rf', submodule.alias], options); + run('git rm -rf .gitmodules', options); + rm('-rf', path.resolve('.git/modules', submodule.alias)); +}; + module.exports.git = new Git(); /**