diff --git a/README.md b/README.md index e04888ef2..63dc6c6dd 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,12 @@ ![Last version](https://img.shields.io/github/tag/microlinkhq/metascraper.svg?style=flat-square) +[![Coverage Status](https://img.shields.io/coveralls/microlinkhq/metascraper.svg?style=flat-square)](https://coveralls.io/github/microlinkhq/metascraper) [![Build Status](https://img.shields.io/travis/microlinkhq/metascraper/master.svg?style=flat-square)](https://travis-ci.org/microlinkhq/metascraper) [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper) [![NPM Status](https://img.shields.io/npm/dm/metascraper.svg?style=flat-square)](https://www.npmjs.org/package/metascraper) + > A library to easily scrape metadata from an article on the web using Open Graph, JSON+LD, regular HTML metadata, and series of fallbacks. ## Table of Contents @@ -57,7 +59,7 @@ const metascraper = require('metascraper')([ require('metascraper-description')(), require('metascraper-image')(), require('metascraper-logo')(), - require('metascraper-clearbit-logo')(), + require('metascraper-clearbit')(), require('metascraper-publisher')(), require('metascraper-title')(), require('metascraper-url')() @@ -162,7 +164,7 @@ const metascraper = require('metascraper')([ require('metascraper-description')(), require('metascraper-image')(), require('metascraper-logo')(), - require('metascraper-clearbit-logo')(), + require('metascraper-clearbit')(), require('metascraper-publisher')(), require('metascraper-title')(), require('metascraper-url')() @@ -175,7 +177,7 @@ Use the first parameter to pass custom options specific per each rules bundle: ```js const metascraper = require('metascraper')([ - require('metascraper-clearbit-logo')({ + require('metascraper-clearbit')({ size: 256, format: 'jpg' }) @@ -191,7 +193,7 @@ const metascraper = require('metascraper')([ | [`metascraper-amazon`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-amazon) | [![npm](https://img.shields.io/npm/v/metascraper-amazon.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-amazon) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-amazon&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-amazon) | | [`metascraper-audio`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-audio) | [![npm](https://img.shields.io/npm/v/metascraper-audio.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-audio) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-audio&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-audio) | | [`metascraper-author`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-author) | [![npm](https://img.shields.io/npm/v/metascraper-author.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-author) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-author&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-author) | -| [`metascraper-clearbit-logo`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-clearbit-logo) | [![npm](https://img.shields.io/npm/v/metascraper-clearbit-logo.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit-logo) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit-logo&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit-logo) | +| [`metascraper-clearbit`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-clearbit) | [![npm](https://img.shields.io/npm/v/metascraper-clearbit.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit) | | [`metascraper-date`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-date) | [![npm](https://img.shields.io/npm/v/metascraper-date.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-date) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-date&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-date) | | [`metascraper-description`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-description) | [![npm](https://img.shields.io/npm/v/metascraper-description.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-description) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-description&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-description) | | [`@metascraper/helpers`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-helpers) | [![npm](https://img.shields.io/npm/v/@metascraper/helpers.svg?style=flat-square)](https://www.npmjs.com/package/@metascraper/helpers) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-helpers&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-helpers) | diff --git a/package.json b/package.json index da0ed7e26..eec23b545 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,10 @@ "name": "Shawn Dempsey", "email": "shwanton@gmail.com" }, + { + "name": "Sampo Juustila", + "email": "sampo.niskanen@iki.fi" + }, { "name": "Steven Frostwick", "email": "steven.frostwick@gmail.com" @@ -30,10 +34,6 @@ { "name": "Jesse Rogers", "email": "djrquazz@gmail.com" - }, - { - "name": "Sampo Juustila", - "email": "sampo.niskanen@iki.fi" } ], "repository": { diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index 78edd6bc9..11277e344 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -19,6 +19,7 @@ "@metascraper/helpers": "^5.4.6" }, "devDependencies": { + "coveralls": "latest", "lodash": "latest", "mocha": "latest", "nyc": "latest", @@ -33,6 +34,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-audio/index.js b/packages/metascraper-audio/index.js index 267e289d6..e04a57e85 100644 --- a/packages/metascraper-audio/index.js +++ b/packages/metascraper-audio/index.js @@ -1,6 +1,6 @@ 'use strict' -const { isMime, url: urlFn, isAudioUrl } = require('@metascraper/helpers') +const { isMime, audio } = require('@metascraper/helpers') /** * Wrap a rule with validation and formatting logic. @@ -14,10 +14,7 @@ const createWrapper = fn => rule => ({ htmlDom, url }) => { return fn(value, url) } -const wrapAudio = createWrapper((value, url) => { - const urlValue = urlFn(value, { url }) - return isAudioUrl(urlValue) && urlValue -}) +const wrapAudio = createWrapper((value, url) => audio(value, { url })) const withContentType = (url, contentType) => isMime(contentType, 'audio') ? url : false diff --git a/packages/metascraper-audio/package.json b/packages/metascraper-audio/package.json index 5b1d0a821..cd9e49c40 100644 --- a/packages/metascraper-audio/package.json +++ b/packages/metascraper-audio/package.json @@ -19,6 +19,7 @@ "@metascraper/helpers": "^5.4.6" }, "devDependencies": { + "coveralls": "latest", "lodash": "latest", "mocha": "latest", "nyc": "latest", @@ -33,6 +34,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json index aef9982b0..c1c8e9e51 100644 --- a/packages/metascraper-author/package.json +++ b/packages/metascraper-author/package.json @@ -29,6 +29,7 @@ "index.js" ], "scripts": { + "coverage": "exit 0", "test": "exit 0" }, "license": "MIT" diff --git a/packages/metascraper-clearbit-logo/CHANGELOG.md b/packages/metascraper-clearbit-logo/CHANGELOG.md deleted file mode 100644 index b6f3a09f6..000000000 --- a/packages/metascraper-clearbit-logo/CHANGELOG.md +++ /dev/null @@ -1,73 +0,0 @@ -# Change Log - -All notable changes to this project will be documented in this file. -See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. - -## [5.0.2](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v5.0.1...v5.0.2) (2019-03-31) - - -### Bug Fixes - -* ensure external requests has early timeout ([#162](https://github.com/microlinkhq/metascraper-clearbit-logo/issues/162)) ([f7a7816](https://github.com/microlinkhq/metascraper-clearbit-logo/commit/f7a7816)) - - - - - -## [5.0.1](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v5.0.0...v5.0.1) (2019-03-17) - -**Note:** Version bump only for package metascraper-clearbit-logo - - - - - -# [5.0.0](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.10.3...v5.0.0) (2019-03-17) - -**Note:** Version bump only for package metascraper-clearbit-logo - - - - - -## [4.10.1](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.10.0...v4.10.1) (2019-01-25) - -**Note:** Version bump only for package metascraper-clearbit-logo - - - - - -# [4.10.0](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.9.0...v4.10.0) (2019-01-13) - - -### Features - -* remove unnecessary dependency ([8dbb1ac](https://github.com/microlinkhq/metascraper-clearbit-logo/commit/8dbb1ac)) - - - - - -## [4.8.5](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.8.4...v4.8.5) (2018-12-25) - -**Note:** Version bump only for package metascraper-clearbit-logo - - - - - -## [4.8.4](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.8.3...v4.8.4) (2018-12-16) - - -### Bug Fixes - -* avoid use generic publisher ([#141](https://github.com/microlinkhq/metascraper-clearbit-logo/issues/141)) ([c13f833](https://github.com/microlinkhq/metascraper-clearbit-logo/commit/c13f833)) - - - - - -## [4.8.2](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.8.1...v4.8.2) (2018-12-12) - -**Note:** Version bump only for package metascraper-clearbit-logo diff --git a/packages/metascraper-clearbit-logo/__snapshots__/index.js.snap-shot b/packages/metascraper-clearbit-logo/__snapshots__/index.js.snap-shot deleted file mode 100644 index db39b3456..000000000 --- a/packages/metascraper-clearbit-logo/__snapshots__/index.js.snap-shot +++ /dev/null @@ -1,11 +0,0 @@ -exports['if logo is not present, fallback to clearbit logo API 1'] = { - "author": null, - "date": null, - "description": null, - "image": null, - "logo": "https://logo.clearbit.com/facebook.com?size=128&format=png", - "publisher": null, - "title": null, - "url": "https://facebook.com" -} - diff --git a/packages/metascraper-clearbit-logo/index.js b/packages/metascraper-clearbit-logo/index.js deleted file mode 100644 index 031b18b49..000000000 --- a/packages/metascraper-clearbit-logo/index.js +++ /dev/null @@ -1,32 +0,0 @@ -'use strict' - -const { stringify } = require('querystring') -const { URL } = require('url') -const got = require('got') - -const ENDPOINT = 'https://logo.clearbit.com' - -const apiUrl = (url, opts) => { - const { hostname } = new URL(url) - const apiUrl = `${ENDPOINT}/${hostname}` - return opts ? `${apiUrl}?${stringify(opts)}` : apiUrl -} - -module.exports = opts => { - const clearbitLogo = async ({ url }) => { - const logoUrl = apiUrl(url, opts) - - try { - await got.head(logoUrl, { retry: 0, timeout: 1000 }) - return logoUrl - } catch (err) { - return null - } - } - - return { - logo: clearbitLogo - } -} - -module.exports.apiUrl = apiUrl diff --git a/packages/metascraper-clearbit-logo/test/index.js b/packages/metascraper-clearbit-logo/test/index.js deleted file mode 100644 index 2f0183b0b..000000000 --- a/packages/metascraper-clearbit-logo/test/index.js +++ /dev/null @@ -1,32 +0,0 @@ -'use strict' - -const should = require('should') - -const metascraper = require('metascraper')([require('..')()]) -const { apiUrl } = require('..') - -describe('metascraper-clearbit-logo', () => { - it('returns when is possible resolve logo', async () => { - const url = 'https://facebook.com' - const meta = await metascraper({ url }) - should(meta.logo.indexOf('clearbit') !== -1).be.true() - }) - - it('otherwise returns null', async () => { - const url = 'https://lolwerhere.com' - const meta = await metascraper({ url }) - should(meta.logo).be.null() - }) - - it('compose urls using opts params', () => { - should(apiUrl('https://facebook.com')).be.equal( - 'https://logo.clearbit.com/facebook.com' - ) - should(apiUrl('https://facebook.com', { format: 'png' })).be.equal( - 'https://logo.clearbit.com/facebook.com?format=png' - ) - should(apiUrl('https://facebook.com', { greyscale: false })).be.equal( - 'https://logo.clearbit.com/facebook.com?greyscale=false' - ) - }) -}) diff --git a/packages/metascraper-clearbit-logo/.npmrc b/packages/metascraper-clearbit/.npmrc similarity index 100% rename from packages/metascraper-clearbit-logo/.npmrc rename to packages/metascraper-clearbit/.npmrc diff --git a/packages/metascraper-clearbit/CHANGELOG.md b/packages/metascraper-clearbit/CHANGELOG.md new file mode 100644 index 000000000..1a922b697 --- /dev/null +++ b/packages/metascraper-clearbit/CHANGELOG.md @@ -0,0 +1,73 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. + +## [5.0.2](https://github.com/microlinkhq/metascraper-clearbit/compare/v5.0.1...v5.0.2) (2019-03-31) + + +### Bug Fixes + +* ensure external requests has early timeout ([#162](https://github.com/microlinkhq/metascraper-clearbit/issues/162)) ([f7a7816](https://github.com/microlinkhq/metascraper-clearbit/commit/f7a7816)) + + + + + +## [5.0.1](https://github.com/microlinkhq/metascraper-clearbit/compare/v5.0.0...v5.0.1) (2019-03-17) + +**Note:** Version bump only for package metascraper-clearbit + + + + + +# [5.0.0](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.10.3...v5.0.0) (2019-03-17) + +**Note:** Version bump only for package metascraper-clearbit + + + + + +## [4.10.1](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.10.0...v4.10.1) (2019-01-25) + +**Note:** Version bump only for package metascraper-clearbit + + + + + +# [4.10.0](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.9.0...v4.10.0) (2019-01-13) + + +### Features + +* remove unnecessary dependency ([8dbb1ac](https://github.com/microlinkhq/metascraper-clearbit/commit/8dbb1ac)) + + + + + +## [4.8.5](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.8.4...v4.8.5) (2018-12-25) + +**Note:** Version bump only for package metascraper-clearbit + + + + + +## [4.8.4](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.8.3...v4.8.4) (2018-12-16) + + +### Bug Fixes + +* avoid use generic publisher ([#141](https://github.com/microlinkhq/metascraper-clearbit/issues/141)) ([c13f833](https://github.com/microlinkhq/metascraper-clearbit/commit/c13f833)) + + + + + +## [4.8.2](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.8.1...v4.8.2) (2018-12-12) + +**Note:** Version bump only for package metascraper-clearbit diff --git a/packages/metascraper-clearbit-logo/README.md b/packages/metascraper-clearbit/README.md similarity index 51% rename from packages/metascraper-clearbit-logo/README.md rename to packages/metascraper-clearbit/README.md index 081002aeb..5f9049abd 100644 --- a/packages/metascraper-clearbit-logo/README.md +++ b/packages/metascraper-clearbit/README.md @@ -1,14 +1,14 @@ -# metascraper-clearbit-logo +# metascraper-clearbit -[![npm](https://img.shields.io/npm/v/metascraper-clearbit-logo.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit-logo) -[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit-logo&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit-logo) +[![npm](https://img.shields.io/npm/v/metascraper-clearbit.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit) +[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit) > metascraper integration with Clearbit Logo API. ## Install ```bash -$ npm install metascraper-clearbit-logo --save +$ npm install metascraper-clearbit --save ``` ## API @@ -40,7 +40,7 @@ Desaturates image if set to `true`. ## License -**metascraper-clearbit-logo** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/metascraper-clearbit-logo/blob/master/LICENSE.md) License.
-Authored and maintained by microlink.io with help from [contributors](https://github.com/microlinkhq/metascraper-clearbit-logo/contributors). +**metascraper-clearbit** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/metascraper-clearbit/blob/master/LICENSE.md) License.
+Authored and maintained by microlink.io with help from [contributors](https://github.com/microlinkhq/metascraper-clearbit/contributors). > [microlink.io](https://microlink.io) · GitHub [@microlink.io](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq) diff --git a/packages/metascraper-clearbit/__snapshots__/index.js.snap-shot b/packages/metascraper-clearbit/__snapshots__/index.js.snap-shot new file mode 100644 index 000000000..0198c4b65 --- /dev/null +++ b/packages/metascraper-clearbit/__snapshots__/index.js.snap-shot @@ -0,0 +1,10 @@ +exports['returns when is possible resolve logo 1'] = { + "logo": "https://logo.clearbit.com/facebook.com", + "publisher": "Facebook" +} + +exports['otherwise returns null 1'] = { + "logo": null, + "publisher": null +} + diff --git a/packages/metascraper-clearbit/index.js b/packages/metascraper-clearbit/index.js new file mode 100644 index 000000000..e885278ee --- /dev/null +++ b/packages/metascraper-clearbit/index.js @@ -0,0 +1,35 @@ +'use strict' + +const { createValidator } = require('@metascraper/helpers') +const memoizeOne = require('memoize-one') +const { getDomain } = require('tldts') +const got = require('got') + +const ENDPOINT = 'https://autocomplete.clearbit.com/v1/companies/suggest' + +const memoFn = (newArgs, oldArgs) => newArgs[0].url === oldArgs[0].url + +const clearbit = memoizeOne(async ({ url }) => { + const domain = getDomain(url) + + try { + const { body } = await got(`${ENDPOINT}?query=${domain}`, { + json: true, + retry: 0, + timeout: 1000 + }) + + return body.find(item => domain === item.domain) + } catch (err) { + return null + } +}, memoFn) + +const getClearbit = createValidator(clearbit) + +module.exports = () => { + return { + logo: getClearbit({ from: 'logo' }), + publisher: getClearbit({ from: 'name', to: 'publisher' }) + } +} diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit/package.json similarity index 64% rename from packages/metascraper-clearbit-logo/package.json rename to packages/metascraper-clearbit/package.json index 7e954eff1..070921d25 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit/package.json @@ -1,28 +1,33 @@ { - "name": "metascraper-clearbit-logo", + "name": "metascraper-clearbit", "description": "metascraper integration with Clearbit Logo API", - "homepage": "https://nicedoc.io/microlinkhq/metascraper/packages/metascraper-clearbit-logo", + "homepage": "https://nicedoc.io/microlinkhq/metascraper/packages/metascraper-clearbit", "version": "5.3.0", "main": "index.js", "repository": { "type": "git", - "url": "git+https://github.com/microlinkhq/metascraper-clearbit-logo.git" + "url": "git+https://github.com/microlinkhq/metascraper-clearbit.git" }, "bugs": { - "url": "https://github.com/microlinkhq/metascraper-clearbit-logo/issues" + "url": "https://github.com/microlinkhq/metascraper-clearbit/issues" }, "keywords": [ + "autocomplete", "clearbit", "logo", "metascraper" ], "dependencies": { - "got": "~9.6.0" + "@metascraper/helpers": "~5.4.6", + "got": "~9.6.0", + "memoize-one": "~5.0.4", + "tldts": "~5.1.0" }, "devDependencies": { + "coveralls": "latest", "mocha": "latest", "nyc": "latest", - "should": "latest", + "snap-shot": "latest", "standard": "latest" }, "engines": { @@ -32,6 +37,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-clearbit/test/index.js b/packages/metascraper-clearbit/test/index.js new file mode 100644 index 000000000..0d180b417 --- /dev/null +++ b/packages/metascraper-clearbit/test/index.js @@ -0,0 +1,19 @@ +'use strict' + +const snapshot = require('snap-shot') + +const metascraper = require('metascraper')([require('..')()]) + +describe('metascraper-clearbit', () => { + it('returns when is possible resolve logo', async () => { + const url = 'https://facebook.com' + const meta = await metascraper({ url }) + snapshot(meta) + }) + + it('otherwise returns null', async () => { + const url = 'https://lolwerhere.com' + const meta = await metascraper({ url }) + snapshot(meta) + }) +}) diff --git a/packages/metascraper-clearbit-logo/test/mocha.opts b/packages/metascraper-clearbit/test/mocha.opts similarity index 78% rename from packages/metascraper-clearbit-logo/test/mocha.opts rename to packages/metascraper-clearbit/test/mocha.opts index bc69e8879..7dae578bb 100644 --- a/packages/metascraper-clearbit-logo/test/mocha.opts +++ b/packages/metascraper-clearbit/test/mocha.opts @@ -1,4 +1,3 @@ ---require should --reporter spec --timeout 120000 --slow 300 diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json index 44c75f6a3..b4e5b5c5e 100644 --- a/packages/metascraper-date/package.json +++ b/packages/metascraper-date/package.json @@ -28,6 +28,7 @@ "index.js" ], "scripts": { + "coverage": "exit 0", "test": "exit 0" }, "license": "MIT" diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json index 578a03a8c..3d9a93485 100644 --- a/packages/metascraper-description/package.json +++ b/packages/metascraper-description/package.json @@ -28,6 +28,7 @@ "index.js" ], "scripts": { + "coverage": "exit 0", "test": "exit 0" }, "license": "MIT" diff --git a/packages/metascraper-helpers/index.js b/packages/metascraper-helpers/index.js index 62afc3ab4..263bf97a4 100644 --- a/packages/metascraper-helpers/index.js +++ b/packages/metascraper-helpers/index.js @@ -16,7 +16,8 @@ const { replace, size, toLower, - trim + trim, + invoke } = require('lodash') const langs = require('iso-639-3').map(({ iso6391 }) => iso6391) @@ -236,6 +237,40 @@ const $jsonld = propName => ($, url) => { return isEmpty(value) ? value : decodeHTML(value) } +const image = url + +const logo = url + +const video = (value, opts) => { + const urlValue = url(value, opts) + return isVideoUrl(urlValue) && urlValue +} + +const audio = (value, opts) => { + const urlValue = url(value, opts) + return isAudioUrl(urlValue) && urlValue +} + +const validator = { + date, + audio, + author, + video, + title, + publisher, + image, + logo, + url, + description, + lang +} + +const createValidator = fn => ({ from, to = from }) => async args => { + const data = await fn(args) + const value = get(data, from) + return invoke(validator, to, value) +} + module.exports = { $filter, $jsonld, @@ -264,5 +299,11 @@ module.exports = { sanetizeUrl, title, titleize, - url + url, + image, + logo, + audio, + video, + validator, + createValidator } diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json index c9edb5b51..0dfec320b 100644 --- a/packages/metascraper-helpers/package.json +++ b/packages/metascraper-helpers/package.json @@ -38,6 +38,7 @@ }, "devDependencies": { "cheerio": "latest", + "coveralls": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -51,6 +52,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-image/index.js b/packages/metascraper-image/index.js index 84ac5360d..c45b24330 100644 --- a/packages/metascraper-image/index.js +++ b/packages/metascraper-image/index.js @@ -1,6 +1,6 @@ 'use strict' -const { $jsonld, $filter, url: urlFn } = require('@metascraper/helpers') +const { $jsonld, $filter, image } = require('@metascraper/helpers') /** * Wrap a rule with validation and formatting logic. @@ -11,7 +11,7 @@ const { $jsonld, $filter, url: urlFn } = require('@metascraper/helpers') const wrap = rule => ({ htmlDom, url }) => { const value = rule(htmlDom, url) - return urlFn(value, { url }) + return image(value, { url }) } const getSrc = el => el.attr('src') diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json index 97115ca99..fd43b4273 100644 --- a/packages/metascraper-image/package.json +++ b/packages/metascraper-image/package.json @@ -28,6 +28,7 @@ "index.js" ], "scripts": { + "coverage": "exit 0", "test": "exit 0" }, "license": "MIT" diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json index cf2bf964f..0e500ec0a 100644 --- a/packages/metascraper-lang/package.json +++ b/packages/metascraper-lang/package.json @@ -19,6 +19,7 @@ "@metascraper/helpers": "^5.4.6" }, "devDependencies": { + "coveralls": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -32,6 +33,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-logo-favicon/index.js b/packages/metascraper-logo-favicon/index.js index 7c5b115fc..3a767daa2 100644 --- a/packages/metascraper-logo-favicon/index.js +++ b/packages/metascraper-logo-favicon/index.js @@ -1,8 +1,8 @@ 'use strict' const { flow, first, toNumber, split, chain, concat } = require('lodash') +const { logo, url: urlFn } = require('@metascraper/helpers') const { resolve: resolveUrl, URL } = require('url') -const { url: urlFn } = require('@metascraper/helpers') const got = require('got') const getSize = flow([str => split(str, 'x'), first, toNumber]) @@ -63,7 +63,7 @@ module.exports = () => ({ try { await got.head(logoUrl, { retry: 0, timeout: 10000 }) - return logoUrl + return logo(logoUrl) } catch (err) { return null } diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json index 685a1a7a6..c63507499 100644 --- a/packages/metascraper-logo-favicon/package.json +++ b/packages/metascraper-logo-favicon/package.json @@ -22,6 +22,7 @@ "lodash": "~4.17.11" }, "devDependencies": { + "coveralls": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -35,6 +36,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json index a5fdc24d0..1ca23eb47 100644 --- a/packages/metascraper-logo/package.json +++ b/packages/metascraper-logo/package.json @@ -31,6 +31,7 @@ "index.js" ], "scripts": { + "coverage": "exit 0", "test": "exit 0" }, "license": "MIT" diff --git a/packages/metascraper-media-provider/package.json b/packages/metascraper-media-provider/package.json index 9ceebec68..4d086f713 100644 --- a/packages/metascraper-media-provider/package.json +++ b/packages/metascraper-media-provider/package.json @@ -26,6 +26,7 @@ "memoize-one": "~5.0.4" }, "devDependencies": { + "coveralls": "latest", "is-ci": "latest", "mocha": "latest", "nyc": "latest", @@ -42,6 +43,7 @@ "src" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index c426fbfdc..8995f4f51 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -19,6 +19,7 @@ "@metascraper/helpers": "^5.4.6" }, "devDependencies": { + "coveralls": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -31,6 +32,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-readability/index.js b/packages/metascraper-readability/index.js index 61069e0ec..bfd6b95a3 100644 --- a/packages/metascraper-readability/index.js +++ b/packages/metascraper-readability/index.js @@ -1,24 +1,19 @@ 'use strict' -const helpers = require('@metascraper/helpers') -const { eq, get, invoke } = require('lodash') +const { createValidator } = require('@metascraper/helpers') const Readability = require('readability') const memoizeOne = require('memoize-one') const { JSDOM } = require('jsdom') -const memoFn = (newArgs, oldArgs) => eq(newArgs[1].url, oldArgs[1].url) +const memoFn = (newArgs, oldArgs) => newArgs[0].url === oldArgs[0].url -const readability = memoizeOne((html, opts) => { - const dom = new JSDOM(html, opts) +const readability = memoizeOne(({ htmlDom, url }) => { + const dom = new JSDOM(htmlDom.html(), { url }) const reader = new Readability(dom.window.document) return reader.parse() }, memoFn) -const getReadbility = ({ from, to = from }) => ({ htmlDom, url }) => { - const data = readability(htmlDom.html(), { url }) - const value = get(data, from) - return invoke(helpers, to, value) -} +const getReadbility = createValidator(readability) module.exports = () => { return { diff --git a/packages/metascraper-readability/package.json b/packages/metascraper-readability/package.json index 74d2cd25a..82a2f3746 100644 --- a/packages/metascraper-readability/package.json +++ b/packages/metascraper-readability/package.json @@ -18,11 +18,11 @@ "dependencies": { "@metascraper/helpers": "^5.4.6", "jsdom": "~15.1.1", - "lodash": "~4.17.11", "memoize-one": "~5.0.4", "readability": "github:mozilla/readability" }, "devDependencies": { + "coveralls": "latest", "metascraper": "latest", "mocha": "latest", "nyc": "latest", @@ -36,6 +36,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index 105f72313..3c5929954 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -20,6 +20,7 @@ "@metascraper/helpers": "^5.4.6" }, "devDependencies": { + "coveralls": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -33,6 +34,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json index aea39106d..dd2c0f46c 100644 --- a/packages/metascraper-title/package.json +++ b/packages/metascraper-title/package.json @@ -29,6 +29,7 @@ "index.js" ], "scripts": { + "coverage": "exit 0", "test": "exit 0" }, "license": "MIT" diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json index c387ed5a1..045f94d53 100644 --- a/packages/metascraper-url/package.json +++ b/packages/metascraper-url/package.json @@ -28,6 +28,7 @@ "index.js" ], "scripts": { + "coverage": "exit 0", "test": "exit 0" }, "license": "MIT" diff --git a/packages/metascraper-video/index.js b/packages/metascraper-video/index.js index ab2708bea..356eb99ad 100644 --- a/packages/metascraper-video/index.js +++ b/packages/metascraper-video/index.js @@ -1,11 +1,6 @@ 'use strict' -const { - isMime, - url: urlFn, - isVideoUrl, - extension -} = require('@metascraper/helpers') +const { isMime, url: urlFn, extension, video } = require('@metascraper/helpers') const { chain } = require('lodash') /** @@ -29,8 +24,8 @@ const wrapVideo = createWrapper((domNodes, url) => { .orderBy(videoUrl => extension(videoUrl) === 'mp4', ['desc']) .first() .value() - const urlValue = urlFn(videoUrl, { url }) - return isVideoUrl(urlValue) && urlValue + + return video(videoUrl, { url }) }) const withContentType = (url, contentType) => diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index 841dd8e73..9812f0f1d 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -20,6 +20,7 @@ "lodash": "~4.17.11" }, "devDependencies": { + "coveralls": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -33,6 +34,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index eee3d4837..377529d9a 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -23,6 +23,7 @@ "p-locate": "~4.1.0" }, "devDependencies": { + "coveralls": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -36,6 +37,7 @@ "index.js" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", diff --git a/packages/metascraper/package.json b/packages/metascraper/package.json index 22e87eded..d70216199 100644 --- a/packages/metascraper/package.json +++ b/packages/metascraper/package.json @@ -66,7 +66,8 @@ }, "devDependencies": { "clear-module": "latest", - "metascraper-clearbit-logo": "latest", + "coveralls": "latest", + "metascraper-clearbit": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -80,6 +81,7 @@ "src" ], "scripts": { + "coverage": "nyc report --reporter=text-lcov | coveralls", "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT",