diff --git a/README.md b/README.md
index e04888ef2..63dc6c6dd 100644
--- a/README.md
+++ b/README.md
@@ -6,10 +6,12 @@
![Last version](https://img.shields.io/github/tag/microlinkhq/metascraper.svg?style=flat-square)
+[![Coverage Status](https://img.shields.io/coveralls/microlinkhq/metascraper.svg?style=flat-square)](https://coveralls.io/github/microlinkhq/metascraper)
[![Build Status](https://img.shields.io/travis/microlinkhq/metascraper/master.svg?style=flat-square)](https://travis-ci.org/microlinkhq/metascraper)
[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper)
[![NPM Status](https://img.shields.io/npm/dm/metascraper.svg?style=flat-square)](https://www.npmjs.org/package/metascraper)
+
> A library to easily scrape metadata from an article on the web using Open Graph, JSON+LD, regular HTML metadata, and series of fallbacks.
## Table of Contents
@@ -57,7 +59,7 @@ const metascraper = require('metascraper')([
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-logo')(),
- require('metascraper-clearbit-logo')(),
+ require('metascraper-clearbit')(),
require('metascraper-publisher')(),
require('metascraper-title')(),
require('metascraper-url')()
@@ -162,7 +164,7 @@ const metascraper = require('metascraper')([
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-logo')(),
- require('metascraper-clearbit-logo')(),
+ require('metascraper-clearbit')(),
require('metascraper-publisher')(),
require('metascraper-title')(),
require('metascraper-url')()
@@ -175,7 +177,7 @@ Use the first parameter to pass custom options specific per each rules bundle:
```js
const metascraper = require('metascraper')([
- require('metascraper-clearbit-logo')({
+ require('metascraper-clearbit')({
size: 256,
format: 'jpg'
})
@@ -191,7 +193,7 @@ const metascraper = require('metascraper')([
| [`metascraper-amazon`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-amazon) | [![npm](https://img.shields.io/npm/v/metascraper-amazon.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-amazon) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-amazon&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-amazon) |
| [`metascraper-audio`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-audio) | [![npm](https://img.shields.io/npm/v/metascraper-audio.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-audio) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-audio&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-audio) |
| [`metascraper-author`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-author) | [![npm](https://img.shields.io/npm/v/metascraper-author.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-author) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-author&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-author) |
-| [`metascraper-clearbit-logo`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-clearbit-logo) | [![npm](https://img.shields.io/npm/v/metascraper-clearbit-logo.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit-logo) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit-logo&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit-logo) |
+| [`metascraper-clearbit`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-clearbit) | [![npm](https://img.shields.io/npm/v/metascraper-clearbit.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit) |
| [`metascraper-date`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-date) | [![npm](https://img.shields.io/npm/v/metascraper-date.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-date) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-date&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-date) |
| [`metascraper-description`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-description) | [![npm](https://img.shields.io/npm/v/metascraper-description.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-description) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-description&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-description) |
| [`@metascraper/helpers`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-helpers) | [![npm](https://img.shields.io/npm/v/@metascraper/helpers.svg?style=flat-square)](https://www.npmjs.com/package/@metascraper/helpers) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-helpers&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-helpers) |
diff --git a/package.json b/package.json
index da0ed7e26..eec23b545 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,10 @@
"name": "Shawn Dempsey",
"email": "shwanton@gmail.com"
},
+ {
+ "name": "Sampo Juustila",
+ "email": "sampo.niskanen@iki.fi"
+ },
{
"name": "Steven Frostwick",
"email": "steven.frostwick@gmail.com"
@@ -30,10 +34,6 @@
{
"name": "Jesse Rogers",
"email": "djrquazz@gmail.com"
- },
- {
- "name": "Sampo Juustila",
- "email": "sampo.niskanen@iki.fi"
}
],
"repository": {
diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json
index 78edd6bc9..11277e344 100644
--- a/packages/metascraper-amazon/package.json
+++ b/packages/metascraper-amazon/package.json
@@ -19,6 +19,7 @@
"@metascraper/helpers": "^5.4.6"
},
"devDependencies": {
+ "coveralls": "latest",
"lodash": "latest",
"mocha": "latest",
"nyc": "latest",
@@ -33,6 +34,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-audio/index.js b/packages/metascraper-audio/index.js
index 267e289d6..e04a57e85 100644
--- a/packages/metascraper-audio/index.js
+++ b/packages/metascraper-audio/index.js
@@ -1,6 +1,6 @@
'use strict'
-const { isMime, url: urlFn, isAudioUrl } = require('@metascraper/helpers')
+const { isMime, audio } = require('@metascraper/helpers')
/**
* Wrap a rule with validation and formatting logic.
@@ -14,10 +14,7 @@ const createWrapper = fn => rule => ({ htmlDom, url }) => {
return fn(value, url)
}
-const wrapAudio = createWrapper((value, url) => {
- const urlValue = urlFn(value, { url })
- return isAudioUrl(urlValue) && urlValue
-})
+const wrapAudio = createWrapper((value, url) => audio(value, { url }))
const withContentType = (url, contentType) =>
isMime(contentType, 'audio') ? url : false
diff --git a/packages/metascraper-audio/package.json b/packages/metascraper-audio/package.json
index 5b1d0a821..cd9e49c40 100644
--- a/packages/metascraper-audio/package.json
+++ b/packages/metascraper-audio/package.json
@@ -19,6 +19,7 @@
"@metascraper/helpers": "^5.4.6"
},
"devDependencies": {
+ "coveralls": "latest",
"lodash": "latest",
"mocha": "latest",
"nyc": "latest",
@@ -33,6 +34,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json
index aef9982b0..c1c8e9e51 100644
--- a/packages/metascraper-author/package.json
+++ b/packages/metascraper-author/package.json
@@ -29,6 +29,7 @@
"index.js"
],
"scripts": {
+ "coverage": "exit 0",
"test": "exit 0"
},
"license": "MIT"
diff --git a/packages/metascraper-clearbit-logo/CHANGELOG.md b/packages/metascraper-clearbit-logo/CHANGELOG.md
deleted file mode 100644
index b6f3a09f6..000000000
--- a/packages/metascraper-clearbit-logo/CHANGELOG.md
+++ /dev/null
@@ -1,73 +0,0 @@
-# Change Log
-
-All notable changes to this project will be documented in this file.
-See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
-
-## [5.0.2](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v5.0.1...v5.0.2) (2019-03-31)
-
-
-### Bug Fixes
-
-* ensure external requests has early timeout ([#162](https://github.com/microlinkhq/metascraper-clearbit-logo/issues/162)) ([f7a7816](https://github.com/microlinkhq/metascraper-clearbit-logo/commit/f7a7816))
-
-
-
-
-
-## [5.0.1](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v5.0.0...v5.0.1) (2019-03-17)
-
-**Note:** Version bump only for package metascraper-clearbit-logo
-
-
-
-
-
-# [5.0.0](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.10.3...v5.0.0) (2019-03-17)
-
-**Note:** Version bump only for package metascraper-clearbit-logo
-
-
-
-
-
-## [4.10.1](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.10.0...v4.10.1) (2019-01-25)
-
-**Note:** Version bump only for package metascraper-clearbit-logo
-
-
-
-
-
-# [4.10.0](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.9.0...v4.10.0) (2019-01-13)
-
-
-### Features
-
-* remove unnecessary dependency ([8dbb1ac](https://github.com/microlinkhq/metascraper-clearbit-logo/commit/8dbb1ac))
-
-
-
-
-
-## [4.8.5](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.8.4...v4.8.5) (2018-12-25)
-
-**Note:** Version bump only for package metascraper-clearbit-logo
-
-
-
-
-
-## [4.8.4](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.8.3...v4.8.4) (2018-12-16)
-
-
-### Bug Fixes
-
-* avoid use generic publisher ([#141](https://github.com/microlinkhq/metascraper-clearbit-logo/issues/141)) ([c13f833](https://github.com/microlinkhq/metascraper-clearbit-logo/commit/c13f833))
-
-
-
-
-
-## [4.8.2](https://github.com/microlinkhq/metascraper-clearbit-logo/compare/v4.8.1...v4.8.2) (2018-12-12)
-
-**Note:** Version bump only for package metascraper-clearbit-logo
diff --git a/packages/metascraper-clearbit-logo/__snapshots__/index.js.snap-shot b/packages/metascraper-clearbit-logo/__snapshots__/index.js.snap-shot
deleted file mode 100644
index db39b3456..000000000
--- a/packages/metascraper-clearbit-logo/__snapshots__/index.js.snap-shot
+++ /dev/null
@@ -1,11 +0,0 @@
-exports['if logo is not present, fallback to clearbit logo API 1'] = {
- "author": null,
- "date": null,
- "description": null,
- "image": null,
- "logo": "https://logo.clearbit.com/facebook.com?size=128&format=png",
- "publisher": null,
- "title": null,
- "url": "https://facebook.com"
-}
-
diff --git a/packages/metascraper-clearbit-logo/index.js b/packages/metascraper-clearbit-logo/index.js
deleted file mode 100644
index 031b18b49..000000000
--- a/packages/metascraper-clearbit-logo/index.js
+++ /dev/null
@@ -1,32 +0,0 @@
-'use strict'
-
-const { stringify } = require('querystring')
-const { URL } = require('url')
-const got = require('got')
-
-const ENDPOINT = 'https://logo.clearbit.com'
-
-const apiUrl = (url, opts) => {
- const { hostname } = new URL(url)
- const apiUrl = `${ENDPOINT}/${hostname}`
- return opts ? `${apiUrl}?${stringify(opts)}` : apiUrl
-}
-
-module.exports = opts => {
- const clearbitLogo = async ({ url }) => {
- const logoUrl = apiUrl(url, opts)
-
- try {
- await got.head(logoUrl, { retry: 0, timeout: 1000 })
- return logoUrl
- } catch (err) {
- return null
- }
- }
-
- return {
- logo: clearbitLogo
- }
-}
-
-module.exports.apiUrl = apiUrl
diff --git a/packages/metascraper-clearbit-logo/test/index.js b/packages/metascraper-clearbit-logo/test/index.js
deleted file mode 100644
index 2f0183b0b..000000000
--- a/packages/metascraper-clearbit-logo/test/index.js
+++ /dev/null
@@ -1,32 +0,0 @@
-'use strict'
-
-const should = require('should')
-
-const metascraper = require('metascraper')([require('..')()])
-const { apiUrl } = require('..')
-
-describe('metascraper-clearbit-logo', () => {
- it('returns when is possible resolve logo', async () => {
- const url = 'https://facebook.com'
- const meta = await metascraper({ url })
- should(meta.logo.indexOf('clearbit') !== -1).be.true()
- })
-
- it('otherwise returns null', async () => {
- const url = 'https://lolwerhere.com'
- const meta = await metascraper({ url })
- should(meta.logo).be.null()
- })
-
- it('compose urls using opts params', () => {
- should(apiUrl('https://facebook.com')).be.equal(
- 'https://logo.clearbit.com/facebook.com'
- )
- should(apiUrl('https://facebook.com', { format: 'png' })).be.equal(
- 'https://logo.clearbit.com/facebook.com?format=png'
- )
- should(apiUrl('https://facebook.com', { greyscale: false })).be.equal(
- 'https://logo.clearbit.com/facebook.com?greyscale=false'
- )
- })
-})
diff --git a/packages/metascraper-clearbit-logo/.npmrc b/packages/metascraper-clearbit/.npmrc
similarity index 100%
rename from packages/metascraper-clearbit-logo/.npmrc
rename to packages/metascraper-clearbit/.npmrc
diff --git a/packages/metascraper-clearbit/CHANGELOG.md b/packages/metascraper-clearbit/CHANGELOG.md
new file mode 100644
index 000000000..1a922b697
--- /dev/null
+++ b/packages/metascraper-clearbit/CHANGELOG.md
@@ -0,0 +1,73 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+## [5.0.2](https://github.com/microlinkhq/metascraper-clearbit/compare/v5.0.1...v5.0.2) (2019-03-31)
+
+
+### Bug Fixes
+
+* ensure external requests has early timeout ([#162](https://github.com/microlinkhq/metascraper-clearbit/issues/162)) ([f7a7816](https://github.com/microlinkhq/metascraper-clearbit/commit/f7a7816))
+
+
+
+
+
+## [5.0.1](https://github.com/microlinkhq/metascraper-clearbit/compare/v5.0.0...v5.0.1) (2019-03-17)
+
+**Note:** Version bump only for package metascraper-clearbit
+
+
+
+
+
+# [5.0.0](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.10.3...v5.0.0) (2019-03-17)
+
+**Note:** Version bump only for package metascraper-clearbit
+
+
+
+
+
+## [4.10.1](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.10.0...v4.10.1) (2019-01-25)
+
+**Note:** Version bump only for package metascraper-clearbit
+
+
+
+
+
+# [4.10.0](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.9.0...v4.10.0) (2019-01-13)
+
+
+### Features
+
+* remove unnecessary dependency ([8dbb1ac](https://github.com/microlinkhq/metascraper-clearbit/commit/8dbb1ac))
+
+
+
+
+
+## [4.8.5](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.8.4...v4.8.5) (2018-12-25)
+
+**Note:** Version bump only for package metascraper-clearbit
+
+
+
+
+
+## [4.8.4](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.8.3...v4.8.4) (2018-12-16)
+
+
+### Bug Fixes
+
+* avoid use generic publisher ([#141](https://github.com/microlinkhq/metascraper-clearbit/issues/141)) ([c13f833](https://github.com/microlinkhq/metascraper-clearbit/commit/c13f833))
+
+
+
+
+
+## [4.8.2](https://github.com/microlinkhq/metascraper-clearbit/compare/v4.8.1...v4.8.2) (2018-12-12)
+
+**Note:** Version bump only for package metascraper-clearbit
diff --git a/packages/metascraper-clearbit-logo/README.md b/packages/metascraper-clearbit/README.md
similarity index 51%
rename from packages/metascraper-clearbit-logo/README.md
rename to packages/metascraper-clearbit/README.md
index 081002aeb..5f9049abd 100644
--- a/packages/metascraper-clearbit-logo/README.md
+++ b/packages/metascraper-clearbit/README.md
@@ -1,14 +1,14 @@
-# metascraper-clearbit-logo
+# metascraper-clearbit
-[![npm](https://img.shields.io/npm/v/metascraper-clearbit-logo.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit-logo)
-[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit-logo&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit-logo)
+[![npm](https://img.shields.io/npm/v/metascraper-clearbit.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit)
+[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit)
> metascraper integration with Clearbit Logo API.
## Install
```bash
-$ npm install metascraper-clearbit-logo --save
+$ npm install metascraper-clearbit --save
```
## API
@@ -40,7 +40,7 @@ Desaturates image if set to `true`.
## License
-**metascraper-clearbit-logo** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/metascraper-clearbit-logo/blob/master/LICENSE.md) License.
-Authored and maintained by microlink.io with help from [contributors](https://github.com/microlinkhq/metascraper-clearbit-logo/contributors).
+**metascraper-clearbit** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/metascraper-clearbit/blob/master/LICENSE.md) License.
+Authored and maintained by microlink.io with help from [contributors](https://github.com/microlinkhq/metascraper-clearbit/contributors).
> [microlink.io](https://microlink.io) · GitHub [@microlink.io](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq)
diff --git a/packages/metascraper-clearbit/__snapshots__/index.js.snap-shot b/packages/metascraper-clearbit/__snapshots__/index.js.snap-shot
new file mode 100644
index 000000000..0198c4b65
--- /dev/null
+++ b/packages/metascraper-clearbit/__snapshots__/index.js.snap-shot
@@ -0,0 +1,10 @@
+exports['returns when is possible resolve logo 1'] = {
+ "logo": "https://logo.clearbit.com/facebook.com",
+ "publisher": "Facebook"
+}
+
+exports['otherwise returns null 1'] = {
+ "logo": null,
+ "publisher": null
+}
+
diff --git a/packages/metascraper-clearbit/index.js b/packages/metascraper-clearbit/index.js
new file mode 100644
index 000000000..e885278ee
--- /dev/null
+++ b/packages/metascraper-clearbit/index.js
@@ -0,0 +1,35 @@
+'use strict'
+
+const { createValidator } = require('@metascraper/helpers')
+const memoizeOne = require('memoize-one')
+const { getDomain } = require('tldts')
+const got = require('got')
+
+const ENDPOINT = 'https://autocomplete.clearbit.com/v1/companies/suggest'
+
+const memoFn = (newArgs, oldArgs) => newArgs[0].url === oldArgs[0].url
+
+const clearbit = memoizeOne(async ({ url }) => {
+ const domain = getDomain(url)
+
+ try {
+ const { body } = await got(`${ENDPOINT}?query=${domain}`, {
+ json: true,
+ retry: 0,
+ timeout: 1000
+ })
+
+ return body.find(item => domain === item.domain)
+ } catch (err) {
+ return null
+ }
+}, memoFn)
+
+const getClearbit = createValidator(clearbit)
+
+module.exports = () => {
+ return {
+ logo: getClearbit({ from: 'logo' }),
+ publisher: getClearbit({ from: 'name', to: 'publisher' })
+ }
+}
diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit/package.json
similarity index 64%
rename from packages/metascraper-clearbit-logo/package.json
rename to packages/metascraper-clearbit/package.json
index 7e954eff1..070921d25 100644
--- a/packages/metascraper-clearbit-logo/package.json
+++ b/packages/metascraper-clearbit/package.json
@@ -1,28 +1,33 @@
{
- "name": "metascraper-clearbit-logo",
+ "name": "metascraper-clearbit",
"description": "metascraper integration with Clearbit Logo API",
- "homepage": "https://nicedoc.io/microlinkhq/metascraper/packages/metascraper-clearbit-logo",
+ "homepage": "https://nicedoc.io/microlinkhq/metascraper/packages/metascraper-clearbit",
"version": "5.3.0",
"main": "index.js",
"repository": {
"type": "git",
- "url": "git+https://github.com/microlinkhq/metascraper-clearbit-logo.git"
+ "url": "git+https://github.com/microlinkhq/metascraper-clearbit.git"
},
"bugs": {
- "url": "https://github.com/microlinkhq/metascraper-clearbit-logo/issues"
+ "url": "https://github.com/microlinkhq/metascraper-clearbit/issues"
},
"keywords": [
+ "autocomplete",
"clearbit",
"logo",
"metascraper"
],
"dependencies": {
- "got": "~9.6.0"
+ "@metascraper/helpers": "~5.4.6",
+ "got": "~9.6.0",
+ "memoize-one": "~5.0.4",
+ "tldts": "~5.1.0"
},
"devDependencies": {
+ "coveralls": "latest",
"mocha": "latest",
"nyc": "latest",
- "should": "latest",
+ "snap-shot": "latest",
"standard": "latest"
},
"engines": {
@@ -32,6 +37,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-clearbit/test/index.js b/packages/metascraper-clearbit/test/index.js
new file mode 100644
index 000000000..0d180b417
--- /dev/null
+++ b/packages/metascraper-clearbit/test/index.js
@@ -0,0 +1,19 @@
+'use strict'
+
+const snapshot = require('snap-shot')
+
+const metascraper = require('metascraper')([require('..')()])
+
+describe('metascraper-clearbit', () => {
+ it('returns when is possible resolve logo', async () => {
+ const url = 'https://facebook.com'
+ const meta = await metascraper({ url })
+ snapshot(meta)
+ })
+
+ it('otherwise returns null', async () => {
+ const url = 'https://lolwerhere.com'
+ const meta = await metascraper({ url })
+ snapshot(meta)
+ })
+})
diff --git a/packages/metascraper-clearbit-logo/test/mocha.opts b/packages/metascraper-clearbit/test/mocha.opts
similarity index 78%
rename from packages/metascraper-clearbit-logo/test/mocha.opts
rename to packages/metascraper-clearbit/test/mocha.opts
index bc69e8879..7dae578bb 100644
--- a/packages/metascraper-clearbit-logo/test/mocha.opts
+++ b/packages/metascraper-clearbit/test/mocha.opts
@@ -1,4 +1,3 @@
---require should
--reporter spec
--timeout 120000
--slow 300
diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json
index 44c75f6a3..b4e5b5c5e 100644
--- a/packages/metascraper-date/package.json
+++ b/packages/metascraper-date/package.json
@@ -28,6 +28,7 @@
"index.js"
],
"scripts": {
+ "coverage": "exit 0",
"test": "exit 0"
},
"license": "MIT"
diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json
index 578a03a8c..3d9a93485 100644
--- a/packages/metascraper-description/package.json
+++ b/packages/metascraper-description/package.json
@@ -28,6 +28,7 @@
"index.js"
],
"scripts": {
+ "coverage": "exit 0",
"test": "exit 0"
},
"license": "MIT"
diff --git a/packages/metascraper-helpers/index.js b/packages/metascraper-helpers/index.js
index 62afc3ab4..263bf97a4 100644
--- a/packages/metascraper-helpers/index.js
+++ b/packages/metascraper-helpers/index.js
@@ -16,7 +16,8 @@ const {
replace,
size,
toLower,
- trim
+ trim,
+ invoke
} = require('lodash')
const langs = require('iso-639-3').map(({ iso6391 }) => iso6391)
@@ -236,6 +237,40 @@ const $jsonld = propName => ($, url) => {
return isEmpty(value) ? value : decodeHTML(value)
}
+const image = url
+
+const logo = url
+
+const video = (value, opts) => {
+ const urlValue = url(value, opts)
+ return isVideoUrl(urlValue) && urlValue
+}
+
+const audio = (value, opts) => {
+ const urlValue = url(value, opts)
+ return isAudioUrl(urlValue) && urlValue
+}
+
+const validator = {
+ date,
+ audio,
+ author,
+ video,
+ title,
+ publisher,
+ image,
+ logo,
+ url,
+ description,
+ lang
+}
+
+const createValidator = fn => ({ from, to = from }) => async args => {
+ const data = await fn(args)
+ const value = get(data, from)
+ return invoke(validator, to, value)
+}
+
module.exports = {
$filter,
$jsonld,
@@ -264,5 +299,11 @@ module.exports = {
sanetizeUrl,
title,
titleize,
- url
+ url,
+ image,
+ logo,
+ audio,
+ video,
+ validator,
+ createValidator
}
diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json
index c9edb5b51..0dfec320b 100644
--- a/packages/metascraper-helpers/package.json
+++ b/packages/metascraper-helpers/package.json
@@ -38,6 +38,7 @@
},
"devDependencies": {
"cheerio": "latest",
+ "coveralls": "latest",
"mocha": "latest",
"nyc": "latest",
"should": "latest",
@@ -51,6 +52,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-image/index.js b/packages/metascraper-image/index.js
index 84ac5360d..c45b24330 100644
--- a/packages/metascraper-image/index.js
+++ b/packages/metascraper-image/index.js
@@ -1,6 +1,6 @@
'use strict'
-const { $jsonld, $filter, url: urlFn } = require('@metascraper/helpers')
+const { $jsonld, $filter, image } = require('@metascraper/helpers')
/**
* Wrap a rule with validation and formatting logic.
@@ -11,7 +11,7 @@ const { $jsonld, $filter, url: urlFn } = require('@metascraper/helpers')
const wrap = rule => ({ htmlDom, url }) => {
const value = rule(htmlDom, url)
- return urlFn(value, { url })
+ return image(value, { url })
}
const getSrc = el => el.attr('src')
diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json
index 97115ca99..fd43b4273 100644
--- a/packages/metascraper-image/package.json
+++ b/packages/metascraper-image/package.json
@@ -28,6 +28,7 @@
"index.js"
],
"scripts": {
+ "coverage": "exit 0",
"test": "exit 0"
},
"license": "MIT"
diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json
index cf2bf964f..0e500ec0a 100644
--- a/packages/metascraper-lang/package.json
+++ b/packages/metascraper-lang/package.json
@@ -19,6 +19,7 @@
"@metascraper/helpers": "^5.4.6"
},
"devDependencies": {
+ "coveralls": "latest",
"mocha": "latest",
"nyc": "latest",
"should": "latest",
@@ -32,6 +33,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-logo-favicon/index.js b/packages/metascraper-logo-favicon/index.js
index 7c5b115fc..3a767daa2 100644
--- a/packages/metascraper-logo-favicon/index.js
+++ b/packages/metascraper-logo-favicon/index.js
@@ -1,8 +1,8 @@
'use strict'
const { flow, first, toNumber, split, chain, concat } = require('lodash')
+const { logo, url: urlFn } = require('@metascraper/helpers')
const { resolve: resolveUrl, URL } = require('url')
-const { url: urlFn } = require('@metascraper/helpers')
const got = require('got')
const getSize = flow([str => split(str, 'x'), first, toNumber])
@@ -63,7 +63,7 @@ module.exports = () => ({
try {
await got.head(logoUrl, { retry: 0, timeout: 10000 })
- return logoUrl
+ return logo(logoUrl)
} catch (err) {
return null
}
diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json
index 685a1a7a6..c63507499 100644
--- a/packages/metascraper-logo-favicon/package.json
+++ b/packages/metascraper-logo-favicon/package.json
@@ -22,6 +22,7 @@
"lodash": "~4.17.11"
},
"devDependencies": {
+ "coveralls": "latest",
"mocha": "latest",
"nyc": "latest",
"should": "latest",
@@ -35,6 +36,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json
index a5fdc24d0..1ca23eb47 100644
--- a/packages/metascraper-logo/package.json
+++ b/packages/metascraper-logo/package.json
@@ -31,6 +31,7 @@
"index.js"
],
"scripts": {
+ "coverage": "exit 0",
"test": "exit 0"
},
"license": "MIT"
diff --git a/packages/metascraper-media-provider/package.json b/packages/metascraper-media-provider/package.json
index 9ceebec68..4d086f713 100644
--- a/packages/metascraper-media-provider/package.json
+++ b/packages/metascraper-media-provider/package.json
@@ -26,6 +26,7 @@
"memoize-one": "~5.0.4"
},
"devDependencies": {
+ "coveralls": "latest",
"is-ci": "latest",
"mocha": "latest",
"nyc": "latest",
@@ -42,6 +43,7 @@
"src"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json
index c426fbfdc..8995f4f51 100644
--- a/packages/metascraper-publisher/package.json
+++ b/packages/metascraper-publisher/package.json
@@ -19,6 +19,7 @@
"@metascraper/helpers": "^5.4.6"
},
"devDependencies": {
+ "coveralls": "latest",
"mocha": "latest",
"nyc": "latest",
"should": "latest",
@@ -31,6 +32,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-readability/index.js b/packages/metascraper-readability/index.js
index 61069e0ec..bfd6b95a3 100644
--- a/packages/metascraper-readability/index.js
+++ b/packages/metascraper-readability/index.js
@@ -1,24 +1,19 @@
'use strict'
-const helpers = require('@metascraper/helpers')
-const { eq, get, invoke } = require('lodash')
+const { createValidator } = require('@metascraper/helpers')
const Readability = require('readability')
const memoizeOne = require('memoize-one')
const { JSDOM } = require('jsdom')
-const memoFn = (newArgs, oldArgs) => eq(newArgs[1].url, oldArgs[1].url)
+const memoFn = (newArgs, oldArgs) => newArgs[0].url === oldArgs[0].url
-const readability = memoizeOne((html, opts) => {
- const dom = new JSDOM(html, opts)
+const readability = memoizeOne(({ htmlDom, url }) => {
+ const dom = new JSDOM(htmlDom.html(), { url })
const reader = new Readability(dom.window.document)
return reader.parse()
}, memoFn)
-const getReadbility = ({ from, to = from }) => ({ htmlDom, url }) => {
- const data = readability(htmlDom.html(), { url })
- const value = get(data, from)
- return invoke(helpers, to, value)
-}
+const getReadbility = createValidator(readability)
module.exports = () => {
return {
diff --git a/packages/metascraper-readability/package.json b/packages/metascraper-readability/package.json
index 74d2cd25a..82a2f3746 100644
--- a/packages/metascraper-readability/package.json
+++ b/packages/metascraper-readability/package.json
@@ -18,11 +18,11 @@
"dependencies": {
"@metascraper/helpers": "^5.4.6",
"jsdom": "~15.1.1",
- "lodash": "~4.17.11",
"memoize-one": "~5.0.4",
"readability": "github:mozilla/readability"
},
"devDependencies": {
+ "coveralls": "latest",
"metascraper": "latest",
"mocha": "latest",
"nyc": "latest",
@@ -36,6 +36,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json
index 105f72313..3c5929954 100644
--- a/packages/metascraper-soundcloud/package.json
+++ b/packages/metascraper-soundcloud/package.json
@@ -20,6 +20,7 @@
"@metascraper/helpers": "^5.4.6"
},
"devDependencies": {
+ "coveralls": "latest",
"mocha": "latest",
"nyc": "latest",
"should": "latest",
@@ -33,6 +34,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json
index aea39106d..dd2c0f46c 100644
--- a/packages/metascraper-title/package.json
+++ b/packages/metascraper-title/package.json
@@ -29,6 +29,7 @@
"index.js"
],
"scripts": {
+ "coverage": "exit 0",
"test": "exit 0"
},
"license": "MIT"
diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json
index c387ed5a1..045f94d53 100644
--- a/packages/metascraper-url/package.json
+++ b/packages/metascraper-url/package.json
@@ -28,6 +28,7 @@
"index.js"
],
"scripts": {
+ "coverage": "exit 0",
"test": "exit 0"
},
"license": "MIT"
diff --git a/packages/metascraper-video/index.js b/packages/metascraper-video/index.js
index ab2708bea..356eb99ad 100644
--- a/packages/metascraper-video/index.js
+++ b/packages/metascraper-video/index.js
@@ -1,11 +1,6 @@
'use strict'
-const {
- isMime,
- url: urlFn,
- isVideoUrl,
- extension
-} = require('@metascraper/helpers')
+const { isMime, url: urlFn, extension, video } = require('@metascraper/helpers')
const { chain } = require('lodash')
/**
@@ -29,8 +24,8 @@ const wrapVideo = createWrapper((domNodes, url) => {
.orderBy(videoUrl => extension(videoUrl) === 'mp4', ['desc'])
.first()
.value()
- const urlValue = urlFn(videoUrl, { url })
- return isVideoUrl(urlValue) && urlValue
+
+ return video(videoUrl, { url })
})
const withContentType = (url, contentType) =>
diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json
index 841dd8e73..9812f0f1d 100644
--- a/packages/metascraper-video/package.json
+++ b/packages/metascraper-video/package.json
@@ -20,6 +20,7 @@
"lodash": "~4.17.11"
},
"devDependencies": {
+ "coveralls": "latest",
"mocha": "latest",
"nyc": "latest",
"should": "latest",
@@ -33,6 +34,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json
index eee3d4837..377529d9a 100644
--- a/packages/metascraper-youtube/package.json
+++ b/packages/metascraper-youtube/package.json
@@ -23,6 +23,7 @@
"p-locate": "~4.1.0"
},
"devDependencies": {
+ "coveralls": "latest",
"mocha": "latest",
"nyc": "latest",
"should": "latest",
@@ -36,6 +37,7 @@
"index.js"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",
diff --git a/packages/metascraper/package.json b/packages/metascraper/package.json
index 22e87eded..d70216199 100644
--- a/packages/metascraper/package.json
+++ b/packages/metascraper/package.json
@@ -66,7 +66,8 @@
},
"devDependencies": {
"clear-module": "latest",
- "metascraper-clearbit-logo": "latest",
+ "coveralls": "latest",
+ "metascraper-clearbit": "latest",
"mocha": "latest",
"nyc": "latest",
"should": "latest",
@@ -80,6 +81,7 @@
"src"
],
"scripts": {
+ "coverage": "nyc report --reporter=text-lcov | coveralls",
"test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
},
"license": "MIT",