From a04231273070f7a54a527e5063d7193fba063b44 Mon Sep 17 00:00:00 2001 From: Sven Date: Fri, 26 Jun 2020 11:34:18 +0200 Subject: [PATCH 01/13] version bump --- changelog.txt | 8 ++++++-- package-lock.json | 2 +- package.json | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/changelog.txt b/changelog.txt index 73c5ae59..9c283168 100755 --- a/changelog.txt +++ b/changelog.txt @@ -1,7 +1,7 @@ ==================== JokeAPI Changelog - - Version 2.1.4 - + - Version 2.1.5 - ==================== @@ -20,7 +20,11 @@ - Daemonize the auth token refreshing -[CURRENT: 2.1.4] - 2020 Q3 general patch #1 +[CURRENT: 2.1.5] - 2020 Q3 general patch #2 + - Ditched the botched rate limiting package for a "commercial" one + + +[2.1.4] - 2020 Q3 general patch #1 - Fixed the IP getter module for like the 500th time now diff --git a/package-lock.json b/package-lock.json index b5b08906..28877a66 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@sv443/jokeapi", - "version": "2.1.4", + "version": "2.1.5", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index c05d9f09..8ad31fad 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sv443/jokeapi", - "version": "2.1.4", + "version": "2.1.5", "description": "A RESTful API that serves jokes from many categories while also offering a lot of filtering methods", "main": "JokeAPI.js", "homepage": "https://sv443.net/jokeapi/v2", From 70ca79d1277dad91213e089ad3ffcfb93944068a Mon Sep 17 00:00:00 2001 From: Sven Date: Fri, 26 Jun 2020 12:40:35 +0200 Subject: [PATCH 02/13] implemented third party rate limiting package --- docs/raw/index.html | 14 ++++- package-lock.json | 5 ++ package.json | 1 + settings.js | 4 +- src/analytics.js | 11 +++- src/fileFormatConverter.js | 3 +- src/httpServer.js | 117 +++++++++++++++++++++++++++++-------- 7 files changed, 124 insertions(+), 31 deletions(-) diff --git a/docs/raw/index.html b/docs/raw/index.html index 8a852288..e8a79916 100755 --- a/docs/raw/index.html +++ b/docs/raw/index.html @@ -538,12 +538,12 @@

By using this website and API you are agreeing to - In all response data, no matter from which endpoint will have an "error" parameter.
+ In all response data, no matter from which endpoint will have a boolean "error" parameter.
Usually, it is set to false, which means there was no error in your request or on the server.
If the "error" parameter is set to true though, there was an error and will add an "internalError", "message", "causedBy", "additionalInfo" and "timestamp" parameter.

The "internalError" parameter will be set to false, if the error was due to a malformed or failed request and it will be true, if the error is 's fault.
- The "message" parameter will contain a short version of the error message and the "timestamp" parameter will contain a 13-character UNIX timestamp in Central European time (GMT+1).
+ The "message" parameter will contain a short version of the error message and the "timestamp" parameter will contain a 13-character Unix timestamp.
The "causedBy" parameter is an array of possible causes of this error and the "addidionalInfo" parameter contains a more descriptive error message.

You can view an example of an error, where an invalid category was used, just below this paragraph.
@@ -848,6 +848,15 @@

By using this website and API you are agreeing to + + + + has a way of whitelisting certain clients. This is achieved through an API token.
+ At the moment, you will only receive one of these tokens temporarily if something breaks or if you are a business and need more than requests per minute.
+ If you do end up receiving a token, using it is as easy as adding an X-API-Token header with the actual token as its value. + + + These are some examples in some commonly used languages to show you how you could implement :

@@ -1185,6 +1194,7 @@

By using this website and API you are agreeing to Filtering Jokes + API Tokens Examples

Dependencies:
- These are the packages depends on (excluding dependencies of dependencies): + These are the packages depends on (excluding dependencies of dependencies and devDependencies):
From 46d01b2d08b0f873cf77d5c6e4ba56e5c31449d5 Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 20:05:30 +0200 Subject: [PATCH 04/13] fixed IP hashing --- settings.js | 2 +- src/resolveIP.js | 83 +++--------------------------------------------- 2 files changed, 5 insertions(+), 80 deletions(-) diff --git a/settings.js b/settings.js index 3b61298a..780ceda1 100755 --- a/settings.js +++ b/settings.js @@ -175,7 +175,7 @@ const settings = { }, auth: { tokenListFile: "./data/tokens.json", // path to the token list file - tokenHeaderName: "x-auth-token", // the name of the token header (lower case) + tokenHeaderName: "authorization", // the name of the token header (in lower case) } } diff --git a/src/resolveIP.js b/src/resolveIP.js index da8a1e63..54631944 100755 --- a/src/resolveIP.js +++ b/src/resolveIP.js @@ -25,88 +25,13 @@ const resolveIP = req => { ipaddr = req.headers["cf-pseudo-ipv4"]; } - return ipaddr; - - /* - Procedure: - 1. HEAD x-forwarded-for - 2. HEAD cf-connecting-ip - 3. HEAD x-proxyuser-ip - 4. HEAD cf-pseudo-ipv4 - 5. VAL HTTP_REMOTE_ADDR - 6. VAL err_no_IP::HEAD_cf-ipcountry - 7. VAL err_no_IP - 8. VAL err_couldnt_hash - */ - - /* - - let usedRevProxy = false; - - try - { - if(req.headers && settings.httpServer.reverseProxy && req.headers["x-forwarded-for"]) // format: , , - { - ipaddr = req.headers["x-forwarded-for"]; // reverse proxy adds this header - - let ipSplit = ipaddr.split(/[,]\s{0,}/gm); - if(ipaddr.includes(",") || isValidIP(ipSplit[0])) - ipaddr = ipSplit[0] || null; // try to get IP from - else if(!isValidIP(ipaddr)) - { - // else if IP invalid: - if((Array.isArray(ipSplit) && ipSplit.length >= 1) && ipaddr.includes(",") && isValidIP(ipSplit[1])) - ipaddr = ipSplit[1]; // if IP is invalid, try instead - else - ipaddr = req.connection.remoteAddress || null; // else just default to the remote IP or if that doesn't exist, null - } - usedRevProxy = true; - } - else - ipaddr = req.connection.remoteAddress || null; // if reverse proxy is disabled, default to the remote IP or if that doesn't exist, null - } - catch(err) - { - ipaddr = null; // if any error is thrown, default to null - } - - ipaddr = ipaddr.toString().trim(); // trim whitespaces - ipaddr = (ipaddr != null && isValidIP(ipaddr)) ? ipaddr : null; // if the IP up to this point is valid, leave it as it is, else set it to null - - if(jsl.isEmpty(ipaddr) || (settings.httpServer.reverseProxy && !usedRevProxy)) // if the reverse proxy didn't work, try getting the IP from the Gateway / Proxy headers + if(ipaddr == null) { - if(!jsl.isEmpty(req.headers["cf-connecting-ip"]) && (isValidIP(req.headers["cf-connecting-ip"]))) // Cloudflare - ipaddr = req.headers["cf-connecting-ip"]; - else if(!jsl.isEmpty(req.headers["x-proxyuser-ip"]) && (isValidIP(req.headers["x-proxyuser-ip"]))) // Google services - ipaddr = req.headers["x-proxyuser-ip"]; - else if(!jsl.isEmpty(req.headers["cf-pseudo-ipv4"]) && isValidIP(req.headers["cf-pseudo-ipv4"])) // Cloudflare pseudo IPv4 replacement if IPv6 isn't recognized - ipaddr = req.headers["cf-pseudo-ipv4"]; - else if(ipaddr == null || !isValidIP(ipaddr)) - ipaddr = "err_no_IP"; + if(req.headers && typeof req.headers["cf_ipcountry"] == "string") + ipaddr = `unknown_${req.headers["cf_ipcountry"]}`; } - if((ipaddr == "err_no_IP" || ipaddr == null) && typeof req.headers["cf-ipcountry"] == "string") - ipaddr = `err_no_IP::${req.headers["cf-ipcountry"]}`; - else if(ipaddr == null) - ipaddr = "err_no_IP" - - ipaddr = (ipaddr.length < 15 ? ipaddr : (ipaddr.substr(0,7) === "::ffff:" ? ipaddr.substr(7) : ipaddr)); - - try - { - if(settings.httpServer.ipHashing.enabled && isValidIP(ipaddr)) - ipaddr = hashIP(ipaddr); - - return typeof ipaddr == "string" ? ipaddr : ipaddr.toString(); - } - catch(err) - { - if(typeof req.headers["cf-ipcountry"] == "string") - return `err_couldnt_hash::${req.headers["cf-ipcountry"]}`; - else - return "err_couldnt_hash"; - } - */ + return hashIP(ipaddr); }; /** From 7a5b778487783aa6188b6b2412d4c784ecc89b57 Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 20:46:41 +0200 Subject: [PATCH 05/13] fixed auth token stuff --- docs/raw/index.css | 19 ++++++++++--------- docs/raw/index.html | 8 ++++++-- endpoints/info.js | 18 +++++++++++++----- settings.js | 9 +++++---- src/auth.js | 16 +++++++++++++--- src/httpServer.js | 15 +++++++-------- src/resolveIP.js | 2 +- tools/add-token.js | 4 ++-- 8 files changed, 57 insertions(+), 34 deletions(-) diff --git a/docs/raw/index.css b/docs/raw/index.css index 82cdb2a3..6afe87be 100755 --- a/docs/raw/index.css +++ b/docs/raw/index.css @@ -22,7 +22,8 @@ --doc-header-font-size: 27px; --doc-sub-header-font-size: 19px; - --sidenav-animation-speed: 0.3s; + --sidenav-animation-speed: 0.15s; + --sidenav-colorblur-speed: 0.5s; --targetblink-animation-delay: 0.5s; --scrollbar-track-color: #222; @@ -54,7 +55,7 @@ body { overflow-y: hidden; - transition: background-color var(--sidenav-animation-speed); + transition: background-color ease-out var(--sidenav-colorblur-speed); /* I had to do this for some unknown reason - don't ask me why: */ -webkit-touch-callout: inherit; @@ -172,7 +173,7 @@ header { background-color: var(--header-bg-color); z-index: 1000; - transition: filter linear var(--sidenav-animation-speed), width linear var(--sidenav-animation-speed); + transition: filter ease-out var(--sidenav-colorblur-speed), width linear var(--sidenav-animation-speed); display: flex; flex-direction: row; @@ -351,7 +352,7 @@ mark { padding: 3px; font-family: "Cascadia Code", "Roboto", "Courier New", monospace; color: #ccc; - transition: background-color linear var(--sidenav-animation-speed); + transition: background-color ease-out var(--sidenav-colorblur-speed); } body[data-sidenav="opened"] mark { @@ -421,7 +422,7 @@ body[data-sidenav="closed"] code, body code { filter: grayscale(0%); - transition: filter linear var(--sidenav-animation-speed); + transition: filter ease-out var(--sidenav-colorblur-speed); } body code > .actualCode { @@ -440,7 +441,7 @@ body[data-sidenav="opened"] code { filter: grayscale(50%); - transition: filter linear var(--sidenav-animation-speed); + transition: filter ease-out var(--sidenav-colorblur-speed); } body code .nocode.codeheader { @@ -504,12 +505,12 @@ ul.lispacer li { padding: 3px; padding-left: 10px; padding-right: 40px; - transition: background-color linear var(--sidenav-animation-speed); + transition: background-color ease-out var(--sidenav-colorblur-speed); } body[data-sidenav="opened"] .requestURLwrapper { background-color: var(--bg-accent-color-darker-sidenav-opened); - transition: background-color linear var(--sidenav-animation-speed); + transition: background-color ease-out var(--sidenav-colorblur-speed); } .requestMethodGET, .requestMethodPUT { @@ -950,4 +951,4 @@ input[disabled]+label { #urlBuilderUrl { display: inline-block; padding: 10px 5px 10px 8px; -} \ No newline at end of file +} diff --git a/docs/raw/index.html b/docs/raw/index.html index 75b947a4..50b82a66 100755 --- a/docs/raw/index.html +++ b/docs/raw/index.html @@ -671,8 +671,11 @@

By using this website and API you are agreeing to This endpoint provides some information on :
- The version number
- The amount of jokes
+ - All the available categories, flags, types and formats
- A 13-character UNIX timestamp
- - The URL to a joke submission form + - The URL to a joke submission form
+ - The minimum and maximum values of an ID range
+ - A string with some information, like a message of the day

Supported URL parameters:
    @@ -853,7 +856,8 @@

    By using this website and API you are agreeing to has a way of whitelisting certain clients. This is achieved through an API token.
    At the moment, you will only receive one of these tokens temporarily if something breaks or if you are a business and need more than requests per minute.
    - If you do end up receiving a token, using it is as easy as adding an X-API-Token header with the actual token as its value. + If you do end up receiving a token, using it is as easy as adding an Authorization header with the actual token as its value.
    + You will receive a response header called Token-Valid which will contain the number 1 or 0, depending on the validity of the token. diff --git a/endpoints/info.js b/endpoints/info.js index b91cfb02..18b550ed 100755 --- a/endpoints/info.js +++ b/endpoints/info.js @@ -57,6 +57,8 @@ const call = (req, res, url, params, format) => { }; } + let totalJokesCount = (!jsl.isEmpty(parseJokes.jokeCount) ? parseJokes.jokeCount : 0); + if(format != "xml") { responseText = convertFileFormat.auto(format, { @@ -64,11 +66,14 @@ const call = (req, res, url, params, format) => { "version": settings.info.version, "jokes": { - "totalCount": (!jsl.isEmpty(parseJokes.jokeCount) ? parseJokes.jokeCount : 0), + "totalCount": totalJokesCount, "categories": [settings.jokes.possible.anyCategoryName, ...settings.jokes.possible.categories], "flags": settings.jokes.possible.flags, - "submissionURL": settings.jokes.jokeSubmissionURL + "types": settings.jokes.possible.types, + "submissionURL": settings.jokes.jokeSubmissionURL, + "idRange": [ 0, --totalJokesCount ] }, + "formats": settings.jokes.possible.formats, "info": settings.info.infoMsg, "timestamp": new Date().getTime() }); @@ -80,11 +85,14 @@ const call = (req, res, url, params, format) => { "version": settings.info.version, "jokes": { - "totalCount": (!jsl.isEmpty(parseJokes.jokeCount) ? parseJokes.jokeCount : 0), + "totalCount": totalJokesCount, "categories": {"category": [settings.jokes.possible.anyCategoryName, ...settings.jokes.possible.categories]}, "flags": {"flag": settings.jokes.possible.flags}, - "submissionURL": settings.jokes.jokeSubmissionURL + "types": {"type": settings.jokes.possible.types}, + "submissionURL": settings.jokes.jokeSubmissionURL, + "idRange": [ 0, --totalJokesCount ] }, + "formats": {"format": settings.jokes.possible.formats}, "info": settings.info.infoMsg, "timestamp": new Date().getTime() }); @@ -93,4 +101,4 @@ const call = (req, res, url, params, format) => { httpServer.pipeString(res, responseText, parseURL.getMimeTypeFromFileFormatString(format)); }; -module.exports = { meta, call }; \ No newline at end of file +module.exports = { meta, call }; diff --git a/settings.js b/settings.js index 780ceda1..320be4dc 100755 --- a/settings.js +++ b/settings.js @@ -97,7 +97,7 @@ const settings = { httpServer: { port: 8076, // http server port allowCORS: true, // whether or not to allow Cross Origin Resource Sharing - rateLimiting: 2, // amount of allowed requests per below defined timeframe + rateLimiting: 60, // amount of allowed requests per below defined timeframe timeFrame: 60, // timeframe in seconds - also supports floating point numbers urlPathOffset: 0, // example: "/jokeapi/info" with an offset of 1 will only start parsing the path beginning at "info" - an Apache reverse proxy will do this automatically though maxPayloadSize: 5120, // max size (in bytes) that will be accepted in a PUT request - if payload exceeds this size, it will abort with status 413 @@ -105,9 +105,9 @@ const settings = { disableCache: true, // whether or not to disable the cache - default: true (setting to false may prevent the users from getting new jokes) infoHeaders: true, // whether or not to add an informational header about JokeAPI to each request reverseProxy: true, // whether or not JokeAPI gets its requests from a reverse proxy - ipSanitization: { // used to sanitize IP addresses so they can be used in file paths + ipSanitization: { // used to sanitize IP addresses so they can be used in file paths regex: /[^A-Za-z0-9\-_./]|^COM[0-9]([/.]|$)|^LPT[0-9]([/.]|$)|^PRN([/.]|$)|^CLOCK\$([/.]|$)|^AUX([/.]|$)|^NUL([/.]|$)|^CON([/.]|$)/gm, - replaceChar: "#", // what character to use instead of illegal characters + replaceChar: "#", // what character to use instead of illegal characters }, ipHashing: { enabled: true, // hashes all IP addresses. If set to false, JokeAPI is not GDPR compliant anymore! @@ -176,7 +176,8 @@ const settings = { auth: { tokenListFile: "./data/tokens.json", // path to the token list file tokenHeaderName: "authorization", // the name of the token header (in lower case) + tokenValidHeader: "Token-Valid", // the name of the token validity response header (normal case, not lower case) } } -module.exports = settings; +module.exports = Object.freeze(settings); // use Object.freeze() to prevent modifications at runtime diff --git a/src/auth.js b/src/auth.js index ed860c06..9ca10c9d 100755 --- a/src/auth.js +++ b/src/auth.js @@ -30,12 +30,19 @@ const init = () => { }); }; +/** + * @typedef {Object} Authorization + * @prop {Boolean} isAuthorized + * @prop {String} token + */ + /** * Checks if the requester has provided an auth header and if the auth header is valid * @param {http.IncomingMessage} req - * @returns {Object} + * @param {http.ServerResponse} [res] If not provided, users will not get the `Token-Valid` response header + * @returns {Authorization} */ -const authByHeader = (req) => { +const authByHeader = (req, res) => { let isAuthorized = false; let requestersToken = ""; @@ -51,6 +58,9 @@ const authByHeader = (req) => { } }); } + + if(res && typeof res.setHeader == "function") + res.setHeader(settings.auth.tokenValidHeader, (isAuthorized ? "1" : "0")); } return { @@ -59,4 +69,4 @@ const authByHeader = (req) => { }; }; -module.exports = { init, authByHeader }; \ No newline at end of file +module.exports = { init, authByHeader }; diff --git a/src/httpServer.js b/src/httpServer.js index 3a2ee734..80828fd1 100755 --- a/src/httpServer.js +++ b/src/httpServer.js @@ -41,7 +41,7 @@ const init = () => { let parsedURL = parseURL(req.url); let ip = resolveIP(req); let localhostIP = resolveIP.isLocal(ip); - let hasHeaderAuth = auth.authByHeader(req); + let headerAuth = auth.authByHeader(req, res); let analyticsObject = { ipAddress: ip, urlPath: parsedURL.pathArray, @@ -128,7 +128,7 @@ const init = () => { { let rlRes = await rl.get(ip); - if((rlRes && rlRes._remainingPoints < 0) && !lists.isWhitelisted(ip) && !hasHeaderAuth) + if((rlRes && rlRes._remainingPoints < 0) && !lists.isWhitelisted(ip) && !headerAuth.isAuthorized) { setRateLimitedHeaders(res, rlRes); analytics.rateLimited(ip); @@ -163,12 +163,11 @@ const init = () => { endpoints.forEach(async (ep) => { if(ep.name == requestedEndpoint) { - let authHeaderObj = auth.authByHeader(req); - let hasHeaderAuth = authHeaderObj.isAuthorized; - let headerToken = authHeaderObj.token; + let isAuthorized = headerAuth.isAuthorized; + let headerToken = headerAuth.token; // now that the request is not a docs / favicon request, the blacklist is checked and the request is made eligible for rate limiting - if(!settings.endpoints.ratelimitBlacklist.includes(ep.name) && !hasHeaderAuth) + if(!settings.endpoints.ratelimitBlacklist.includes(ep.name) && !isAuthorized) { try { @@ -180,7 +179,7 @@ const init = () => { } } - if(hasHeaderAuth) + if(isAuthorized) { debug("HTTP", `Requester has valid token ${jsl.colors.fg.green}${req.headers[settings.auth.tokenHeaderName] || null}${jsl.colors.rst}`); analytics({ @@ -221,7 +220,7 @@ const init = () => { { let rlRes = await rl.get(ip); - if((rlRes &&rlRes._remainingPoints < 0) && !lists.isWhitelisted(ip) && !hasHeaderAuth) + if((rlRes &&rlRes._remainingPoints < 0) && !lists.isWhitelisted(ip) && !isAuthorized) { setRateLimitedHeaders(res, rlRes); logRequest("ratelimited", `IP: ${ip}`, analyticsObject); diff --git a/src/resolveIP.js b/src/resolveIP.js index 54631944..a0439249 100755 --- a/src/resolveIP.js +++ b/src/resolveIP.js @@ -31,7 +31,7 @@ const resolveIP = req => { ipaddr = `unknown_${req.headers["cf_ipcountry"]}`; } - return hashIP(ipaddr); + return settings.httpServer.ipHashing.enabled ? hashIP(ipaddr) : ipaddr; }; /** diff --git a/tools/add-token.js b/tools/add-token.js index a57a49b5..a78c06a0 100755 --- a/tools/add-token.js +++ b/tools/add-token.js @@ -20,7 +20,7 @@ try for(let i = 0; i < amount; i++) { - let tok = jsl.generateUUID.custom("xxxxyyyy_xxxxyyyyxxxxyyyy_xxxxyyyyxxxxyyyy", "0123456789abcdefghijklmnopqrstuvwxyz!?$ยง%"); + let tok = jsl.generateUUID.custom("xxxxyyyyxxxxyyyy_xxxxyyyyxxxxyyyy_xxxxyyyyxxxxyyyy_xxxxyyyyxxxxyyyy", "0123456789abcdefghijklmnopqrstuvwxyz!?$ยง%*.~"); let oldFile = []; if(fs.existsSync(settings.auth.tokenListFile)) @@ -48,4 +48,4 @@ try catch(err) { return process.exit(1); -} \ No newline at end of file +} From 2db484acf633d4c70d183c29c1aaee5f2e62d69a Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 21:10:45 +0200 Subject: [PATCH 06/13] cleaned up callback hell in main.js --- docs/raw/index.html | 4 +-- package-lock.json | 5 +++ package.json | 1 + src/main.js | 84 +++++++++++++++++++++++++-------------------- 4 files changed, 55 insertions(+), 39 deletions(-) diff --git a/docs/raw/index.html b/docs/raw/index.html index 50b82a66..074bf459 100755 --- a/docs/raw/index.html +++ b/docs/raw/index.html @@ -250,8 +250,8 @@

    By using this website and API you are agreeing to
    - - + +
    diff --git a/package-lock.json b/package-lock.json index 158ed3df..19b6df60 100755 --- a/package-lock.json +++ b/package-lock.json @@ -2978,6 +2978,11 @@ "asap": "~2.0.3" } }, + "promise-all-sequential": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/promise-all-sequential/-/promise-all-sequential-1.0.0.tgz", + "integrity": "sha512-XQPGPgQZERb1mYEpqSR5tIsC2E5amQZgnqVUPK6gsDtcOv/Yjxmz99cOA1MxLWcuWXz1lZZiD3hERA+rFVdP2w==" + }, "protocols": { "version": "1.4.7", "resolved": "https://registry.npmjs.org/protocols/-/protocols-1.4.7.tgz", diff --git a/package.json b/package.json index b24e770b..54d54db8 100755 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "json-to-pretty-yaml": "^1.2.2", "mysql": "^2.18.1", "node-wrap": "^0.2.0", + "promise-all-sequential": "^1.0.0", "rate-limiter-flexible": "^2.1.7", "request-ip": "^2.1.3", "snyk": "^1.321.0", diff --git a/src/main.js b/src/main.js index 291ef8bc..66790377 100755 --- a/src/main.js +++ b/src/main.js @@ -17,13 +17,12 @@ const docs = require("./docs"); const analytics = require("./analytics"); const logRequest = require("./logRequest"); const auth = require("./auth"); +const promiseAllSequential = require("promise-all-sequential"); const col = jsl.colors.fg; process.debuggerActive = jsl.inDebugger(); const noDbg = process.debuggerActive || false; -let pb; - settings.init.exitSignals.forEach(sig => { process.on(sig, () => softExit(0)); }); @@ -36,41 +35,52 @@ const initAll = () => { process.jokeapi = {}; initializeDirs(); - //#SECTION parse jokes + let initPromises = []; + let initStages = [ + { + name: "Initializing joke parser", + fn: parseJokes.init + }, + { + name: "Initializing lists", + fn: lists.init + }, + { + name: "Initializing documentation", + fn: docs.init + }, + { + name: "Initializing Authorization module", + fn: auth.init + }, + { + name: "Initializing HTTP listener", + fn: httpServer.init + }, + { + name: "Initializing analytics module", + fn: analytics.init + } + ]; + + let pb; if(!noDbg && !settings.debug.progressBarDisabled) - pb = new jsl.ProgressBar(6, "Parsing Jokes..."); - - parseJokes.init().then(() => { - - //#SECTION init lists - if(!jsl.isEmpty(pb)) pb.next("Initializing lists..."); - lists.init().then(() => { - - //#SECTION init documentation page - if(!jsl.isEmpty(pb)) pb.next("Initializing documentation..."); - docs.init().then(() => { - - //#SECTION init auth - if(!jsl.isEmpty(pb)) pb.next("Initializing Authorization module..."); - auth.init().then(() => { - - //#SECTION init HTTP server - if(!jsl.isEmpty(pb)) pb.next("Initializing HTTP listener..."); - httpServer.init().then(() => { - - //#SECTION init analytics - if(!jsl.isEmpty(pb)) pb.next("Initializing analytics module..."); - analytics.init().then(() => { - if(!jsl.isEmpty(pb)) pb.next("Done."); - logRequest.initMsg(initTimestamp); - - // done. - }).catch(err => initError("initializing the analytics module", err)); - }).catch(err => initError("initializing the HTTP server", err)); - }).catch(err => initError("initializing the Auth module", err)); - }).catch(err => initError("initializing documentation", err)); - }).catch(err => initError("initializing the lists", err)); - }).catch(err => initError("parsing jokes", err)); + pb = new jsl.ProgressBar(initStages.length, initStages[0].name); + + initStages.forEach(stage => { + initPromises.push(stage.fn); + }); + + promiseAllSequential(initPromises).then((res) => { + jsl.unused(res); + + if(!jsl.isEmpty(pb)) + pb.next("Done."); + + logRequest.initMsg(initTimestamp); + }).catch(err => { + initError("initializing", err); + }); }; @@ -117,4 +127,4 @@ const softExit = code => { module.exports = { softExit }; -initAll(); \ No newline at end of file +initAll(); From 53af59a7b7aaa1f73a9294f7f2227fdfcd7fdb7a Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 21:26:05 +0200 Subject: [PATCH 07/13] rephrased api token section in docs --- docs/raw/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/raw/index.html b/docs/raw/index.html index 074bf459..5e327c72 100755 --- a/docs/raw/index.html +++ b/docs/raw/index.html @@ -857,7 +857,8 @@

    By using this website and API you are agreeing to has a way of whitelisting certain clients. This is achieved through an API token.
    At the moment, you will only receive one of these tokens temporarily if something breaks or if you are a business and need more than requests per minute.
    If you do end up receiving a token, using it is as easy as adding an Authorization header with the actual token as its value.
    - You will receive a response header called Token-Valid which will contain the number 1 or 0, depending on the validity of the token. + You will receive a response header called Token-Valid which will contain the number 1 if the token is valid.
    + If the token is invalid, the Token-Valid header will be set to 0 and will treat your request as if you didn't supply a token in the first place. From 8d3b13b53a9dc6c7444463a21ce3b3a7421a0c08 Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 21:44:25 +0200 Subject: [PATCH 08/13] updated changelog --- changelog.txt | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/changelog.txt b/changelog.txt index 9c283168..fa6bdb6f 100755 --- a/changelog.txt +++ b/changelog.txt @@ -8,20 +8,27 @@ [PLANNED: 2.2.0] - - Allow definition of max requests per minute per each client (GitHub issue #37) - - Add support for jokes and error messages of different languages (GitHub issue #75) + - Add joke category "Pun" (issue #105) + - Allow definition of max requests per minute per each client (issue #37) + - Add support for jokes and error messages of different languages (issue #75) - /langcode/{LANGUAGE} endpoint - /languages endpoint - "?lang=code" URL parameter - - Fix ID caching (again, sigh) - - Add pm2 custom metrics - - Fix HTTP 403 errors - - Remake the URL parser using a package - - Daemonize the auth token refreshing + - Fix ID caching (again, sigh) (issue #80) + - Add pm2 custom metrics (issue #91) + - Fix HTTP 403 errors (issue #96) + - Remake the URL parser using a "commercial" package (issue #97) + - Daemonize the API token refreshing (issue #102) + - Rate limit joke submissions more harshly (issue #104) + - Fix resources not being served with Brotli even though it is supported by the client (issue #106) [CURRENT: 2.1.5] - 2020 Q3 general patch #2 - - Ditched the botched rate limiting package for a "commercial" one + - Ditched my botched rate limiting package for a "commercial" one + - Added API token section to documentation + - Client now receives a "Token-Valid" header with the value 0 or 1 depending on token validity + - Renamed "X-Auth-Token" header to "Authorization" so requests don't get blocked by Cloudflare + - Cleaned up a lot of code [2.1.4] - 2020 Q3 general patch #1 From b3cf07f85542d5cdc0cee52d7b82ba77724dd72a Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 21:45:23 +0200 Subject: [PATCH 09/13] updated changelog --- changelog.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/changelog.txt b/changelog.txt index fa6bdb6f..78b47916 100755 --- a/changelog.txt +++ b/changelog.txt @@ -24,10 +24,10 @@ [CURRENT: 2.1.5] - 2020 Q3 general patch #2 - - Ditched my botched rate limiting package for a "commercial" one - - Added API token section to documentation - - Client now receives a "Token-Valid" header with the value 0 or 1 depending on token validity - - Renamed "X-Auth-Token" header to "Authorization" so requests don't get blocked by Cloudflare + - Ditched my botched rate limiting package for a "commercial" one (issue #113) + - Added API token section to documentation (issue #114) + - Client now receives a "Token-Valid" header with the value 0 or 1 depending on token validity (issue #115) + - Renamed "X-Auth-Token" header to "Authorization" so requests don't get blocked by Cloudflare (issue #117) - Cleaned up a lot of code @@ -36,7 +36,7 @@ [2.1.3] - 2020 Q2 general patch #1 - - Added option to disable all console output but error messages (GitHub isse #72) + - Added option to disable all console output but error messages (GitHub issue #72) - The content of jokes in the joke submission form is now correctly escaped and can no longer mess up the page (GitHub issue #68) - Fixed crash when parsing a malformatted URI (GitHub issue #69 (nice)) - Re-flagged some jokes From 3d8db1fd2d4a5dbadde194d3384734ab6d2d5d54 Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 21:57:55 +0200 Subject: [PATCH 10/13] modified docs and readme --- README.md | 12 +++++++++++- docs/raw/index.css | 2 +- docs/raw/index.html | 13 ++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 41324073..ed3817fd 100755 --- a/README.md +++ b/README.md @@ -25,11 +25,18 @@

    + + +
    + # Official API Wrappers: - [Rust](https://github.com/canarado/sv443_jokeapi_wrapper) - [Python](https://github.com/thenamesweretakenalready/Sv443s-JokeAPI-Python-Wrapper) -
    +    		 + +      + + # Projects (I'm aware of) that use JokeAPI: - [dozens Advices](https://github.com/ZephyrVentum/dozens-Advices) by [ZephyrVentum](https://github.com/ZephyrVentum) @@ -40,6 +47,9 @@ ([Contact me](https://sv443.net/discord) to get your project added here) +
    +

    diff --git a/docs/raw/index.css b/docs/raw/index.css index 6afe87be..b85a69f0 100755 --- a/docs/raw/index.css +++ b/docs/raw/index.css @@ -322,7 +322,7 @@ label { } #statusCodeTable tr td.dr, span.col.dr { - color: #df0909; + color: #f52727; } .colB { diff --git a/docs/raw/index.html b/docs/raw/index.html index 5e327c72..3fadc056 100755 --- a/docs/raw/index.html +++ b/docs/raw/index.html @@ -615,11 +615,18 @@

    By using this website and API you are agreeing to Internal Server Error There was a general internal error within . You can get more info from the properties in the response text + + 523 + Origin Unreachable + My server is temporarily offline due to maintenance or a dynamic IP update. Please be patient in this case. +
    - Success • Client Error • Server Error + + Success • Client Error • Server Error + @@ -1137,10 +1144,14 @@

    By using this website and API you are agreeing to
  • json-to-pretty-yaml
  • MySQL
  • node-wrap
    • +
  • promise-all-sequential
  • rate-limiter-flexible
  • request-ip
  • JSLib
  • xss
    • +

From e648df12002c41fa60769e8fbae54fcc00357786 Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 21:58:38 +0200 Subject: [PATCH 11/13] reverted readme --- README.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/README.md b/README.md index ed3817fd..41324073 100755 --- a/README.md +++ b/README.md @@ -25,18 +25,11 @@

- - -
- # Official API Wrappers: - [Rust](https://github.com/canarado/sv443_jokeapi_wrapper) - [Python](https://github.com/thenamesweretakenalready/Sv443s-JokeAPI-Python-Wrapper) - - -      - - +
# Projects (I'm aware of) that use JokeAPI: - [dozens Advices](https://github.com/ZephyrVentum/dozens-Advices) by [ZephyrVentum](https://github.com/ZephyrVentum) @@ -47,9 +40,6 @@ ([Contact me](https://sv443.net/discord) to get your project added here) -
-

From 1edcd11cd1313b604e86f939569eca929554b67a Mon Sep 17 00:00:00 2001 From: Sv443 Date: Wed, 1 Jul 2020 22:02:09 +0200 Subject: [PATCH 12/13] fixed readme icon --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 41324073..80b75b6f 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-# [![icon](https://sv443.net/cdn/jokeapi/icon_150x150.png)](#readme)
JokeAPI +# [![icon](https://sv443.net/cdn/jokeapi/icon_readme.png)](#readme)
JokeAPI ### A REST API that serves uniformly and well formatted jokes and offers a great variety of filtering methods and response customization [![GitHub](https://img.shields.io/github/license/Sv443/JokeAPI)](https://sv443.net/LICENSE) [![Uptime / 7 Days](https://img.shields.io/uptimerobot/ratio/7/m782998549-afcc0d10c27c87df47e73289)](https://status.sv443.net/) From 6e59b5ae1ac3b083f0a666b98fb36b57077bd1d6 Mon Sep 17 00:00:00 2001 From: Sven Fehler Date: Thu, 2 Jul 2020 12:41:50 +0200 Subject: [PATCH 13/13] Delete ipBlacklist.json --- data/lists/ipBlacklist.json | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 data/lists/ipBlacklist.json diff --git a/data/lists/ipBlacklist.json b/data/lists/ipBlacklist.json deleted file mode 100644 index f055698d..00000000 --- a/data/lists/ipBlacklist.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - "2001:e68:5427:51d6:d8d2:8c0e:767a:4b6b", - "2001:e68:5427:51d6:2d66:94f6:2f8b:f695", - "2001:e68:5427:c50e:b852:2ee9:c30e:61e7" -] \ No newline at end of file