From c6c7b88c8538f1cd6b6d5f9b8f7d3090366409f7 Mon Sep 17 00:00:00 2001 From: Wenzhi Ding Date: Wed, 8 Jan 2025 16:16:58 +0800 Subject: [PATCH 1/8] Fix: Foreign Affairs update front-end. --- Foreign Affairs.js | 110 +++++++++++++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 33 deletions(-) diff --git a/Foreign Affairs.js b/Foreign Affairs.js index b5c7e833952..38a5da36ef4 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -1,15 +1,15 @@ { "translatorID": "4ab6d49c-d94e-4a9c-ae9a-3310c44ba612", "label": "Foreign Affairs", - "creator": "Sebastian Karcher, Philipp Zumstein", + "creator": "Sebastian Karcher, Philipp Zumstein, Wenzhi Dave Ding", "target": "^https?://www\\.foreignaffairs\\.com", "minVersion": "3.0", "maxVersion": "", - "priority": 100, + "priority": 200, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-08-23 12:37:06" + "lastUpdated": "2025-01-08 08:08:54" } /* @@ -37,7 +37,7 @@ function detectWeb(doc, _url) { - if (doc.getElementsByClassName('article-body-text').length) { + if (doc.getElementsByClassName('article').length) { return "magazineArticle"; } else if (getSearchResults(doc, true)) { @@ -47,19 +47,23 @@ function detectWeb(doc, _url) { } function getSearchResults(doc, checkOnly) { - let isIssues = /^https:\/\/[^/]+\/issues\/.+/.test(doc.location.href); var items = {}; var found = false; - var rows = doc.querySelectorAll('article.article-card > a, div.article-data > h2.title > a'); + + let isIssues = /^https:\/\/[^/]+\/issues\/.+/.test(doc.location.href); + let isSearch = /^https:\/\/[^/]+\/search\/.+/.test(doc.location.href); + + let rows = []; + if (isIssues) { + rows = doc.querySelectorAll('h3 > a'); + } + if (isSearch) { + rows = doc.querySelectorAll('h2 > a'); + } + for (let row of rows) { let href = row.href; - let title; - if (isIssues) { - title = text(row, ".article-card-title"); - } - else { - title = ZU.trimInternal(row.textContent); - } + let title = ZU.trimInternal(row.textContent); if (!href || !title) continue; if (checkOnly) return true; found = true; @@ -83,22 +87,21 @@ async function doWeb(doc, url) { async function scrape(doc, url = doc.location.href) { var item = new Zotero.Item("magazineArticle"); - var author = text(doc, '.article-byline-author'); - var tags = doc.querySelectorAll('.article-footer--tag-item'); - let issueNode = doc.querySelector(".article-header--metadata-date > a"); + + let issueNode = doc.querySelector(".topper__issue"); if (issueNode) { var volumeTitle = ZU.trimInternal(issueNode.textContent.trim()); - // the digits are yyyy/vol/num - let issueMatch = issueNode.href.match(/\/issues\/\d+\/(\d+)\/(\d+)$/); + if (volumeTitle) { + if (!item.extra) item.extra = ""; + item.extra += `\nVolume Title: ${volumeTitle}`; + } + + let issueMatch = issueNode.parentNode.href.match(/\/issues\/\d+\/(\d+)\/(\d+)$/); if (issueMatch) { item.volume = issueMatch[1]; item.issue = issueMatch[2]; } } - if (volumeTitle) { - if (!item.extra) item.extra = ""; - item.extra += `\nVolume Title: ${volumeTitle}`; - } item.date = attr(doc, 'meta[property="article:published_time"]', 'content'); item.title = attr(doc, 'meta[property="og:title"]', 'content'); @@ -107,15 +110,18 @@ async function scrape(doc, url = doc.location.href) { item.date = ZU.strToISO(item.date); } + var author = doc.querySelector('.topper__byline').textContent.trim(); + author = author.replace("Reviewed by ", ""); let authors = author.split(/, and|and |, /); for (let aut of authors) { item.creators.push(ZU.cleanAuthor(aut, "author")); } + var tags = doc.querySelectorAll('.mr-15'); for (let tag of tags) { item.tags.push(tag.textContent); } - item.url = url; + item.url = url.split('?')[0]; item.attachments.push({ document: doc, title: "Snapshot" }); item.publicationTitle = "Foreign Affairs"; item.ISSN = "0015-7120"; @@ -123,7 +129,6 @@ async function scrape(doc, url = doc.location.href) { item.complete(); } - /** BEGIN TEST CASES **/ var testCases = [ { @@ -141,21 +146,21 @@ var testCases = [ "title": "A History of Argentina in the Twentieth Century", "creators": [ { - "firstName": "Kenneth", - "lastName": "Maxwell", - "creatorType": "author" + "firstName":"Kenneth", + "lastName":"Maxwell", + "creatorType":"author" } ], + "extra": "Volume Title: May/June 2003", + "volume": "82", + "issue": "3", "date": "2003-05-01", "ISSN": "0015-7120", "abstractNote": "A fascinating and well-translated account of Argentina's misadventures over the last century by one of that country's brightest historians. Absorbing vast amounts of British capital and tens of thousands of European immigrants, Argentina began the century with great promise. In 1914, with half of its population still foreign, a dynamic society had emerged that was both open and mobile.", - "extra": "Volume Title: May/June 2003", - "issue": "3", "language": "en-US", "libraryCatalog": "Foreign Affairs", "publicationTitle": "Foreign Affairs", "url": "https://www.foreignaffairs.com/reviews/capsule-review/2003-05-01/history-argentina-twentieth-century", - "volume": "82", "attachments": [ { "title": "Snapshot", @@ -164,10 +169,13 @@ var testCases = [ ], "tags": [ { - "tag": "Argentina" + "tag": " Americas" + }, + { + "tag": " Argentina" }, { - "tag": "Western Hemisphere" + "tag": " South America" } ], "notes": [], @@ -209,6 +217,9 @@ var testCases = [ { "tag": "Arms Control & Disarmament" }, + { + "tag": "Intelligence" + }, { "tag": "Defense & Military" }, @@ -228,11 +239,14 @@ var testCases = [ "tag": "Nuclear Weapons & Proliferation" }, { - "tag": "Obama Administration" + "tag": "Barack Obama Administration" }, { "tag": "Persian Gulf" }, + { + "tag": "Sanctions" + }, { "tag": "Security" }, @@ -302,6 +316,21 @@ var testCases = [ { "tag": "North America" }, + { + "tag": "World" + }, + { + "tag": "Globalization" + }, + { + "tag": "Finance" + }, + { + "tag": "Politics & Society" + }, + { + "tag": "Inequality" + }, { "tag": "United States" } @@ -347,6 +376,21 @@ var testCases = [ }, { "tag": "South Asia" + }, + { + "tag": "Politics & Society" + }, + { + "tag": "Demography" + }, + { + "tag": "Gender" + }, + { + "tag": "Narendra Modi" + }, + { + "tag": "Business" } ], "notes": [], From 323dbace08017f137bfb18e6d972c01abc5a7c99 Mon Sep 17 00:00:00 2001 From: Wenzhi Ding Date: Wed, 8 Jan 2025 20:22:03 +0800 Subject: [PATCH 2/8] Fix: Try to solve time out for test case "search" --- Foreign Affairs.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Foreign Affairs.js b/Foreign Affairs.js index 38a5da36ef4..834be5e7364 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -5,11 +5,11 @@ "target": "^https?://www\\.foreignaffairs\\.com", "minVersion": "3.0", "maxVersion": "", - "priority": 200, + "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2025-01-08 08:08:54" + "lastUpdated": "2025-01-08 12:21:03" } /* @@ -137,6 +137,12 @@ var testCases = [ "defer": true, "items": "multiple" }, + { + "type": "web", + "url": "https://www.foreignaffairs.com/search/arkansas", + "defer": true, + "items": "multiple" + }, { "type": "web", "url": "https://www.foreignaffairs.com/reviews/capsule-review/2003-05-01/history-argentina-twentieth-century", @@ -146,9 +152,9 @@ var testCases = [ "title": "A History of Argentina in the Twentieth Century", "creators": [ { - "firstName":"Kenneth", - "lastName":"Maxwell", - "creatorType":"author" + "firstName": "Kenneth", + "lastName": "Maxwell", + "creatorType": "author" } ], "extra": "Volume Title: May/June 2003", @@ -471,12 +477,6 @@ var testCases = [ "seeAlso": [] } ] - }, - { - "type": "web", - "url": "https://www.foreignaffairs.com/search/argentina", - "defer": true, - "items": "multiple" } ] /** END TEST CASES **/ From 593ea52172a9830920e575537df5204d3ca0a1e3 Mon Sep 17 00:00:00 2001 From: Wenzhi Ding Date: Fri, 10 Jan 2025 15:29:39 +0800 Subject: [PATCH 3/8] fix: Improve detection robustness. --- Foreign Affairs.js | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Foreign Affairs.js b/Foreign Affairs.js index 834be5e7364..cf76128c012 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2025-01-08 12:21:03" + "lastUpdated": "2025-01-10 07:28:35" } /* @@ -50,14 +50,16 @@ function getSearchResults(doc, checkOnly) { var items = {}; var found = false; - let isIssues = /^https:\/\/[^/]+\/issues\/.+/.test(doc.location.href); - let isSearch = /^https:\/\/[^/]+\/search\/.+/.test(doc.location.href); + // let isIssues = /^https:\/\/[^/]+\/issues\/.+/.test(doc.location.href); + let isIssues = /^\/issues\/.+/.test(doc.location.pathname); + // let isSearch = /^https:\/\/[^/]+\/search\/.+/.test(doc.location.href); + let isSearch = /^\/search\/.+/.test(doc.location.pathname); let rows = []; if (isIssues) { rows = doc.querySelectorAll('h3 > a'); } - if (isSearch) { + else if (isSearch) { rows = doc.querySelectorAll('h2 > a'); } @@ -92,8 +94,9 @@ async function scrape(doc, url = doc.location.href) { if (issueNode) { var volumeTitle = ZU.trimInternal(issueNode.textContent.trim()); if (volumeTitle) { - if (!item.extra) item.extra = ""; - item.extra += `\nVolume Title: ${volumeTitle}`; + // if (!item.extra) item.extra = ""; + // item.extra += `\nVolume Title: ${volumeTitle}`; + item.setExtra('Volume Title', volumeTitle); } let issueMatch = issueNode.parentNode.href.match(/\/issues\/\d+\/(\d+)\/(\d+)$/); @@ -117,7 +120,12 @@ async function scrape(doc, url = doc.location.href) { item.creators.push(ZU.cleanAuthor(aut, "author")); } - var tags = doc.querySelectorAll('.mr-15'); + // var tags = doc.querySelectorAll('.mr-15'); + var tags = doc.querySelectorAll( + 'a[href^="/regions/"].text-decoration-none, ' + + 'a[href^="/topics/"].text-decoration-none, ' + + 'a[href^="/tags/"].text-decoration-none' + ); for (let tag of tags) { item.tags.push(tag.textContent); } From 92799476435fc89fe8491b269bbcbc79dccb2d7e Mon Sep 17 00:00:00 2001 From: Wenzhi Ding Date: Fri, 10 Jan 2025 15:30:47 +0800 Subject: [PATCH 4/8] fix: Copyrights update --- Foreign Affairs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Foreign Affairs.js b/Foreign Affairs.js index cf76128c012..ffc78d772e2 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -15,7 +15,7 @@ /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2016-2022 Sebastian Karcher & Philipp Zumstein + Copyright © 2016-2025 Sebastian Karcher & Philipp Zumstein & Wenzhi Dave Ding This file is part of Zotero. From ef89bdc1482b034ca29c6cb73d11ec54a326810e Mon Sep 17 00:00:00 2001 From: Wenzhi Ding Date: Fri, 10 Jan 2025 15:31:49 +0800 Subject: [PATCH 5/8] fix: Housekeeping --- Foreign Affairs.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Foreign Affairs.js b/Foreign Affairs.js index ffc78d772e2..2cec39be45b 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -50,9 +50,7 @@ function getSearchResults(doc, checkOnly) { var items = {}; var found = false; - // let isIssues = /^https:\/\/[^/]+\/issues\/.+/.test(doc.location.href); let isIssues = /^\/issues\/.+/.test(doc.location.pathname); - // let isSearch = /^https:\/\/[^/]+\/search\/.+/.test(doc.location.href); let isSearch = /^\/search\/.+/.test(doc.location.pathname); let rows = []; @@ -94,8 +92,6 @@ async function scrape(doc, url = doc.location.href) { if (issueNode) { var volumeTitle = ZU.trimInternal(issueNode.textContent.trim()); if (volumeTitle) { - // if (!item.extra) item.extra = ""; - // item.extra += `\nVolume Title: ${volumeTitle}`; item.setExtra('Volume Title', volumeTitle); } @@ -120,7 +116,6 @@ async function scrape(doc, url = doc.location.href) { item.creators.push(ZU.cleanAuthor(aut, "author")); } - // var tags = doc.querySelectorAll('.mr-15'); var tags = doc.querySelectorAll( 'a[href^="/regions/"].text-decoration-none, ' + 'a[href^="/topics/"].text-decoration-none, ' + From d044d2a42c24c4a52587bf1038f3b2caf7e7522d Mon Sep 17 00:00:00 2001 From: Wenzhi Ding Date: Fri, 10 Jan 2025 15:34:26 +0800 Subject: [PATCH 6/8] fix: linting --- Foreign Affairs.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Foreign Affairs.js b/Foreign Affairs.js index 2cec39be45b..a06236acb92 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -117,10 +117,10 @@ async function scrape(doc, url = doc.location.href) { } var tags = doc.querySelectorAll( - 'a[href^="/regions/"].text-decoration-none, ' + - 'a[href^="/topics/"].text-decoration-none, ' + - 'a[href^="/tags/"].text-decoration-none' - ); + 'a[href^="/regions/"].text-decoration-none, ' + + 'a[href^="/topics/"].text-decoration-none, ' + + 'a[href^="/tags/"].text-decoration-none' + ); for (let tag of tags) { item.tags.push(tag.textContent); } From 870eccb0dd231485771a8f10f02c1538830e143b Mon Sep 17 00:00:00 2001 From: Wenzhi Ding Date: Mon, 20 Jan 2025 23:13:11 +0800 Subject: [PATCH 7/8] fix selector --- Foreign Affairs.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Foreign Affairs.js b/Foreign Affairs.js index a06236acb92..161563db0c9 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -9,13 +9,13 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2025-01-10 07:28:35" + "lastUpdated": "2025-01-20 15:10:36" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2016-2025 Sebastian Karcher & Philipp Zumstein & Wenzhi Dave Ding + Copyright © 2016-2022 Sebastian Karcher & Philipp Zumstein This file is part of Zotero. @@ -117,10 +117,10 @@ async function scrape(doc, url = doc.location.href) { } var tags = doc.querySelectorAll( - 'a[href^="/regions/"].text-decoration-none, ' - + 'a[href^="/topics/"].text-decoration-none, ' - + 'a[href^="/tags/"].text-decoration-none' - ); + 'main > div > div > div > div > ul > li > a[href^="/regions/"], ' + + 'main > div > div > div > div > ul > li > a[href^="/topics/"], ' + + 'main > div > div > div > div > ul > li > a[href^="/tags/"]' + ); for (let tag of tags) { item.tags.push(tag.textContent); } From 773cee66a30ee4cf0369f6ff646cf6ee50aa8248 Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Tue, 21 Jan 2025 11:26:30 -0500 Subject: [PATCH 8/8] Correct tag selector --- Foreign Affairs.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Foreign Affairs.js b/Foreign Affairs.js index 161563db0c9..be140952610 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2025-01-20 15:10:36" + "lastUpdated": "2025-01-21 16:26:22" } /* @@ -116,11 +116,11 @@ async function scrape(doc, url = doc.location.href) { item.creators.push(ZU.cleanAuthor(aut, "author")); } - var tags = doc.querySelectorAll( - 'main > div > div > div > div > ul > li > a[href^="/regions/"], ' + - 'main > div > div > div > div > ul > li > a[href^="/topics/"], ' + - 'main > div > div > div > div > ul > li > a[href^="/tags/"]' - ); + var tags = doc.querySelectorAll(` + #content ul > li > a[href^="/regions/"], + #content ul > li > a[href^="/topics/"], + #content ul > li > a[href^="/tags/"] + `); for (let tag of tags) { item.tags.push(tag.textContent); }