Skip to content

Commit

Permalink
Fix missing crawler timeout config option
Browse files Browse the repository at this point in the history
  • Loading branch information
ptrumpis committed Feb 4, 2025
1 parent bbf523c commit 31aa0c0
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 1 deletion.
3 changes: 3 additions & 0 deletions config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ import:

# search configuration
search:
# web search connection timeout in milliseconds
timeout: 9000

# caching is required to activate web lenses within search results
# web caching requires 'app.flag.enable_web_source' to be 'true'
web_cache:
Expand Down
1 change: 1 addition & 0 deletions src/utils/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ async function loadConfig() {
zip_archive: true,
},
search: {
timeout: 9000,
web_cache: {
ttl: 1800,
check: 600,
Expand Down
4 changes: 3 additions & 1 deletion src/utils/web.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@ import { Config } from './config.js';
import * as DB from './db.js';
import * as Util from './helper.js';

const crawler = new LensWebCrawler();
const Cache = new NodeCache({
stdTTL: Config.search.web_cache.ttl,
checkperiod: Config.search.web_cache.check,
});

const creatorUrl = Config.search.creator_url;
const searchTimeout = Config.search.timeout;

const crawler = new LensWebCrawler(searchTimeout);

async function search(searchTerm) {
let result = [];
Expand Down

0 comments on commit 31aa0c0

Please sign in to comment.