diff --git a/package-lock.json b/package-lock.json index cceb243c..3abc14c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -62,6 +62,7 @@ "eslint-plugin-storybook": "^0.11.1", "file-loader": "^6.2.0", "globals": "^15.12.0", + "gzip-size": "^7.0.0", "html-webpack-plugin": "^5.6.3", "jest": "^29.7.0", "jsdom": "^25.0.1", @@ -11339,16 +11340,16 @@ "license": "MIT" }, "node_modules/gzip-size": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", - "integrity": "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-7.0.0.tgz", + "integrity": "sha512-O1Ld7Dr+nqPnmGpdhzLmMTQ4vAsD+rHwMm1NLUmoUFFymBOMKxCCrtDxqdBRYXdeEPEi3SyoR4TizJLQrnKBNA==", "dev": true, "license": "MIT", "dependencies": { "duplexer": "^0.1.2" }, "engines": { - "node": ">=10" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -21141,6 +21142,22 @@ "node": ">= 10" } }, + "node_modules/webpack-bundle-analyzer/node_modules/gzip-size": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", + "integrity": "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "duplexer": "^0.1.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/webpack-bundle-analyzer/node_modules/ws": { "version": "7.5.10", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", diff --git a/package.json b/package.json index 78db44fd..ed4652b3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "interslavic", - "version": "1.25.2", + "version": "1.25.3", "description": "Interslavic Dictionary", "license": "MIT", "author": { @@ -84,6 +84,7 @@ "eslint-plugin-storybook": "^0.11.1", "file-loader": "^6.2.0", "globals": "^15.12.0", + "gzip-size": "^7.0.0", "html-webpack-plugin": "^5.6.3", "jest": "^29.7.0", "jsdom": "^25.0.1", diff --git a/src/components/Textarea/Textarea.scss b/src/components/Textarea/Textarea.scss index 5919f2ab..9b46188a 100644 --- a/src/components/Textarea/Textarea.scss +++ b/src/components/Textarea/Textarea.scss @@ -37,4 +37,8 @@ border: 2px solid var(--invalid-color); } } + + &:not(.error) textarea:focus { + outline: var(--outline); + } } diff --git a/src/services/dictionary-test/dictionary-test.ts b/src/services/dictionary-test/dictionary-test.ts index 0ceb233a..8598fdd3 100644 --- a/src/services/dictionary-test/dictionary-test.ts +++ b/src/services/dictionary-test/dictionary-test.ts @@ -1,11 +1,14 @@ import * as fs from 'fs' -import { Dictionary } from 'services' +import { Dictionary, IBasicData } from 'services' -import basicData from './basic.json' +import testData from './basic.json' import { deepDiffMapper } from './deepDiffMapper' import testSnapshot from './snapshot.json' +const basicData = testData as unknown as IBasicData + + Dictionary.init(basicData.wordList, basicData.searchIndex) const snapshot = {} diff --git a/src/services/dictionary.ts b/src/services/dictionary.ts index 176b27ed..f5b5fa15 100644 --- a/src/services/dictionary.ts +++ b/src/services/dictionary.ts @@ -4,6 +4,7 @@ import { IAlphabets } from 'reducers' import { convertCases, + deduplicate, filterLatin, filterNiqqud, getCaseTips, @@ -154,6 +155,12 @@ export interface ITranslateResult { remove?: boolean; } +export const ISV_SRC = 'isv-src' +export const ISV = 'isv' + +export type WordList = string[][] +export type SearchIndex = Record> + class DictionaryClass { public static getInstance(): DictionaryClass { if (!DictionaryClass.instance) { @@ -169,7 +176,7 @@ class DictionaryClass { private langsList: string[] private headerIndexes: Map private percentsOfChecked: {[lang: string]: string} - private words: string[][] + private words: WordList private splittedMap: {[lang: string]: Map} private isvSearchLetters: { from: string[], to: string[] } private isvSearchByWordForms: boolean @@ -184,9 +191,9 @@ class DictionaryClass { } public init( - wordList: string[][], - searchIndex?: any | false, - percentsOfChecked?: any, + wordList: WordList, + searchIndex?: SearchIndex, + percentsOfChecked?: Record, ): number { let startInitTime = 0 @@ -197,7 +204,7 @@ class DictionaryClass { this.header = wordList[0] this.langsList = [ - 'isv', + ISV, 'en', ...langs, ...addLangs, @@ -209,7 +216,7 @@ class DictionaryClass { const needIndex = []; [ - 'isv-src', + ISV_SRC, ...this.langsList, ].forEach((lang) => { if (searchIndex && searchIndex[lang]) { @@ -225,7 +232,7 @@ class DictionaryClass { const id = `${this.getField(item, 'id')}` needIndex.forEach((lang) => { - let fromField = this.getField(item, lang === 'isv-src' ? 'isv' : lang) + let fromField = this.getField(item, lang === ISV_SRC ? ISV : lang) fromField = removeBrackets(fromField, '[', ']') fromField = removeBrackets(fromField, '(', ')') @@ -234,19 +241,23 @@ class DictionaryClass { let splittedField switch (lang) { - case 'isv': - splittedField = this - .splittedMap['isv-src'] - .get(id) - .map((word) => this.searchPrepare(lang, word)) - + case ISV: + splittedField = deduplicate( + this + .splittedMap[ISV_SRC] + .get(id) + .map((word) => this.searchPrepare(lang, word)) + ) + break - case 'isv-src': - splittedField = this - .splitWords(fromField) - .concat(getWordForms(item)) - .map((word) => this.searchPrepare('isv-src', getLatin(word, '2'))) - + case ISV_SRC: + splittedField = deduplicate( + this + .splitWords(fromField) + .concat(getWordForms(item)) + .map((word) => this.searchPrepare(ISV_SRC, getLatin(word, '2'))) + ) + break default: splittedField = this.splitWords(fromField).map((word) => this.searchPrepare(lang, word)) @@ -276,7 +287,7 @@ class DictionaryClass { return initTime } - public addLang(wordList: string[], searchIndex?: any) { + public addLang(wordList: string[], searchIndex?: SearchIndex) { const lang = wordList[0] if (this.hasLang(lang)) { @@ -295,7 +306,7 @@ class DictionaryClass { public hasLang(lang): boolean { return this.headerIndexes.has(lang) } - public getWordList(): string[][] { + public getWordList(): WordList { return this.words } public getWord(wordId: string) { @@ -303,11 +314,11 @@ class DictionaryClass { return this.words.filter((line) => this.getField(line, 'id') === wordId)[0] } } - public getIndex() { + public getIndex(): SearchIndex { const searchIndex = {}; [ - 'isv-src', + ISV_SRC, ...this.langsList, ].forEach((lang) => { searchIndex[lang] = Array.from(this.splittedMap[lang].keys()).map((key: string) => [ @@ -318,7 +329,7 @@ class DictionaryClass { return searchIndex } - public translate(translateParams: ITranslateParams, showTime = true): [string[][], number] { + public translate(translateParams: ITranslateParams, showTime = true): [WordList, number] { const { inputText, from, @@ -343,11 +354,11 @@ class DictionaryClass { const inputOptions = inputText.split(' -').map((option) => option.trim()) const inputWord = inputOptions.shift() - const lang = from === 'isv' ? to : from - const inputIsvPrepared = this.inputPrepare('isv', inputWord) + const lang = from === ISV ? to : from + const inputIsvPrepared = this.inputPrepare(ISV, inputWord) const inputLangPrepared = this.inputPrepare(lang, inputWord) - if (from === 'isv' && !inputIsvPrepared) { + if (from === ISV && !inputIsvPrepared) { return [[], 0] } @@ -361,10 +372,10 @@ class DictionaryClass { const twoWaySearch = inputOptions.some((o) => o === 'b') let isvText = '' - if (from === 'isv' || twoWaySearch) { + if (from === ISV || twoWaySearch) { isvText = inputWord isvText = this.applyIsvSearchLetters(getLatin(isvText, flavorisationType, true), flavorisationType) - isvText = this.inputPrepare('isv-src', isvText) + isvText = this.inputPrepare(ISV_SRC, isvText) } // option -end - search by ending of word if (inputOptions.some((option) => option.trim() === 'end')) { @@ -372,7 +383,7 @@ class DictionaryClass { } // option -etym - hard search by etymological orthography for Isv - const hardEtymSearch = from === 'isv' && (inputOptions.some((o) => o === 'etym')) + const hardEtymSearch = from === ISV && (inputOptions.some((o) => o === 'etym')) // filter by part of speech let filterPartOfSpeech = [] @@ -397,19 +408,19 @@ class DictionaryClass { } let filterResult = false - if (from === 'isv' || twoWaySearch) { + if (from === ISV || twoWaySearch) { // hardEtymSearch - hard etymological search for isv, otherwise - simple search // when isvSearchByWordForms = false OR entered 1 symbol - searching without word forms - let splittedField = this.getSplittedField(hardEtymSearch ? 'isv-src' : 'isv', item) + let splittedField = this.getSplittedField(hardEtymSearch ? ISV_SRC : ISV, item) if ( !this.isvSearchByWordForms || inputIsvPrepared.length === 1 ) { - const wordsCount = this.getField(item, 'isv').split(',').length + const wordsCount = this.getField(item, ISV).split(',').length splittedField = splittedField.slice(0, wordsCount) } filterResult = splittedField.some((chunk) => ( searchTypes[searchType](chunk, hardEtymSearch ? inputWord : inputIsvPrepared) )) } - if (to === 'isv' || twoWaySearch) { + if (to === ISV || twoWaySearch) { const splittedField = this.getSplittedField(lang, item) filterResult = filterResult || splittedField.some((chunk) => searchTypes[searchType](chunk, inputLangPrepared)) @@ -432,19 +443,19 @@ class DictionaryClass { .filter((item) => { let filterResult = true // search in isv with search sensitive letters - if ((from === 'isv' || twoWaySearch) && + if ((from === ISV || twoWaySearch) && !hardEtymSearch && (flavorisationType === '2' || flavorisationType === '3') && this.isvSearchLetters.to.some((letter) => inputIsvPrepared.includes(letter))) { - let splittedField = this.getSplittedField('isv-src', item) + let splittedField = this.getSplittedField(ISV_SRC, item) if ( !this.isvSearchByWordForms || inputIsvPrepared.length === 1 ) { - const wordsCount = this.getField(item, 'isv').split(',').length + const wordsCount = this.getField(item, ISV).split(',').length splittedField = splittedField.slice(0, wordsCount) } filterResult = splittedField.some((chunk) => ( searchTypes[searchType](this.applyIsvSearchLetters(chunk, flavorisationType), isvText) )) } - if (!filterResult && (to === 'isv' || twoWaySearch)) { + if (!filterResult && (to === ISV || twoWaySearch)) { const splittedField = this.getSplittedField(lang, item) filterResult = filterResult || splittedField.some((chunk) => searchTypes[searchType](chunk, inputLangPrepared)) @@ -462,7 +473,7 @@ class DictionaryClass { } let dist = splittedField .reduce((acc, item) => { - const lDist = levenshteinDistance(from === 'isv' ? inputIsvPrepared : inputLangPrepared, + const lDist = levenshteinDistance(from === ISV ? inputIsvPrepared : inputLangPrepared, this.searchPrepare(from, item)) if (lDist < acc) { return lDist @@ -480,7 +491,7 @@ class DictionaryClass { } const dist2 = splittedField .reduce((acc, item) => { - const lDist = levenshteinDistance(from === 'isv' ? inputLangPrepared : inputIsvPrepared, + const lDist = levenshteinDistance(from === ISV ? inputLangPrepared : inputIsvPrepared, this.searchPrepare(to, item)) if (lDist < acc) { return lDist @@ -509,7 +520,7 @@ class DictionaryClass { } public formatTranslate( - results: string[][], + results: WordList, from: string, to: string, flavorisationType: string, @@ -517,7 +528,7 @@ class DictionaryClass { caseQuestions?: boolean ): ITranslateResult[] { return results.map((item) => { - const isvRaw = this.getField(item, 'isv') + const isvRaw = this.getField(item, ISV) const remove = isvRaw.startsWith('!') const isv = removeBrackets( removeExclamationMark(isvRaw), '[', ']' @@ -530,7 +541,7 @@ class DictionaryClass { if(caseInfo && caseQuestions) { caseInfo = getCaseTips(caseInfo.slice(1),'nounShort') } - const translate = this.getField(item, (from === 'isv' ? to : from)) + const translate = this.getField(item, (from === ISV ? to : from)) const formattedItem: ITranslateResult = { translate: removeExclamationMark(translate), original: getLatin(isv, flavorisationType), @@ -611,7 +622,7 @@ class DictionaryClass { .replace(/,/g, '') .replace(/[ʼ’]/g, "'") - if (lang !== 'isv-src') { + if (lang !== ISV_SRC) { lowerCaseText = lowerCaseText.replace(/[\u0300-\u036f]/g, '') } else { lowerCaseText = lowerCaseText @@ -621,9 +632,9 @@ class DictionaryClass { } switch (lang) { - case 'isv-src': + case ISV_SRC: return lowerCaseText - case 'isv': + case ISV: return this.isvToEngLatin(lowerCaseText) case 'cs': case 'pl': @@ -652,7 +663,7 @@ class DictionaryClass { } } public splitWords(text: string): string[] { - return text.includes(';') ? text.split(';') : text.split(',') + return (text.includes(';') ? text.split(';') : text.split(',')).map(e => e.trim()) } private getSplittedField(from: string, item: string[]): string[] { const key = this.getField(item, 'id') @@ -660,7 +671,7 @@ class DictionaryClass { return this.splittedMap[from].get(key) } private applyIsvSearchLetters(text: string, flavorisationType: string): string { - text = this.searchPrepare('isv-src', text) + text = this.searchPrepare(ISV_SRC, text) isvReplacebleLetters .filter((replacement) => !this.isvSearchLetters.from.includes(replacement[0]) || diff --git a/src/services/fetchDictionary.ts b/src/services/fetchDictionary.ts index bd90d281..25758efd 100644 --- a/src/services/fetchDictionary.ts +++ b/src/services/fetchDictionary.ts @@ -2,7 +2,7 @@ import { addLangs } from 'consts' import { isLoadingAction, runSearch } from 'actions' -import { Dictionary } from 'services' +import { Dictionary, SearchIndex, WordList } from 'services' async function fetchStat() { return fetch('data/translateStatistic.json').then((res) => res.json()).then((data) => data) @@ -14,7 +14,12 @@ async function fetchLangs(langList: string[]) { ) } -async function fetchBasic() { +export interface IBasicData { + wordList: WordList, + searchIndex?: SearchIndex, +} + +async function fetchBasic(): Promise { return await fetch('data/basic.json').then((res) => res.json()) } diff --git a/src/services/generateDictionary.ts b/src/services/generateDictionary.ts index 84682b22..8ede0f84 100644 --- a/src/services/generateDictionary.ts +++ b/src/services/generateDictionary.ts @@ -2,10 +2,12 @@ import * as fs from 'fs' import { addLangs, initialAddFields, initialFields,langs } from 'consts' -import { Dictionary, loadTablesData } from 'services' +import { Dictionary, ISV, ISV_SRC, loadTablesData } from 'services' import { getColumnName, transposeMatrix } from 'utils' +import { gzipSizeSync } from 'gzip-size' + loadTablesData.then(({ data, columns }) => { Dictionary.init(data) @@ -24,11 +26,16 @@ loadTablesData.then(({ data, columns }) => { } }) - const basicData = transposeMatrix(basicDataTransposed) + let basicData = transposeMatrix(basicDataTransposed) + + basicData = [ + basicData[0], + ...basicData.slice(1).sort((a, b) => a[1].localeCompare(b[1])), + ] const searchIndexBasic = [ - 'isv-src', - 'isv', + ISV_SRC, + ISV, 'en', ...langs, ].reduce((searchIndexObj, lang) => { @@ -49,6 +56,10 @@ loadTablesData.then(({ data, columns }) => { fs.writeFileSync('./static/data/basic.json', jsonDataStr) fs.writeFileSync('./static/data/translateStatistic.json', translateStatisticStr) + const sizeMB = gzipSizeSync(jsonDataStr) / 1000000 + // eslint-disable-next-line no-console + console.info(`basic.json g-zip size: ${sizeMB.toFixed(2)} MB`) + addLangs.forEach((lang) => { const langDataTransposed = [ columns.find(([fieldName]) => fieldName === lang), diff --git a/src/utils/deduplicate/deduplicate.ts b/src/utils/deduplicate/deduplicate.ts new file mode 100644 index 00000000..6e35addc --- /dev/null +++ b/src/utils/deduplicate/deduplicate.ts @@ -0,0 +1 @@ +export const deduplicate = (arr) => Array.from(new Set(arr)) diff --git a/src/utils/deduplicate/index.ts b/src/utils/deduplicate/index.ts new file mode 100644 index 00000000..dc2f319c --- /dev/null +++ b/src/utils/deduplicate/index.ts @@ -0,0 +1 @@ +export * from './deduplicate' diff --git a/src/utils/index.ts b/src/utils/index.ts index 7268e426..efd28cad 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,6 +1,7 @@ export * from './bcp47' export * from './convertCases' export * from './createTaggedTemplate' +export * from './deduplicate' export * from './filterLatin' export * from './filterNiqqud' export * from './getAllDataFromResults'