diff --git a/helm/charts/aleph/values.yaml b/helm/charts/aleph/values.yaml index 3806089113..3576eb3571 100644 --- a/helm/charts/aleph/values.yaml +++ b/helm/charts/aleph/values.yaml @@ -162,7 +162,7 @@ ingestfile: image: repository: ghcr.io/alephdata/ingest-file - tag: "3.16.1" + tag: "3.16.3" pullPolicy: Always containerSecurityContext: diff --git a/requirements.txt b/requirements.txt index f6661d20ee..99754a4d57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Dependencies maintained by OCCRP banal==1.0.6 -followthemoney==2.9.3 +followthemoney==2.9.5 followthemoney-store[postgresql]==3.0.3 followthemoney-compare==0.4.3 fingerprints==1.0.3 @@ -16,21 +16,21 @@ Flask-Migrate==3.1.0 Flask-Cors==3.0.10 Flask-Babel==2.0.0 flask-talisman==1.0.0 -SQLAlchemy==1.4.36 -alembic==1.7.7 +SQLAlchemy==1.4.37 +alembic==1.8.0 authlib==0.15.5 elasticsearch==7.17.0 marshmallow==2.19.2 gunicorn[eventlet]==20.1.0 -jsonschema==4.5.1 +jsonschema==4.6.0 apispec==5.2.2 apispec-webframeworks==0.5.2 blinker==1.4 Babel==2.10.1 PyYAML==5.4.1 python-frontmatter==1.0.0 -pyjwt >= 2.0.1, < 2.4.0 +pyjwt >= 2.0.1, < 2.5.0 cryptography >= 36.0.0, < 38.0.0 requests[security] >= 2.25.1, < 3.0.0 urllib3==1.26.9 diff --git a/ui/package.json b/ui/package.json index f8ccea8c7b..4078ec9ec0 100644 --- a/ui/package.json +++ b/ui/package.json @@ -3,8 +3,8 @@ "version": "3.12.3", "private": true, "dependencies": { - "@alephdata/followthemoney": "2.9.3", - "@alephdata/react-ftm": "2.6.0", + "@alephdata/followthemoney": "2.9.4", + "@alephdata/react-ftm": "2.6.5", "@blueprintjs/colors": "^3.0.0", "@blueprintjs/core": "3.54.0", "@blueprintjs/icons": "3.31.0", @@ -14,7 +14,7 @@ "@formatjs/cli": "^4.2.2", "@formatjs/intl-locale": "^2.4.14", "@formatjs/intl-pluralrules": "^4.0.6", - "@formatjs/intl-relativetimeformat": "^10.0.1", + "@formatjs/intl-relativetimeformat": "^11.0.1", "@formatjs/intl-utils": "^3.8.4", "@types/jest": "^27.0.1", "@types/node": "^17.0.8", @@ -26,6 +26,7 @@ "js-file-download": "^0.4.9", "jwt-decode": "^3.0.0", "lodash": "^4.17.11", + "moment": "^2.29.1", "node-sass": "6.0.1", "numeral": "^2.0.6", "papaparse": "^5.1.0", diff --git a/ui/src/components/Facet/DateFacet.jsx b/ui/src/components/Facet/DateFacet.jsx index c0bba7ecff..e3e9246468 100644 --- a/ui/src/components/Facet/DateFacet.jsx +++ b/ui/src/components/Facet/DateFacet.jsx @@ -1,14 +1,15 @@ import React, { Component } from 'react'; import { compose } from 'redux'; import { connect } from 'react-redux'; -import { defineMessages, FormattedMessage, injectIntl } from 'react-intl'; +import { defineMessages, FormattedMessage, FormattedDate, injectIntl } from 'react-intl'; import queryString from 'query-string'; import { Button, Card, Icon, Intent, Spinner } from '@blueprintjs/core'; import { Histogram } from '@alephdata/react-ftm'; +import moment from 'moment'; import withRouter from 'app/withRouter' -import { DEFAULT_START_INTERVAL, filterDateIntervals, formatDateQParam, timestampToYear } from 'components/Facet/util'; - +import { DEFAULT_START_INTERVAL, filterDateIntervals, formatDateQParam, timestampToLabel, isDateIntervalUncertain } from 'components/Facet/util'; +import { selectEntitiesResult, selectLocale } from 'selectors' import './DateFacet.scss'; @@ -19,6 +20,18 @@ const messages = defineMessages({ id: 'search.screen.dates_label', defaultMessage: 'results', }, + uncertainMonth: { + id: 'search.screen.dates_uncertain_month', + defaultMessage: '* this count includes dates in {year} where no month is specified', + }, + uncertainDay: { + id: 'search.screen.dates_uncertain_day', + defaultMessage: '* this count includes dates where no day is specified', + }, + uncertainDayMonth: { + id: 'search.screen.dates_uncertain_day_month', + defaultMessage: '* this count includes dates in {year} where no day or month is specified', + }, }); export class DateFilter extends Component { @@ -29,16 +42,28 @@ export class DateFilter extends Component { } onSelect(selected) { - const { field, query, updateQuery } = this.props; + const { facetInterval, field, query, updateQuery } = this.props; let newRange; + let newQuery = query; + if (Array.isArray(selected)) { - newRange = selected.sort().map(formatDateQParam); + newRange = selected.sort().map(val => formatDateQParam(val, facetInterval)); } else { - const year = formatDateQParam(selected); - newRange = [year, year]; + + if (facetInterval === 'year') { + newQuery = newQuery.set(`facet_interval:${field}`, 'month') + const end = moment.utc(selected).endOf('year').format('YYYY-MM-DD') + newRange = [formatDateQParam(selected, 'month'), formatDateQParam(end, 'month')] + } else if (facetInterval === 'month') { + newQuery = newQuery.set(`facet_interval:${field}`, 'day') + const end = moment.utc(selected).endOf('month').format('YYYY-MM-DD') + newRange = [formatDateQParam(selected, 'day'), formatDateQParam(end, 'day')] + } else { + newRange = [formatDateQParam(selected, 'day'), formatDateQParam(selected, 'day')] + } } - const newQuery = query.setFilter(`gte:${field}`, newRange[0]) + newQuery = newQuery.setFilter(`gte:${field}`, newRange[0]) .setFilter(`lte:${field}`, newRange[1]); updateQuery(newQuery) @@ -88,8 +113,54 @@ export class DateFilter extends Component { ); } + renderParentLabel() { + const { facetInterval, filteredIntervals } = this.props; + + const sampleDate = filteredIntervals[0].label + + const content = facetInterval === 'month' + ? moment.utc(sampleDate).year() + : ( + + ) + return {content} + } + + formatUncertainWarning(timestamp) { + const { facetInterval, intl } = this.props; + const year = moment.utc(timestamp).year() + + if (facetInterval === 'month') { + return intl.formatMessage(messages.uncertainMonth, { year }) + } else { + const isFirstMonth = moment.utc(timestamp).month() === 0 + return intl.formatMessage(messages[isFirstMonth ? 'uncertainDayMonth' : 'uncertainDay'], { year }); + } + } + + formatData(dataPropName) { + const { facetInterval, filteredIntervals, locale } = this.props; + + return filteredIntervals.map(({ label, count, id }) => { + const isUncertain = facetInterval !== 'year' && isDateIntervalUncertain(label, facetInterval) + const uncertainWarning = isUncertain && this.formatUncertainWarning(label) + + return ({ + ...timestampToLabel(label, facetInterval, locale), + [dataPropName]: count, + isUncertain, + uncertainWarning, + id + }) + }) + } + render() { - const { dataLabel, emptyComponent, filteredIntervals, intl, displayShowHiddenToggle, showAll, showLabel = true } = this.props; + const { dataLabel, emptyComponent, facetInterval, filteredIntervals, intl, displayShowHiddenToggle, showLabel = true } = this.props; let content; if (filteredIntervals) { @@ -97,17 +168,19 @@ export class DateFilter extends Component { content = emptyComponent; } else { const dataPropName = dataLabel || intl.formatMessage(messages.results); + content = ( <> + {facetInterval !== 'year' && this.renderParentLabel()} ({ label: timestampToYear(label), [dataPropName]: count, ...rest }))} + data={this.formatData(dataPropName)} dataPropName={dataPropName} onSelect={this.onSelect} containerProps={{ height: DATE_FACET_HEIGHT, }} /> - {(displayShowHiddenToggle || showAll) && this.renderShowHiddenToggle()} + {displayShowHiddenToggle && this.renderShowHiddenToggle()} ) } @@ -137,17 +210,21 @@ export class DateFilter extends Component { } const mapStateToProps = (state, ownProps) => { - const { location, intervals, query } = ownProps; + const { field, location, intervals, query } = ownProps; const hashQuery = queryString.parse(location.hash); const showAll = hashQuery.show_all_dates === 'true'; + const result = selectEntitiesResult(state, query); - if (intervals) { - const { filteredIntervals, hasOutOfRange } = filterDateIntervals({ query, intervals, useDefaultBounds: !showAll }) + if (intervals && !result.isPending) { + const { filteredIntervals, hasOutOfRange } = filterDateIntervals({ field, query, intervals, useDefaultBounds: !showAll }) + const locale = selectLocale(state); return { filteredIntervals, displayShowHiddenToggle: hasOutOfRange, - showAll + facetInterval: query.getString(`facet_interval:${field}`), + showAll, + locale: locale === 'en' ? 'en-gb' : locale }; } return {}; diff --git a/ui/src/components/Facet/DateFacet.scss b/ui/src/components/Facet/DateFacet.scss index 534cbd628a..943de38105 100644 --- a/ui/src/components/Facet/DateFacet.scss +++ b/ui/src/components/Facet/DateFacet.scss @@ -38,6 +38,12 @@ } } + &__parent-label { + font-weight: bold; + color: $blue2; + margin-bottom: $aleph-grid-size/2; + } + .ErrorSection { min-height: unset; diff --git a/ui/src/components/Facet/Facet.jsx b/ui/src/components/Facet/Facet.jsx index 870372bedc..e966e931c1 100644 --- a/ui/src/components/Facet/Facet.jsx +++ b/ui/src/components/Facet/Facet.jsx @@ -58,7 +58,8 @@ class Facet extends Component { const { field, query } = this.props; const newQuery = query .clearFilter(`lte:${field}`) - .clearFilter(`gte:${field}`); + .clearFilter(`gte:${field}`) + .set(`facet_interval:${field}`, 'year') this.props.updateQuery(newQuery); } diff --git a/ui/src/components/Facet/util.js b/ui/src/components/Facet/util.js index 4d208b89de..8efc440342 100644 --- a/ui/src/components/Facet/util.js +++ b/ui/src/components/Facet/util.js @@ -1,40 +1,70 @@ -const DEFAULT_START_INTERVAL = 1950; -const ES_SUFFIX = '||/y'; +import moment from 'moment'; -const formatDateQParam = (datetime) => { - return `${new Date(datetime).getFullYear()}||/y` +const DEFAULT_START_INTERVAL = '1950'; + +const formatDateQParam = (datetime, granularity) => { + const date = moment.utc(datetime).format("YYYY-MM-DD") + if (granularity === 'month') { + return `${date}||/M` + } else if (granularity === 'day') { + return `${date}||/d` + } + return `${date}||/y` }; const cleanDateQParam = (value) => { - return value.replace(ES_SUFFIX, ''); + if (!value) { return; } + const [date, suffix] = value.split('||/'); + + if (suffix === 'y') { + return moment.utc(date).format('YYYY') + } else if (suffix === 'M') { + return moment.utc(date).format('YYYY-MM') + } else { + return date; + } }; -const timestampToYear = timestamp => { - return new Date(timestamp).getFullYear(); +const timestampToLabel = (timestamp, granularity, locale) => { + const dateObj = new Date(timestamp) + let label, tooltipLabel; + + if (granularity === 'month') { + label = new Intl.DateTimeFormat(locale, { month: 'short' }).format(dateObj) + tooltipLabel = new Intl.DateTimeFormat(locale, { month: 'short', year: 'numeric' }).format(dateObj) + } else if (granularity === 'day') { + label = dateObj.getDate() + tooltipLabel = new Intl.DateTimeFormat(locale, { month: 'short', year: 'numeric', day: 'numeric' }).format(dateObj) + } else { + label = tooltipLabel = dateObj.getFullYear(); + } + + return ({ label, tooltipLabel }) } -const filterDateIntervals = ({ query, intervals, useDefaultBounds }) => { - const defaultEndInterval = new Date().getFullYear(); - const hasGtFilter = query.hasFilter('gte:dates'); - const hasLtFilter = query.hasFilter('lte:dates'); +const filterDateIntervals = ({ field, query, intervals, useDefaultBounds }) => { + const defaultEndInterval = moment.utc().format('YYYY-MM-DD'); + const hasGtFilter = query.hasFilter(`gte:${field}`); + const hasLtFilter = query.hasFilter(`lte:${field}`); - const gt = hasGtFilter - ? cleanDateQParam(query.getFilter('gte:dates')[0]) + const gtRaw = hasGtFilter + ? cleanDateQParam(query.getFilter(`gte:${field}`)[0]) : (useDefaultBounds && DEFAULT_START_INTERVAL); - const lt = hasLtFilter - ? cleanDateQParam(query.getFilter('lte:dates')[0]) + const ltRaw = hasLtFilter + ? cleanDateQParam(query.getFilter(`lte:${field}`)[0]) : (useDefaultBounds && defaultEndInterval); - let gtOutOfRange, ltOutOfRange = false; + const gt = gtRaw && moment(gtRaw) + const lt = ltRaw && moment(ltRaw) + let gtOutOfRange, ltOutOfRange = false; const filteredIntervals = intervals.filter(({ id }) => { - const year = timestampToYear(id); - if (gt && year < gt) { + if (gt && gt.isAfter(id)) { gtOutOfRange = true; return false; } - if (lt && year > lt) { + if (lt && lt.isBefore(id)) { ltOutOfRange = true; return false; } @@ -42,13 +72,27 @@ const filterDateIntervals = ({ query, intervals, useDefaultBounds }) => { }) const hasOutOfRange = useDefaultBounds && ((!hasGtFilter && gtOutOfRange) || (!hasLtFilter && ltOutOfRange)); + return { filteredIntervals, hasOutOfRange }; } +const isDateIntervalUncertain = (timestamp, granularity) => { + const dateObj = moment.utc(timestamp) + + if (granularity === 'month' && dateObj.month() === 0) { + return true; + } else if (granularity === 'day' && dateObj.date() === 1) { + return true; + } + + return false; +} + export { cleanDateQParam, DEFAULT_START_INTERVAL, formatDateQParam, - timestampToYear, + timestampToLabel, + isDateIntervalUncertain, filterDateIntervals } diff --git a/ui/src/components/QueryTags/QueryFilterTag.jsx b/ui/src/components/QueryTags/QueryFilterTag.jsx index f71a5e32cf..9c2c5b6605 100644 --- a/ui/src/components/QueryTags/QueryFilterTag.jsx +++ b/ui/src/components/QueryTags/QueryFilterTag.jsx @@ -1,10 +1,9 @@ import React, { PureComponent } from 'react'; import { FormattedMessage } from 'react-intl'; import { Icon, Tag as TagWidget } from '@blueprintjs/core'; -import { cleanDateQParam } from 'components/Facet/util'; import { - Schema, Tag, Country, Language, Category, Collection, Date, Entity, + Schema, Tag, Country, Language, Category, Collection, Entity, } from 'components/common'; import './QueryFilterTag.scss'; @@ -17,11 +16,11 @@ class QueryFilterTag extends PureComponent { } onRemove() { - const { filter, value, remove } = this.props; - remove(filter, value); + const { filter, type, value, remove } = this.props; + remove(filter, type, value); } - label = (filter, type, value) => { + label = (query, filter, type, value) => { switch (type || filter) { case 'schema': return ( @@ -98,22 +97,11 @@ class QueryFilterTag extends PureComponent { {value} ); - case 'eq:dates': - case 'lte:dates': - case 'gte:dates': case 'date': - let prefix; - if (filter.includes('gte')) { - prefix = - } else if (filter.includes('lte')) { - prefix = - } - return ( <> - {prefix} - + {value} ); default: @@ -122,7 +110,7 @@ class QueryFilterTag extends PureComponent { } render() { - const { filter, type, value } = this.props; + const { filter, type, value, query } = this.props; return ( - {this.label(filter, type, value)} + {this.label(query, filter, type, value)} ); } diff --git a/ui/src/components/QueryTags/QueryTags.jsx b/ui/src/components/QueryTags/QueryTags.jsx index 1ca88f3c4c..7d025c1442 100644 --- a/ui/src/components/QueryTags/QueryTags.jsx +++ b/ui/src/components/QueryTags/QueryTags.jsx @@ -1,8 +1,11 @@ import React, { Component } from 'react'; import { FormattedMessage } from 'react-intl'; import _ from 'lodash'; +import moment from 'moment'; + import { Button } from '@blueprintjs/core'; import QueryFilterTag from './QueryFilterTag'; +import { cleanDateQParam } from 'components/Facet/util'; const HIDDEN_TAGS_CUTOFF = 10; @@ -15,16 +18,18 @@ class QueryTags extends Component { this.state = { showHidden: false }; } - removeFilterValue(filter, value) { + removeFilterValue(filter, type, value) { const { query, updateQuery } = this.props; let newQuery; - if (filter.includes('eq:')) { - const field = filter.replace('eq:', '') - newQuery = query.removeFilter(`gte:${field}`, value) - .removeFilter(`lte:${field}`, value); + + if (type === 'date') { + newQuery = query.clearFilter(`gte:${filter}`) + .clearFilter(`lte:${filter}`) + .set(`facet_interval:${filter}`, 'year') } else { newQuery = query.removeFilter(filter, value); } + updateQuery(newQuery); } @@ -33,40 +38,54 @@ class QueryTags extends Component { updateQuery(query.removeAllFilters()); } + getTagsList(tagsList) { + const { query } = this.props; + + const tags = _.flatten( + query.filters() + .map(filter => query.getFilter(filter).map(value => ({ filter, value, type: query.getFacetType(filter) }))) + ); + + const [dateTags, otherTags] = _.partition(tags, tag => tag.filter.includes(":")); + const dateProps = _.groupBy(dateTags, tag => tag.filter.split(':')[1]) + + // combines "greater than" and "less than" filters into a single tag + const processedDateTags = Object.entries(dateProps).map(([propName, values]) => { + const gt = cleanDateQParam(values.find(({ filter }) => filter.includes('gte'))?.value) + const lt = cleanDateQParam(values.find(({ filter }) => filter.includes('lte'))?.value) + let combinedValue; + if (!gt || !lt) { + return null; + } else if (gt === lt) { + combinedValue = gt + // if timespan between greater than and less than is exactly a year, displays year in query tag + } else if (moment.utc(gt).month() === 0 && moment.utc(lt).month() === 11) { + combinedValue = moment.utc(gt).year() + } else if (moment.utc(gt).isSame(moment.utc(gt).startOf('month'), 'day') && moment.utc(lt).isSame(moment.utc(lt).endOf('month'), 'day')) { + combinedValue = moment.utc(gt).format('yyyy-MM') + } else { + combinedValue = `${gt} - ${lt}` + } + + return ({ filter: propName, value: combinedValue, type: 'date' }) + }) + + return [...processedDateTags, ...otherTags]; + } + render() { const { query } = this.props; const { showHidden } = this.state; - let activeFilters = query ? query.filters() : []; - if (activeFilters.length === 0) { + if (!query || query.filters().length === 0) { return null; } - // if gte and lte are equal to the same value for a field, remove and replace with a single equals tag - let addlTags = []; - activeFilters - .filter(f => f.includes('gte:')) - .forEach(f => { - const field = f.replace('gte:', ''); - if (query.hasFilter(`lte:${field}`)) { - const gte = query.getFilter(`gte:${field}`)[0]; - const lte = query.getFilter(`lte:${field}`)[0]; - if (gte === lte) { - activeFilters = activeFilters.filter(f => (f !== `gte:${field}` && f !== `lte:${field}`)) - addlTags.push({ filter: `eq:${field}`, value: gte, type: 'date' }); - } - } - }) - - const filterTags = _.flatten( - activeFilters - .map(filter => query.getFilter(filter).map(value => ({ filter, value, type: query.getFacetType(filter) }))) - ); - const allTags = [...filterTags, ...addlTags]; - const visibleTags = showHidden ? allTags : allTags.slice(0, HIDDEN_TAGS_CUTOFF); + const filterTags = this.getTagsList(); + const visibleTags = showHidden ? filterTags : filterTags.slice(0, HIDDEN_TAGS_CUTOFF); - const showHiddenToggle = !showHidden && allTags.length > HIDDEN_TAGS_CUTOFF; - const showClearAll = allTags.length > 1; + const showHiddenToggle = !showHidden && filterTags.length > HIDDEN_TAGS_CUTOFF; + const showClearAll = filterTags.length > 1; // @FIXME This should still selectively display filters for the following: // "?exclude={id}" @@ -76,6 +95,7 @@ class QueryTags extends Component {
{visibleTags.map(({ filter, type, value }) => ( )}