diff --git a/.prettierrc b/.prettierrc deleted file mode 100644 index a20502b7..00000000 --- a/.prettierrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "singleQuote": true, - "trailingComma": "all" -} diff --git a/README.md b/README.md index 860403ec..02716ec1 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,8 @@ In order to work, your user **must have** `admin` profile sets with the `-authPr With help of different sidecars, Fibr can generate image, video and PDF thumbnails. These sidecars can be self hosted with ease. It can also extract and enrich content displayed by looking at [EXIF Data](https://en.wikipedia.org/wiki/Exif), also with the help of a little sidecar. These behaviours are opt-out (if you remove the `url` of the service, Fibr will do nothing). +For the last mile, Fibr can try to reverse geocoding the GPS data found in EXIF, using [Open Street Map](https://wiki.openstreetmap.org/wiki/Nominatim). Self-hosting this kind of service can be complicated and calling a third-party party with such sensible datas is an opt-in decision. + ### Metrics Fibr exposes a lot of metrics via OpenTelemetry gRPC mode. Common metrics are exposed: Golang statistics, HTTP statuses and response time, AMQP statuses and sidecars/metadatas actions. diff --git a/cmd/fibr/templates/exif.html b/cmd/fibr/templates/exif.html index ea0044b8..3ce51583 100644 --- a/cmd/fibr/templates/exif.html +++ b/cmd/fibr/templates/exif.html @@ -23,15 +23,16 @@

{{ end }} - {{ if or .Exif.Coordinates .Exif.HasAddress }} + {{ if or .Exif.Geocode.HasAddress .Exif.Geocode.HasCoordinates }}

- {{ if .Exif.Coordinates }} - Coordinates: {{ index .Exif.Coordinates 0 }},{{ index .Exif.Coordinates 1 }}
+ {{ if .Exif.Geocode.HasCoordinates }} + Coordinates: {{ .Exif.Geocode.Latitude }},{{ .Exif.Geocode.Longitude }}
{{ end }} Location
-   City: {{ .Exif.GetCity }}
-   Country: {{ .Exif.GetCountry }}
+ {{ range $key, $value := .Exif.Geocode.Address }} +   {{ $key }}: {{ $value }}
+ {{ end }}

{{ end }} diff --git a/cmd/fibr/templates/story.html b/cmd/fibr/templates/story.html index cebeda97..e3f1ab0c 100644 --- a/cmd/fibr/templates/story.html +++ b/cmd/fibr/templates/story.html @@ -193,19 +193,27 @@
- {{ if or (.Exif.HasAddress) (not .Exif.Date.IsZero) }} + {{ if or (.Exif.Geocode.HasAddress) (not .Exif.Date.IsZero) }}

- {{ if .Exif.HasAddress }} + {{ if .Exif.Geocode.HasAddress }} location - {{ if .Exif.GetCity }} - {{ .Exif.GetCity }}, - {{- end }} {{ .Exif.GetCountry }} + {{ if index .Exif.Geocode.Address "city_district" }} + {{ index .Exif.Geocode.Address "city_district" }}, + {{ else }} + {{ if index .Exif.Geocode.Address "village" }} + {{ index .Exif.Geocode.Address "village" }}, + {{ else }} + {{ if index .Exif.Geocode.Address "county" }} + {{ index .Exif.Geocode.Address "county" }}, + {{- end }} + {{- end }} + {{- end }} {{ index .Exif.Geocode.Address "country" }} {{ end }} {{ if not .Exif.Date.IsZero }} - calendar + calendar {{ .Exif.Date.Format "2006-01-02T15:04:05Z07:00" }} {{ end }}

diff --git a/compose.yaml b/compose.yaml index 0e6312de..eb4cccb8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -48,6 +48,7 @@ services: environment: EXAS_AMQP_URI: 'amqp://guest:guest@rabbit:5672/' EXAS_AMQP_WAIT_TIMEOUT: '30s' + EXAS_GEOCODE_URL: 'https://nominatim.openstreetmap.org' EXAS_STORAGE_FILE_SYSTEM_DIRECTORY: '/data' volumes: - '${DATA_DIR}:/data' diff --git a/go.mod b/go.mod index b0d2dc9d..9127d28c 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ViBiOh/ChatPotte v0.4.6 github.com/ViBiOh/absto v1.7.10 github.com/ViBiOh/auth/v2 v2.18.4 - github.com/ViBiOh/exas v0.8.0 + github.com/ViBiOh/exas v0.7.1 github.com/ViBiOh/flags v1.5.0 github.com/ViBiOh/httputils/v4 v4.76.0 github.com/ViBiOh/vith v0.6.0 diff --git a/go.sum b/go.sum index a310d078..7bba7fbf 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/ViBiOh/absto v1.7.10 h1:8z+F/46Y+uyddacBS9PQvHUJDev0K0LZLv4L4YgZXRQ= github.com/ViBiOh/absto v1.7.10/go.mod h1:MWd8MRDAAUOl6wGqte7U4wugXfRGageltfchOK7K6AQ= github.com/ViBiOh/auth/v2 v2.18.4 h1:XV825Y3oTJZrg9yBe74rLvqaCJ7bjUBoPdp4SN2CVXI= github.com/ViBiOh/auth/v2 v2.18.4/go.mod h1:vZ00r7g+aQ8RF6fApTruQRZyKZ1n7ZSrIpo02uLrZk4= -github.com/ViBiOh/exas v0.8.0 h1:C+3zeKH+VAK2HQxR3wvLqLxOwUXM+R3iYFlR6aJta/k= -github.com/ViBiOh/exas v0.8.0/go.mod h1:aldybvPY4yJQlvUpq6GJluGxH+7prpaJjDQ03t3VvCw= +github.com/ViBiOh/exas v0.7.1 h1:nzh6E1668OFRgoTIJZJa1RybCW64Gd918qi0Z+0Q4WA= +github.com/ViBiOh/exas v0.7.1/go.mod h1:MvIWQZXpWgg45Vsde9bMV48Kb0J8frya8vimmrkQ04E= github.com/ViBiOh/flags v1.5.0 h1:nwuFS8tAwtV6rTPpv2pCB+r12WjZYLjluW7yT+SeVpQ= github.com/ViBiOh/flags v1.5.0/go.mod h1:39UMuTnKsIp6walgD8dK99KRCb4DJt9vPtbWehHh1T0= github.com/ViBiOh/httputils/v4 v4.76.0 h1:fNMACQxmyPsSHFVeXitLw7BpO1Pm0yrWrTfKk2hSdvQ= diff --git a/pkg/crud/get.go b/pkg/crud/get.go index 687a6a36..b714600b 100644 --- a/pkg/crud/get.go +++ b/pkg/crud/get.go @@ -252,7 +252,7 @@ func (s Service) generateGeoJSON(ctx context.Context, w io.Writer, request provi return } - if exif.Coordinates == nil { + if !exif.Geocode.HasCoordinates() { continue } @@ -267,8 +267,8 @@ func (s Service) generateGeoJSON(ctx context.Context, w io.Writer, request provi commaNeeded = true } - point.Coordinates.Latitude = exif.Coordinates[0] - point.Coordinates.Longitude = exif.Coordinates[1] + point.Coordinates.Latitude = exif.Geocode.Latitude + point.Coordinates.Longitude = exif.Geocode.Longitude feature.Properties["url"] = request.RelativeURL(item) feature.Properties["date"] = exif.Date.Format(time.RFC850) diff --git a/pkg/crud/get_test.go b/pkg/crud/get_test.go index a5d52f8b..b7b5cc84 100644 --- a/pkg/crud/get_test.go +++ b/pkg/crud/get_test.go @@ -45,20 +45,29 @@ func BenchmarkServeGeoJSON(b *testing.B) { mockExif.EXPECT().GetAllMetadataFor(gomock.Any(), gomock.Any()).Return(map[string]provider.Metadata{ "9012": { Exif: exas.Exif{ - Coordinates: &exas.LatLng{1.0, 1.0}, - Date: time.Date(2022, 0o2, 22, 22, 0o2, 22, 0, time.UTC), + Geocode: exas.Geocode{ + Latitude: 1.0, + Longitude: 1.0, + }, + Date: time.Date(2022, 0o2, 22, 22, 0o2, 22, 0, time.UTC), }, }, "5678": { Exif: exas.Exif{ - Coordinates: &exas.LatLng{1.0, 1.0}, - Date: time.Date(2022, 0o2, 22, 22, 0o2, 22, 0, time.UTC), + Geocode: exas.Geocode{ + Latitude: 1.0, + Longitude: 1.0, + }, + Date: time.Date(2022, 0o2, 22, 22, 0o2, 22, 0, time.UTC), }, }, "1234": { Exif: exas.Exif{ - Coordinates: &exas.LatLng{1.0, 1.0}, - Date: time.Date(2022, 0o2, 22, 22, 0o2, 22, 0, time.UTC), + Geocode: exas.Geocode{ + Latitude: 1.0, + Longitude: 1.0, + }, + Date: time.Date(2022, 0o2, 22, 22, 0o2, 22, 0, time.UTC), }, }, }, nil).AnyTimes() diff --git a/pkg/crud/search.go b/pkg/crud/search.go index 61b7df4a..6b927db1 100644 --- a/pkg/crud/search.go +++ b/pkg/crud/search.go @@ -38,7 +38,7 @@ func (s Service) search(r *http.Request, request provider.Request, item absto.It items[i] = renderItem - if !hasMap && metadata.Coordinates != nil { + if !hasMap && metadata.Geocode.Longitude != 0 && metadata.Geocode.Latitude != 0 { hasMap = true } } diff --git a/pkg/crud/story.go b/pkg/crud/story.go index f330d37a..328e18dc 100644 --- a/pkg/crud/story.go +++ b/pkg/crud/story.go @@ -53,7 +53,7 @@ func (s Service) story(r *http.Request, request provider.Request, item absto.Ite exif := exifs[file.ID] - if !hasMap && exif.Coordinates != nil { + if !hasMap && exif.Geocode.HasCoordinates() { hasMap = true } diff --git a/pkg/metadata/aggregate.go b/pkg/metadata/aggregate.go index 873722d6..1f024691 100644 --- a/pkg/metadata/aggregate.go +++ b/pkg/metadata/aggregate.go @@ -11,6 +11,12 @@ import ( "github.com/ViBiOh/httputils/v4/pkg/telemetry" ) +var ( + aggregateRatio = 0.4 + + levels = []string{"city", "state", "country"} +) + func redisKey(item absto.Item) string { return version.Redis("exif:" + item.ID) } @@ -105,6 +111,7 @@ func (s Service) aggregate(ctx context.Context, item absto.Item) error { } func (s Service) computeAndSaveAggregate(ctx context.Context, dir absto.Item) error { + directoryAggregate := newAggregate() var minDate, maxDate time.Time previousAggregate, _ := s.GetAggregateFor(ctx, dir) @@ -127,6 +134,10 @@ func (s Service) computeAndSaveAggregate(ctx context.Context, dir absto.Item) er minDate, maxDate = aggregateDate(minDate, maxDate, exifData.Date) } + if exifData.Geocode.HasAddress() { + directoryAggregate.ingest(exifData.Geocode) + } + return nil }) if err != nil { @@ -134,9 +145,10 @@ func (s Service) computeAndSaveAggregate(ctx context.Context, dir absto.Item) er } return s.SaveAggregateFor(ctx, dir, provider.Aggregate{ - Cover: previousAggregate.Cover, - Start: minDate, - End: maxDate, + Cover: previousAggregate.Cover, + Location: directoryAggregate.value(), + Start: minDate, + End: maxDate, }) } diff --git a/pkg/metadata/model.go b/pkg/metadata/model.go new file mode 100644 index 00000000..ed085711 --- /dev/null +++ b/pkg/metadata/model.go @@ -0,0 +1,70 @@ +package metadata + +import ( + "strings" + + "github.com/ViBiOh/exas/pkg/model" +) + +type locationAggregate map[string]map[string]int64 + +func newAggregate() locationAggregate { + return make(map[string]map[string]int64) +} + +func (a locationAggregate) ingest(geocoding model.Geocode) { + for _, level := range levels { + a.inc(level, geocoding.Address[level]) + } +} + +func (a locationAggregate) inc(key, value string) { + if len(value) == 0 { + return + } + + if level, ok := a[key]; ok { + level[value]++ + } else { + a[key] = map[string]int64{ + value: 1, + } + } +} + +func (a locationAggregate) value() string { + if len(a) == 0 { + return "" + } + + for _, level := range levels { + if val := a.valueOf(level); len(val) > 0 { + return val + } + } + + return "Worldwide" +} + +func (a locationAggregate) valueOf(key string) string { + values, ok := a[key] + if !ok { + return "" + } + + var sum int64 + for _, v := range values { + sum += v + } + + var names []string + minSum := int64(float64(sum) * aggregateRatio) + + for k, v := range values { + if v > minSum { + names = append(names, k) + } + } + + return strings.Join(names, ", ") +} diff --git a/pkg/provider/metadata.go b/pkg/provider/metadata.go index 2e8a3628..f007cc99 100644 --- a/pkg/provider/metadata.go +++ b/pkg/provider/metadata.go @@ -16,9 +16,9 @@ type ExifResponse struct { } type Metadata struct { - exas.Exif Description string `json:"description,omitempty"` Tags []string `json:"tags,omitempty"` + exas.Exif } type Aggregate struct { diff --git a/pkg/version/version.go b/pkg/version/version.go index c64a52b9..e5e1dc28 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -7,7 +7,7 @@ import ( ) var ( - cacheVersion = provider.Hash("vibioh/fibr/4")[:8] + cacheVersion = provider.Hash("vibioh/fibr/3")[:8] cachePrefix = "fibr:" + cacheVersion )