Merge pull request #6330 from hotosm/develop

Publish Data quality and other changes to staging site

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
   schedule:
     interval: daily
     time: "13:00"
-  open-pull-requests-limit: 10
+  open-pull-requests-limit: 0
   ignore:
   - dependency-name: python-dotenv
     versions:
@@ -31,7 +31,7 @@ updates:
   schedule:
     interval: daily
     time: "13:00"
-  open-pull-requests-limit: 10
+  open-pull-requests-limit: 0
   ignore:
   - dependency-name: "@testing-library/user-event"
     versions:
@@ -275,7 +275,7 @@ updates:
   schedule:
     interval: daily
     time: "13:00"
-  open-pull-requests-limit: 10
+  open-pull-requests-limit: 0
   ignore:
   - dependency-name: python-dotenv
     versions:

backend/__init__.py

@@ -360,6 +360,7 @@ def add_api_endpoints(app):
         UsersStatisticsAPI,
         UsersStatisticsInterestsAPI,
         UsersStatisticsAllAPI,
+        OhsomeProxyAPI,
     )
 
     # System API endpoint
@@ -900,6 +901,9 @@ def add_api_endpoints(app):
         UsersStatisticsAllAPI,
         format_url("users/statistics/"),
     )
+    api.add_resource(
+        OhsomeProxyAPI, format_url("users/statistics/ohsome/"), methods=["GET"]
+    )
     # User RecommendedProjects endpoint
     api.add_resource(
         UsersRecommendedProjectsAPI,

backend/api/users/statistics.py

@@ -1,12 +1,14 @@
 from json import JSONEncoder
 from datetime import date, timedelta
 from flask_restful import Resource, request
+import requests
 
 from backend.services.users.user_service import UserService
 from backend.services.stats_service import StatsService
 from backend.services.interests_service import InterestService
 from backend.services.users.authentication_service import token_auth
 from backend.api.utils import validate_date_input
+from backend.config import EnvironmentConfig
 
 
 class UsersStatisticsAPI(Resource, JSONEncoder):
@@ -138,3 +140,43 @@ def get(self):
             return stats.to_primitive(), 200
         except (KeyError, ValueError) as e:
             return {"Error": str(e).split("-")[1], "SubCode": str(e).split("-")[0]}, 400
+
+
+class OhsomeProxyAPI(Resource):
+    @token_auth.login_required
+    def get(self):
+        """
+        Get HomePage Stats
+        ---
+        tags:
+          - system
+        produces:
+          - application/json
+        parameters:
+        - in: header
+          name: Authorization
+          description: Base64 encoded session token
+          required: true
+          type: string
+          default: Token sessionTokenHere==
+        - in: query
+          name: url
+          type: string
+          description: get user stats for osm contributions
+        responses:
+            200:
+                description: User stats
+            500:
+                description: Internal Server Error
+        """
+        url = request.args.get("url")
+        if not url:
+            return {"Error": "URL is None", "SubCode": "URL not provided"}, 400
+        try:
+            headers = {"Authorization": f"Basic {EnvironmentConfig.OHSOME_STATS_TOKEN}"}
+
+            # Make the GET request with headers
+            response = requests.get(url, headers=headers)
+            return response.json(), 200
+        except Exception as e:
+            return {"Error": str(e), "SubCode": "Error fetching data"}, 400

backend/config.py

@@ -237,6 +237,9 @@ class EnvironmentConfig:
     # Sentry backend DSN
     SENTRY_BACKEND_DSN = os.getenv("TM_SENTRY_BACKEND_DSN", None)
 
+    # Ohsome Stats Token
+    OHSOME_STATS_TOKEN = os.getenv("OHSOME_STATS_TOKEN", None)
+
 
 class TestEnvironmentConfig(EnvironmentConfig):
     POSTGRES_TEST_DB = os.getenv("POSTGRES_TEST_DB", None)

backend/services/messaging/message_service.py

@@ -1,6 +1,7 @@
 import re
 import time
 import datetime
+import bleach
 
 from cachetools import TTLCache, cached
 from typing import List
@@ -243,6 +244,34 @@ def send_message_after_comment(
             task_link = MessageService.get_task_link(project_id, task_id)
             project_link = MessageService.get_project_link(project_id, project_name)
 
+            # Clean comment and convert to html
+            allowed_tags = [
+                "a",
+                "b",
+                "blockquote",
+                "br",
+                "code",
+                "em",
+                "h1",
+                "h2",
+                "h3",
+                "img",
+                "i",
+                "li",
+                "ol",
+                "p",
+                "pre",
+                "strong",
+                "ul",
+            ]
+            allowed_atrributes = {"a": ["href", "rel"], "img": ["src", "alt"]}
+            clean_comment = bleach.clean(
+                markdown(comment, output_format="html"),
+                tags=allowed_tags,
+                attributes=allowed_atrributes,
+            )  # Bleach input to ensure no nefarious script tags etc
+            clean_comment = bleach.linkify(clean_comment)
+
             messages = []
             for username in usernames:
                 try:
@@ -260,7 +289,7 @@ def send_message_after_comment(
                     f"You were mentioned in a comment in {task_link} "
                     + f"of Project {project_link}"
                 )
-                message.message = comment
+                message.message = clean_comment
                 messages.append(
                     dict(message=message, user=user, project_name=project_name)
                 )

example.env

@@ -206,6 +206,8 @@ TM_DEFAULT_LOCALE=en
 # TM_SENTRY_BACKEND_DSN=https://foo.ingest.sentry.io/1234567
 # TM_SENTRY_FRONTEND_DSN=https://bar.ingest.sentry.io/8901234
 
+# Underpass API URL (for project live monitoring feature)
+UNDERPASS_URL=https://underpass.hotosm.org
 
 #EXPORT TOOL Integration with 0(Disable) and 1(Enable) and S3 URL for Export Tool
 #EXPORT_TOOL_S3_URL=https://foorawdataapi.s3.amazonaws.com

frontend/package.json

@@ -9,6 +9,7 @@
     "@formatjs/intl-relativetimeformat": "^11.2.4",
     "@formatjs/macro": "^0.2.8",
     "@hotosm/id": "^2.21.1",
+    "@hotosm/underpass-ui": "^0.0.4",
     "@hotosm/iso-countries-languages": "^1.1.2",
     "@mapbox/mapbox-gl-draw": "^1.4.1",
     "@mapbox/mapbox-gl-geocoder": "^5.0.1",

frontend/src/api/projects.js

@@ -1,9 +1,11 @@
+import axios from 'axios';
 import { subMonths, format } from 'date-fns';
 import { useQuery } from '@tanstack/react-query';
 import { useSelector } from 'react-redux';
 
 import { remapParamsToAPI } from '../utils/remapParamsToAPI';
 import api from './apiClient';
+import { UNDERPASS_URL } from '../config';
 
 export const useProjectsQuery = (fullProjectsQuery, action) => {
   const token = useSelector((state) => state.auth.token);
@@ -187,6 +189,18 @@ export const submitValidationTask = (projectId, payload, token, locale) => {
   );
 };
 
+export const useAvailableCountriesQuery = () => {
+  const fetchGeojsonData = () => {
+    return axios.get(`${UNDERPASS_URL}/availability`);
+  };
+
+  return useQuery({
+    queryKey: ['priority-geojson'],
+    queryFn: fetchGeojsonData,
+    select: (res) => res.data,
+  });
+};
+
 const backendToQueryConversion = {
   difficulty: 'difficulty',
   campaign: 'campaign',

frontend/src/api/stats.js

@@ -1,9 +1,14 @@
 import { useQuery } from '@tanstack/react-query';
-import { fetchExternalJSONAPI } from '../network/genericJSONRequest';
 
+import { fetchExternalJSONAPI } from '../network/genericJSONRequest';
 import api from './apiClient';
 import { OHSOME_STATS_BASE_URL } from '../config';
 
+const ohsomeProxyAPI = (url) => {
+  const token = localStorage.getItem('token');
+  return api(token).get(`users/statistics/ohsome/?url=${url}`);
+};
+
 export const useSystemStatisticsQuery = () => {
   const fetchSystemStats = ({ signal }) => {
     return api().get(`system/statistics/`, {
@@ -65,17 +70,17 @@ export const useOsmHashtagStatsQuery = (defaultComment) => {
 
 export const useUserOsmStatsQuery = (id) => {
   const fetchUserOsmStats = () => {
-    return fetchExternalJSONAPI(
+    return ohsomeProxyAPI(
       `${OHSOME_STATS_BASE_URL}/topic/poi,highway,building,waterway/user?userId=${id}`,
-      true,
     );
   };
 
   return useQuery({
     queryKey: ['user-osm-stats'],
     queryFn: fetchUserOsmStats,
-    useErrorBoundary: true,
-    select: (data) => data.result,
+    // userDetail.test.js fails on CI when useErrorBoundary=true
+    useErrorBoundary: process.env.NODE_ENV !== 'test',
+    select: (data) => data.data.result,
     enabled: !!id,
   });
 };

frontend/src/components/footer/index.js

@@ -23,7 +23,7 @@ import {
 import './styles.scss';
 
 const socialNetworks = [
-  { link: ORG_TWITTER, icon: <TwitterIcon style={{ height: '20px', width: '20px' }} /> },
+  { link: ORG_TWITTER, icon: <TwitterIcon style={{ height: '20px', width: '20px' }} noBg /> },
   { link: ORG_FB, icon: <FacebookIcon style={{ height: '20px', width: '20px' }} /> },
   { link: ORG_YOUTUBE, icon: <YoutubeIcon style={{ height: '20px', width: '20px' }} /> },
   { link: ORG_INSTAGRAM, icon: <InstagramIcon style={{ height: '20px', width: '20px' }} /> },
@@ -38,6 +38,7 @@ export function Footer() {
     'projects/:id/tasks',
     'projects/:id/map',
     'projects/:id/validate',
+    'projects/:id/live',
     'manage/organisations/new/',
     'manage/teams/new',
     'manage/campaigns/new',

frontend/src/components/projectDetail/index.js

@@ -21,7 +21,9 @@ import { PermissionBox } from './permissionBox';
 import { CustomButton } from '../button';
 import { ProjectInfoPanel } from './infoPanel';
 import { OSMChaButton } from './osmchaButton';
+import { LiveViewButton } from './liveViewButton';
 import { useSetProjectPageTitleTag } from '../../hooks/UseMetaTags';
+import useHasLiveMonitoringFeature from '../../hooks/UseHasLiveMonitoringFeature';
 import { useProjectContributionsQuery, useProjectTimelineQuery } from '../../api/projects';
 import { Alert } from '../alert';
 
@@ -144,6 +146,8 @@ export const ProjectDetail = (props) => {
     props.project.projectId,
   );
 
+  const hasLiveMonitoringFeature = useHasLiveMonitoringFeature();
+
   const htmlDescription =
     props.project.projectInfo && htmlFromMarkdown(props.project.projectInfo.description);
   const h2Classes = 'pl4 f3 f2-ns fw5 mt2 mb3 mb4-ns ttu barlow-condensed blue-dark';
@@ -346,6 +350,15 @@ export const ProjectDetail = (props) => {
             project={props.project}
             className="bg-white blue-dark ba b--grey-light pa3"
           />
+
+          {/* show live view button only when the project has live monitoring feature */}
+          {hasLiveMonitoringFeature && (
+            <LiveViewButton
+              projectId={props.project.projectId}
+              className="bg-white blue-dark ba b--grey-light pa3"
+            />
+          )}
+
           <DownloadAOIButton
             projectId={props.project.projectId}
             className="bg-white blue-dark ba b--grey-light pa3"

frontend/src/components/projectDetail/liveViewButton.js

@@ -0,0 +1,20 @@
+import React from 'react';
+import { Link } from 'react-router-dom';
+import { FormattedMessage } from 'react-intl';
+
+import messages from './messages';
+import { CustomButton } from '../button';
+
+export const LiveViewButton = ({ projectId, className, compact = false }) => (
+  <Link to={`/projects/${projectId}/live`} className="pr2">
+    {
+      <CustomButton className={className}>
+        {compact ? (
+          <FormattedMessage {...messages.live} />
+        ) : (
+          <FormattedMessage {...messages.liveMonitoring} />
+        )}
+      </CustomButton>
+    }
+  </Link>
+);

frontend/src/components/projectDetail/messages.js

@@ -240,6 +240,14 @@ export default defineMessages({
     id: 'project.detail.sections.contributions.osmcha',
     defaultMessage: 'Changesets in OSMCha',
   },
+  live: {
+    id: 'project.detail.sections.contributions.live',
+    defaultMessage: 'Live',
+  },
+  liveMonitoring: {
+    id: 'project.detail.sections.contributions.liveMonitoring',
+    defaultMessage: 'Live monitoring',
+  },
   changesets: {
     id: 'project.detail.sections.contributions.changesets',
     defaultMessage: 'Changesets',

frontend/src/components/projectDetail/shareButton.js

@@ -6,7 +6,7 @@ import messages from './messages';
 import { ORG_CODE } from '../../config';
 import { createPopup } from '../../utils/login';
 import { getTwitterLink, getLinkedInLink, getFacebookLink } from '../../utils/shareFunctions';
-import { TwitterIconNoBg, FacebookIcon, LinkedinIcon, ShareIcon } from '../svgIcons';
+import { TwitterIcon, FacebookIcon, LinkedinIcon, ShareIcon } from '../svgIcons';
 
 export function ShareButton({ projectId }: Object) {
   const iconStyle = { width: '1.4em', height: '1.4em' };
@@ -41,7 +41,7 @@ export function ShareButton({ projectId }: Object) {
                 className="link no-underline base-font f6 pointer pv1"
                 onClick={() => twitterPopup(msg)}
               >
-                <TwitterIconNoBg style={iconStyle} className="light-blue v-mid pb1 pr2" />
+                <TwitterIcon style={iconStyle} className="light-blue v-mid pb1 pr2" />
                 Tweet
               </div>
               <div

frontend/src/components/svgIcons/index.js

@@ -69,7 +69,6 @@ export { LoadingIcon } from './loading';
 export { EnvelopeIcon } from './envelope';
 export { LinkedinIcon } from './linkedin';
 export { MarkerIcon } from './marker';
-export { TwitterIconNoBg } from './twitterNoBg';
 export { ZoomPlusIcon } from './zoomPlus';
 export { SidebarIcon } from './sidebar';
 export { QuestionCircleIcon } from './questionCircle';

frontend/src/components/svgIcons/twitter.js

@@ -3,15 +3,20 @@ import React from 'react';
 export class TwitterIcon extends React.PureComponent {
   render() {
     return (
-      <svg width="15" height="14" viewBox="0 0 15 14" {...this.props} aria-label="Twitter">
-        <g fill="none" fillRule="evenodd">
-          <path d="M-13-13h40v40h-40z" />
-          <path
-            d="M12.394 14h-10.5a1.75 1.75 0 0 1-1.75-1.75V1.75C.144.784.928 0 1.894 0h10.5c.966 0 1.75.784 1.75 1.75v10.5a1.75 1.75 0 0 1-1.75 1.75zM5.52 10.556c3.3 0 5.107-2.737 5.107-5.106 0-.078 0-.154-.004-.232.35-.252.655-.57.896-.93a3.637 3.637 0 0 1-1.032.283c.37-.22.654-.574.79-.994a3.57 3.57 0 0 1-1.14.434 1.797 1.797 0 0 0-3.059 1.638 5.096 5.096 0 0 1-3.7-1.876 1.8 1.8 0 0 0 .556 2.398 1.815 1.815 0 0 1-.811-.224v.024c0 .868.619 1.596 1.438 1.76a1.75 1.75 0 0 1-.808.032 1.795 1.795 0 0 0 1.676 1.246 3.6 3.6 0 0 1-2.656.745 5.105 5.105 0 0 0 2.747.802z"
-            fill="currentColor"
-            fillRule="nonzero"
-          />
-        </g>
+      <svg
+        width="15"
+        height="15"
+        viewBox="0 0 15 15"
+        fill="none"
+        xmlns="http://www.w3.org/2000/svg"
+        {...this.props}
+        aria-label="Twitter"
+      >
+        {!this.props.noBg && <rect width="15" height="15" rx="1" fill="black" />}
+        <path
+          d="M8.54822 6.66248L12.644 2H11.6735L8.1156 6.04753L5.27595 2H2L6.29505 8.12111L2 13.01H2.97043L6.72541 8.7347L9.72488 13.01H13.0008L8.54822 6.66248ZM7.21864 8.17485L6.7828 7.56494L3.32038 2.71647H4.81116L7.60626 6.63082L8.04027 7.24073L11.6731 12.3285H10.1823L7.21864 8.17485Z"
+          fill="white"
+        />
       </svg>
     );
   }