Merge pull request #306 from caktus/develop

Production release v1.20.0

.github/workflows/deploy.yaml

@@ -24,7 +24,7 @@ jobs:
           echo "ENV_URL=https://nccopwatch.org/" >> $GITHUB_ENV
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pip'
           cache-dependency-path: 'requirements/*/*.txt'
       - name: Install dependencies

.github/workflows/test.yaml

@@ -28,7 +28,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pip'
           cache-dependency-path: 'requirements/*/*.txt'
       - name: Install dependencies

.gitignore

@@ -16,6 +16,7 @@ traffic_stops.log.1
 reports
 env
 venv
+bin
 jmeter.log
 npm-debug.log
 .transifexrc
@@ -51,6 +52,7 @@ env-local.sh
 jgdc.yml
 aws_ec2.yml
 deploy/roles
+deploy/.kube
 docker-compose.override.yml
 htmlcov
 nc/notebooks/**/**.html

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
-        language_version: python3.8
+        language_version: python3.10
         exclude: migrations
   - repo: https://github.com/PyCQA/flake8
     rev: 4.0.1

Dockerfile

@@ -8,7 +8,7 @@ RUN npm install --silent
 COPY frontend/ /code/
 RUN npm run build
 
-FROM python:3.8-slim-bullseye as base
+FROM python:3.10-slim-bullseye as base
 
 # Create a group and user to run our app
 ARG APP_USER=appuser
@@ -93,7 +93,7 @@ ENTRYPOINT ["/code/docker-entrypoint.sh"]
 # Start uWSGI
 CMD ["newrelic-admin", "run-program", "uwsgi", "--single-interpreter", "--enable-threads", "--show-config"]
 
-FROM python:3.8-slim-bullseye AS dev
+FROM python:3.10-slim-bullseye AS dev
 
 ARG USERNAME=appuser
 ARG USER_UID=1000
@@ -113,8 +113,8 @@ RUN groupadd --gid $USER_GID $USERNAME \
 #   openssh-client -- for git over SSH
 #   sudo -- to run commands as superuser
 #   vim -- enhanced vi editor for commits
-ENV KUBE_CLIENT_VERSION="v1.25.10"
-ENV HELM_VERSION="3.12.0"
+ENV KUBE_CLIENT_VERSION="v1.29.4"
+ENV HELM_VERSION="3.14.4"
 RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
     --mount=type=cache,mode=0755,target=/root/.cache/pip \
     set -ex \

deploy/group_vars/all.yml

@@ -41,7 +41,7 @@ cloudformation_stack:
     MaxScale: 4
     UseAES256Encryption: "true"
     CustomerManagedCmkArn: ""
-    ContainerInstanceType: t3a.medium
+    ContainerInstanceType: t3a.large
     ContainerVolumeSize: 40
     DatabaseAllocatedStorage: 100
     DatabaseClass: db.t3.large
@@ -64,7 +64,7 @@ k8s_install_descheduler: yes
 # You must set the k8s_descheduler_chart_version to match the Kubernetes
 # node version (0.23.x -> K8s 1.23.x); see:
 # https://github.com/kubernetes-sigs/descheduler#compatibility-matrix
-k8s_descheduler_chart_version: v0.25.2
+k8s_descheduler_chart_version: v0.29.0
 # See values.yaml for options:
 # https://github.com/kubernetes-sigs/descheduler/blob/master/charts/descheduler/values.yaml#L63
 k8s_descheduler_release_values:
@@ -94,9 +94,9 @@ k8s_iam_users: [copelco]
 # Pin ingress-nginx and cert-manager to current versions so future upgrades of this
 # role will not upgrade these charts without your intervention:
 # https://github.com/kubernetes/ingress-nginx/releases
-k8s_ingress_nginx_chart_version: "4.4.2"
+k8s_ingress_nginx_chart_version: "4.9.1"
 # https://github.com/jetstack/cert-manager/releases
-k8s_cert_manager_chart_version: "v1.11.1"
+k8s_cert_manager_chart_version: "v1.14.3"
 # AWS only:
 # Use the newer load balancer type (NLB). DO NOT edit k8s_aws_load_balancer_type after
 # creating your Service.
@@ -106,11 +106,11 @@ k8s_aws_load_balancer_type: nlb
 # caktus.k8s-hosting-services: Logging and monitoring configuration
 # ----------------------------------------------------------------------------
 
-k8s_papertrail_logspout_destination: "syslog+tls://logs2.papertrailapp.com:20851"
-k8s_papertrail_logspout_memory_limit: 128Mi
+# k8s_papertrail_logspout_destination: "syslog+tls://logs2.papertrailapp.com:20851"
+# k8s_papertrail_logspout_memory_limit: 128Mi
 
 # New Relic Account: [email protected]
-k8s_newrelic_chart_version: "5.0.4"
+k8s_newrelic_chart_version: "5.0.68"
 k8s_newrelic_logging_enabled: true
 k8s_newrelic_license_key: !vault |
   $ANSIBLE_VAULT;1.1;AES256

deploy/group_vars/k8s.yml

@@ -51,6 +51,15 @@ k8s_worker_command:
   - worker
   - --loglevel=info
 k8s_worker_beat_enabled: true
+k8s_worker_beat_command:
+  - celery
+  - --app={{ k8s_worker_celery_app }}
+  - --workdir=/code
+  - beat
+  - --loglevel=info
+  # Avoid setting PID file that might get stale in case of a hard crash (StatefulSet will ensure only 1 copy is ever running)
+  - --pidfile=
+  - --schedule=/data/schedulefile.db
 k8s_memcached_enabled: false
 k8s_redis_enabled: true
 

deploy/requirements.yml

@@ -2,16 +2,16 @@
 
 - src: https://github.com/caktus/ansible-role-django-k8s
   name: caktus.django-k8s
-  version: v1.6.0
+  version: v1.9.0
 
 - src: https://github.com/caktus/ansible-role-aws-web-stacks
   name: caktus.aws-web-stacks
   version: ''
 
 - src: https://github.com/caktus/ansible-role-k8s-web-cluster
   name: caktus.k8s-web-cluster
-  version: v1.5.0
+  version: v1.6.0
 
 - src: https://github.com/caktus/ansible-role-k8s-hosting-services
   name: caktus.k8s-hosting-services
-  version: v0.11.0
+  version: v0.12.0

docs/dev-setup.rst

@@ -5,7 +5,7 @@ Below you will find basic setup and deployment instructions for the NC Traffic
 Stops project. To begin you should have the following applications installed on
 your local development system:
 
-- Python 3.8
+- Python 3.10
 - NodeJS >= 12.6.0
 - `pip >= 8 or so <http://www.pip-installer.org/>`_
 - Postgres >= 12
@@ -85,8 +85,8 @@ To use ``psql`` locally, make sure you have the following env variables loaded
 To setup your local environment you should create a virtualenv and install the
 necessary requirements::
 
-    $ which python3.8  # make sure you have Python 3.8 installed
-    $ mkvirtualenv --python=`which python3.8` traffic-stops
+    $ which python3.11  # make sure you have Python 3.11 installed
+    $ mkvirtualenv --python=`which python3.11` traffic-stops
     (traffic-stops)$ pip install -U pip
     (traffic-stops)$ make setup
 

docs/hosting-services.md

@@ -4,7 +4,7 @@ The services configured for this project are:
 * PostgreSQL database backups to S3 (within Caktus AWS account)
     * Currently, this is only `traffic_stops`, which contains users, census data, etc.
     * `traffic_stops_nc` is not backed up since the entire dataset is re-imported daily.
-* Papertrail logging (to Caktus account)
+* New Relic logging (to Caktus account)
 * New Relic Infrastructure monitoring (Account: `[email protected]`)
 
 

frontend/src/Components/AgencyData/AgencyData.js

@@ -15,6 +15,8 @@ import AgencyHeader from './AgencyHeader';
 import Sidebar from '../Sidebar/Sidebar';
 import ChartRoutes from '../Charts/ChartRoutes';
 import { CompareAlertBox } from '../Elements/Alert/Alert';
+import { YEARS_DEFAULT } from '../Charts/chartUtils';
+import axios from '../../Services/Axios';
 
 function AgencyData(props) {
   let { agencyId } = useParams();
@@ -27,6 +29,10 @@ function AgencyData(props) {
   const [chartsOpen, setChartsOpen] = useState(false);
   const [chartState] = useDataset(agencyId, AGENCY_DETAILS);
 
+  const [yearRange, setYearRange] = useState([YEARS_DEFAULT]);
+  const [year, setYear] = useState(YEARS_DEFAULT);
+  const [yearIdx, setYearIdx] = useState(null);
+
   useEffect(() => {
     if (chartState.data[AGENCY_DETAILS]) setSidebarOpen(true);
   }, [chartState.data[AGENCY_DETAILS]]);
@@ -39,6 +45,18 @@ function AgencyData(props) {
     if (chartState.data[AGENCY_DETAILS]) setChartsOpen(true);
   }, [chartState.data[AGENCY_DETAILS]]);
 
+  useEffect(() => {
+    axios.get(`/api/agency/${agencyId}/year-range/`).then((res) => {
+      setYearRange([YEARS_DEFAULT].concat(res.data.year_range));
+    });
+  }, [agencyId]);
+
+  const handleYearSelect = (y, idx) => {
+    if (y === year) return;
+    setYear(y);
+    setYearIdx(idx); // Used for some pie chart graphs
+  };
+
   return (
     <S.AgencyData data-testid="AgencyData" {...props}>
       {props.showCompare && !props.agencyId && <CompareAlertBox />}
@@ -48,6 +66,9 @@ function AgencyData(props) {
         toggleShowCompare={props.toggleShowCompare}
         showCompareDepartments={props.showCompare}
         showCloseButton={!!props?.agencyId}
+        yearRange={yearRange}
+        year={year}
+        handleYearSelect={handleYearSelect}
       />
       <S.ContentWrapper showCompare={props.showCompare}>
         <AnimatePresence>
@@ -67,7 +88,16 @@ function AgencyData(props) {
             </motion.div>
           )}
         </AnimatePresence>
-        {chartsOpen && <ChartRoutes agencyId={agencyId} showCompare={props.showCompare} />}
+        {chartsOpen && (
+          <ChartRoutes
+            agencyId={agencyId}
+            showCompare={props.showCompare}
+            agencyName={chartState.data[AGENCY_DETAILS].name}
+            yearRange={yearRange}
+            year={year}
+            yearIdx={yearIdx}
+          />
+        )}
       </S.ContentWrapper>
     </S.AgencyData>
   );

frontend/src/Components/AgencyData/AgencyHeader.js

@@ -18,13 +18,17 @@ import BackButton from '../Elements/BackButton';
 import Button from '../Elements/Button';
 import * as ChartHeaderStyles from '../Charts/ChartSections/ChartHeader.styled';
 import CensusData from './CensusData';
+import DataSubsetPicker from '../Charts/ChartSections/DataSubsetPicker/DataSubsetPicker';
 
 function AgencyHeader({
   agencyHeaderOpen,
   agencyDetails,
   toggleShowCompare,
   showCompareDepartments,
   showCloseButton,
+  yearRange,
+  year,
+  handleYearSelect,
 }) {
   const history = useHistory();
   const { agencyId } = useParams();
@@ -92,6 +96,13 @@ function AgencyHeader({
               showCompareDepartments={showCompareDepartments}
             />
           </S.SubHeaderContentRow>
+          <DataSubsetPicker
+            label="Year"
+            value={year}
+            onChange={handleYearSelect}
+            options={yearRange}
+            dropDown
+          />
           {!showCloseButton && (
             <S.AgencyHeaderButton>
               <Button

frontend/src/Components/Charts/Arrest/Arrests.js

@@ -0,0 +1,93 @@
+import React, { useEffect, useState } from 'react';
+import ArrestsStyled from './Arrests.styles';
+
+// Hooks
+import useMetaTags from '../../../Hooks/useMetaTags';
+import useTableModal from '../../../Hooks/useTableModal';
+
+// Children
+import PercentageOfStops from './Charts/PercentageOfStops';
+import PercentageOfSearches from './Charts/PercentageOfSearches';
+import CountOfStopsAndArrests from './Charts/CountOfStopsAndArrests';
+import PercentageOfStopsForStopPurposeGroup from './Charts/PercentageOfStopsForPurposeGroup';
+import PercentageOfStopsForStopPurpose from './Charts/PercentageOfStopsPerStopPurpose';
+import PercentageOfSearchesForStopPurposeGroup from './Charts/PercentageOfSearchesForPurposeGroup';
+import PercentageOfSearchesPerStopPurpose from './Charts/PercentageOfSearchesPerStopPurpose';
+import PercentageOfStopsPerContrabandType from './Charts/PercentageOfStopsPerContrabandType';
+import Switch from 'react-switch';
+import { SwitchContainer } from '../TrafficStops/TrafficStops.styled';
+
+function Arrests(props) {
+  const { year } = props;
+  const [togglePercentageOfStops, setTogglePercentageOfStops] = useState(true);
+  const [togglePercentageOfSearches, setTogglePercentageOfSearches] = useState(true);
+
+  const renderMetaTags = useMetaTags();
+  const [renderTableModal] = useTableModal();
+
+  useEffect(() => {
+    if (window.location.hash) {
+      document.querySelector(`${window.location.hash}`).scrollIntoView();
+    }
+  }, []);
+
+  const stopGraphToggle = () => (
+    <SwitchContainer>
+      <span>
+        Switch to view {togglePercentageOfStops ? 'all stop purposes' : 'grouped stop purposes '}
+      </span>
+      <Switch
+        onChange={() => setTogglePercentageOfStops(!togglePercentageOfStops)}
+        checked={togglePercentageOfStops}
+        className="react-switch"
+      />
+    </SwitchContainer>
+  );
+
+  const searchGraphToggle = () => (
+    <SwitchContainer>
+      <span>
+        Switch to view {togglePercentageOfSearches ? 'all stop purposes' : 'grouped stop purposes '}
+      </span>
+      <Switch
+        onChange={() => setTogglePercentageOfSearches(!togglePercentageOfSearches)}
+        checked={togglePercentageOfSearches}
+        className="react-switch"
+      />
+    </SwitchContainer>
+  );
+
+  return (
+    <ArrestsStyled>
+      {renderMetaTags()}
+      {renderTableModal()}
+      <PercentageOfStops {...props} year={year} />
+      <PercentageOfSearches {...props} year={year} />
+      <CountOfStopsAndArrests {...props} year={year} />
+
+      {togglePercentageOfStops ? (
+        <PercentageOfStopsForStopPurposeGroup {...props} year={year}>
+          {stopGraphToggle()}
+        </PercentageOfStopsForStopPurposeGroup>
+      ) : (
+        <PercentageOfStopsForStopPurpose {...props} year={year}>
+          {stopGraphToggle()}
+        </PercentageOfStopsForStopPurpose>
+      )}
+
+      {togglePercentageOfSearches ? (
+        <PercentageOfSearchesForStopPurposeGroup {...props} year={year}>
+          {searchGraphToggle()}
+        </PercentageOfSearchesForStopPurposeGroup>
+      ) : (
+        <PercentageOfSearchesPerStopPurpose {...props} year={year}>
+          {searchGraphToggle()}
+        </PercentageOfSearchesPerStopPurpose>
+      )}
+
+      <PercentageOfStopsPerContrabandType {...props} year={year} />
+    </ArrestsStyled>
+  );
+}
+
+export default Arrests;