Frontend: Fix Vaccine Provider (#3933)

frontend/playwright-tests/vaccination.ci.spec.ts

@@ -0,0 +1,75 @@
+import { test } from '@playwright/test'
+
+test('National Vaccination Full Test', async ({ page }) => {
+  await page.goto('/exploredata?mls=1.covid_vaccinations-3.00&group1=All')
+  await page
+    .locator('#rate-map')
+    .getByRole('heading', { name: 'COVID-19 vaccination rates in' })
+    .click()
+  await page.getByText('Percentages Over 100%').click()
+  await page
+    .locator('#rate-chart')
+    .getByRole('heading', { name: 'COVID-19 vaccination rates in' })
+    .click()
+  await page.getByLabel('Bar Chart Showing COVID-19').click()
+  await page
+    .getByLabel('Bar Chart Showing COVID-19')
+    .getByText('% vaccinated (at least one')
+    .click()
+  await page.getByRole('heading', { name: 'Share of total COVID-19' }).click()
+  await page.getByText('% unknown').click()
+  await page.locator('#unknown-demographic-map').getByText('no data').click()
+  await page
+    .getByRole('heading', { name: 'Population vs. distribution' })
+    .click()
+  await page
+    .getByLabel('light green bars represent %')
+    .getByText('% of population')
+    .click()
+  await page
+    .getByLabel('dark green bars represent %')
+    .getByText('% of all vaccinations')
+    .click()
+  await page.getByLabel('Comparison bar chart showing').locator('svg').click()
+  await page.getByRole('heading', { name: 'Summary for COVID-19' }).click()
+  await page
+    .getByRole('columnheader', { name: 'COVID-19 vaccination rates' })
+    .click()
+  await page
+    .getByRole('columnheader', { name: 'Share of total COVID-19' })
+    .click()
+  await page.getByRole('columnheader', { name: 'Population share' }).click()
+  await page
+    .getByRole('columnheader', { name: 'Population percentage' })
+    .click()
+})
+
+test('State Vaccination Quick Test', async ({ page }) => {
+  await page.goto('/exploredata?mls=1.covid_vaccinations-3.06&group1=All')
+  await page
+    .locator('#rate-map')
+    .getByRole('heading', { name: 'COVID-19 vaccination rates in' })
+    .click()
+  await page.getByText('Percentages Over 100%').click()
+  await page
+    .locator('#madlib-box')
+    .getByRole('button', { name: 'California' })
+    .click()
+  await page.getByRole('combobox').click()
+  await page.getByRole('combobox').fill('los')
+  await page
+    .getByRole('option', { name: 'Los Angeles County, California' })
+    .click()
+})
+
+test('County Vaccination Quick Test', async ({ page }) => {
+  await page.goto('/exploredata?mls=1.covid_vaccinations-3.06037&group1=All')
+  await page
+    .getByRole('heading', {
+      name: 'COVID-19 vaccination rates in Los Angeles County, California',
+      exact: true,
+    })
+    .click()
+  await page.getByLabel('Map showing COVID-19').locator('path').nth(1).click()
+  await page.getByText('This county has a social').click()
+})

frontend/src/data/config/DatasetMetadata.ts

@@ -132,10 +132,10 @@ export type DatasetId =
   | 'cdc_restricted_data-by_sex_national_processed'
   | 'cdc_restricted_data-by_sex_state_processed_time_series'
   | 'cdc_restricted_data-by_sex_state_processed'
-  | 'cdc_vaccination_county-alls_county'
-  | 'cdc_vaccination_national-age_processed' // TODO: rm "processed" on the backend and use the actual geography "national"
-  | 'cdc_vaccination_national-race_processed' // TODO: rm "processed" on the backend and use the actual geography "national"
-  | 'cdc_vaccination_national-sex_processed' // TODO: rm "processed" on the backend and use the actual geography "national"
+  | 'cdc_vaccination_county-alls_county_current'
+  | 'cdc_vaccination_national-age_national_current'
+  | 'cdc_vaccination_national-race_national_current'
+  | 'cdc_vaccination_national-sex_national_current'
   | 'cdc_wisqars_data-age_national_current'
   | 'cdc_wisqars_data-age_national_historical'
   | 'cdc_wisqars_data-age_state_current'
@@ -210,8 +210,8 @@ export type DatasetId =
   | 'geo_context-national'
   | 'geo_context-state'
   | 'geo_context-county'
-  | 'kff_vaccination-alls_state'
-  | 'kff_vaccination-race_and_ethnicity_state'
+  | 'kff_vaccination-alls_state_current'
+  | 'kff_vaccination-race_and_ethnicity_state_current'
   | 'maternal_mortality_data-by_race_national_current'
   | 'maternal_mortality_data-by_race_national_historical'
   | 'maternal_mortality_data-by_race_state_current'
@@ -649,35 +649,35 @@ export const DatasetMetadataMap: Record<DatasetId, DatasetMetadata> = {
     original_data_sourced: 'January 2020 - May 2024',
     source_id: 'cdc_restricted',
   },
-  'cdc_vaccination_county-alls_county': {
+  'cdc_vaccination_county-alls_county_current': {
     name: 'COVID-19 vaccinations by county',
     contains_nh: true,
     original_data_sourced: 'March 2023',
     source_id: 'cdc_vaccination_county',
   },
-  'cdc_vaccination_national-age_processed': {
+  'cdc_vaccination_national-age_national_current': {
     name: 'COVID-19 vaccinations by age, nationally',
     original_data_sourced: 'March 2023',
     source_id: 'cdc_vaccination_national',
   },
-  'cdc_vaccination_national-sex_processed': {
+  'cdc_vaccination_national-sex_national_current': {
     name: 'COVID-19 vaccinations by sex, nationally',
     original_data_sourced: 'March 2023',
     source_id: 'cdc_vaccination_national',
   },
-  'cdc_vaccination_national-race_processed': {
+  'cdc_vaccination_national-race_national_current': {
     name: 'COVID-19 vaccinations by race and ethnicity, nationally',
     original_data_sourced: 'March 2023',
     contains_nh: true,
     source_id: 'cdc_vaccination_national',
   },
-  'kff_vaccination-race_and_ethnicity_state': {
+  'kff_vaccination-race_and_ethnicity_state_current': {
     name: 'COVID-19 vaccinations by race and ethnicity by state/territory',
     original_data_sourced: 'July 2022',
     contains_nh: true,
     source_id: 'kff_vaccination',
   },
-  'kff_vaccination-alls_state': {
+  'kff_vaccination-alls_state_current': {
     name: 'COVID-19 vaccinations by state/territory',
     original_data_sourced: 'July 2022',
     contains_nh: true,

frontend/src/data/config/MetadataMap.ts

@@ -213,7 +213,7 @@ export const dataSourceMetadataMap: Record<DataSourceId, DataSourceMetadata> = {
     update_frequency: 'Daily',
     description:
       'Overall US COVID-19 Vaccine administration and vaccine equity data at county level Data represents all vaccine partners including jurisdictional partner clinics, retail pharmacies, long-term care facilities, dialysis centers, Federal Emergency Management Agency and Health Resources and Services Administration partner sites, and federal entity facilities.',
-    dataset_ids: ['cdc_vaccination_county-alls_county'],
+    dataset_ids: ['cdc_vaccination_county-alls_county_current'],
     downloadable: true,
     time_period_range: null,
   },
@@ -231,9 +231,9 @@ export const dataSourceMetadataMap: Record<DataSourceId, DataSourceMetadata> = {
     description:
       'Overall Demographic Characteristics of People Receiving COVID-19 Vaccinations in the United States at national level. Data represents all vaccine partners including jurisdictional partner clinics, retail pharmacies, long-term care facilities, dialysis centers, Federal Emergency Management Agency and Health Resources and Services Administration partner sites, and federal entity facilities. (CDC 2021)',
     dataset_ids: [
-      'cdc_vaccination_national-age_processed',
-      'cdc_vaccination_national-race_processed',
-      'cdc_vaccination_national-sex_processed',
+      'cdc_vaccination_national-age_national_current',
+      'cdc_vaccination_national-race_national_current',
+      'cdc_vaccination_national-sex_national_current',
     ],
     downloadable: true,
     time_period_range: null,
@@ -251,8 +251,8 @@ export const dataSourceMetadataMap: Record<DataSourceId, DataSourceMetadata> = {
     description:
       "State level vaccination information based off of Kaiser Family Foundation analysis of publicly available data from state websites. Per 100k metrics are found on 'COVID-19 Vaccinations by Race/Ethnicity', percent share metrics are found on 'Percent of Total Population that has Received a COVID-19 Vaccine by Race/Ethnicity' and the All metric is found on 'COVID-19 Vaccines Delivered and Administered'",
     dataset_ids: [
-      'kff_vaccination-race_and_ethnicity_state',
-      'kff_vaccination-alls_state',
+      'kff_vaccination-race_and_ethnicity_state_current',
+      'kff_vaccination-alls_state_current',
     ],
     downloadable: true,
     time_period_range: null,

frontend/src/data/providers/VaccineProvider.test.ts

@@ -27,7 +27,13 @@ async function ensureCorrectDatasetsDownloaded(
 
   // Evaluate the response with requesting "All" field
   const responseIncludingAll = await vaccineProvider.getData(
-    new MetricQuery([], baseBreakdown.addBreakdown(demographicType)),
+    new MetricQuery(
+      [],
+      baseBreakdown.addBreakdown(demographicType),
+      undefined,
+      undefined,
+      'rate-map', // needed to trigger fallback to ALLs
+    ),
   )
 
   expect(dataFetcher.getNumLoadDatasetCalls()).toBe(1)
@@ -46,41 +52,41 @@ describe('VaccineProvider', () => {
     dataFetcher.setFakeMetadataLoaded(DatasetMetadataMap)
   })
 
-  test('State and Race Breakdown', async () => {
-    await ensureCorrectDatasetsDownloaded(
-      'kff_vaccination-race_and_ethnicity_state',
-      Breakdowns.forFips(new Fips(NC.code)),
-      RACE,
-    )
-  })
-
   test('National and Race Breakdown', async () => {
     await ensureCorrectDatasetsDownloaded(
-      'cdc_vaccination_national-race_processed',
+      'cdc_vaccination_national-race_national_current',
       Breakdowns.forFips(new Fips(USA.code)),
       RACE,
     )
   })
 
   test('National and Sex Breakdown', async () => {
     await ensureCorrectDatasetsDownloaded(
-      'cdc_vaccination_national-sex_processed',
+      'cdc_vaccination_national-sex_national_current',
       Breakdowns.forFips(new Fips(USA.code)),
       SEX,
     )
   })
 
   test('National and Age Breakdown', async () => {
     await ensureCorrectDatasetsDownloaded(
-      'cdc_vaccination_national-age_processed',
+      'cdc_vaccination_national-age_national_current',
       Breakdowns.forFips(new Fips(USA.code)),
       AGE,
     )
   })
 
+  test('State and Race Breakdown', async () => {
+    await ensureCorrectDatasetsDownloaded(
+      'kff_vaccination-race_and_ethnicity_state_current',
+      Breakdowns.forFips(new Fips(NC.code)),
+      RACE,
+    )
+  })
+
   test('County and Race Breakdown', async () => {
     await ensureCorrectDatasetsDownloaded(
-      'cdc_vaccination_county-alls_county',
+      'cdc_vaccination_county-alls_county_current',
       Breakdowns.forFips(new Fips(MARIN.code)),
       RACE,
     )

frontend/src/data/providers/VaccineProvider.ts

@@ -1,7 +1,10 @@
 import { getDataManager } from '../../utils/globals'
-import type { DatasetId } from '../config/DatasetMetadata'
-import type { Breakdowns } from '../query/Breakdowns'
-import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
+import type { Breakdowns, GeographicBreakdown } from '../query/Breakdowns'
+import {
+  type MetricQuery,
+  MetricQueryResponse,
+  resolveDatasetId,
+} from '../query/MetricQuery'
 import { appendFipsIfNeeded } from '../utils/datasetutils'
 import { GetAcsDatasetId } from './AcsPopulationProvider'
 import VariableProvider from './VariableProvider'
@@ -13,6 +16,14 @@ export const COVID_VACCINATION_RESTRICTED_DEMOGRAPHIC_DETAILS = [
   ['Sex', reason],
 ]
 
+const datasetNameMappings: Record<GeographicBreakdown, string> = {
+  national: 'cdc_vaccination_national',
+  state: 'kff_vaccination',
+  territory: 'kff_vaccination',
+  'state/territory': 'kff_vaccination',
+  county: 'cdc_vaccination_county',
+}
+
 class VaccineProvider extends VariableProvider {
   constructor() {
     super('vaccine_provider', [
@@ -24,35 +35,21 @@ class VaccineProvider extends VariableProvider {
     ])
   }
 
-  getDatasetId(breakdowns: Breakdowns): DatasetId | undefined {
-    if (breakdowns.geography === 'national') {
-      if (breakdowns.hasOnlyRace())
-        return 'cdc_vaccination_national-race_processed'
-      if (breakdowns.hasOnlySex())
-        return 'cdc_vaccination_national-sex_processed'
-      if (breakdowns.hasOnlyAge())
-        return 'cdc_vaccination_national-age_processed'
-    }
-    if (breakdowns.geography === 'state') {
-      if (breakdowns.hasOnlyRace())
-        return 'kff_vaccination-race_and_ethnicity_state'
-      // WE HAVE THE ALLS SO CAN AT LEAST SHOW THOSE FOR AGE OR SEX REPORTS
-      if (breakdowns.hasOnlySex() || breakdowns.hasOnlyAge())
-        return 'kff_vaccination-alls_state'
-    }
-    if (breakdowns.geography === 'county') {
-      return 'cdc_vaccination_county-alls_county'
-    }
-  }
-
   async getDataInternal(
     metricQuery: MetricQuery,
   ): Promise<MetricQueryResponse> {
-    const breakdowns = metricQuery.breakdowns
-    const datasetId = this.getDatasetId(breakdowns)
+    const bqDatasetName = datasetNameMappings[metricQuery.breakdowns.geography]
+
+    const { datasetId, isFallbackId, breakdowns } = resolveDatasetId(
+      bqDatasetName,
+      '',
+      metricQuery,
+    )
+
     if (!datasetId) {
       return new MetricQueryResponse([], [])
     }
+
     const specificDatasetId = appendFipsIfNeeded(datasetId, breakdowns)
     const vaxData = await getDataManager().loadDataset(specificDatasetId)
     let df = vaxData.toDataFrame()
@@ -86,8 +83,12 @@ class VaccineProvider extends VariableProvider {
       consumedDatasetIds.push('acs_population-by_race_county')
     }
 
-    df = this.applyDemographicBreakdownFilters(df, breakdowns)
-    df = this.removeUnrequestedColumns(df, metricQuery)
+    if (isFallbackId) {
+      df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns)
+    } else {
+      df = this.applyDemographicBreakdownFilters(df, breakdowns)
+      df = this.removeUnrequestedColumns(df, metricQuery)
+    }
     return new MetricQueryResponse(df.toArray(), consumedDatasetIds)
   }