handles all gun deaths

SatcherInstitute · Feb 7, 2025 · 8cbb27b · 8cbb27b
1 parent 283a962
commit 8cbb27b
Show file tree

Hide file tree

Showing 15 changed files with 27,052 additions and 3,940 deletions.
diff --git a/data/cdc_wisqars/gun_deaths-national-age.csv b/data/cdc_wisqars/gun_deaths-national-age.csv
diff --git a/data/cdc_wisqars/gun_deaths-national-sex.csv b/data/cdc_wisqars/gun_deaths-national-sex.csv
@@ -0,0 +1,61 @@
+"Year","Sex","Deaths","Population","Crude Rate","Age-Adjusted Rate","Years of Potential Life Lost"
+"2022","Males","41,302","165,283,553","24.99","24.52","992,318"
+"2022","Females","6,902","168,004,004","4.11","4.13","169,559"
+"2021","Males","41,866","164,722,995","25.42","25.09","1,050,307"
+"2021","Females","6,964","167,308,559","4.16","4.19","174,808"
+"2020","Males","38,981","164,525,109","23.69","23.40","977,338"
+"2020","Females","6,241","166,986,403","3.74","3.75","153,767"
+"2019","Males","34,041","161,692,336","21.05","20.63","796,133"
+"2019","Females","5,666","166,637,617","3.40","3.38","128,890"
+"2018","Males","33,955","160,960,513","21.10","20.65","785,393"
+"2018","Females","5,785","165,877,686","3.49","3.45","133,792"
+"2017","Males","34,062","160,113,445","21.27","20.90","807,929"
+"2017","Females","5,711","165,008,683","3.46","3.43","132,035"
+"2016","Males","32,994","159,085,693","20.74","20.40","785,176"
+"2016","Females","5,664","163,986,062","3.45","3.43","131,693"
+"2015","Males","31,032","157,906,843","19.65","19.31","725,275"
+"2015","Females","5,220","162,832,151","3.21","3.16","117,196"
+"2014","Males","28,715","156,695,810","18.33","18.01","650,838"
+"2014","Females","4,879","161,690,519","3.02","2.97","108,755"
+"2013","Males","28,794","155,514,054","18.52","18.26","660,720"
+"2013","Females","4,842","160,545,893","3.02","2.95","106,986"
+"2012","Males","28,838","154,397,027","18.68","18.43","671,823"
+"2012","Females","4,725","159,480,635","2.96","2.93","108,063"
+"2011","Males","27,738","153,212,980","18.10","17.92","651,378"
+"2011","Females","4,613","158,370,501","2.91","2.87","105,834"
+"2010","Males","27,356","151,788,777","18.02","17.90","651,295"
+"2010","Females","4,316","156,969,328","2.75","2.71","97,730"
+"2009","Males","26,921","150,807,454","17.85","17.78","641,715"
+"2009","Females","4,426","155,964,075","2.84","2.80","102,481"
+"2008","Males","27,336","149,489,864","18.29","18.22","670,228"
+"2008","Females","4,257","154,603,927","2.75","2.73","100,939"
+"2007","Males","27,047","148,064,715","18.27","18.25","681,804"
+"2007","Females","4,177","153,166,315","2.73","2.71","100,946"
+"2006","Males","26,712","146,647,158","18.22","18.20","686,197"
+"2006","Females","4,184","151,732,602","2.76","2.73","101,083"
+"2005","Males","26,657","145,196,982","18.36","18.42","677,596"
+"2005","Females","4,037","150,319,506","2.69","2.67","97,347"
+"2004","Males","25,498","143,827,954","17.73","17.83","645,627"
+"2004","Females","4,071","148,977,257","2.73","2.71","97,685"
+"2003","Males","26,124","142,428,853","18.34","18.47","668,380"
+"2003","Females","4,012","147,678,992","2.72","2.70","97,805"
+"2002","Males","26,098","141,230,565","18.48","18.69","664,285"
+"2002","Females","4,144","146,394,561","2.83","2.82","104,671"
+"2001","Males","25,480","139,891,523","18.21","18.44","655,528"
+"2001","Females","4,093","145,077,405","2.82","2.81","101,056"
+"Total","","776,476","6,851,096,884","11.33","11.13","18,770,404"
+"","","","","","",""
+"Injury Outcome: Fatal","","","","","",""
+"Injury Type: All Injury","","","","","",""
+"Data Years: 2001 to 2022","","","","","",""
+"Geography: United States","","","","","",""
+"Intent: All Intents","","","","","",""
+"Mechanism: Firearm","","","","","",""
+"Age: All Ages","","","","","",""
+"Sex: All Sexes","","","","","",""
+"Race: All Races","","","","","",""
+"Ethnicity: All Ethnicities","","","","","",""
+"Metro / Non-Metro Indicator: None Selected","","","","","",""
+"YPLL Age: 65","","","","","",""
+"Year and Race Options: 2001 - 2021 with No Race","","","","","",""
+"Produced by: National Center for Injury Prevention and Control, CDC.","","","","","",""
diff --git a/data/cdc_wisqars/gun_deaths-state-age.csv b/data/cdc_wisqars/gun_deaths-state-age.csv
diff --git a/data/cdc_wisqars/gun_deaths-state-sex.csv b/data/cdc_wisqars/gun_deaths-state-sex.csv
diff --git a/python/datasources/cdc_wisqars.py b/python/datasources/cdc_wisqars.py
@@ -37,6 +37,18 @@
 `Year`, `Race`, `State`, `None`
 "gun_deaths-state-race_and_ethnicity"
 
+ALL GUN DEATHS - BY SEX
+`Year`, `Sex`, `None`, `None`
+"gun_deaths-national-sex"
+`Year`, `Sex`, `State`, `None`
+"gun_deaths-state-sex"
+
+ALL GUN DEATHS - BY AGE
+`Year`, `Age Group`, `None`, `None`
+"gun_deaths-national-age"
+`Year`, `Age Group`, `State`, `None`
+"gun_deaths-state-age"
+
 GUN HOMICIDES AND SUICIDES - ALL PEOPLE
 `Year`, `Intent`, `None`, `None`
 "gun_homicides_suicides-national-all"
@@ -114,6 +126,14 @@
     # pylint: disable-next=line-too-long
     "gun_deaths-state-all": "https://wisqars.cdc.gov/reports/?o=MORT&y1=2001&y2=2022&t=0&d=&i=0&m=20890&g=00&me=0&s=0&r=0&ry=0&e=0&yp=65&a=ALL&g1=0&g2=199&a1=0&a2=199&r1=YEAR&r2=STATE&r3=NONE&r4=NONE",  # noqa: E501
     # pylint: disable-next=line-too-long
+    "gun_deaths-national-sex": "https://wisqars.cdc.gov/reports/?o=MORT&y1=2001&y2=2022&t=0&d=&i=0&m=20890&g=00&me=0&s=0&r=0&ry=0&e=0&yp=65&a=ALL&g1=0&g2=199&a1=0&a2=199&r1=YEAR&r2=SEX&r3=NONE&r4=NONE",  # noqa: E501
+    # pylint: disable-next=line-too-long
+    "gun_deaths-state-sex": "https://wisqars.cdc.gov/reports/?o=MORT&y1=2001&y2=2022&t=0&d=&i=0&m=20890&g=00&me=0&s=0&r=0&ry=0&e=0&yp=65&a=ALL&g1=0&g2=199&a1=0&a2=199&r1=YEAR&r2=SEX&r3=STATE&r4=NONE",  # noqa: E501
+    # pylint: disable-next=line-too-long
+    "gun_deaths-national-age": "https://wisqars.cdc.gov/reports/?o=MORT&y1=2001&y2=2022&t=0&d=&i=0&m=20890&g=00&me=0&s=0&r=0&ry=0&e=0&yp=65&a=ALL&g1=0&g2=199&a1=0&a2=199&r1=YEAR&r2=AGEGP&r3=NONE&r4=NONE",  # noqa: E501
+    # pylint: disable-next=line-too-long
+    "gun_deaths-state-age": "https://wisqars.cdc.gov/reports/?o=MORT&y1=2001&y2=2022&t=0&d=&i=0&m=20890&g=00&me=0&s=0&r=0&ry=0&e=0&yp=65&a=ALL&g1=0&g2=199&a1=0&a2=199&r1=YEAR&r2=AGEGP&r3=STATE&r4=NONE",  # noqa: E501
+    # pylint: disable-next=line-too-long
     "gun_deaths-national-ethnicity": "https://wisqars.cdc.gov/reports/?o=MORT&y1=2001&y2=2022&t=0&i=0&m=20890&g=00&me=0&s=0&r=0&ry=0&e=0&yp=65&a=ALL&g1=0&g2=199&a1=0&a2=199&r1=YEAR&r2=ETHNICTY&r3=NONE&r4=NONE",  # noqa: E501
     # pylint: disable-next=line-too-long
     "gun_deaths-state-ethnicity": "https://wisqars.cdc.gov/reports/?o=MORT&y1=2001&y2=2022&t=0&d=&i=0&m=20890&g=00&me=0&s=0&r=0&ry=0&e=0&yp=65&a=ALL&g1=0&g2=199&a1=0&a2=199&r1=YEAR&r2=ETHNICTY&r3=STATE&r4=NONE",  # noqa: E501
@@ -178,9 +198,15 @@
         "denominator_col": std_col.FATAL_POPULATION,
         "rate_col": std_col.GUN_VIOLENCE_SUICIDE_PER_100K,
     },
+    {
+        "numerator_col": "gun_violence_all_intents_estimated_total",
+        "denominator_col": std_col.FATAL_POPULATION,
+        "rate_col": "gun_violence_all_intents_per_100k",
+    },
 ]
 
 GUN_DEATHS_BY_INTENT: WISQARS_VAR_TYPE = "gun_homicides_suicides"
+GUN_DEATHS_OVERALL: WISQARS_VAR_TYPE = "gun_deaths"
 
 
 class CDCWisqarsData(DataSource):
@@ -224,7 +250,12 @@ def write_to_bq(self, dataset, gcs_bucket, **attrs):
         for time_view in (CURRENT, HISTORICAL):
             table_id = gcs_to_bq_util.make_bq_table_id(demographic, geo_level, time_view)
             time_cols = TIME_MAP[time_view]
+
+            time_cols = [col.replace("gun_violence_all_intents", "gun_deaths") for col in time_cols]
+            df.columns = [col.replace("gun_violence_all_intents", "gun_deaths") for col in df.columns]
+
             df_for_bq, col_types = generate_time_df_with_cols_and_types(df, time_cols, time_view, demographic)
+
             gcs_to_bq_util.add_df_to_bq(df_for_bq, dataset, table_id, column_types=col_types)
 
     def generate_breakdown_df(self, demographic: WISQARS_DEMO_TYPE, geo_level: GEO_TYPE, alls_df: pd.DataFrame):
@@ -285,10 +316,17 @@ def process_wisqars_df(demographic: WISQARS_DEMO_TYPE, geo_level: GEO_TYPE):
     """
     output_df = pd.DataFrame(columns=[std_col.TIME_PERIOD_COL])
 
-    df = load_wisqars_as_df_from_data_dir(GUN_DEATHS_BY_INTENT, geo_level, demographic)
+    df_each_intent = load_wisqars_as_df_from_data_dir(GUN_DEATHS_BY_INTENT, geo_level, demographic)
+    df_all_intent_combined = load_wisqars_as_df_from_data_dir(GUN_DEATHS_OVERALL, geo_level, demographic)
+    df_all_intent_combined[WISQARS_INTENT] = "All Intents"
+
+    df = pd.concat([df_each_intent, df_all_intent_combined], axis=0)
 
     if demographic == std_col.RACE_OR_HISPANIC_COL:
-        df_eth = load_wisqars_as_df_from_data_dir(GUN_DEATHS_BY_INTENT, geo_level, "ethnicity")
+        df_eth_each_intent = load_wisqars_as_df_from_data_dir(GUN_DEATHS_BY_INTENT, geo_level, "ethnicity")
+        df_eth_all_intent_combined = load_wisqars_as_df_from_data_dir(GUN_DEATHS_OVERALL, geo_level, "ethnicity")
+        df_eth_all_intent_combined[WISQARS_INTENT] = "All Intents"
+        df_eth = pd.concat([df_eth_each_intent, df_eth_all_intent_combined], axis=0)
         df_eth = df_eth[df_eth[WISQARS_ETH] != "Non-Hispanic"]
         df_eth = df_eth.rename(columns={WISQARS_ETH: std_col.RACE_CATEGORY_ID_COL})
         df_eth = df_eth.replace({std_col.RACE_CATEGORY_ID_COL: ETHNICITY_NAMES_MAPPING})
@@ -298,7 +336,7 @@ def process_wisqars_df(demographic: WISQARS_DEMO_TYPE, geo_level: GEO_TYPE):
 
         df = pd.concat([df, df_eth], axis=0)
 
-    df = df[df[WISQARS_INTENT].isin(["Homicide", "Suicide"])].copy()
+    df = df[df[WISQARS_INTENT].isin(["Homicide", "Suicide", "All Intents"])].copy()
 
     # Reshapes df to add the intent rows as columns
     pivot_df = df.pivot(
@@ -307,7 +345,7 @@ def process_wisqars_df(demographic: WISQARS_DEMO_TYPE, geo_level: GEO_TYPE):
         values=[WISQARS_DEATHS, WISQARS_CRUDE_RATE],
     )
 
-    new_columns = [
+    flat_columns = [
         (
             f"gun_violence_{col[1].lower().replace(' ', '_')}_{std_col.RAW_SUFFIX}"
             if col[0] == WISQARS_DEATHS
@@ -316,8 +354,7 @@ def process_wisqars_df(demographic: WISQARS_DEMO_TYPE, geo_level: GEO_TYPE):
         for col in pivot_df.columns
     ]
 
-    # need to be an Index to pass linter
-    pivot_df.columns = pd.Index(new_columns)
+    pivot_df.columns = pd.Index(flat_columns)
     df = pivot_df.reset_index()
 
     df.rename(

diff --git a/python/ingestion/cdc_wisqars_utils.py b/python/ingestion/cdc_wisqars_utils.py
@@ -22,7 +22,7 @@
 
 INJ_OUTCOMES = [std_col.FATAL_PREFIX]
 
-INJ_INTENTS = [std_col.GUN_VIOLENCE_HOMICIDE_PREFIX, std_col.GUN_VIOLENCE_SUICIDE_PREFIX]
+INJ_INTENTS = [std_col.GUN_VIOLENCE_HOMICIDE_PREFIX, std_col.GUN_VIOLENCE_SUICIDE_PREFIX, "gun_violence_all_intents"]
 
 WISQARS_INTENT = "Intent"
 WISQARS_URBANICITY = "Metro / Non-Metro"

diff --git a/python/tests/data/cdc_wisqars/golden_data/age_national_current.csv b/python/tests/data/cdc_wisqars/golden_data/age_national_current.csv
@@ -1,10 +1,10 @@
-state_name,state_fips,age,gun_violence_homicide_per_100k,gun_violence_suicide_per_100k,gun_violence_homicide_pct_share,gun_violence_suicide_pct_share,fatal_population_pct,gun_violence_homicide_estimated_total,gun_violence_suicide_estimated_total,fatal_population
-United States,00,0-14,0.82,0.29,2.5,0.6,17.8,488.0,175.0,59437387.0
-United States,00,15-19,12.2,4.91,13.4,3.9,6.5,2639.0,1063.0,21635792.0
-United States,00,20-24,13.98,9.61,16.2,8.1,6.8,3175.0,2183.0,22705779.0
-United States,00,25-29,13.26,9.47,15.0,7.8,6.7,2943.0,2102.0,22193164.0
-United States,00,30-34,12.41,9.13,14.7,7.9,7.0,2893.0,2128.0,23308136.0
-United States,00,35-44,8.84,8.6,19.6,13.9,13.1,3861.0,3759.0,43695365.0
-United States,00,45-64,3.67,10.0,15.4,30.5,24.8,3027.0,8250.0,82517082.0
-United States,00,65+,1.08,12.75,3.2,27.3,17.3,625.0,7367.0,57794852.0
-United States,00,All,5.9,8.11,100.0,100.0,100.0,19651.0,27032.0,333287557.0
+state_name,state_fips,age,gun_violence_homicide_per_100k,gun_violence_suicide_per_100k,gun_deaths_per_100k,gun_violence_homicide_pct_share,gun_violence_suicide_pct_share,gun_deaths_pct_share,fatal_population_pct,gun_violence_homicide_estimated_total,gun_violence_suicide_estimated_total,gun_deaths_estimated_total,fatal_population
+United States,00,0-14,0.82,0.29,1.32,2.5,0.6,1.6,17.8,488.0,175.0,784.0,59437387.0
+United States,00,15-19,12.2,4.91,17.73,13.4,3.9,8.0,6.5,2639.0,1063.0,3835.0,21635792.0
+United States,00,20-24,13.98,9.61,24.35,16.2,8.1,11.5,6.8,3175.0,2183.0,5528.0,22705779.0
+United States,00,25-29,13.26,9.47,23.51,15.0,7.8,10.8,6.7,2943.0,2102.0,5218.0,22193164.0
+United States,00,30-34,12.41,9.13,22.39,14.7,7.9,10.8,7.0,2893.0,2128.0,5218.0,23308136.0
+United States,00,35-44,8.84,8.6,18.09,19.6,13.9,16.4,13.1,3861.0,3759.0,7904.0,43695365.0
+United States,00,45-64,3.67,10.0,14.06,15.4,30.5,24.1,24.8,3027.0,8250.0,11606.0,82517082.0
+United States,00,65+,1.08,12.75,14.03,3.2,27.3,16.8,17.3,625.0,7367.0,8111.0,57794852.0
+United States,00,All,5.9,8.11,14.46,100.0,100.0,100.0,100.0,19651.0,27032.0,48204.0,333287557.0