Tests to see that we are getting big TIGER files when we expect to. (#…

…169)

censusdis/datasets.py

@@ -595,7 +595,7 @@
     CPS_VOLUNTEER: "http://api.census.gov/data/2021/cps/volunteer/sep",
     CPS_VOTING: "http://api.census.gov/data/2022/cps/voting/nov",
     CPS_WORKSCHED: "http://api.census.gov/data/2004/cps/worksched/may",
-    CRE: "http://api.census.gov/data/2019/cre",
+    CRE: "http://api.census.gov/data/2021/cre",
     DECENNIAL_AIAN: "http://api.census.gov/data/2010/dec/aian",
     DECENNIAL_AIANPROFILE: "http://api.census.gov/data/2000/dec/aianprofile",
     DECENNIAL_AS: "http://api.census.gov/data/2010/dec/as",

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "censusdis"
-version = "0.16.0"
+version = "0.17.0"
 description = "US Census utilities for a variety of data loading and mapping purposes."
 license = "HL3-CL-ECO-EXTR-FFD-LAW-MIL-SV"
 authors = ["Darren Vengroff"]

tests/test_cb_tiger.py

@@ -0,0 +1,55 @@
+"""Tests for fetching geometry and getting TIGER or CB files as available."""
+
+import unittest
+from typing import Union
+
+from shapely import Polygon, MultiPolygon
+
+import censusdis.data as ced
+
+
+def _shape_size(shape: Union[Polygon, MultiPolygon]) -> int:
+    """How many points in a Polygon or MultiPolygon?"""
+    if isinstance(shape, Polygon):
+        return len(shape.exterior.coords) + sum(
+            len(hole.coords) for hole in shape.interiors
+        )
+    elif isinstance(shape, MultiPolygon):
+        return sum(_shape_size(poly) for poly in shape.geoms)
+    else:
+        return 0
+
+
+class TigerTestCase(unittest.TestCase):
+    """Test falling back on TIGER maps, which are much bigger and higher resolution."""
+
+    def test_single_years(self):
+        for year in range(2010, 2022):
+            gdf = ced.download(
+                "acs/acs5",
+                year,
+                ["NAME"],
+                state="01",
+                county="*",
+                tract="*",
+                with_geometry=True,
+            )
+
+            gdf["geometry_size"] = gdf.geometry.map(_shape_size)
+
+            total_size = gdf["geometry_size"].sum()
+
+            # Old tiger files from before CB files started in 2013 are much
+            # bigger. But there was a CB in 2010.
+            if year < 2013 and year != 2010:
+                # TIGER
+                self.assertGreater(total_size, 1_000_000)
+                self.assertLess(total_size, 1_050_000)
+            else:
+                # CB 500k
+                self.assertGreater(total_size, 120_000)
+                self.assertLess(total_size, 160_000)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_old_geos.py

@@ -19,6 +19,9 @@ def test_geo_tract_across_years(self):
         data changed between 2013 and 2014. They changed both
         the directory structure and the name of some columns, e.g.
         'TRACT' vs. 'TRACTCE'.
+
+        For some cases we will automatically fall back on TIGER
+        files instead of CB files.
         """
         # These are the years for which the shape files exist.
         for year in range(2010, 2022):