Merge pull request #122 from Congress-Dev/vote-importer

Vote Importer Module

backend/.gitignore

@@ -11,4 +11,6 @@ usc/
 
 **.egg-info/
 dist/
-build/
+build/
+
+bioguide.json

backend/alembic/versions/79a29914ef4a_votes.py

@@ -0,0 +1,71 @@
+"""votes
+
+Revision ID: 79a29914ef4a
+Revises: d01322760f6d
+Create Date: 2025-01-18 20:57:35.036173
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '79a29914ef4a'
+down_revision: Union[str, None] = 'c941abf22042'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('legislation_vote',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('number', sa.Integer(), nullable=False),
+    sa.Column('date', sa.Date(), nullable=True),
+    sa.Column('legislation_id', sa.Integer(), nullable=True),
+    sa.Column('question', sa.String(), nullable=True),
+    sa.Column('independent', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('republican', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('democrat', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('total', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('passed', sa.Boolean(), nullable=True),
+    sa.ForeignKeyConstraint(['legislation_id'], ['legislation.legislation_id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_legislation_vote_id'), 'legislation_vote', ['id'], unique=False)
+    op.create_index(op.f('ix_legislation_vote_legislation_id'), 'legislation_vote', ['legislation_id'], unique=False)
+    op.create_index(op.f('ix_legislation_vote_number'), 'legislation_vote', ['number'], unique=False)
+    op.create_index(op.f('ix_legislation_vote_question'), 'legislation_vote', ['question'], unique=False)
+    op.create_table('legislator_vote',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('legislation_vote_id', sa.Integer(), nullable=True),
+    sa.Column('legislator_bioguide_id', sa.String(), nullable=True),
+    sa.Column('vote', sa.Enum('yay', 'nay', 'present', 'abstain', name='legislatorvotetype'), nullable=True),
+    sa.ForeignKeyConstraint(['legislation_vote_id'], ['legislation_vote.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['legislator_bioguide_id'], ['legislator.bioguide_id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_legislator_vote_id'), 'legislator_vote', ['id'], unique=False)
+    op.create_index(op.f('ix_legislator_vote_legislation_vote_id'), 'legislator_vote', ['legislation_vote_id'], unique=False)
+    op.create_index(op.f('ix_legislator_vote_legislator_bioguide_id'), 'legislator_vote', ['legislator_bioguide_id'], unique=False)
+    op.add_column('legislator', sa.Column('lis_id', sa.String(), nullable=True))
+    op.create_index(op.f('ix_legislator_lis_id'), 'legislator', ['lis_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_legislator_lis_id'), table_name='legislator')
+    op.drop_column('legislator', 'lis_id')
+    op.drop_index(op.f('ix_legislator_vote_legislator_bioguide_id'), table_name='legislator_vote')
+    op.drop_index(op.f('ix_legislator_vote_legislation_vote_id'), table_name='legislator_vote')
+    op.drop_index(op.f('ix_legislator_vote_id'), table_name='legislator_vote')
+    op.drop_table('legislator_vote')
+    op.drop_index(op.f('ix_legislation_vote_question'), table_name='legislation_vote')
+    op.drop_index(op.f('ix_legislation_vote_number'), table_name='legislation_vote')
+    op.drop_index(op.f('ix_legislation_vote_legislation_id'), table_name='legislation_vote')
+    op.drop_index(op.f('ix_legislation_vote_id'), table_name='legislation_vote')
+    op.drop_table('legislation_vote')
+    # ### end Alembic commands ###

backend/alembic/versions/b3b426df69a6_votes_datetime.py

@@ -0,0 +1,32 @@
+"""votes-datetime
+
+Revision ID: b3b426df69a6
+Revises: c54322bdb307
+Create Date: 2025-01-19 02:52:36.012847
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'b3b426df69a6'
+down_revision: Union[str, None] = 'c54322bdb307'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('legislation_vote', sa.Column('datetime', sa.DateTime(), nullable=True))
+    op.drop_column('legislation_vote', 'date')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('legislation_vote', sa.Column('date', sa.DATE(), autoincrement=False, nullable=True))
+    op.drop_column('legislation_vote', 'datetime')
+    # ### end Alembic commands ###

backend/alembic/versions/c54322bdb307_votes_metadata.py

@@ -0,0 +1,37 @@
+"""votes-metadata
+
+
+Revision ID: c54322bdb307
+Revises: 79a29914ef4a
+Create Date: 2025-01-19 00:40:01.827121
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'c54322bdb307'
+down_revision: Union[str, None] = '79a29914ef4a'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('legislation_vote', sa.Column('chamber', sa.Enum('House', 'Senate', name='legislationchamber'), nullable=True))
+    op.add_column('legislation_vote', sa.Column('congress_id', sa.Integer(), nullable=True))
+    op.create_index(op.f('ix_legislation_vote_congress_id'), 'legislation_vote', ['congress_id'], unique=False)
+    op.create_foreign_key(None, 'legislation_vote', 'congress', ['congress_id'], ['congress_id'], ondelete='CASCADE')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'legislation_vote', type_='foreignkey')
+    op.drop_index(op.f('ix_legislation_vote_congress_id'), table_name='legislation_vote')
+    op.drop_column('legislation_vote', 'congress_id')
+    op.drop_column('legislation_vote', 'chamber')
+    # ### end Alembic commands ###

backend/billparser/bioguide/manager.py

@@ -4,6 +4,7 @@
 import zipfile
 import json
 import io
+import json
 
 from billparser.db.handler import Session
 from billparser.db.models import Legislator
@@ -13,14 +14,17 @@
 CURRENT_LEGIS_URL = (
     "https://theunitedstates.io/congress-legislators/legislators-current.json"
 )
-
+HISTORICAL_LEGIS_URL = (
+    "https://theunitedstates.io/congress-legislators/legislators-historical.json"
+)
 
 class BioGuideImporter:
     def __init__(
-        self, bulk_bioguide_url=BULK_BIOGUIDE_URL, current_legis_url=CURRENT_LEGIS_URL
+        self, bulk_bioguide_url=BULK_BIOGUIDE_URL, current_legis_url=CURRENT_LEGIS_URL, historical_legis_url=HISTORICAL_LEGIS_URL
     ):
         self.bulk_bioguide_url = bulk_bioguide_url
         self.current_legis_url = current_legis_url
+        self.historical_legis_url = historical_legis_url
         self.session = Session()
 
     def _download_zip(self) -> zipfile.ZipFile:
@@ -49,10 +53,33 @@ def run_import(self) -> List[BioGuideMember]:
                     else:
                         legis = BioGuideMember(**jdata.get('data'))
                     items.append(legis)
+
+        with open('bioguide.json', 'r') as jdata:
+            for item in json.load(jdata):
+                item['usCongressBioId'] = item['id']
+                items.append(BioGuideMember(**item))
+
         return items
 
+    def run_metadata(self):
+        member_lis_lookup = {}
+
+        r = requests.get(self.current_legis_url)
+        for legislator in r.json():
+            if legislator['id'].get('lis') is not None:
+                member_lis_lookup[legislator['id'].get('bioguide')] = legislator['id'].get('lis')
+
+        r = requests.get(self.historical_legis_url)
+        for legislator in r.json():
+            if legislator['id'].get('lis') is not None:
+                member_lis_lookup[legislator['id'].get('bioguide')] = legislator['id'].get('lis')
+
+        return member_lis_lookup
+
     def download_to_database(self) -> None:
         legislators = self.run_import()
+        lis_lookup = self.run_metadata()
+
         db_items = []
         with self.session.begin():
             for legislator in legislators:
@@ -77,17 +104,28 @@ def download_to_database(self) -> None:
                     image_url = None
                     image_source = None
 
-                record_data = {
-                    'bioguide_id': legislator.usCongressBioId,
-                    'first_name': legislator.nickName or legislator.unaccentedGivenName or legislator.givenName,
-                    'last_name': legislator.unaccentedFamilyName or legislator.familyName,
-                    'middle_name': legislator.unaccentedMiddleName or legislator.middleName,
-                    'party': party,
-                    'state': state,
-                    'image_url': image_url,
-                    'image_source': image_source,
-                    'profile': legislator.profileText
-                }
+                record_data = {}
+
+                if lis_lookup.get(legislator.usCongressBioId, None):
+                    record_data['lis_id'] = lis_lookup.get(legislator.usCongressBioId, None)
+                if legislator.usCongressBioId:
+                    record_data['bioguide_id'] = legislator.usCongressBioId
+                if legislator.nickName or legislator.unaccentedGivenName or legislator.givenName:
+                    record_data['first_name'] = legislator.nickName or legislator.unaccentedGivenName or legislator.givenName
+                if legislator.unaccentedFamilyName or legislator.familyName:
+                    record_data['last_name'] = legislator.unaccentedFamilyName or legislator.familyName
+                if legislator.unaccentedMiddleName or legislator.middleName:
+                    record_data['middle_name'] = legislator.unaccentedMiddleName or legislator.middleName
+                if party:
+                    record_data['party'] = party
+                if state:
+                    record_data['state'] = state
+                if image_url:
+                    record_data['image_url'] = image_url
+                if image_source:
+                    record_data['image_source'] = image_source
+                if legislator.profileText:
+                    record_data['profile'] = legislator.profileText
 
                 existing_record = self.session.query(Legislator).filter(Legislator.bioguide_id == legislator.usCongressBioId).first()
                 if existing_record is None:

backend/billparser/bioguide/types.py

@@ -168,11 +168,11 @@ class BioGuideMember(BaseModel):
     birthCirca: Optional[bool] = None
     deathDate: Optional[str] = None
     deathCirca: Optional[bool] = None
-    profileText: str
-    relationship: List[RelationshipItem]
-    jobPositions: List[JobPosition]
-    creativeWork: List[CreativeWorkItem]
-    researchRecord: List[ResearchRecordItem]
+    profileText: Optional[str] = None
+    relationship: Optional[List[RelationshipItem]] = []
+    jobPositions: Optional[List[JobPosition]] = []
+    creativeWork: Optional[List[CreativeWorkItem]] = []
+    researchRecord: Optional[List[ResearchRecordItem]] = []
     nickName: Optional[str] = None
     image: Optional[List[ImageItem]] = None
     middleName: Optional[str] = None

backend/billparser/db/models.py

@@ -33,6 +33,29 @@ def bind_expression(self, bindvalue):
         return sa.cast(bindvalue, self)
 
 
+class LegislatorVoteType(str, enum.Enum):
+    yay = "yay"
+    nay = "nay"
+    present = "present"
+    abstain = "abstain"
+
+    @classmethod
+    def from_string(cls, string: str) -> "LegislationChamber":
+        if string.lower() == 'yea':
+            return cls.yay
+        elif string.lower() == 'aye':
+            return cls.yay
+        elif string.lower() == 'nay':
+            return cls.nay
+        elif string.lower() == 'no':
+            return cls.nay
+        elif string.lower() == 'present':
+            return cls.present
+        elif string.lower() == 'not voting':
+            return cls.abstain
+        else:
+            raise ValueError(f"Invalid LegislatorVoteType: {string}")
+
 class LegislationType(str, enum.Enum):
     Bill = "Bill"
     CRes = "Continuing Resolution"
@@ -126,6 +149,47 @@ def from_string(cls, string: str) -> "LegislationVersionEnum":
             raise ValueError(f"Invalid version: {string}")
 
 
+class LegislationVote(Base):
+    __tablename__ = "legislation_vote"
+
+    id = Column(Integer, primary_key=True, index=True)
+    number = Column(Integer, index=True, nullable=False)
+    datetime = Column(DateTime)
+
+    chamber = Column(Enum(LegislationChamber))
+    congress_id = Column(
+        Integer, ForeignKey("congress.congress_id", ondelete="CASCADE"), index=True
+    )
+
+    legislation_id = Column(
+        Integer, ForeignKey("legislation.legislation_id", ondelete="CASCADE"), index=True
+    )
+
+    question = Column(String, index=True)
+    independent = Column(JSONB)
+    republican = Column(JSONB)
+    democrat = Column(JSONB)
+    total = Column(JSONB)
+
+    passed = Column(Boolean)
+
+
+class LegislatorVote(Base):
+    __tablename__ = "legislator_vote"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    legislation_vote_id = Column(
+        Integer, ForeignKey("legislation_vote.id", ondelete="CASCADE"), index=True
+    )
+
+    legislator_bioguide_id = Column(
+        String, ForeignKey("legislator.bioguide_id", ondelete="CASCADE"), index=True
+    )
+
+    vote = Column(Enum(LegislatorVoteType))
+
+
 class User(Base):
     """
     Holds the relationships for the website user
@@ -858,6 +922,8 @@ class Legislator(Base):
         String, index=True, unique=True
     )  # https://bioguideretro.congress.gov/
 
+    lis_id = Column(String, index=True, nullable=True)
+
     first_name = Column(String)
     middle_name = Column(String)
     last_name = Column(String)