Update to newer repo format

.gitignore

@@ -8,4 +8,5 @@ docs/_site
 data/raw/scrapedxml
 data/interim/regmem/
 docs/data
-docs/_*
+docs/_*
+data/vol_interests

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM ghcr.io/mysociety/data_common:sha-a3b8e93
+FROM ghcr.io/mysociety/data_common:sha-d5aad46
 
 # Make an empty project directory so the 'self' setup doesn't fail and scripts can be setup
 # Override the .pth created at previous stages to point to where the working directory will land

data/data_packages/all_time_register/register_of_interests.resource.yaml

@@ -47,6 +47,12 @@ schema:
     constraints:
       unique: false
     example: '2000-11-10'
+  - name: source_order
+    type: integer
+    description:
+    constraints:
+      unique: false
+    example: 200012001
   - name: new_in_latest
     type: boolean
     description: Whether this interest is new in the latest register (true/false)
@@ -77,4 +83,4 @@ schema:
     constraints:
       unique: false
     example: ''
-hash: 626236f5f439e8821824688b9d311296
+hash: 17c0f4d4d02e563197b2a0ac90fb6452

data/data_packages/parliament_2019/register_of_interests.resource.yaml

@@ -48,6 +48,12 @@ schema:
     constraints:
       unique: false
     example: '2020-01-11'
+  - name: source_order
+    type: integer
+    description:
+    constraints:
+      unique: false
+    example: 202002101
   - name: new_in_latest
     type: boolean
     description: Whether this interest is new in the latest register (true/false)
@@ -77,4 +83,4 @@ schema:
     constraints:
       unique: false
     example: ''
-hash: 5ec0ab7f0e769a874131e58c98a7b2c1
+hash: 548e6d60c64e80c6a474542a3b07337a

data/data_packages/parliament_2024/register_of_interests.resource.yaml

@@ -68,6 +68,12 @@ schema:
       - '2024-08-04'
       - '2024-09-02'
     example: '2024-08-04'
+  - name: source_order
+    type: integer
+    description:
+    constraints:
+      unique: false
+    example: 202408401
   - name: new_in_latest
     type: boolean
     description: Whether this interest is new in the latest register (true/false)
@@ -99,4 +105,4 @@ schema:
     constraints:
       unique: false
     example: ''
-hash: 7f45aa90cae1f8a4a9a9fb0d0d9442d9
+hash: dd132cdb9e6eee811bc8e491b7f60627

pyproject.toml

@@ -10,7 +10,8 @@ data_common = { path = "src/data_common/", develop = true }
 spacy = "^3.3.0"
 tqdm = "^4.64.1"
 typing-extensions = "^4.4.0"
-mysoc-validator = {git = "https://github.com/mysociety/mysoc-validator"}
+pandas-stubs = "^2.2.2.240909"
+mysoc-validator = "^0.4.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.1"
@@ -38,6 +39,6 @@ credit_url = "https://survey.alchemer.com/s3/6876792/Data-usage"
 [tool.duck]
 query_dir = "src/parl_register_interests/sql"
 
-[tool.ruff]
+[tool.ruff.lint]
 select = ["E", "F"]
 ignore = ["E501"]

script/lint

@@ -0,0 +1,4 @@
+#!/bin/bash 
+
+ruff check . --fix
+ruff format .

script/test

@@ -3,24 +3,27 @@
 echo "--------------"
 echo "Running pytest"
 echo "--------------"
-pytest
+poetry run pytest
 pytestexit=$?
 echo "--------------"
-echo "Running black"
+echo "Running ruff"
 echo "--------------"
-black . --check
-blackexit=$?
+poetry run ruff check .
+ruffcheckexit=$?
+poetry run ruff format . --check
+ruffformatexit=$?
 echo "--------------"
 echo "Running pyright"
 echo "--------------"
-pyright
+poetry run pyright
 pyrightexit=$?
 
 echo "Pytest status: $pytestexit"
-echo "Black status: $blackexit"
+echo "Ruff Check status: $ruffcheckexit"
+echo "Ruff Format status: $ruffformatexit"
 echo "Pyright status: $pyrightexit"
 
-sum="$(($pytestexit + $blackexit + $pyrightexit))"
+sum="$(($pytestexit + $pyrightexit + $ruffcheckexit + $ruffformatexit))"
 
 if [ "$sum" != "0" ]; then
     exit 1

src/parl_register_interests/official_data.py

@@ -1,8 +1,10 @@
+from datetime import date as date
+from datetime import datetime
 from pathlib import Path
+
 import pandas as pd
 from mysoc_validator import Popolo
 from mysoc_validator.models.popolo import Chamber, IdentifierScheme
-from datetime import date as date, datetime
 
 RAW_DATA = Path("data", "raw", "external", "official_data")
 
@@ -141,8 +143,8 @@ def get_sort_tuple(row: pd.Series):
 
 
 def process_all_regmem():
-    for date in known_dates:
-        process_register(date)
+    for regmem_date in known_dates:
+        process_register(regmem_date)
 
 
 if __name__ == "__main__":

tests/test_parl_register_experiment.py

@@ -1,7 +1,2 @@
-import parl_register_interests
-
-import pytest
-
-
 def test_true_is_true():
     assert True is True