Run pipelines

open-innovations · Oct 29, 2024 · 2668a91 · 2668a91
1 parent d7ab864
commit 2668a91
Show file tree

Hide file tree

Showing 8 changed files with 200 additions and 200 deletions.
diff --git a/pipelines/dvc.lock b/pipelines/dvc.lock
@@ -22,7 +22,7 @@ stages:
       size: 77
     - path: src/themes/sustainable-growth/b-corporations/_data/updated.yaml
       hash: md5
-      md5: 8d3399781bf4d39a08385d943307d3a8
+      md5: 0dc0e1f3227367affe3affec6b9e3b54
       size: 16
   metadata:
     cmd: python pipelines/metadata.py
@@ -45,7 +45,7 @@ stages:
     deps:
     - path: https://raw.githubusercontent.com/economic-analytics/edd/main/data/edd_dict.csv
       hash: md5
-      checksum: '"d60d0affb2305787fa43205773f4ff876e9ccb1af0635f288b909598ef2e3ef8"'
+      checksum: '"c9a6a77ecb61171f7a51794bad680de557de3f93ddb1e5812076c4b373a8345f"'
       size: 10438
     - path: pipelines/edd_dictionary.py
       hash: md5
@@ -54,5 +54,5 @@ stages:
     outs:
     - path: src/_data/edd_dictionary.yaml
       hash: md5
-      md5: 19794c92b1947540b701c671281edd2f
+      md5: 19cc68b25988a079e4776a955e6de726
       size: 7791
diff --git a/pipelines/people/dvc.lock b/pipelines/people/dvc.lock
@@ -9,22 +9,22 @@ stages:
       size: 3763
     - path: https://github.com/economic-analytics/edd/raw/main/data/parquet/LMS.parquet
       hash: md5
-      checksum: '"5c480b7ca56516bde94aef71e91cee537a4680ea347640caa661d4ddeb1ce753"'
-      size: 3230806
+      checksum: '"5c0d82ed1653d6c5ca9be522cfc36f8126124f8551f305a783229584ef3ed967"'
+      size: 3261273
     - path: vacancies.py
       hash: md5
-      md5: e7202141bcf8de5b8e50329eda6d0b45
-      size: 2278
+      md5: 90fc207f6ddc2eead9ae40a6ccdbe324
+      size: 2263
     outs:
     - path: ../../src/themes/people-skills-future/vacancies/_data/vacancies_by_sector.csv
       hash: md5
-      md5: 9d5a19693e8bbe37eac96277df204c14
-      size: 13530
+      md5: ead133c4016e1e623670960b9044d9c5
+      size: 12576
     - path: 
         ../../src/themes/people-skills-future/vacancies/_data/vacancies_yearly_change_by_sector.csv
       hash: md5
-      md5: a9e124bcd89ab6f76329f1299880e107
-      size: 1504
+      md5: abc60b2147df631b8265b85b437e2f4c
+      size: 1543
   employment:
     cmd: PYTHONPATH=../.. python employment.py
     deps:

diff --git a/pipelines/people/vacancies.py b/pipelines/people/vacancies.py
@@ -17,10 +17,10 @@ def vacancies_by_sector():
     data['variable.name'] = data['variable.name'].str.replace(pat="UK Job Vacancies (thousands) - ", repl="")
 
     data['unix'] = pd.to_datetime(data['date'], format=f'%Y-%m-%d').astype(int).div(10**6).astype(int)
-    data['decimal_date'] = data['unix'].div((86400*365.25)).add(1970).round(2)
-
+    # data['decimal_date'] = data['unix'].div((86400*365.25)).add(1970).round(2)
+    print(data)
     # pivot data to wide format for visualisation
-    data = data.pivot(index=['decimal_date', 'date'], columns='variable.name', values='value')
+    data = data.pivot(index='date', columns='variable.name', values='value')
 
     # limit the time series to last 10 years -> 12months x 10 years = 120 values.
     data = data.tail(120)
@@ -31,14 +31,14 @@ def vacancies_by_sector():
 def yearly_change_by_sector(data):
     # Get yearly data going back from most recent data, then flip to put in correct order.
     data = data.iloc[-1::-12, :].copy()
-    data.sort_values(by='decimal_date', ascending=True, inplace=True)
+    data.sort_values(by='date', ascending=True, inplace=True)
     # iterate through each column to work out pct change.
     for col in data.columns.to_list():
         data[f'{col}'] = data[f'{col}'].pct_change().mul(100).round(1)
 
     data.reset_index(inplace=True)
-    data.drop(columns='date', inplace=True)
-    data.set_index('decimal_date', inplace=True)
+    # data.drop(columns='date', inplace=True)
+    data.set_index('date', inplace=True)
     data = data.T
     data.index.rename('sector', inplace=True)
     data.to_csv(os.path.join(SRC_DIR, 'themes/people-skills-future/vacancies/_data/vacancies_yearly_change_by_sector.csv'))

diff --git a/pipelines/true-north/dvc.lock b/pipelines/true-north/dvc.lock
@@ -13,44 +13,44 @@ stages:
       size: 9712
     - path: clean_members_list.ipynb
       hash: md5
-      md5: 38a2af2fc156c5ac9b60144b98b6384e
-      size: 14515
+      md5: 509af1d26c3b24e578a84ff6ff57164b
+      size: 12250
     outs:
     - path: ../../working/true-north/true_north_clean.csv
       hash: md5
-      md5: bbc2db00c2f66e95bdfb08104a9886a4
-      size: 76323
+      md5: 5a021a64f031b64faf8363fee2b68fa3
+      size: 79110
   analyse:
     cmd: papermill --no-progress-bar --no-report-mode analyse_members_list.ipynb ../../output/analyse_members_list.ipynb
     deps:
     - path: ../../working/true-north/true_north_clean.csv
       hash: md5
-      md5: bbc2db00c2f66e95bdfb08104a9886a4
-      size: 76323
+      md5: 5a021a64f031b64faf8363fee2b68fa3
+      size: 79110
     - path: analyse_members_list.ipynb
       hash: md5
-      md5: 1d417f023c4c12e0a26c417d092e1dff
-      size: 67720
+      md5: 52716f7d5516e39748bd4e483c719d97
+      size: 13167
     outs:
     - path: ../../src/overview/membership/_data/cumsum.csv
       hash: md5
-      md5: 4eba2a13cff76ba3821da7df2dcb227b
-      size: 642
+      md5: 12cc6d2d0ff0d498ccacc9f1b610b418
+      size: 677
     - path: ../../src/overview/membership/_data/sector_word_counts.csv
       hash: md5
-      md5: b1018dd25a05c99b01120baddc708267
-      size: 318
+      md5: 8132951d832786b44c8311db8e84672b
+      size: 326
     - path: ../../src/overview/membership/_data/top_locations.csv
       hash: md5
-      md5: 5c455423c1895bd32f4e8acfc23bfbb1
+      md5: 2c610b37b2b6e6484814bdb82bc0954b
       size: 96
     - path: ../../src/overview/membership/_data/true_north_members_list.csv
       hash: md5
-      md5: b5dff2598785e7bbbde9c861ccba3dcb
+      md5: da3ac476e40b4f9f80724ecb93e646b2
       size: 719
     - path: ../../src/overview/membership/_data/updated.yaml
       hash: md5
-      md5: cd14408154993b809ccc0a6d1ee14b83
+      md5: fb78ad7e5a350a30f736429ffcae03bb
       size: 20
   northern-stars:
     cmd: papermill --no-progress-bar --no-report-mode northern_stars.ipynb ../../output/northern_stars.ipynb

diff --git a/src/_data/edd_dictionary.yaml b/src/_data/edd_dictionary.yaml
@@ -19,53 +19,53 @@ BB:
   desc: 'UK National Accounts, The Blue Book'
   url: 'https://www.ons.gov.uk/file?uri=/economy/grossdomesticproductgdp/datasets/bluebook/current/bb.csv'
 CXNV:
-  last_update: '2024-08-15'
-  next_update: '2024-09-30'
+  last_update: '2024-09-30'
+  next_update: '2024-11-15'
   desc: 'Business Investment'
   url: 'https://www.ons.gov.uk/file?uri=/economy/grossdomesticproductgdp/datasets/businessinvestment/current/cxnv.csv'
 DIOP:
-  last_update: '2024-09-11'
-  next_update: '2024-10-11'
+  last_update: '2024-10-11'
+  next_update: '2024-11-15'
   desc: 'Index of Production'
   url: 'https://www.ons.gov.uk/file?uri=/economy/economicoutputandproductivity/output/datasets/indexofproduction/current/diop.csv'
 DRSI:
-  last_update: '2024-08-16'
-  next_update: '2024-09-20'
+  last_update: '2024-10-18'
+  next_update: '2024-11-22'
   desc: 'Retail Sales Index'
   url: 'https://www.ons.gov.uk/file?uri=/businessindustryandtrade/retailindustry/datasets/retailsales/current/drsi.csv'
 EMP:
-  last_update: '2024-09-10'
-  next_update: '2024-10-15'
+  last_update: '2024-10-15'
+  next_update: '2024-11-12'
   desc: 'Average Weekly Earnings'
   url: 'https://www.ons.gov.uk/file?uri=/employmentandlabourmarket/peopleinwork/earningsandworkinghours/datasets/averageweeklyearnings/current/emp.csv'
 IOS1:
-  last_update: '2024-09-11'
-  next_update: '2024-10-11'
+  last_update: '2024-10-11'
+  next_update: '2024-11-15'
   desc: 'Index of Services'
   url: 'https://www.ons.gov.uk/file?uri=/economy/economicoutputandproductivity/output/datasets/indexofservices/current/ios1.csv'
 LMS:
-  last_update: '2024-09-10'
-  next_update: '2024-10-15'
+  last_update: '2024-10-15'
+  next_update: '2024-11-12'
   desc: 'Labour Market Statistics'
   url: 'https://www.ons.gov.uk/file?uri=/employmentandlabourmarket/peopleinwork/employmentandemployeetypes/datasets/labourmarketstatistics/current/lms.csv'
 MGDP:
-  last_update: '2024-09-11'
-  next_update: '2024-10-11'
+  last_update: '2024-10-11'
+  next_update: '2024-11-15'
   desc: 'Monthly GDP'
   url: 'https://www.ons.gov.uk/file?uri=/economy/grossdomesticproductgdp/datasets/gdpmonthlyestimateuktimeseriesdataset/current/mgdp.csv'
 MM22:
-  last_update: '2024-09-18'
-  next_update: '2024-10-16'
+  last_update: '2024-10-16'
+  next_update: '2024-11-20'
   desc: 'Producer Price Inflation'
   url: 'https://www.ons.gov.uk/file?uri=/economy/inflationandpriceindices/datasets/producerpriceindex/current/mm22.csv'
 MM23:
-  last_update: '2024-09-18'
-  next_update: '2024-10-16'
+  last_update: '2024-10-16'
+  next_update: '2024-11-20'
   desc: 'Consumer Price Inflation'
   url: 'https://www.ons.gov.uk/file?uri=/economy/inflationandpriceindices/datasets/consumerpriceindices/current/mm23.csv'
 MRET:
-  last_update: '2024-09-11'
-  next_update: '2024-10-11'
+  last_update: '2024-10-11'
+  next_update: '2024-11-15'
   desc: 'UK Trade'
   url: 'https://www.ons.gov.uk/file?uri=/economy/nationalaccounts/balanceofpayments/datasets/tradeingoodsmretsallbopeu2013timeseriesspreadsheet/current/mret.csv'
 PDGP:
@@ -89,8 +89,8 @@ PSE:
   desc: 'Public Sector Employment'
   url: 'https://www.ons.gov.uk/file?uri=/employmentandlabourmarket/peopleinwork/publicsectorpersonnel/datasets/publicsectoremploymenttimeseriesdataset/current/pse.csv'
 PUSF:
-  last_update: '2024-08-21'
-  next_update: '2024-09-20'
+  last_update: '2024-10-22'
+  next_update: '2024-11-21'
   desc: 'Public Sector Finances'
   url: 'https://www.ons.gov.uk/file?uri=/economy/governmentpublicsectorandtaxes/publicsectorfinance/datasets/publicsectorfinances/current/pusf.csv'
 QNA:
@@ -114,13 +114,13 @@ UCST:
   desc: 'Unit Labour Cost and Unit Wage Cost'
   url: 'https://www.ons.gov.uk/file?uri=/employmentandlabourmarket/peopleinwork/labourproductivity/datasets/unitlabourcostandunitwagecosttimeseries/current/ucst.csv'
 UKEA:
-  last_update: '2024-06-28'
-  next_update: '2024-09-30'
+  last_update: '2024-09-30'
+  next_update: '2024-12-23'
   desc: 'UK Economic Accounts'
   url: 'https://www.ons.gov.uk/file?uri=/economy/grossdomesticproductgdp/datasets/unitedkingdomeconomicaccounts/current/ukea.csv'
 UNEM:
-  last_update: '2024-09-10'
-  next_update: '2024-10-15'
+  last_update: '2024-10-15'
+  next_update: '2024-11-12'
   desc: 'Claimant Count and Vacancies'
   url: 'https://www.ons.gov.uk/file?uri=/employmentandlabourmarket/peoplenotinwork/unemployment/datasets/claimantcountandvacanciesdataset/current/unem.csv'
 PROD_LAD:
@@ -214,8 +214,8 @@ RFDI:
   desc: 'Regional Foreign Direct Investment'
   url: 'NA'
 CONS:
-  last_update: '2024-09-11'
-  next_update: '2024-10-11'
+  last_update: '2024-10-11'
+  next_update: '2024-11-15'
   desc: 'Output in the Construction Industry'
   url: 'NA'
 INS1:

diff --git a/src/overview/membership/_data/updated.yaml b/src/overview/membership/_data/updated.yaml
@@ -1 +1 @@
-2024-09-18 09:53:11
+2024-10-29 14:11:15