Update to latest LCIP stats

scripts/lcip/transform.py

@@ -8,7 +8,7 @@
 DATA_DIR = os.path.join('data', 'lcip')
 WARD_DATA = os.path.join('data', 'leeds_wards.csv')
 
-LCIP_DATA = os.path.join('data', 'lcip', 'LCIP GRANT STATS FINAL Q1 & 2 2024.xlsx')
+LCIP_DATA = os.path.join('data', 'lcip', 'LCIP GRANT STATS REVISED FINAL Q1 & 2 2024 inc EDI 111024.xlsx')
 
 
 diversity_metrics = [
@@ -29,15 +29,12 @@
     'RELIGIOUS BELIEF (FUNDED)',
     'CARER (FUNDED)'
 ]
-
 ward_themes = [
     'WARDS - APPLICANT BASED', 
     'WARDS - FUNDED',
     'WARDS - RECEIVING ACTIVITY'
 ]
 
-project_files = ['inspire', 'grow_project', 'activate']
-revenue_files = ['grow_revenue', 'thrive', 'cultural_anchors']
 
 def fuzzy_merge(df1, df2, key1, key2, threshold=90):
     s = df2[key2].tolist()
@@ -91,7 +88,6 @@ def process_diversity_metrics(data, out_path, name):
             'TOTAL_FUNDED': 'FUNDED'
         })
 
-        # Remove trailing underscore from filename
         theme_filename = theme.replace(" ", "_").replace("/", "_").replace('(', '').replace(')', '').replace('_-_', '_').lower().rstrip('_') + '.csv'
         theme_out_path = os.path.join(out_path, 'diversity', name, theme_filename)
 
@@ -101,16 +97,24 @@ def process_diversity_metrics(data, out_path, name):
 
 def clean_name(name):
     name_clean = name.replace(' 2024', '')
-    name_clean = (name_clean.lower().rstrip().replace(' ', '_'))
+    name_clean = (name_clean.upper().rstrip().replace(' ', '_'))
     return name_clean
 
 def clean_sheet(sheet):
-    sheet = sheet.dropna(axis='columns', how='all')
+    sheet = sheet.dropna(axis='columns', how='all').copy()
+
+    sheet['THEME'] = sheet['THEME'].str.replace(r'\(APPLIED\)', '- APPLIED', regex=True)
+    sheet['THEME'] = sheet['THEME'].str.replace(r'\(FUNDED\)', '- FUNDED', regex=True)
+
     period_cols = [col for col in sheet.columns if col.startswith('PERIOD')]
+    sheet[period_cols] = sheet[period_cols].apply(pd.to_numeric, errors='coerce')
+
     sheet['TOTAL'] = sheet[period_cols].sum(axis=1)
     sheet = sheet.drop(columns=period_cols)
+
     sheet.loc[:, 'THEME'] = sheet['THEME'].str.rstrip()
     sheet.loc[:, 'METRIC'] = sheet['METRIC'].str.rstrip()
+
     return sheet
 
 def clean_theme(theme):
@@ -128,30 +132,69 @@ def clean_theme(theme):
     lcip_data = pd.read_excel(LCIP_DATA, sheet_name=['Inspire 2024', 'Grow Project 2024', 'Activate 2024', 'Grow Revenue 2024', 'Thrive 2024', 'Cultural Anchors 2024'])
     ward_data = pd.read_csv(WARD_DATA)
 
+    project_df = lcip_data['Inspire 2024'][['THEME', 'METRIC']]
+    revenue_df = lcip_data['Inspire 2024'][['THEME', 'METRIC']]
+
     for name, sheet in lcip_data.items():
-            # Clean up sheet name
-            name = clean_name(name)
-
-            # Define output path
-            theme_path = os.path.join(OUT_DIR, name)
-            os.makedirs(theme_path, exist_ok=True)
-
-            # Clean up data
-            sheet = clean_sheet(sheet)
-
-            process_diversity_metrics(sheet, OUT_DIR, name)
-            # Create datasets for each theme:
-            themes = sheet['THEME'].unique()
-            for theme in themes:
-                if theme in ward_themes:
-                    process_wards(ward_data, sheet, theme, name, f'{clean_theme(theme)}')
-                elif not theme in diversity_metrics:
-                    theme_df = sheet[sheet['THEME'] == theme]
-                    # theme_df = theme_df.fillna(0)
-                    theme_df = theme_df.pivot_table(index='THEME', columns='METRIC', values='TOTAL')
-                    theme_df = theme_df.round(0).astype(int)
-                    theme_df.columns = theme_df.columns.str.replace(',',' ')
-                    theme_filename = clean_theme(theme)
+
+        project_files = ['INSPIRE', 'GROW_PROJECT', 'ACTIVATE']
+        revenue_files = ['GROW_REVENUE', 'THRIVE', 'CULTURAL_ANCHORS']
+
+        name = clean_name(name)
+
+        theme_path = os.path.join(OUT_DIR, name)
+        os.makedirs(theme_path, exist_ok=True)
+
+        sheet = clean_sheet(sheet)
+
+        if name in project_files:
+            sheet['TOTAL'] = pd.to_numeric(sheet['TOTAL'], errors='coerce')
+            project_df = project_df.merge(sheet[['METRIC', 'TOTAL']], on='METRIC', how='left', suffixes=('', f'_{name}'))
+            project_df = project_df.rename(columns={'TOTAL': f'TOTAL_{name}'})
+        else:
+            sheet['TOTAL'] = pd.to_numeric(sheet['TOTAL'], errors='coerce')
+            revenue_df = revenue_df.merge(sheet[['METRIC', 'TOTAL']], on='METRIC', how='left', suffixes=('', f'_{name}'))
+            revenue_df = revenue_df.rename(columns={'TOTAL': f'TOTAL_{name}'})
+
+        process_diversity_metrics(sheet, OUT_DIR, name)
+
+        themes = sheet['THEME'].unique()
+        for theme in themes:
+            if theme in ward_themes:
+                process_wards(ward_data, sheet, theme, name, f'{clean_theme(theme)}')
+            elif theme in diversity_metrics: 
+                process_diversity_metrics(sheet, OUT_DIR, name)
+            elif not theme in diversity_metrics:
+                theme_df = sheet[sheet['THEME'] == theme]
+                # theme_df = theme_df.fillna(0)
+                theme_df = theme_df.pivot_table(index='THEME', columns='METRIC', values='TOTAL')
+                theme_df = theme_df.round(0).astype(int)
+                theme_df.columns = theme_df.columns.str.replace(',',' ')
+                theme_filename = clean_theme(theme)
+                theme_df = theme_df.fillna(0)
+                # theme_df = theme_df.loc[~(theme_df == 0).all(axis=1)]
+                if not theme_df.empty:
                     theme_df.to_csv(os.path.join(theme_path, theme_filename), index=True)
 
 
+
+
+    project_df = project_df.set_index(['THEME', 'METRIC']).fillna(0)
+    revenue_df = revenue_df.set_index(['THEME', 'METRIC']).fillna(0)
+
+
+    project_df['TOTAL_PROJECT'] = (
+        project_df['TOTAL_INSPIRE'].fillna(0) + 
+        project_df['TOTAL_GROW_PROJECT'].fillna(0) + 
+        project_df['TOTAL_ACTIVATE'].fillna(0)
+    )
+
+    project_df.reset_index().to_csv(os.path.join(OUT_DIR, 'project_summary.csv'), index=False)
+
+    revenue_df['TOTAL_REVENUE'] = (
+        revenue_df['TOTAL_GROW_REVENUE'].fillna(0) + 
+        revenue_df['TOTAL_THRIVE'].fillna(0) + 
+        revenue_df['TOTAL_CULTURAL_ANCHORS'].fillna(0)
+    )
+
+    revenue_df.reset_index().to_csv(os.path.join(OUT_DIR, 'revenue_summary.csv'), index=False)

src/_data/viz/lcip/activate/age_applied.csv

@@ -1,2 +1,2 @@
 THEME,18 - 24,25 - 34,35 - 44,45 - 54,55 - 64,65 - 74,75 - 84,85+,Not known,Prefer not to say
-AGE - APPLIED,0,0,0,0,0,0,0,0,0,0
+AGE - APPLIED,1,3,3,0,0,0,0,0,0,0

src/_data/viz/lcip/activate/age_funded.csv

@@ -1,2 +1,2 @@
 THEME,18 - 24,25 - 34,35 - 44,45 - 54,55 - 64,65 - 74,75 - 84,85+,Not known,Prefer not to say
-AGE - FUNDED,0,0,0,0,0,0,0,0,0,0
+AGE - FUNDED,1,3,3,0,0,0,0,0,0,0

src/_data/viz/lcip/activate/carer_applied.csv

@@ -1,2 +1,2 @@
 THEME,Do you consider yourself to be a carer?,No,Not known,Prefer not to say,Yes
-CARER - APPLIED,0,0,0,0,0
+CARER - APPLIED,0,0,6,0,0

src/_data/viz/lcip/activate/carer_funded.csv

@@ -1,2 +1,2 @@
 THEME,Do you consider yourself to be a carer?,No,Not known,Prefer not to say,Yes
-CARER - FUNDED,0,0,0,0,0
+CARER - FUNDED,0,0,6,0,0

src/_data/viz/lcip/activate/disability_applied.csv

@@ -1,2 +1,2 @@
 THEME,Hearing impairment,Learning disability,Long-standing illness or health condition,Mental health condition,Neuro Diverse,Not known,Other,Physical impairment,Prefer not to say,Visual impairment
-DISABILITY - APPLIED,0,0,0,0,0,0,0,0,0,0
+DISABILITY - APPLIED,0,1,0,1,0,0,0,0,1,0

src/_data/viz/lcip/activate/disability_funded.csv

@@ -1,2 +1,2 @@
 THEME,Hearing impairment,Learning disability,Long-standing illness or health condition,Mental health condition,Neuro Diverse,Not known,Other,Physical impairment,Prefer not to say,Visual impairment
-DISABILITY - FUNDED,0,0,0,0,0,0,0,0,0,0
+DISABILITY - FUNDED,0,1,0,1,0,0,0,0,1,0

src/_data/viz/lcip/activate/ethnic_origin_applied.csv

@@ -1,2 +1,2 @@
 THEME,African,Any other Asian background,Any other Black background,Any other background,Any other mixed/multiple ethnic group,Any other white background,Arab,Asian or Asian British,Bangladeshi,Black or Black British,Caribbean,Chinese,Indian,Kashmiri,Mixed/ multiple ethnic group,Other ethnic groups,Pakistani,Prefer not to say,White,White British,White English,White Gypsy or Traveller,White Irish,White Northern Irish,White Scottish,White Welsh,White and Asian,White and Black African,White and Black Caribbean
-ETHNIC ORIGIN - APPLIED,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+ETHNIC ORIGIN - APPLIED,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,1,0,0,0,0

src/_data/viz/lcip/activate/ethnic_origin_funded.csv

@@ -1,2 +1,2 @@
 THEME,African,Any other Asian background,Any other Black background,Any other background,Any other mixed/multiple ethnic group,Any other white background,Arab,Asian or Asian British,Bangladeshi,Black or Black British,Caribbean,Chinese,Indian,Kashmiri,Mixed/ multiple ethnic group,Other ethnic groups,Pakistani,Prefer not to say,White,White British,White English,White Gypsy or Traveller,White Irish,White Northern Irish,White Scottish,White Welsh,White and Asian,White and Black African,White and Black Caribbean
-ETHNIC ORIGIN - FUNDED,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+ETHNIC ORIGIN - FUNDED,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,1,0,0,0,0

src/_data/viz/lcip/activate/funding_advice_sessions.csv

@@ -0,0 +1,2 @@
+THEME,No. of funding surgeries booked,No. of funding surgeries offered,No. of in person sessions,No. of online/Telephone sessions
+FUNDING ADVICE SESSIONS,0,0,0,0

src/_data/viz/lcip/activate/religious_belief_applied.csv

@@ -1,2 +1,2 @@
 THEME,Buddhist,Christian,Hindu,Jewish,Muslim,No religion/belief,Not known,Other,Prefer not to say,Sikh
-RELIGIOUS BELIEF - APPLIED,0,0,0,0,0,0,0,0,0,0
+RELIGIOUS BELIEF - APPLIED,0,0,0,0,0,6,0,0,0,0

src/_data/viz/lcip/activate/religious_belief_funded.csv

@@ -1,2 +1,2 @@
 THEME,Buddhist,Christian,Hindu,Jewish,Muslim,No religion/belief,Not known,Other,Prefer not to say,Sikh
-RELIGIOUS BELIEF - FUNDED,0,0,0,0,0,0,0,0,0,0
+RELIGIOUS BELIEF - FUNDED,0,0,0,0,0,6,0,0,0,0

src/_data/viz/lcip/activate/results.csv

@@ -1,2 +1,2 @@
-THEME,No. of funded organisations,No. of unsuccessful organisation applications,Total amount awarded to organisations,Total amount awarded to organisations (£)
-RESULTS,5,0,17600,17600
+THEME,No. of  withdrawn by applicant,No. of  withdrawn by organisation,No. of funded individual artists,No. of funded organisations,No. of unsucessful artist applications,No. of unsucessful organisations applications,Total amount awarded to individual artists,Total amount awarded to organisations (£)
+RESULTS,0,0,0,5,0,0,0,17600

src/_data/viz/lcip/activate/sex_applied.csv

@@ -1,2 +1,2 @@
 THEME,Female,Male,Non Binary,Not known,Other,Prefer not to say
-SEX - APPLIED,0,0,0,0,0,0
+SEX - APPLIED,2,4,0,0,0,0

src/_data/viz/lcip/activate/sex_funded.csv

@@ -1,2 +1,2 @@
 THEME,Female,Male,Non Binary,Not known,Other,Prefer not to say
-SEX - FUNDED,0,0,0,0,0,0
+SEX - FUNDED,2,4,0,0,0,0

src/_data/viz/lcip/activate/sexual_orientation_applied.csv

@@ -1,2 +1,2 @@
 THEME,Bi-sexual,Gay,Heterosexual or Straight,Lesbian,Not known,Other,Prefer not to say
-SEXUAL ORIENTATION - APPLIED,0,0,0,0,0,0,0
+SEXUAL ORIENTATION - APPLIED,0,1,5,0,0,0,0

src/_data/viz/lcip/activate/sexual_orientation_funded.csv

@@ -1,2 +1,2 @@
 THEME,Bi-sexual,Gay,Heterosexual or Straight,Lesbian,Not known,Other,Prefer not to say
-SEXUAL ORIENTATION - FUNDED,0,0,0,0,0,0,0
+SEXUAL ORIENTATION - FUNDED,0,1,5,0,0,0,0

src/_data/viz/lcip/cultural_anchors/age_applied.csv

@@ -1,2 +1,2 @@
 THEME,18 - 24,25 - 34,35 - 44,45 - 54,55 - 64,65 - 74,75 - 84,85+,Not known,Prefer not to say
-AGE - APPLIED,0,0,0,0,0,0,0,0,0,0
+AGE - APPLIED,31,88,71,71,66,11,1,0,47,4

src/_data/viz/lcip/cultural_anchors/age_funded.csv

@@ -1,2 +1,2 @@
 THEME,18 - 24,25 - 34,35 - 44,45 - 54,55 - 64,65 - 74,75 - 84,85+,Not known,Prefer not to say
-AGE - FUNDED,0,0,0,0,0,0,0,0,0,0
+AGE - FUNDED,31,88,71,71,66,11,1,0,47,4

src/_data/viz/lcip/cultural_anchors/applications_received.csv

@@ -0,0 +1,2 @@
+THEME,No. of eligible applications,No. of ineligible,Total amount of funding requested (£),from an organisation
+APPLICATIONS RECEIVED,0,0,1065611,3

src/_data/viz/lcip/cultural_anchors/artform_invited.csv → src/_data/viz/lcip/cultural_anchors/artform_applied.csv

@@ -1,2 +1,2 @@
 THEME,Circus,Combined Arts,Crafts,Dance,Design,Digital Media,Festival/Carnival,Film,Literature / Poetry / Spoken Word,Live Art,Music,Photography,Theatre,Visual Art
-ARTFORM (INVITED),0,0,0,1,0,0,0,0,0,0,1,0,1,0
+ARTFORM - APPLIED,0,0,0,1,0,0,0,0,0,0,1,0,1,0

src/_data/viz/lcip/cultural_anchors/carer_applied.csv

@@ -1,2 +1,2 @@
 THEME,Do you consider yourself to be a carer?,No,Not known,Prefer not to say,Yes
-CARER - APPLIED,0,0,0,0,0
+CARER - APPLIED,0,0,375,0,0

src/_data/viz/lcip/cultural_anchors/carer_funded.csv

@@ -1,2 +1,2 @@
 THEME,Do you consider yourself to be a carer?,No,Not known,Prefer not to say,Yes
-CARER - FUNDED,0,0,0,0,0
+CARER - FUNDED,0,0,375,0,0

src/_data/viz/lcip/cultural_anchors/disability_applied.csv

@@ -1,2 +1,2 @@
 THEME,Hearing impairment,Learning disability,Long-standing illness or health condition,Mental health condition,Neuro Diverse,Not known,Other,Physical impairment,Prefer not to say,Visual impairment
-DISABILITY - APPLIED,0,0,0,0,0,0,0,0,0,0
+DISABILITY - APPLIED,0,0,0,0,21,58,62,0,21,0

src/_data/viz/lcip/cultural_anchors/disability_funded.csv

@@ -1,2 +1,2 @@
 THEME,Hearing impairment,Learning disability,Long-standing illness or health condition,Mental health condition,Neuro Diverse,Not known,Other,Physical impairment,Prefer not to say,Visual impairment
-DISABILITY - FUNDED,0,0,0,0,0,0,0,0,0,0
+DISABILITY - FUNDED,0,0,0,0,21,58,62,0,21,0

src/_data/viz/lcip/cultural_anchors/ethnic_origin_applied.csv

@@ -1,2 +1,2 @@
 THEME,African,Any other Asian background,Any other Black background,Any other background,Any other mixed/multiple ethnic group,Any other white background,Arab,Asian or Asian British,Bangladeshi,Black or Black British,Caribbean,Chinese,Indian,Kashmiri,Mixed/ multiple ethnic group,Other ethnic groups,Pakistani,Prefer not to say,White,White British,White English,White Gypsy or Traveller,White Irish,White Northern Irish,White Scottish,White Welsh,White and Asian,White and Black African,White and Black Caribbean
-ETHNIC ORIGIN - APPLIED,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+ETHNIC ORIGIN - APPLIED,2,3,0,5,1,29,0,1,0,0,4,3,3,0,0,0,2,56,0,276,0,0,3,0,0,0,1,0,2

src/_data/viz/lcip/cultural_anchors/ethnic_origin_funded.csv

@@ -1,2 +1,2 @@
 THEME,African,Any other Asian background,Any other Black background,Any other background,Any other mixed/multiple ethnic group,Any other white background,Arab,Asian or Asian British,Bangladeshi,Black or Black British,Caribbean,Chinese,Indian,Kashmiri,Mixed/ multiple ethnic group,Other ethnic groups,Pakistani,Prefer not to say,White,White British,White English,White Gypsy or Traveller,White Irish,White Northern Irish,White Scottish,White Welsh,White and Asian,White and Black African,White and Black Caribbean
-ETHNIC ORIGIN - FUNDED,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+ETHNIC ORIGIN - FUNDED,2,3,0,5,1,29,0,1,0,0,4,3,3,0,0,0,2,56,0,276,0,0,3,0,0,0,1,0,2

src/_data/viz/lcip/cultural_anchors/funding_advice_sessions.csv

@@ -0,0 +1,2 @@
+THEME,No. of funding surgeries booked,No. of funding surgeries offered,No. of in person sessions,No. of online/Telephone sessions
+FUNDING ADVICE SESSIONS,0,0,0,0

src/_data/viz/lcip/cultural_anchors/gender_registered_at_birth_applied.csv

@@ -1,2 +1,2 @@
 THEME,Not known,Prefer not to say,Total
-GENDER REGISTERED AT BIRTH - APPLIED,0,0,0
+GENDER REGISTERED AT BIRTH - APPLIED,121,0,0

src/_data/viz/lcip/cultural_anchors/gender_registered_at_birth_funded.csv

@@ -1,2 +1,2 @@
 THEME,Not known,Prefer not to say,Total
-GENDER REGISTERED AT BIRTH - FUNDED,0,0,0
+GENDER REGISTERED AT BIRTH - FUNDED,121,0,0

src/_data/viz/lcip/cultural_anchors/project_outputs_applied.csv

@@ -1,2 +1,2 @@
-THEME,Days employment for freelance artists and creatives,Learning or participation sessions,New cultural activities/products,Performances or exhibition days,Training opportunities,Volunteer opportunities
-PROJECT OUTPUTS - APPLIED,24480,9492,138,845,42,4348
+THEME,Days employment for freelance artists and creatives,Training opportunities,Volunteer opportunities
+PROJECT OUTPUTS - APPLIED,24480,42,4348

src/_data/viz/lcip/cultural_anchors/religious_belief_applied.csv

@@ -1,2 +1,2 @@
 THEME,Buddhist,Christian,Hindu,Jewish,Muslim,No religion/belief,Not known,Other,Prefer not to say,Sikh
-RELIGIOUS BELIEF - APPLIED,0,0,0,0,0,0,0,0,0,0
+RELIGIOUS BELIEF - APPLIED,0,0,0,0,0,0,375,0,0,0

src/_data/viz/lcip/cultural_anchors/religious_belief_funded.csv

@@ -1,2 +1,2 @@
 THEME,Buddhist,Christian,Hindu,Jewish,Muslim,No religion/belief,Not known,Other,Prefer not to say,Sikh
-RELIGIOUS BELIEF - FUNDED,0,0,0,0,0,0,0,0,0,0
+RELIGIOUS BELIEF - FUNDED,0,0,0,0,0,0,375,0,0,0

src/_data/viz/lcip/cultural_anchors/results.csv

@@ -0,0 +1,2 @@
+THEME,No. of  withdrawn by applicant,No. of  withdrawn by organisation,No. of funded individual artists,No. of funded organisations,No. of unsucessful artist applications,No. of unsucessful organisations applications,Total amount awarded to individual artists,Total amount awarded to organisations (£)
+RESULTS,0,0,0,3,0,0,0,1035000

src/_data/viz/lcip/cultural_anchors/sex_applied.csv

@@ -1,2 +1,2 @@
 THEME,Female,Male,Non Binary,Not known,Other,Prefer not to say
-SEX - APPLIED,0,0,0,0,0,0
+SEX - APPLIED,191,131,3,48,0,8

src/_data/viz/lcip/cultural_anchors/sex_funded.csv

@@ -1,2 +1,2 @@
 THEME,Female,Male,Non Binary,Not known,Other,Prefer not to say
-SEX - FUNDED,0,0,0,0,0,0
+SEX - FUNDED,191,131,3,48,0,8

src/_data/viz/lcip/cultural_anchors/sexual_orientation_applied.csv

@@ -1,2 +1,2 @@
 THEME,Bi-sexual,Gay,Heterosexual or Straight,Lesbian,Not known,Other,Prefer not to say
-SEXUAL ORIENTATION - APPLIED,0,0,0,0,0,0,0
+SEXUAL ORIENTATION - APPLIED,23,23,257,10,48,6,24

src/_data/viz/lcip/cultural_anchors/sexual_orientation_funded.csv

@@ -1,2 +1,2 @@
 THEME,Bi-sexual,Gay,Heterosexual or Straight,Lesbian,Not known,Other,Prefer not to say
-SEXUAL ORIENTATION - FUNDED,0,0,0,0,0,0,0
+SEXUAL ORIENTATION - FUNDED,23,23,257,10,48,6,24

src/_data/viz/lcip/cultural_anchors/total_artform_applied.csv

@@ -0,0 +1,2 @@
+THEME,Total
+TOTAL ARTFORM - APPLIED,3