dvc.yaml

stages:

  # INGEST DATA
  # In our case, this is simply copying local csv files
  ingest_data:
    foreach:
      - addresses
      - properties
      - users
    do:
      cmd: python scripts/ingest_data.py ${item}
      deps:
        - scripts/ingest_data.py
      outs:
        - data/source/src_${item}.csv
  
  # CREATE STAGE
  # Transform the csv files into dataframes and do some
  # basic cleaning like removing duplicates, handling null
  # values and renaming columns.
  create_stage:
    foreach:
      - addresses
      - properties
      - users
    do:
      cmd: python stg_${item}.py
      wdir: scripts
      deps:
        - stg_${item}.py
        - ../data/source/src_${item}.csv
      outs:
        - ../data/stage/stg_${item}.pickle

  # CREATE FACTS
  # Prepare fact tables with appropriate foreign keys
  # to our dimension tables
  create_facts:
    cmd: python fct_co2_emissions.py
    wdir: scripts
    deps:
      - stg_properties.py
      - ../data/stage/stg_properties.pickle
    outs:
      - ../data/facts/fct_co2_emissions.csv

  # CREATE DIMENSIONS
  # Prepare dimension tables
  create_dimensions:
    foreach:
      - addresses
      - properties
      - users
    do:
      cmd: python dim_${item}.py
      wdir: scripts
      deps:
        - dim_${item}.py
        - ../data/stage/stg_${item}.pickle
      outs:
        - ../data/dimensions/dim_${item}.csv

  # CREATE PLOTS
  # Use the analysis script to further explore
  # the data and create a number of relevant plots
  analyze:
    cmd: python analysis.py
    wdir: scripts
    deps:
      - dim_addresses.py
      - dim_properties.py
      - dim_users.py
      - fct_co2_emissions.py
      - ../data/dimensions/
      - ../data/facts/fct_co2_emissions.csv
    outs:
      - ../plots/properties_by_city.png
      - ../plots/properties_by_plz.png
      - ../plots/box_plots.png
      - ../plots/scatter_co2_intensity_vs_construction_year.png
      - ../plots/bar_charts_text_and_boolean_categories.png
      - ../plots/boxplot_co2_emissions_by_year.png