main.py

import os
import sys
import argparse
from importlib import import_module
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

# Load configuration
parser = argparse.ArgumentParser()
parser.add_argument("job", type=str, nargs='?', default="Job1.MyMethod")
parser.add_argument("slot", type=str, nargs='?', default="2018/11/19")
args = parser.parse_args()
job_module, job_method = args.job.rsplit('.',1)
slot = args.slot


if "local" in spark.sparkContext.master:
    dirname = os.path.dirname(__file__)
    sys.path.insert(0, (os.path.join(dirname, 'Utils')))
    sys.path.insert(0, (os.path.join(dirname, 'Jobs')))
    spark.conf.set("ADLS",os.path.join(dirname, 'DataLake'))
else:
    spark.sparkContext.addPyFile("dbfs:/MyApplication/Code/scripts.zip")
    spark.conf.set("ADLS",'adl://myazuredatalake.azuredatalakestore.net/')
    spark.conf.set("dfs.adls.oauth2.access.token.provider.type", "ClientCredential")
    spark.conf.set("dfs.adls.oauth2.client.id", dbutils.secrets.get(scope = "SparkADLS - Secrets", key = "clientid"))
    spark.conf.set("dfs.adls.oauth2.credential", dbutils.secrets.get(scope = "SparkADLS - Secrets", key = "credential"))
    spark.conf.set("dfs.adls.oauth2.refresh.url", "https://login.microsoftonline.com/[tenantid]/oauth2/token")


# Execute Job
mod = import_module(job_module)
met = getattr(mod, job_method)
met(slot)