From 981930c97a7b762afe4f1743d5c3e05e42d0841e Mon Sep 17 00:00:00 2001 From: Maroun Touma Date: Mon, 18 Nov 2024 09:01:59 -0500 Subject: [PATCH] bug fixes Signed-off-by: Maroun Touma --- .../dpk_html2parquet/transform_python.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/transforms/language/html2parquet/dpk_html2parquet/transform_python.py b/transforms/language/html2parquet/dpk_html2parquet/transform_python.py index 323295344..2ffe02cd6 100644 --- a/transforms/language/html2parquet/dpk_html2parquet/transform_python.py +++ b/transforms/language/html2parquet/dpk_html2parquet/transform_python.py @@ -42,18 +42,25 @@ def __init__(self): class Html2ParquetRuntime(): def __init__(self, **kwargs): self.params={} - for key, value in kwargs: - self.params[key]=value - + for key in kwargs: + self.params[key]=kwargs[key] + # if input_folder and output_folder are specified, then assume it is represent data_local_config + try: + local_conf={k:self.params[k] for k in ('input_folder', 'output_folder')} + self.params['data_local_config']= ParamsUtils.convert_to_ast(local_conf) + del self.params['input_folder'] + del self.params['output_folder'] + except: + pass - def ingest(self): + def transform(self): sys.argv = ParamsUtils.dict_to_req(d=(self.params)) # create launcher launcher = PythonTransformLauncher(Html2ParquetPythonTransformConfiguration()) # launch return_code = launcher.launch() return return_code - + if __name__ == "__main__": launcher = PythonTransformLauncher(Html2ParquetPythonTransformConfiguration())