diff --git a/src/core/fetch_and_organize_data.py b/src/core/fetch_and_organize_data.py index ae2934c..e552e20 100644 --- a/src/core/fetch_and_organize_data.py +++ b/src/core/fetch_and_organize_data.py @@ -2,7 +2,6 @@ import subprocess import shutil from urllib.parse import urlparse -import time from utils import load_yaml_file def download_content(urls, folder_path): @@ -39,7 +38,6 @@ def count_downloaded_files(base_path): print(f"Number of files downloaded: {count}") except Exception as e: print(f"An error occurred while counting files in {base_path}: {e}") - time.sleep(5) def reorganize_files(src_path, dst_path): diff --git a/src/core/ingest.py b/src/core/ingest.py index a627377..28d59d4 100644 --- a/src/core/ingest.py +++ b/src/core/ingest.py @@ -6,7 +6,6 @@ from langchain_community.document_loaders.merge import MergedDataLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores import Chroma -import time from embeddings import embedding_function config_data = load_yaml_file("config.yaml")