Crawl news mfa

src/data/openthaigpt_pretraining_data/web_crawls_mfa/crawl_news.py

@@ -0,0 +1,66 @@
+import requests
+import time
+from openthaigpt_pretraining_data.web_crawls_mfa.crawl_gov_achievements import (
+    process_response,
+    process_info,
+)
+
+ROOT = "https://www.mfa.go.th"
+DIV_TAG = "div"
+P_TAG = "p"
+A_TAG = "a"
+DATE_CLASS = "date"
+INFO_CLASS = "p-3 col-md-4"
+DETAIL_CLASS = "ContentDetailstyled__ContentDescription-sc-150bmwg-4 jWrYsI mb-3"
+
+
+def get_title_date(cur_url, page_no, time_delay):
+    """
+    Description:
+        Get data processed by the function process_response.
+    Args:
+        cur_url: The desired URL to be used as a root.
+        page_no: The total number of pages.
+        time_delay: Delay before another request (in second).
+    Returns:
+        news_list: A list containing titles and dates.
+    """
+    news_list = []
+
+    for page in range(1, page_no + 1):
+        url = f"{cur_url}&p={page}"
+        res = requests.get(url)
+        res.encoding = "utf-8"
+
+        if res.status_code == 200:
+            processed_data = process_response(res.text, time_delay)
+            news_list.extend(processed_data)
+
+        time.sleep(0.5)
+
+    return news_list
+
+
+def get_info(cur_url, page_no, time_delay):
+    """
+    Description:
+        get data inside a link for every pafe
+    Args:
+        desired url and total of pages.
+    Returns:
+        info_list contains details of the news
+    """
+    info_list = []
+
+    for page in range(1, page_no + 1):
+        url = f"{cur_url}&p={page}"
+        res = requests.get(url)
+        res.encoding = "utf-8"
+
+        if res.status_code == 200:
+            processed_info = process_info(res.text, time_delay)
+            info_list.extend(processed_info)
+
+        time.sleep(0.5)
+
+    return info_list

src/data/scripts/crawl_mfa/embassy_cosulate.py

@@ -0,0 +1,21 @@
+import pandas as pd
+from datasets import Dataset, load_from_disk
+from openthaigpt_pretraining_data.web_crawls_mfa.crawl_news import (
+    get_title_date,
+    get_info,
+)
+
+EMBASSY_CONSULATE_URL = "https://www.mfa.go.th/th/page/%E0%B8%82%E0%B9%88%E0%B8%B2%E0%B8%A7%E0%B8%81%E0%B8%B4%E0%B8%88%E0%B8%81%E0%B8%A3%E0%B8%A3%E0%B8%A1%E0%B8%AA%E0%B8%96%E0%B8%B2%E0%B8%99%E0%B9%80%E0%B8%AD%E0%B8%81%E0%B8%AD%E0%B8%B1%E0%B8%84%E0%B8%A3%E0%B8%A3%E0%B8%B2%E0%B8%8A%E0%B8%97%E0%B8%B9%E0%B8%95%E0%B9%81%E0%B8%A5%E0%B8%B0%E0%B8%AA%E0%B8%96%E0%B8%B2%E0%B8%99%E0%B8%81%E0%B8%87%E0%B8%AA%E0%B8%B8%E0%B8%A5%E0%B9%83%E0%B8%AB%E0%B8%8D%E0%B9%88?menu=5f2110a3c1d7dc1b17651cb2"
+
+news_title_date = get_title_date(cur_url=EMBASSY_CONSULATE_URL, page_no=501)
+news_details = get_info(cur_url=EMBASSY_CONSULATE_URL, page_no=501)
+
+for i, data_dict in enumerate(news_title_date):
+    if i < len(news_details):
+        data_dict.update({"detail": news_details[i]})
+
+all_news = pd.DataFrame(news_title_date)
+dataset = Dataset.from_pandas(all_news)
+dataset.save_to_disk("mfa_embassy_consulate.arrow")
+loaded_dataset = load_from_disk("mfa_embassy_consulate.arrow")
+

src/data/scripts/crawl_mfa/other_news.py

@@ -0,0 +1,20 @@
+import pandas as pd
+from datasets import Dataset, load_from_disk
+from openthaigpt_pretraining_data.web_crawls_mfa.crawl_news import (
+    get_title_date,
+    get_info,
+)
+
+OTHER_NEWS_URL = "https://www.mfa.go.th/th/page/%E0%B8%82%E0%B9%88%E0%B8%B2%E0%B8%A7%E0%B8%AD%E0%B8%B7%E0%B9%88%E0%B8%99%E0%B9%86?menu=5d5bd3d815e39c306002aac7"
+
+news_title_date = get_title_date(cur_url=OTHER_NEWS_URL, page_no=10)
+news_details = get_info(cur_url=OTHER_NEWS_URL, page_no=10)
+
+for i, data_dict in enumerate(news_title_date):
+    if i < len(news_details):
+        data_dict.update({"detail": news_details[i]})
+
+all_news = pd.DataFrame(news_title_date)
+dataset = Dataset.from_pandas(all_news)
+dataset.save_to_disk("mfa_other_news.arrow")
+loaded_dataset = load_from_disk("mfa_other_news.arrow")

src/data/scripts/crawl_mfa/press_release.py

@@ -0,0 +1,20 @@
+import pandas as pd
+from datasets import Dataset, load_from_disk
+from openthaigpt_pretraining_data.web_crawls_mfa.crawl_news import (
+    get_title_date,
+    get_info,
+)
+
+PRESS_RELEASE_URL = "https://www.mfa.go.th/th/page/%E0%B8%82%E0%B9%88%E0%B8%B2%E0%B8%A7%E0%B8%AA%E0%B8%B2%E0%B8%A3%E0%B8%99%E0%B8%B4%E0%B9%80%E0%B8%97%E0%B8%A8?menu=5d5bd3d815e39c306002aac5"
+
+news_title_date = get_title_date(cur_url=PRESS_RELEASE_URL, page_no=313)
+news_details = get_info(cur_url=PRESS_RELEASE_URL, page_no=313)
+
+for i, data_dict in enumerate(news_title_date):
+    if i < len(news_details):
+        data_dict.update({"detail": news_details[i]})
+
+all_news = pd.DataFrame(news_title_date)
+dataset = Dataset.from_pandas(all_news)
+dataset.save_to_disk("mfa_press_release.arrow")
+loaded_dataset = load_from_disk("mfa_press_release.arrow")

src/data/scripts/crawl_mfa/speeches.py

@@ -0,0 +1,21 @@
+import pandas as pd
+from datasets import Dataset, load_from_disk
+from openthaigpt_pretraining_data.web_crawls_mfa.crawl_news import (
+    get_title_date,
+    get_info,
+)
+
+SPEECHES_URL = "https://www.mfa.go.th/th/page/%E0%B8%AA%E0%B8%B8%E0%B8%99%E0%B8%97%E0%B8%A3%E0%B8%9E%E0%B8%88%E0%B8%99%E0%B9%8C?menu=5d5bd3d815e39c306002aacd"
+
+news_title_date = get_title_date(cur_url=SPEECHES_URL, page_no=8)
+news_details = get_info(cur_url=SPEECHES_URL, page_no=8)
+
+for i, data_dict in enumerate(news_title_date):
+    if i < len(news_details):
+        data_dict.update({"detail": news_details[i]})
+
+all_news = pd.DataFrame(news_title_date)
+dataset = Dataset.from_pandas(all_news)
+dataset.save_to_disk("MFA_speeches.arrow")
+loaded_dataset = load_from_disk("MFA_speeches.arrow")
+

src/data/scripts/crawl_mfa/top_news.py

@@ -0,0 +1,20 @@
+import pandas as pd
+from datasets import Dataset, load_from_disk
+from openthaigpt_pretraining_data.web_crawls_mfa.crawl_news import (
+    get_title_date,
+    get_info,
+)
+
+TOP_STORIES_URL = "https://www.mfa.go.th/th/page/%E0%B8%82%E0%B9%88%E0%B8%B2%E0%B8%A7%E0%B9%80%E0%B8%94%E0%B9%88%E0%B8%99?menu=5d5bd3d815e39c306002aac4"
+
+news_title_date = get_title_date(cur_url=TOP_STORIES_URL, page_no=216)
+news_details = get_info(cur_url=TOP_STORIES_URL, page_no=216)
+
+for i, data_dict in enumerate(news_title_date):
+    if i < len(news_details):
+        data_dict.update({"detail": news_details[i]})
+
+all_news = pd.DataFrame(news_title_date)
+dataset = Dataset.from_pandas(all_news)
+dataset.save_to_disk("MFA_top_news.arrow")
+loaded_dataset = load_from_disk("MFA_top_news.arrow")