From 963035a296be983ef0fbf5cac660f57c33523c3b Mon Sep 17 00:00:00 2001 From: Sermet Pekin Date: Mon, 2 Dec 2024 22:43:39 +0300 Subject: [PATCH 1/2] refactored --- .gitignore | 2 + docs/source/conf.py | 1 - evdschat/__init__.py | 4 +- evdschat/common/akeys.py | 89 +++++++++++++++++++----- evdschat/common/bridge.py | 31 +++++---- evdschat/common/github_actions.py | 8 ++- evdschat/common/globals.py | 3 + evdschat/core/builder.py | 1 - evdschat/core/chat.py | 19 ++++-- evdschat/model/chatters.py | 108 +++++++++++++++++++----------- main.py | 2 +- pyproject.toml | 2 +- tests/test_chat.py | 4 +- tests/test_req.py | 8 +-- 14 files changed, 191 insertions(+), 91 deletions(-) diff --git a/.gitignore b/.gitignore index 29bfff8..96d1fc2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ __pycache__/ --*.* --*/ +ignore*.py + !example.env test_pri_*.* diff --git a/docs/source/conf.py b/docs/source/conf.py index 9885008..e33f560 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -64,7 +64,6 @@ } - logger = logging.getLogger(__name__) diff --git a/evdschat/__init__.py b/evdschat/__init__.py index c322bf7..f140f7b 100644 --- a/evdschat/__init__.py +++ b/evdschat/__init__.py @@ -1,5 +1,3 @@ from evdschat.core.chat import chat, chat_console -__all__ = [ - chat, chat_console -] +__all__ = [chat, chat_console] diff --git a/evdschat/common/akeys.py b/evdschat/common/akeys.py index f60ca0c..1fae705 100644 --- a/evdschat/common/akeys.py +++ b/evdschat/common/akeys.py @@ -17,7 +17,10 @@ from abc import ABC import os from pathlib import Path -from typing import Union +from typing import Union, Dict +import time + +from evdschat.common.globals import WARNING_SLEEP_SECONDS class ErrorApiKey(Exception): @@ -33,13 +36,21 @@ def __init__(self, message="There is an issue with the provided API key."): class ApiKey(ABC): - def __init__(self, key: str) -> None: + def __init__(self, key: str, key_name: str = 'ApiKey') -> None: self.key = key + self.key_name = key_name self.check() + def __str__(self): + return self.key + + def msg_before_raise(self): + showApiKeyMessage(self.__class__.__name__) + # create_env_example_file() + def check(self): if isinstance(self.key, type(None)): - raise ErrorApiKey('Api key not set. Please see the documentation.') + raise ErrorApiKey("Api key not set. Please see the documentation.") if not isinstance(self.key, str) or len(str(self.key)) < 5: raise ErrorApiKey(f"Api key {self.key} is not a valid key") return True @@ -51,16 +62,52 @@ def set_key(self, key: str): self.key = key +def sleep(number: int): + time.sleep(number) + + +def showApiKeyMessage(cls_name: str) -> None: + msg = f""" + {cls_name} not found. + + create `.env` file and put necessary API keys for EVDS and {cls_name} + see documentation for details. + + """ + + print(msg) + sleep(WARNING_SLEEP_SECONDS) + + +def write_env_example(file_name: Path): + content = ( + "\nOPENAI_API_KEY=sk-proj-ABCDEFGIJKLMNOPQRSTUXVZ\nEVDS_API_KEY=ABCDEFGIJKLMNOP" + ) + with open(file_name, "w") as f: + f.write(content) + print("Example .env file was created.") + sleep(WARNING_SLEEP_SECONDS) + + +def create_env_example_file(): + file_name = Path(".env") + if not file_name.exists(): + write_env_example(file_name) + + class OpenaiApiKey(ApiKey): def __init__(self, key: str) -> None: super().__init__(key) self.key = key - self.check() - - def check(self) -> Union[bool, None]: + self.key_name = 'openai_api_key' + # self.check() + def check(self, raise_=True) -> Union[bool, None]: if not str(self.key).startswith("sk-") and len(str(self.key)) < 6: - raise ErrorApiKey(f"{self.key} is not a valid key") + self.msg_before_raise() + if raise_: + raise ErrorApiKey(f"{self.key} is not a valid key") + return False return True @@ -70,7 +117,7 @@ class EvdsApiKey(ApiKey): ... class MistralApiKey(ApiKey): ... -def load_api_keys() -> Union[dict[str, str], None]: +def load_api_keys() -> Dict[str, OpenaiApiKey | EvdsApiKey]: from dotenv import load_dotenv env_file = Path(".env") @@ -83,19 +130,31 @@ def load_api_keys() -> Union[dict[str, str], None]: } -def get_openai_key(): +def load_api_keys_string() -> Dict[str, str]: + from dotenv import load_dotenv + + env_file = Path(".env") + load_dotenv(env_file) + openai_api_key = os.getenv("OPENAI_API_KEY") + evds_api_key = os.getenv("EVDS_API_KEY") + return { + "OPENAI_API_KEY": openai_api_key, + "EVDS_API_KEY": evds_api_key, + } + + +def get_openai_key() -> OpenaiApiKey: d = load_api_keys() - return d["OPENAI_API_KEY"].key + return d["OPENAI_API_KEY"] + +def get_openai_key_string() -> str | None: + d = load_api_keys_string() + return d["OPENAI_API_KEY"] -# @dataclass class ApiKeyManager(BaseModel): api_key: ApiKey = Field(default_factory=lambda: ApiKey()) class Config: arbitrary_types_allowed = True - - # def __get_pydantic_core_schema__(cls, handler): - # # Generate a schema if necessary, or skip it - # return handler.generate_schema(cls) \ No newline at end of file diff --git a/evdschat/common/bridge.py b/evdschat/common/bridge.py index fe8f2ae..a76e543 100644 --- a/evdschat/common/bridge.py +++ b/evdschat/common/bridge.py @@ -4,17 +4,19 @@ from pathlib import Path from importlib import resources from typing import Union -from .github_actions import PytestTesting +from .github_actions import PytestTesting + class PostParams(ctypes.Structure): _fields_ = [ ("url", ctypes.c_char_p), ("prompt", ctypes.c_char_p), ("api_key", ctypes.c_char_p), - ("proxy_url", ctypes.c_char_p) + ("proxy_url", ctypes.c_char_p), ] -def get_exec_file(test = False ) -> Path : + +def get_exec_file(test=False) -> Path: executable_name = "libpost_request.so" if platform.system() == "Windows": @@ -22,14 +24,15 @@ def get_exec_file(test = False ) -> Path : if test or PytestTesting().is_testing(): executable_path = Path(".") / executable_name - if executable_path.is_file() : + if executable_path.is_file(): return executable_path - return False - -def check_c_executable(test = False ) -> Union[Path, bool]: - executable_name= get_exec_file(test ) + return False + + +def check_c_executable(test=False) -> Union[Path, bool]: + executable_name = get_exec_file(test) if not executable_name: - return False + return False try: with resources.path("evdschat", executable_name) as executable_path: if executable_path.is_file() and os.access(executable_path, os.X_OK): @@ -37,10 +40,11 @@ def check_c_executable(test = False ) -> Union[Path, bool]: except FileNotFoundError: return False + lib_path = check_c_executable() if lib_path: lib = ctypes.CDLL(lib_path) - + lib.post_request.argtypes = [ctypes.POINTER(PostParams)] lib.post_request.restype = ctypes.c_char_p @@ -54,16 +58,15 @@ def c_caller(params): def c_caller_main(prompt, api_key, url, proxy=None): prompt = prompt.replace("\n", " ") - + params = PostParams( url=url.encode("utf-8"), prompt=prompt.encode("utf-8"), api_key=api_key.encode("utf-8"), - proxy_url=proxy.encode("utf-8") if proxy else None + proxy_url=proxy.encode("utf-8") if proxy else None, ) return c_caller(params) - - + else: c_caller_main = None diff --git a/evdschat/common/github_actions.py b/evdschat/common/github_actions.py index 72e46bb..e1a3545 100644 --- a/evdschat/common/github_actions.py +++ b/evdschat/common/github_actions.py @@ -13,17 +13,23 @@ # limitations under the License. import sys + + class GithubActions: def is_testing(self): return "hostedtoolcache" in sys.argv[0] + + class PytestTesting: def is_testing(self): # print(" sys.argv[0]" , sys.argv[0]) return "pytest" in sys.argv[0] + + def get_input(msg, default=None): if GithubActions().is_testing() or PytestTesting().is_testing(): if not default: print("currently testing with no default ") return False return default - return input(msg) \ No newline at end of file + return input(msg) diff --git a/evdschat/common/globals.py b/evdschat/common/globals.py index 1436136..6fcc3a3 100644 --- a/evdschat/common/globals.py +++ b/evdschat/common/globals.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +WARNING_SLEEP_SECONDS = 6 +DEFAULT_CHAT_API_URL = "https://evdspychat-dev2-1.onrender.com/api/ask" + def global_mock(): template = """ diff --git a/evdschat/core/builder.py b/evdschat/core/builder.py index cd4a8bc..6242875 100644 --- a/evdschat/core/builder.py +++ b/evdschat/core/builder.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass, field from pydantic import BaseModel, Field diff --git a/evdschat/core/chat.py b/evdschat/core/chat.py index fb82e47..534bbec 100644 --- a/evdschat/core/chat.py +++ b/evdschat/core/chat.py @@ -29,11 +29,11 @@ class GotUndefinedResult(BaseException): ... def chat( - prompt: str, - getter: ModelAbstract = OpenAI(), - debug=False, - test=False, - force=False, + prompt: str, + getter: ModelAbstract = None, + debug=False, + test=False, + force=False, ) -> Union[Tuple[ResultChat, Notes], None]: """ Function to process the chat prompt and return the result. @@ -45,6 +45,11 @@ def chat( :return: DataFrame or Result Instance with .data (DataFrame), .metadata (DataFrame), and .to_excel (Callable). """ + if getter is None: + getter = OpenAI() + + + if not force and PytestTesting().is_testing(): test = True @@ -74,8 +79,8 @@ def chat( raise GotUndefinedResult() result, notes = res if isinstance(result, ResultChat): - return result, notes - raise NotImplementedError("Unknown Result type ") + return result, notes + raise NotImplementedError("Unknown Result type ") def chat_console() -> None: diff --git a/evdschat/model/chatters.py b/evdschat/model/chatters.py index be4c0e6..06a88ee 100644 --- a/evdschat/model/chatters.py +++ b/evdschat/model/chatters.py @@ -14,19 +14,18 @@ import json import traceback -from typing import Callable, Any, Tuple, Union -from abc import ABC +from typing import Callable, Any, Tuple, Union, Dict +from abc import ABC, abstractmethod import requests from evdspy import get_series_exp -from evdschat.common.akeys import get_openai_key +from evdschat.common.akeys import get_openai_key, get_openai_key_string, ApiKey from dataclasses import dataclass -from evdschat.common.globals import global_mock -from ..common.bridge import c_caller_main +from evdschat.common.globals import global_mock, DEFAULT_CHAT_API_URL +from evdschat.common.bridge import c_caller_main import os from pathlib import Path - from evdschat.core.result import ResultChat, create_result from evdschat.core.result import Status @@ -37,7 +36,7 @@ def get_myapi_url(): load_dotenv(Path(".") / ".env") CHAT_API_URL = os.getenv( - "CHAT_API_URL", "https://evdspychat-dev2-1.onrender.com/api/ask" + "CHAT_API_URL", DEFAULT_CHAT_API_URL ) return CHAT_API_URL @@ -47,43 +46,49 @@ class ModelAbstract(ABC): retrieve_fnc: Callable = get_series_exp request_fnc: Callable = requests.post model: str = "gpt-4" - api_key: str = get_openai_key() + api_key: ApiKey | None = None # get_openai_key() debug = True test = False - def parse(self, prompt) -> dict[str, str]: + def __post_init__(self): + ... + + @abstractmethod + def load_api_keys(self): + ... - return {"prompt": prompt, "model": self.model, "openai_api_key": self.api_key} + def get_api_key_str(self): + return str(self.api_key) - def defaultOptions(self) -> str: + def parse(self, prompt) -> dict[str, str]: + return {"prompt": prompt, "model": self.model, self.api_key.key_name: str(self.api_key)} + + @staticmethod + def defaultOptions() -> str: return get_myapi_url() def post(self, prompt: str) -> Union[dict, bool]: if self.debug: return str(self) + self.load_api_keys() response = self.request_fnc(self.defaultOptions(), json=self.parse(prompt)) return self.post_helper(response) def obscure(self, string: str) -> str: - nstr = [] - for i, char in enumerate(string.split()): - if i % 3 == 1: - nstr.append(char) - else: - nstr.append("*") - - return "".join(nstr) + return ''.join( + char if i % 3 == 1 else '*' for i, char in enumerate(string) + ) def __str__(self): - api_key = self.obscure(self.api_key) + api_key = self.obscure(str(self.api_key)) return f""" prompt : {self.parse('')} fnc : {self.request_fnc} key : {api_key} -api url : {get_myapi_url() } +api url : {get_myapi_url()} """ def mock_req(self, prompt) -> dict[str, str]: @@ -94,13 +99,12 @@ def check_permitted(self, key: str, permitted=None) -> bool: permitted = ["start_date", "aggregate", "frequency" "cache"] return key in permitted - def permitted_dict(self, kw: dict, permitted: None) -> Tuple[dict, str]: + def permitted_dict(self, kw: dict, permitted: None) -> Dict[str, str | int]: new_dict = {} for k, v in kw.items(): if self.check_permitted(k, permitted): new_dict[k] = v - return new_dict def eval_real(self, kw, permitted=None) -> tuple[ResultChat, str]: @@ -126,7 +130,7 @@ def eval(self, kw: dict, permitted=None) -> Tuple[Any, str]: if not index: return self.failed_result(), str("") - return self.eval_real(kw, permitted) + return self.eval_real(kw, permitted) def decide_caller(self): """decide_caller""" @@ -154,28 +158,54 @@ def json_loads(self, result) -> dict[str, str]: return res def post_helper(self, response) -> dict[str, str]: - if response.status_code == 200: - data = response.json() - result_code = data.get("result") - - if result_code: - result_code = result_code.replace("'", '"').replace("=", ":") - result_dict = self.json_loads(result_code) - result_dict["cache"] = False - return result_dict - self._raise() - - def _raise(self, *args): - raise ValueError("Could not read return content form Node") + try: + data = response.json() + result_code = data.get("result") + if result_code: + result_code = result_code.replace("'", '"').replace("=", ":") + result_dict = self.json_loads(result_code) + result_dict["cache"] = False + return result_dict + except (KeyError, json.JSONDecodeError) as e: + raise ValueError(f"Error parsing response: {e}, Content: {response.text}") + else: + self._raise(response) + + # def post_helper(self, response) -> dict[str, str]: + # + # if response.status_code == 200: + # data = response.json() + # result_code = data.get("result") + # + # if result_code: + # result_code = result_code.replace("'", '"').replace("=", ":") + # result_dict = self.json_loads(result_code) + # result_dict["cache"] = False + # return result_dict + # self._raise(response) + + def _raise(self, response=None, *args): + error_message = "Could not read return content from Node" + if response: + error_message += f". Response code: {response.status_code}, Content: {response.text}" + raise ValueError(error_message) @dataclass class OpenAI(ModelAbstract): """OpenAI""" - def post_c(self, prompt: str , caller = c_caller_main ) -> dict[str, str]: - resp = caller(prompt, get_openai_key(), self.defaultOptions()) + def check_initial(self) -> bool: + k = get_openai_key_string() + return isinstance(k, str) + + def load_api_keys(self): + self.api_key: ApiKey = get_openai_key() + self.api_key.check() # may raise + + def post_c(self, prompt: str, caller=c_caller_main) -> dict[str, str]: + resp = caller(prompt, str(self.api_key), self.defaultOptions()) result_dict = json.loads(resp) r = result_dict["result"] res = json.loads(r) diff --git a/main.py b/main.py index afd2b00..d692645 100644 --- a/main.py +++ b/main.py @@ -17,7 +17,7 @@ prompt = """ - Can I get reserves data please ? frequence must be monthly and you may aggregate by averages. + Can I get reserves data please ? Frequency must be monthly and you may aggregate by averages. """ diff --git a/pyproject.toml b/pyproject.toml index d249740..47d1b74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "evdschat" -version = "0.1.9" +version = "0.1.10" description = "evdschat is an open-source Python package designed to enhance the evdspy package by allowing users to interact with the evdschat Application. This Node.js project aims to provide the most specific and accurate data users request during conversations, based on arguments such as start date, end date, and aggregation type, as described in the evdspy." authors = [ { name = "Sermet Pekin", email = "Sermet.Pekin@gmail.com" } diff --git a/tests/test_chat.py b/tests/test_chat.py index 7e658de..39f0464 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -18,7 +18,7 @@ def test_chat(capsys): - with capsys.disabled() : + with capsys.disabled(): prompt = """ Can I get reserves data please ? Aylık frekans istiyorum. ortalama olarak toplulaştırır mısın? @@ -26,6 +26,6 @@ def test_chat(capsys): """ with capsys.disabled(): - res, notes = chat(prompt, debug=False, force=False ) + res, notes = chat(prompt, debug=False, force=False) print(res) assert isinstance(res.data, pd.DataFrame) diff --git a/tests/test_req.py b/tests/test_req.py index d627937..71fd128 100644 --- a/tests/test_req.py +++ b/tests/test_req.py @@ -4,14 +4,10 @@ from pathlib import Path import platform -from evdschat.model.chatters import ( - get_openai_key, - TestAI, - get_myapi_url -) +from evdschat.model.chatters import get_openai_key, TestAI, get_myapi_url -def get_exec_file(test=False) -> Path: +def get_exec_file() -> Path: executable_name = "libpost_request.so" if platform.system() == "Windows": executable_name = "libpost_request.dll" From adeffab211671157564464856a46f67aba90a475 Mon Sep 17 00:00:00 2001 From: Sermet Pekin Date: Mon, 2 Dec 2024 23:06:49 +0300 Subject: [PATCH 2/2] toml file hatchling uv#9513 --- evdschat/model/chatters.py | 13 +------------ pyproject.toml | 5 +++-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/evdschat/model/chatters.py b/evdschat/model/chatters.py index 06a88ee..59aa084 100644 --- a/evdschat/model/chatters.py +++ b/evdschat/model/chatters.py @@ -172,18 +172,7 @@ def post_helper(self, response) -> dict[str, str]: else: self._raise(response) - # def post_helper(self, response) -> dict[str, str]: - # - # if response.status_code == 200: - # data = response.json() - # result_code = data.get("result") - # - # if result_code: - # result_code = result_code.replace("'", '"').replace("=", ":") - # result_dict = self.json_loads(result_code) - # result_dict["cache"] = False - # return result_dict - # self._raise(response) + def _raise(self, response=None, *args): error_message = "Could not read return content from Node" diff --git a/pyproject.toml b/pyproject.toml index 47d1b74..95dfd1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dev-dependencies = [ "tox>=4.18.1", ] + [build-system] -requires = ["setuptools", "wheel", "uv"] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" \ No newline at end of file