From e3a3f4a7edaa722801bf954e86a41d0d92644a4c Mon Sep 17 00:00:00 2001 From: Kazuyuki-Takase Date: Thu, 26 Sep 2024 04:25:41 +0900 Subject: [PATCH] Automate csv generation --- README.md | 33 ++++++++ package.json | 3 +- sdk/.gitignore | 167 +++++++++++++++++++++++++++++++++++++ sdk/add_url_column.py | 32 ------- sdk/generate_stocks_csv.py | 63 ++++++++++++++ sdk/requirements.txt | 14 ++++ 6 files changed, 278 insertions(+), 34 deletions(-) create mode 100644 README.md create mode 100644 sdk/.gitignore delete mode 100644 sdk/add_url_column.py create mode 100644 sdk/generate_stocks_csv.py create mode 100644 sdk/requirements.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d26ed9 --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +Getting Started +------------------------------------------------------------------------------- + +In **git-root** : + +```sh +npm ci +``` + +In **sdk/** : + +```sh +python3 -m venv . +source ./bin/activate +pip install -r requirements.txt +``` + + +How to Use +------------------------------------------------------------------------------- + +In **sdk/** : + +```sh +source ./bin/activate +python3 generate_stocks_csv.py {{ KEYCHAIN_LABEL }} +``` + +In **git-root** : + +```sh +npm run start +``` diff --git a/package.json b/package.json index 7d84091..f5b41ab 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,7 @@ "version": "0.0.1", "private": true, "scripts": { - "csv": "python3 sdk/add_url_column.py", - "start": "node --loader ts-node/esm src/index.mts" + "start": "node --loader ts-node/esm src/index.mts sdk/stocks.csv" }, "author": "TAKASE Kazuyuki (@Guvalif)", "license": "UNLICENSED", diff --git a/sdk/.gitignore b/sdk/.gitignore new file mode 100644 index 0000000..c737ca0 --- /dev/null +++ b/sdk/.gitignore @@ -0,0 +1,167 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Project specific ignores +bin/ +include/ +pyvenv.cfg diff --git a/sdk/add_url_column.py b/sdk/add_url_column.py deleted file mode 100644 index c7031cb..0000000 --- a/sdk/add_url_column.py +++ /dev/null @@ -1,32 +0,0 @@ -from csv import reader, writer -from urllib.parse import quote - - -def rows_gen(path): - with open(path, 'r') as csv_file: - csv_reader = reader(csv_file) - - yield [ 'century', 'rarity', 'number', 'color', 'stock', 'name', 'memo', 'url' ] - - next(csv_reader) - - for row in csv_reader: - row.append(f"https://www.wikihouse.com/dim0wiki/index.php?{quote(row[5].encode('euc-jp'))}") - - yield row - - -def main(path): - from os.path import splitext - - name, ext = splitext(path) - - with open(f'{name}.fix{ext}', 'w') as csv_file: - csv_writer = writer(csv_file) - csv_writer.writerows(rows_gen(path)) - - -if __name__ == '__main__': - from sys import argv - - main(argv[1]) diff --git a/sdk/generate_stocks_csv.py b/sdk/generate_stocks_csv.py new file mode 100644 index 0000000..a9197bd --- /dev/null +++ b/sdk/generate_stocks_csv.py @@ -0,0 +1,63 @@ +import subprocess +from xml.etree.ElementTree import fromstring as xml_fromstring +from json import loads as json_loads +from gspread import authorize +from urllib.parse import quote +from google.oauth2.service_account import Credentials +from csv import writer + + +SPREADSHEET_ID = '1N4yco9qmRAMu7uS5pa-_vH55D2ZCahHIC-4wiFwxqGI' +WORKSHEET_NAME = 'STOCKS' + + +def get_service_account_from_keychain(label): + result = subprocess.run( + [ 'security', 'find-generic-password', '-l', label, '-w' ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + hex_data = result.stdout.strip() + raw_data = bytes.fromhex(hex_data).decode('utf-8') + xml = xml_fromstring(raw_data) + json_str = xml.find('dict/string').text + json = json_loads(json_str) + + return json + + +def get_latest_stocks(credentials): + client = authorize(credentials) + spreadsheet = client.open_by_key(SPREADSHEET_ID) + worksheet = spreadsheet.worksheet(WORKSHEET_NAME) + + header, *body = worksheet.get_all_values() + + yield [ 'century', 'rarity', 'number', 'color', 'stock', 'name', 'memo', 'url' ] + + for row in body: + url = f"https://www.wikihouse.com/dim0wiki/index.php?{quote(row[5].encode('euc-jp'))}" + + yield [ *row, url ] + + +def main(label): + credentials = Credentials.from_service_account_info( + get_service_account_from_keychain(label) + ).with_scopes([ + 'https://www.googleapis.com/auth/spreadsheets', + ]) + + stocks = get_latest_stocks(credentials) + + with open('stocks.csv', 'w') as csv_file: + csv_writer = writer(csv_file) + csv_writer.writerows(stocks) + + +if __name__ == '__main__': + from sys import argv + + main(argv[1]) diff --git a/sdk/requirements.txt b/sdk/requirements.txt new file mode 100644 index 0000000..1a6f1b4 --- /dev/null +++ b/sdk/requirements.txt @@ -0,0 +1,14 @@ +cachetools==5.5.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +google-auth==2.35.0 +google-auth-oauthlib==1.2.1 +gspread==6.1.2 +idna==3.10 +oauthlib==3.2.2 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 +requests==2.32.3 +requests-oauthlib==2.0.0 +rsa==4.9 +urllib3==2.2.3