From 069732600cabd2a6340d5f2dbe85d8a8fe77cd91 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 6 May 2020 08:09:20 +0100 Subject: [PATCH 1/7] cleanup for reddit data provider --- my/reddit.py | 77 ++++++++++++++----------------------------------- tests/reddit.py | 59 +++++++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 58 deletions(-) diff --git a/my/reddit.py b/my/reddit.py index 2a341f71..b5293eda 100755 --- a/my/reddit.py +++ b/my/reddit.py @@ -1,8 +1,6 @@ """ Reddit data: saved items/comments/upvotes/etc. """ -from . import init - from pathlib import Path from typing import List, Sequence, Mapping, Iterator @@ -13,14 +11,14 @@ import my.config.repos.rexport.dal as rexport -def get_sources() -> Sequence[Path]: +def inputs() -> Sequence[Path]: # TODO rename to export_path? files = get_files(config.export_dir) + # TODO Cpath better be automatic by get_files... res = list(map(CPath, files)); assert len(res) > 0 # todo move the assert to get_files? return tuple(res) - logger = LazyLogger(__name__, level='debug') @@ -32,30 +30,30 @@ def get_sources() -> Sequence[Path]: def dal() -> rexport.DAL: - # TODO lru cache? but be careful when it runs continuously - return rexport.DAL(get_sources()) + return rexport.DAL(inputs()) -@mcachew(hashf=lambda: get_sources()) +@mcachew(hashf=lambda: inputs()) def saved() -> Iterator[Save]: return dal().saved() -@mcachew(hashf=lambda: get_sources()) +@mcachew(hashf=lambda: inputs()) def comments() -> Iterator[Comment]: return dal().comments() -@mcachew(hashf=lambda: get_sources()) +@mcachew(hashf=lambda: inputs()) def submissions() -> Iterator[Submission]: return dal().submissions() -@mcachew(hashf=lambda: get_sources()) +@mcachew(hashf=lambda: inputs()) def upvoted() -> Iterator[Upvote]: return dal().upvoted() +### the rest of the file is some elaborate attempt of restoring favorite/unfavorite times from typing import Dict, Union, Iterable, Iterator, NamedTuple, Any from functools import lru_cache @@ -115,10 +113,11 @@ def _get_state(bfile: Path) -> Dict[Sid, SaveWithDt]: key=lambda s: s.save.sid, ) +# TODO hmm. think about it.. if we set default backups=inputs() +# it's called early so it ends up as a global variable that we can't monkey patch easily @mcachew('/L/data/.cache/reddit-events.cache') -def _get_events(backups: Sequence[Path]=get_sources(), parallel: bool=True) -> Iterator[Event]: +def _get_events(backups: Sequence[Path], parallel: bool=True) -> Iterator[Event]: # TODO cachew: let it transform return type? so you don't have to write a wrapper for lists? - # parallel = False # NOTE: eh, not sure if still necessary? I think glumov didn't like it? prev_saves: Mapping[Sid, SaveWithDt] = {} # TODO suppress first batch?? @@ -168,55 +167,18 @@ def _get_events(backups: Sequence[Path]=get_sources(), parallel: bool=True) -> I # TODO a bit awkward, favorited should compare lower than unfavorited? @lru_cache(1) -def get_events(*args, **kwargs) -> List[Event]: - evit = _get_events(*args, **kwargs) +def events(*args, **kwargs) -> List[Event]: + evit = _get_events(inputs(), *args, **kwargs) return list(sorted(evit, key=lambda e: e.cmp_key)) - -def test() -> None: - get_events(backups=get_sources()[-1:]) - list(saved()) - - -def test_unfav() -> None: - events = get_events() - url = 'https://reddit.com/r/QuantifiedSelf/comments/acxy1v/personal_dashboard/' - uevents = [e for e in events if e.url == url] - assert len(uevents) == 2 - ff = uevents[0] - assert ff.text == 'favorited' - uf = uevents[1] - assert uf.text == 'unfavorited' - -# TODO move out.. -def test_get_all_saves() -> None: - # TODO not sure if this is necesasry anymore? - saves = list(saved()) - # just check that they are unique.. - make_dict(saves, key=lambda s: s.sid) - - -def test_disappearing() -> None: - # eh. so for instance, 'metro line colors' is missing from reddit-20190402005024.json for no reason - # but I guess it was just a short glitch... so whatever - saves = get_events() - favs = [s.kind for s in saves if s.text == 'favorited'] - [deal_with_it] = [f for f in favs if f.title == '"Deal with it!"'] - assert deal_with_it.backup_dt == datetime(2019, 4, 1, 23, 10, 25, tzinfo=pytz.utc) - - -def test_unfavorite() -> None: - events = get_events() - unfavs = [s for s in events if s.text == 'unfavorited'] - [xxx] = [u for u in unfavs if u.eid == 'unf-19ifop'] - assert xxx.dt == datetime(2019, 1, 28, 8, 10, 20, tzinfo=pytz.utc) +## def main() -> None: # TODO eh. not sure why but parallel on seems to mess glumov up and cause OOM... - events = get_events(parallel=False) - print(len(events)) - for e in events: + el = events(parallel=False) + print(len(el)) + for e in el: print(e.text, e.url) # for e in get_ # 509 with urls.. @@ -226,3 +188,8 @@ def main() -> None: if __name__ == '__main__': main() + +# TODO deprecate... + +get_sources = inputs +get_events = events diff --git a/tests/reddit.py b/tests/reddit.py index 45be4879..1068038c 100644 --- a/tests/reddit.py +++ b/tests/reddit.py @@ -1,4 +1,57 @@ -# ugh. workaround for https://github.com/pytest-dev/pytest/issues/1927 -from my.reddit import * +from datetime import datetime +import pytz -# TODO for reddit test, patch up to take every 10th archive or something; but make sure it's deterministic +from my.reddit import events, inputs, saved +from my.common import make_dict + + +def test() -> None: + list(events()) + list(saved()) + + +def test_unfav() -> None: + ev = events() + url = 'https://reddit.com/r/QuantifiedSelf/comments/acxy1v/personal_dashboard/' + uev = [e for e in ev if e.url == url] + assert len(uev) == 2 + ff = uev[0] + # TODO could recover these from takeout perhaps? + assert ff.text == 'favorited [initial]' + uf = uev[1] + assert uf.text == 'unfavorited' + + +def test_saves() -> None: + # TODO not sure if this is necesasry anymore? + saves = list(saved()) + # just check that they are unique.. + make_dict(saves, key=lambda s: s.sid) + + +def test_disappearing() -> None: + # eh. so for instance, 'metro line colors' is missing from reddit-20190402005024.json for no reason + # but I guess it was just a short glitch... so whatever + saves = events() + favs = [s.kind for s in saves if s.text == 'favorited'] + [deal_with_it] = [f for f in favs if f.title == '"Deal with it!"'] + assert deal_with_it.backup_dt == datetime(2019, 4, 1, 23, 10, 25, tzinfo=pytz.utc) + + +def test_unfavorite() -> None: + evs = events() + unfavs = [s for s in evs if s.text == 'unfavorited'] + [xxx] = [u for u in unfavs if u.eid == 'unf-19ifop'] + assert xxx.dt == datetime(2019, 1, 28, 8, 10, 20, tzinfo=pytz.utc) + + +import pytest # type: ignore +@pytest.fixture(autouse=True, scope='module') +def prepare(): + from my.common import get_files + from my.config import reddit as config + files = get_files(config.export_dir) + # use less files for the test to make it faster + # first bit is for 'test_unfavorite, the second is for test_disappearing + files = files[300:330] + files[500:520] + config.export_dir = files # type: ignore From 9d5d36889176e51addeaa4ae6541dd94d3bb2269 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 6 May 2020 22:05:16 +0100 Subject: [PATCH 2/7] get rid of unnecessary .init imports --- my/books/kobo.py | 2 -- my/calendar/holidays.py | 2 +- my/cfg.py | 2 +- my/coding/codeforces.py | 2 -- my/coding/github.py | 3 --- my/coding/topcoder.py | 2 -- my/hypothesis.py | 2 -- my/materialistic.py | 2 -- my/media/imdb.py | 3 --- my/pdfs.py | 3 --- my/pinboard.py | 2 -- my/smscalls.py | 2 -- tox.ini | 6 +++--- 13 files changed, 5 insertions(+), 28 deletions(-) diff --git a/my/books/kobo.py b/my/books/kobo.py index 09fa8c91..e5603a0e 100644 --- a/my/books/kobo.py +++ b/my/books/kobo.py @@ -1,8 +1,6 @@ """ [[https://uk.kobobooks.com/products/kobo-aura-one][Kobo]] e-ink reader: annotations and reading stats """ -from .. import init - from typing import Callable, Union, List from my.config import kobo as config diff --git a/my/calendar/holidays.py b/my/calendar/holidays.py index 5759ec8c..4f45a932 100644 --- a/my/calendar/holidays.py +++ b/my/calendar/holidays.py @@ -13,7 +13,7 @@ # pip3 install workalendar from workalendar.europe import UnitedKingdom # type: ignore -cal = UnitedKingdom() # TODO FIXME specify in config +cal = UnitedKingdom() # TODO # TODO that should depend on country/'location' of residence I suppose? diff --git a/my/cfg.py b/my/cfg.py index ddc102f6..da1a83ee 100644 --- a/my/cfg.py +++ b/my/cfg.py @@ -14,7 +14,7 @@ """ # TODO later, If I have config stubs that might be unnecessary too.. -from . import init +from . import init # todo not sure if this line is necessary? the stub will trigger it anyway import my.config as config diff --git a/my/coding/codeforces.py b/my/coding/codeforces.py index fbbf586f..138cc732 100644 --- a/my/coding/codeforces.py +++ b/my/coding/codeforces.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -from .. import init - from my.config import codeforces as config from datetime import datetime diff --git a/my/coding/github.py b/my/coding/github.py index 0126e478..508801f5 100644 --- a/my/coding/github.py +++ b/my/coding/github.py @@ -1,9 +1,6 @@ """ Github events and their metadata: comments/issues/pull requests """ - -from .. import init - from typing import Dict, Any, NamedTuple, Tuple, Optional, Iterator, TypeVar, Set from datetime import datetime import json diff --git a/my/coding/topcoder.py b/my/coding/topcoder.py index de98114b..c370b5da 100644 --- a/my/coding/topcoder.py +++ b/my/coding/topcoder.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -from .. import init - from my.config import topcoder as config from datetime import datetime diff --git a/my/hypothesis.py b/my/hypothesis.py index 46e00bc5..94d4edf0 100644 --- a/my/hypothesis.py +++ b/my/hypothesis.py @@ -1,8 +1,6 @@ """ [[https://hypothes.is][Hypothes.is]] highlights and annotations """ -from . import init - from .common import get_files from .error import Res, sort_res_by diff --git a/my/materialistic.py b/my/materialistic.py index 79cd4487..36ff1dc2 100644 --- a/my/materialistic.py +++ b/my/materialistic.py @@ -1,8 +1,6 @@ """ [[https://play.google.com/store/apps/details?id=io.github.hidroh.materialistic][Materialistic]] app for Hackernews """ -from . import init - from datetime import datetime from typing import Any, Dict, Iterator, NamedTuple diff --git a/my/media/imdb.py b/my/media/imdb.py index 42a1bc0f..23bccd77 100644 --- a/my/media/imdb.py +++ b/my/media/imdb.py @@ -1,7 +1,4 @@ #!/usr/bin/env python3 - -from .. import init - import csv import json from datetime import datetime diff --git a/my/pdfs.py b/my/pdfs.py index 5e7a36fa..3c04196a 100755 --- a/my/pdfs.py +++ b/my/pdfs.py @@ -2,9 +2,6 @@ ''' PDF documents and annotations on your filesystem ''' - -from . import init - from concurrent.futures import ProcessPoolExecutor from datetime import datetime import re diff --git a/my/pinboard.py b/my/pinboard.py index 8685d502..d37ba07b 100644 --- a/my/pinboard.py +++ b/my/pinboard.py @@ -1,8 +1,6 @@ """ [[https://pinboard.in][Pinboard]] bookmarks """ -from . import init - from .common import get_files from my.config.repos.pinbexport import dal as pinbexport diff --git a/my/smscalls.py b/my/smscalls.py index e2d80f1e..91d9af57 100644 --- a/my/smscalls.py +++ b/my/smscalls.py @@ -2,8 +2,6 @@ Phone calls and SMS messages """ # TODO extract SMS as well? I barely use them though.. -from . import init - from datetime import datetime from pathlib import Path from typing import NamedTuple, Iterator, Set diff --git a/tox.ini b/tox.ini index db241384..d11c30d4 100644 --- a/tox.ini +++ b/tox.ini @@ -9,10 +9,10 @@ passenv = CI CI_* setenv = MY_CONFIG = nonexistent commands = pip install -e .[testing] - # TODO ?? # python -m pytest {posargs} - python3 -c 'import my.init; from my.config import stub as config; print(config.key)' - python3 -c 'import my.init; import my.config; import my.config.repos' # shouldn't fail at least + # todo these are probably not necessary anymore? + python3 -c 'from my.config import stub as config; print(config.key)' + python3 -c 'import my.config; import my.config.repos' # shouldn't fail at least python3 -m pytest tests/misc.py tests/get_files.py tests/config.py::test_set_repo tests/config.py::test_environment_variable # TODO add; once I figure out porg depdencency?? tests/config.py # TODO run demo.py? just make sure with_my is a bit cleverer? From b7e5640f3548592ae94c7fef4e1a61f830a99566 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 6 May 2020 22:20:00 +0100 Subject: [PATCH 3/7] move init.py to my.core --- my/cfg.py | 7 ++----- my/config/__init__.py | 2 +- my/{ => core}/init.py | 0 3 files changed, 3 insertions(+), 6 deletions(-) rename my/{ => core}/init.py (100%) diff --git a/my/cfg.py b/my/cfg.py index da1a83ee..97268dac 100644 --- a/my/cfg.py +++ b/my/cfg.py @@ -12,15 +12,12 @@ export_path='/path/to/twitter/exports', ) """ -# TODO later, If I have config stubs that might be unnecessary too.. - -from . import init # todo not sure if this line is necessary? the stub will trigger it anyway - +# todo why do we bring this into scope? don't remember.. import my.config as config def set_repo(name: str, repo): - from .init import assign_module + from .core.init import assign_module from . common import import_from module = import_from(repo, name) diff --git a/my/config/__init__.py b/my/config/__init__.py index da9d781d..333ae6e6 100644 --- a/my/config/__init__.py +++ b/my/config/__init__.py @@ -1,5 +1,5 @@ # TODO ok, this thing should trigger .cfg import presumably?? -from .. import init +from ..core import init # TODO maybe, reuse mycfg_template here? diff --git a/my/init.py b/my/core/init.py similarity index 100% rename from my/init.py rename to my/core/init.py From eb97021b8e9837f4c874a4f7a93b03618b6e3097 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 6 May 2020 22:28:37 +0100 Subject: [PATCH 4/7] improve lint script, explore subpackages --- lint | 22 +++++++++++++++------- my/kython/__init__.py | 0 2 files changed, 15 insertions(+), 7 deletions(-) delete mode 100644 my/kython/__init__.py diff --git a/lint b/lint index 5d91b6ec..8cdca2ba 100755 --- a/lint +++ b/lint @@ -31,25 +31,31 @@ def package_name(p: Path) -> str: else: return mname(p) +def subpackages(package: str) -> Iterable[str]: + ppath = package.replace('.', '/') + yield from sorted({ + package_name(p.relative_to(DIR)) for p in (DIR / ppath).rglob('*.py') + }) + + # TODO meh.. think how to check _everything_ on CI def core_modules() -> Iterable[str]: return [ + *subpackages('my.core'), + *subpackages('my.kython'), 'my.common', 'my.config', - 'my.core', 'my.cfg', 'my.error', - 'my.init', 'tests/misc.py', 'tests/get_files.py', # 'tests/config.py', TODO hmm. unclear how to type check this module ] + def all_modules() -> Iterable[str]: - yield from sorted(set( - package_name(p.relative_to(DIR)) for p in (DIR / 'my').rglob('*.py') - )) + yield from subpackages('my') yield from sorted( str(f.relative_to(DIR)) for f in (DIR / 'tests').rglob('*.py') ) @@ -63,11 +69,13 @@ def pylint(): def mypy(thing: str): is_package = Path(thing).suffix != '.py' - return run([ + cmd = [ 'mypy', '--color-output', # TODO eh? doesn't work.. *(['-p'] if is_package else []), thing, - ], stdout=PIPE, stderr=PIPE) + ] + print(' '.join(cmd), file=sys.stderr) + return run(cmd, stdout=PIPE, stderr=PIPE) def mypy_all() -> Iterable[Exception]: diff --git a/my/kython/__init__.py b/my/kython/__init__.py deleted file mode 100644 index e69de29b..00000000 From 15444c7b1f02109a64e1c22e28ca2425a2bbb230 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 6 May 2020 22:36:29 +0100 Subject: [PATCH 5/7] move common/error to my.core --- my/{ => core}/common.py | 0 my/{ => core}/error.py | 0 my/core/init.py | 7 +++++-- 3 files changed, 5 insertions(+), 2 deletions(-) rename my/{ => core}/common.py (100%) rename my/{ => core}/error.py (100%) diff --git a/my/common.py b/my/core/common.py similarity index 100% rename from my/common.py rename to my/core/common.py diff --git a/my/error.py b/my/core/error.py similarity index 100% rename from my/error.py rename to my/core/error.py diff --git a/my/core/init.py b/my/core/init.py index 54686c30..e3a5e7a9 100644 --- a/my/core/init.py +++ b/my/core/init.py @@ -8,9 +8,10 @@ Please let me know if you are aware of a better way of dealing with this! ''' +from types import ModuleType # TODO not ideal to keep it here, but this should really be a leaf in the import tree -def assign_module(parent: str, name: str, module): +def assign_module(parent: str, name: str, module: ModuleType) -> None: import sys import importlib parent_module = importlib.import_module(parent) @@ -20,13 +21,15 @@ def assign_module(parent: str, name: str, module): # TODO that crap should be tested... I guess will get it for free when I run rest of tests in the matrix setattr(parent_module, name, module) +del ModuleType # separate function to present namespace pollution -def setup_config(): +def setup_config() -> None: from pathlib import Path import sys import os import warnings + from typing import Optional # not sure if that's necessary, i.e. could rely on PYTHONPATH instead # on the other hand, by using MY_CONFIG we are guaranteed to load it from the desired path? From 6ecb9536755e4ebbe2d6b9f7966c3e31baefed7e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 6 May 2020 22:50:18 +0100 Subject: [PATCH 6/7] cleanup, mypy coverage & add common/error stubs --- README.org | 11 +++++------ lint | 2 -- my/common.py | 2 ++ my/core/common.py | 4 ++-- my/core/error.py | 2 +- my/error.py | 2 ++ tests/misc.py | 43 +++++++++++++++++++++++-------------------- 7 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 my/common.py create mode 100644 my/error.py diff --git a/README.org b/README.org index 1d62c84d..579fd4f7 100644 --- a/README.org +++ b/README.org @@ -5,6 +5,10 @@ #+macro: map @@html:@@$1@@html:@@ +If you're in a hurry, feel free to jump straight to the [[#usecases][demos]]. + +For *installation/configuration/development guide*, see [[https://github.com/karlicoss/HPI/tree/master/doc/SETUP.org][SETUP.org]]. + *TLDR*: I'm using [[https://github.com/karlicoss/HPI][HPI]] (Human Programming Interface) package as a means of unifying, accessing and interacting with all of my personal data. It's a Python library (named ~my~), a collection of modules for: @@ -48,11 +52,6 @@ and that's why I'm sharing this. Imagine if all your life was reflected digitally and available at your fingertips. This library is my attempt to achieve this vision. -If you're in a hurry, feel free to jump straight to the [[#usecases][demos]]. - -For *installation/configuration/development guide*, see [[https://github.com/karlicoss/HPI/tree/master/doc/SETUP.org][SETUP.org]]. - - #+toc: headlines 2 @@ -593,4 +592,4 @@ In some near future I will write more about: - challenges I had so solve - more use-cases and demos -- it's impossible to fit everything in one post! -, but happy to answer any questions on these topics now! \ No newline at end of file +, but happy to answer any questions on these topics now! diff --git a/lint b/lint index 8cdca2ba..bb2f0975 100755 --- a/lint +++ b/lint @@ -43,10 +43,8 @@ def core_modules() -> Iterable[str]: return [ *subpackages('my.core'), *subpackages('my.kython'), - 'my.common', 'my.config', 'my.cfg', - 'my.error', 'tests/misc.py', 'tests/get_files.py', # 'tests/config.py', TODO hmm. unclear how to type check this module diff --git a/my/common.py b/my/common.py new file mode 100644 index 00000000..bbda576f --- /dev/null +++ b/my/common.py @@ -0,0 +1,2 @@ +# will be deprecated. please add stuff to my.core +from .core.common import * diff --git a/my/core/common.py b/my/core/common.py index 063f5550..1557654a 100644 --- a/my/core/common.py +++ b/my/core/common.py @@ -95,14 +95,14 @@ def listify_helper(*args, **kw): return listify_return(fn) -# TODO FIXME use in bluemaestro +# todo use in bluemaestro # def dictify(fn=None, key=None, value=None): # def md(it): # return make_dict(it, key=key, value=value) # return listify(fn=fn, wrapper=md) -from .kython.klogging import setup_logger, LazyLogger +from ..kython.klogging import setup_logger, LazyLogger Paths = Union[Sequence[PathIsh], PathIsh] diff --git a/my/core/error.py b/my/core/error.py index 721cb630..4423940e 100644 --- a/my/core/error.py +++ b/my/core/error.py @@ -67,7 +67,7 @@ def sort_res_by(items: Iterable[ResT], key) -> List[ResT]: return results -def test_sort_res_by(): +def test_sort_res_by() -> None: class Exc(Exception): def __eq__(self, other): return self.args == other.args diff --git a/my/error.py b/my/error.py new file mode 100644 index 00000000..596c90e6 --- /dev/null +++ b/my/error.py @@ -0,0 +1,2 @@ +# will be deprecated. please add stuff to my.core +from .core.error import * diff --git a/tests/misc.py b/tests/misc.py index 73d1255d..40d63a41 100644 --- a/tests/misc.py +++ b/tests/misc.py @@ -7,24 +7,7 @@ from my.kython.kompress import kopen, kexists, CPath - -import pytest # type: ignore - -@pytest.fixture -def prepare(tmp_path: Path): - (tmp_path / 'file').write_text('just plaintext') - with (tmp_path / 'file.xz').open('wb') as f: - with lzma.open(f, 'w') as lzf: - lzf.write(b'compressed text') - with zipfile.ZipFile(tmp_path / 'file.zip', 'w') as zf: - zf.writestr('path/in/archive', 'data in zip') - try: - yield None - finally: - pass - - -def test_kopen(prepare, tmp_path: Path) -> None: +def test_kopen(tmp_path: Path) -> None: "Plaintext handled transparently" assert kopen(tmp_path / 'file' ).read() == 'just plaintext' assert kopen(tmp_path / 'file.xz').read() == 'compressed text' @@ -33,7 +16,7 @@ def test_kopen(prepare, tmp_path: Path) -> None: assert kopen(tmp_path / 'file.zip', 'path/in/archive').read() == 'data in zip' -def test_kexists(prepare, tmp_path: Path) -> None: +def test_kexists(tmp_path: Path) -> None: assert kexists(str(tmp_path / 'file.zip'), 'path/in/archive') assert not kexists(str(tmp_path / 'file.zip'), 'path/notin/archive') @@ -41,7 +24,27 @@ def test_kexists(prepare, tmp_path: Path) -> None: assert not kexists(tmp_path / 'nosuchzip.zip', 'path/in/archive') -def test_cpath(prepare, tmp_path: Path) -> None: +def test_cpath(tmp_path: Path) -> None: CPath(str(tmp_path / 'file' )).read_text() == 'just plaintext' CPath( tmp_path / 'file.xz').read_text() == 'compressed text' # TODO not sure about zip files?? + + +import pytest # type: ignore + +@pytest.fixture(autouse=True) +def prepare(tmp_path: Path): + (tmp_path / 'file').write_text('just plaintext') + with (tmp_path / 'file.xz').open('wb') as f: + with lzma.open(f, 'w') as lzf: + lzf.write(b'compressed text') + with zipfile.ZipFile(tmp_path / 'file.zip', 'w') as zf: + zf.writestr('path/in/archive', 'data in zip') + try: + yield None + finally: + pass + + +# meh +from my.core.error import test_sort_res_by From d4a430e12e68594380db45a5a952c4e8249e9b57 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 6 May 2020 23:20:12 +0100 Subject: [PATCH 7/7] update dev docs --- README.org | 3 ++- doc/DEVELOPMENT.org | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/README.org b/README.org index 579fd4f7..5df33830 100644 --- a/README.org +++ b/README.org @@ -7,7 +7,8 @@ If you're in a hurry, feel free to jump straight to the [[#usecases][demos]]. -For *installation/configuration/development guide*, see [[https://github.com/karlicoss/HPI/tree/master/doc/SETUP.org][SETUP.org]]. +- see [[https://github.com/karlicoss/HPI/tree/master/doc/SETUP.org][SETUP]] for the *installation/configuration guide* +- see [[https://github.com/karlicoss/HPI/tree/master/doc/DEVELOPMENT.org][DEVELOPMENT]] for the *development guide* *TLDR*: I'm using [[https://github.com/karlicoss/HPI][HPI]] (Human Programming Interface) package as a means of unifying, accessing and interacting with all of my personal data. diff --git a/doc/DEVELOPMENT.org b/doc/DEVELOPMENT.org index dd78c57b..f3387601 100644 --- a/doc/DEVELOPMENT.org +++ b/doc/DEVELOPMENT.org @@ -1,13 +1,45 @@ +* Running tests +I'm using =tox= to run test/lint. You can check out [[file:../.github/workflows/main.yml][Github Actions]] config +and [[file:../scripts/ci/run]] for the up to date info on the specifics. + * IDE setup: make sure my.config is in your package search path In runtime, ~my.config~ is imported from the user config directory dynamically. However, Pycharm/Emacs/whatever you use won't be able to figure that out, so you'd need to adjust your IDE configuration. -- Pycharm: basically, follow the instruction [[https://stackoverflow.com/a/55278260/706389][here]] +- Pycharm: basically, follow the instructions [[https://stackoverflow.com/a/55278260/706389][here]] i.e. create a new interpreter configuration (e.g. name it "Python 3.7 (for HPI)"), and add =~/.config/my=. * Linting -You should be able to use ~./lint~ script to run mypy checks. +You should be able to use [[file:../lint]] script to run mypy checks. + +[[file:../mypy.ini]] points at =~/.config/my= by default. + + +* Modifying/adding modules + +The easiest is just to run HPI via [[file:SETUP.org::#use-without-installing][with_my]] wrapper or with an editable PIP install. +That way your changes will be reflected immediately, and you will be able to quickly iterate/fix bugs/add new methods. + +The "proper way" (unless you want to contribute to the upstream) is to create a separate hierarchy and add your module to =PYTHONPATH=. + +For example, if you want to add an =awesomedatasource=, it could be: + +: custom_module +: └── my +: └──awesomedatasource.py + +You can use all existing HPI modules in =awesomedatasource.py=, for example, =my.config=, or everything from =my.core=. + +But also, you can use all the previously defined HPI modules too. This could be useful to *shadow/override* existing HPI module: + +: custom_reddit_overlay +: └── my +: └──reddit.py + +Now if you add =my_reddit_overlay= *in the front* of ~PYTHONPATH~, all the downstream scripts using =my.reddit= will load it from =custom_reddit_overlay= instead. + +This could be useful to monkey patch some behaviours, or dynamically add some extra data sources -- anything that comes to your mind. -~mypy.ini~ file points at =~/.config/my= by default. +I'll put up a better guide on this, in the meantime see [[https://packaging.python.org/guides/packaging-namespace-packages]["namespace packages"]] for more info.