and a comment too
multiline!
" - # TODO ok, it's annoying... not sure what to do with nonpritable crap - html = "and a comment too
multiline!
" - assert html2org(html, logger=logging) == r''' -and a /comment/ too - -*multiline*! -'''.lstrip() - if __name__ == '__main__': PolarView.main() diff --git a/modules/reddit.py b/modules/reddit2org.py similarity index 94% rename from modules/reddit.py rename to modules/reddit2org.py index 834b3bd..0ac6edb 100755 --- a/modules/reddit.py +++ b/modules/reddit2org.py @@ -2,13 +2,14 @@ """ Better interface for reading saved reddit posts/comments """ -from orger import InteractiveView +from orger import Mirror from orger.inorganic import node, link from orger.common import dt_heading from my.reddit import saved -class RedditView(InteractiveView): + +class RedditView(Mirror): def get_items(self): for s in saved(): yield s.sid, node( @@ -21,6 +22,7 @@ def get_items(self): body=s.text, ) + # todo this could be generic, i.e. checking all urls? def is_dead_url(self, url: str) -> bool: assert self.cmdline_args is not None if not self.cmdline_args.mark_dead: diff --git a/modules/roamresearch.py b/modules/roamresearch.py index bc2d7e0..fe49364 100755 --- a/modules/roamresearch.py +++ b/modules/roamresearch.py @@ -2,26 +2,14 @@ from itertools import chain from typing import Iterable -from orger import StaticView +from orger import Mirror from orger.inorganic import node, link, OrgNode from orger.common import dt_heading +from orger import pandoc import my.roamresearch as roamresearch -from subprocess import run, PIPE - -def md2org(text: str) -> str: - # TODO use batch?? or talk to a process - r = run( - ['pandoc', '-f', 'markdown', '-t', 'org', '--wrap=none'], - check=True, - input=text.encode('utf8'), - stdout=PIPE, - ) - return r.stdout.decode('utf8') - - # todo ^^ ^^ things are highlight? def roam_text_to_org(text: str) -> str: """ @@ -31,7 +19,7 @@ def roam_text_to_org(text: str) -> str: ('{{[[slider]]}}', ''), ]: text = text.replace(f, t) - org = md2org(text) + org = pandoc.to_org(text, from_='markdown') org = org.replace(r'\_', '_') # unescape, it's a bit aggressive.. return org @@ -87,7 +75,7 @@ def roam_note_to_org(node: roamresearch.Node, top=False) -> Iterable[OrgNode]: ) -class RoamView(StaticView): +class RoamView(Mirror): def get_items(self): rr = roamresearch.roam() from concurrent.futures import ThreadPoolExecutor diff --git a/modules/youtube.py b/modules/youtube.py index f8ae0d0..37c5f13 100755 --- a/modules/youtube.py +++ b/modules/youtube.py @@ -3,25 +3,26 @@ from orger.inorganic import node, link from orger.common import dt_heading -from my.media.youtube import get_watched +from my.media.youtube import watched from itertools import groupby class YoutubeView(Mirror): def get_items(self) -> Mirror.Results: - watched = get_watched() by_url = lambda w: w.url by_when = lambda w: w.when items = [ max(group, key=by_when) - for _, group in groupby(sorted(watched, key=by_url), key=by_url) + for _, group in groupby(sorted(watched(), key=by_url), key=by_url) ] items = sorted(items, key=by_when) # TODO for each url only take latest? for item in items: + deleted = item.url == item.title # todo move to HPI? + l = link(title=item.title + (' (DELETED)' if deleted else ''), url=item.url) yield (item.url, node( - heading=dt_heading(item.when, link(title=item.title, url=item.url)), + heading=dt_heading(item.when, l), )) diff --git a/scripts/ci/run b/scripts/ci/run index 82d691f..0d4e0e0 100755 --- a/scripts/ci/run +++ b/scripts/ci/run @@ -21,7 +21,8 @@ if ! [ -z "$CI" ]; then fi # vim is used in one of the tests -command -v vim || sudo apt install vim +command -v vim || sudo apt install vim +command -v pandoc || sudo apt install pandoc pip3 install --user tox tox diff --git a/setup.py b/setup.py index 5cfde5d..91d8e72 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,10 @@ def main(): author_email='karlicoss@gmail.com', description='Converts data into org-mode', - install_requires=['atomicwrites'], + install_requires=[ + 'appdirs' , # to keep state files + 'atomicwrites', # to safely append data to a file + ], extras_require={ 'testing': ['pytest'], 'linting': [ diff --git a/src/orger/common.py b/src/orger/common.py index 59004b5..e334c1b 100644 --- a/src/orger/common.py +++ b/src/orger/common.py @@ -1,11 +1,13 @@ from datetime import datetime from typing import Optional +from pathlib import Path from .inorganic import OrgNode, timestamp, timestamp_with_style, TimestampStyle class settings: DEFAULT_TIMESTAMP_STYLE = TimestampStyle.INACTIVE + USE_PANDOC: bool = True def dt_heading(dt: Optional[datetime], heading: str) -> str: @@ -44,3 +46,8 @@ def todo(dt: datetime, **kwargs): # todo use klogging2? from .klogging import LazyLogger, setup_logger + + +def orger_user_dir() -> Path: + import appdirs # type: ignore[import] + return Path(appdirs.user_config_dir('orger')) diff --git a/src/orger/org_view.py b/src/orger/org_view.py index e573cbd..74017e9 100644 --- a/src/orger/org_view.py +++ b/src/orger/org_view.py @@ -11,7 +11,7 @@ from .inorganic import OrgNode, TimestampStyle from .state import JsonState from .atomic_append import PathIsh, atomic_append_check, assert_not_edited -from .common import setup_logger +from .common import setup_logger, orger_user_dir # TODO tests for determinism? not sure where should they be... # think of some generic thing to test that? @@ -61,10 +61,20 @@ def main_common(self) -> None: settings.DEFAULT_TIMESTAMP_STYLE = _style_map[timestamp_style] setup_logger(self.logger, level=logging.DEBUG) + pandoc = self.args.pandoc + settings.USE_PANDOC = pandoc + @classmethod def parser(cls) -> ArgumentParser: - p = ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + F = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=120) + p = argparse.ArgumentParser(formatter_class=F) # type: ignore + p.add_argument( + '--disable-pandoc', + action='store_false', + dest='pandoc', + help='Pass to disable pandoc conversions to org-mode (it might be slow in some cases)', + ) p.add_argument( '--timestamps', type=str, @@ -93,7 +103,7 @@ class Mirror(OrgView): @classmethod def main(cls, setup_parser=None) -> None: p = cls.parser() - p.add_argument('--to', type=Path, default=Path(cls.name() + '.org')) + p.add_argument('--to', type=Path, default=Path(cls.name() + '.org'), help='Filename to output') if setup_parser is not None: setup_parser(p) @@ -163,9 +173,9 @@ def test(): class Queue(OrgView): """ - *Queue* (old name =InteractiveView=): works as a queue, *only previously unseen items* from the data source are appended to the output org-mode file. + *Queue* (old name =InteractiveView=): works as a queue, *only previously unseen items* from the data source are added to the output org-mode file. - To keep track of old/new items, it's using a separate JSON =state= file. + To keep track of previously seen iteems, it's using a separate JSON =state= file. A typical usecase is a todo list, or a content processing queue. You can use such a module as you use any other org-mode file: schedule/refile/comment/set priorities, etc. @@ -183,8 +193,16 @@ def _run( dry_run: bool=False, ) -> None: if not to.exists() and not init: - raise RuntimeError(f"target {to} doesn't exist! Try running with --init") + err = RuntimeError(f"{to} doesn't exist! Try running with --init") + import sys + if sys.stdin.isatty(): + resp = input(f"{to} doesn't exist. Create empty file? y/n ").strip().lower() + if resp != 'y': + raise err + else: + raise err + state_path.parent.mkdir(parents=True, exist_ok=True) # not sure... state = JsonState( path=state_path, logger=self.logger, @@ -221,11 +239,12 @@ def get_items(self) -> Iterable[OrgWithKey]: @classmethod def main(cls, setup_parser=None) -> None: + default_state = orger_user_dir() / 'states' / (cls.name() + '.state.json') p = cls.parser() - p.add_argument('--to' , type=Path, default=Path(cls.name() + '.org') , help='file where new items are appended') - p.add_argument('--state', type=Path, default=Path(cls.name() + '.state.json'), help='state file for keeping track of handled items') - p.add_argument('--init', action='store_true') - p.add_argument('--dry-run', action='store_true') + p.add_argument('--to' , type=Path, default=Path(cls.name() + '.org') , help='file where new items are added') + p.add_argument('--state', type=Path, default=default_state, help='state file for keeping track of handled items') + p.add_argument('--init', action='store_true') # todo not sure if I really need it? + p.add_argument('--dry-run', action='store_true', help='Run without modifying the state file') if setup_parser is not None: setup_parser(p) diff --git a/src/orger/pandoc.py b/src/orger/pandoc.py new file mode 100644 index 0000000..0257e62 --- /dev/null +++ b/src/orger/pandoc.py @@ -0,0 +1,52 @@ +""" +Helper for converting stuff to pandoc +""" +import logging +import shutil +from subprocess import run, PIPE +from typing import Optional + + +from .common import settings + +if settings.USE_PANDOC: + has_pandoc = shutil.which('pandoc') is not None + + if not has_pandoc: + import warnings + warnings.warn("Please install 'pandoc' to convert HTML to org-mode. See https://pandoc.org/installing.html") + settings.USE_PANDOC = False + + +def to_org(data: str, *, from_: str, logger=logging) -> str: + if not settings.USE_PANDOC: + return data + # TODO batch?? + + # meh. for some reason they are converted to \\ otherwise + if from_ == 'html': + data = data.replace('and a comment too
multiline!
" + # TODO ok, it's annoying... not sure what to do with nonpritable crap + html = "and a comment too
multiline!
" + assert to_org(data=html, from_='html') == r''' +and a /comment/ too + +*multiline*! +'''.lstrip()