From d34d6e68f709bc7c3a23351c319f93484ea35d80 Mon Sep 17 00:00:00 2001 From: karlicoss Date: Tue, 9 Jan 2024 19:52:33 +0000 Subject: [PATCH] move away code for old org and html exports --- exobrain/{src => old}/filter_org.py | 0 exobrain/{src => old}/publish.el | 0 exobrain/src/build.py | 113 +++++--------- exobrain/src/compile_org.py | 1 - exobrain/src/fix-links.py | 147 ------------------ .../src/{publish_new.el => publish_html.el} | 0 .../export.el => src/publish_org.el} | 0 exobrain/src/test.py | 40 +---- exobrain/src/utils.py | 17 +- 9 files changed, 46 insertions(+), 272 deletions(-) rename exobrain/{src => old}/filter_org.py (100%) rename exobrain/{src => old}/publish.el (100%) delete mode 120000 exobrain/src/compile_org.py delete mode 100755 exobrain/src/fix-links.py rename exobrain/src/{publish_new.el => publish_html.el} (100%) rename exobrain/{testdata/export.el => src/publish_org.el} (100%) diff --git a/exobrain/src/filter_org.py b/exobrain/old/filter_org.py similarity index 100% rename from exobrain/src/filter_org.py rename to exobrain/old/filter_org.py diff --git a/exobrain/src/publish.el b/exobrain/old/publish.el similarity index 100% rename from exobrain/src/publish.el rename to exobrain/old/publish.el diff --git a/exobrain/src/build.py b/exobrain/src/build.py index 0626363..4aae58f 100755 --- a/exobrain/src/build.py +++ b/exobrain/src/build.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import argparse from concurrent.futures import ProcessPoolExecutor from dataclasses import dataclass from pathlib import Path @@ -21,9 +22,6 @@ def ccall(*args, **kwargs): # TODO wonder if I can integrate it with blog's header? # TODO export txt files as md as well? -from compile_org import emacs -from utils import classproperty - _root = Path(__file__).absolute().parent.parent src = _root / 'src' @@ -31,23 +29,25 @@ def ccall(*args, **kwargs): DATA: Path # note: need to resolve, otherwise relative links might end up weird during org-publish-cache-ctime-of-src -class cfg: - @classproperty - def input_dir(cls) -> Path: +class Cfg: + @property + def input_dir(self) -> Path: return (DATA / 'input' ).resolve() - @classproperty - def public_dir(cls) -> Path: + @property + def public_dir(self) -> Path: return (DATA / 'public').resolve() - @classproperty - def md_dir(cls) -> Path: + @property + def md_dir(self) -> Path: return (DATA / 'md' ).resolve() - @classproperty - def html_dir(cls) -> Path: + @property + def html_dir(self) -> Path: return (DATA / 'html' ).resolve() +cfg = Cfg() + def clean_dir(path: Path) -> None: assert path.is_dir(), path @@ -88,7 +88,6 @@ def clean(*, skip_org_export: bool) -> None: # TODO check noexport tag; remove "hide" def main() -> None: - import argparse p = argparse.ArgumentParser() p.add_argument('--add' , action='store_true') p.add_argument('--filter', type=str, default=None) @@ -97,7 +96,6 @@ def main() -> None: p.add_argument('--watch', action='store_true') p.add_argument('--under-entr', action='store_true') # ugh. p.add_argument('--data-dir', type=Path) - p.add_argument('--use-new-org-export', action='store_true') p.add_argument('--skip-org-export', action='store_true') p.add_argument('--use-new-html-export', action='store_true') args = p.parse_args() @@ -128,7 +126,7 @@ def main() -> None: preprocess(args, skip_org_export=skip_org_export) if args.html: - postprocess_html(use_new_html_export=args.use_new_html_export) + postprocess_html() @dataclass @@ -154,8 +152,9 @@ def compile_org_to_org(ctx: Context, paths: list[Path]) -> None: (output_dir / rpath).parent.mkdir(parents=True, exist_ok=True) print('exporting', list(map(str, rpaths)), 'to', output_dir) check_call([ - 'emacs', '--batch', '-l', Path('testdata') / 'export.el', + 'emacs', '--batch', '-l', src / 'publish_org.el', input_dir, output_dir, *rpaths, + # '-f', 'toggle-debug-on-error', # dumps stacktrace on error ]) for rpath in rpaths: path = output_dir / rpath @@ -183,7 +182,7 @@ def compile_org_to_html(ctx: Context, paths: list[Path]) -> None: (output_dir / rpath).parent.mkdir(parents=True, exist_ok=True) print('exporting', list(map(str, rpaths)), 'to', output_dir) check_call([ - 'emacs', '--batch', '-l', src / 'publish_new.el', + 'emacs', '--batch', '-l', src / 'publish_html.el', input_dir, output_dir, *rpaths, ]) @@ -279,22 +278,11 @@ def preprocess(args, *, skip_org_export: bool) -> None: input_dir = cfg.input_dir filter = args.filter - efilter = 'nil' if filter is None else rf"""'(:exclude "\\.*" :include ("{filter}"))""" + assert filter is None # FIXME support later assert input_dir.exists(), input_dir - eargs = [ - '--eval', f'''(progn - (setq exobrain/input-dir "{input_dir}" ) - (setq exobrain/public-dir "{public_dir}") - (setq exobrain/md-dir "{cfg.md_dir}" ) - (setq exobrain/html-dir "{cfg.html_dir}" ) - (setq exobrain/filter {efilter} ) - )''', - '--directory', src / 'advice-patch', - # '-f', 'toggle-debug-on-error', # dumps stacktrace on error - ] ctx = Context(input_dir=input_dir, output_dir=public_dir) - if not skip_org_export and args.use_new_org_export: + if not skip_org_export: inputs = sorted(input_dir.rglob('*.org')) with ProcessPoolExecutor() as pool: workers = pool._max_workers @@ -306,17 +294,6 @@ def preprocess(args, *, skip_org_export: bool) -> None: fut.result() except Exception as e: raise RuntimeError(f'error while processing {group}') from e - elif not skip_org_export: - # TODO get rid of this.. - with emacs( - *eargs, - '--load' , src / 'publish.el', # old file - '--eval', - f'''(let ((org-publish-project-alist `(,exobrain/project-preprocess-org))) - (org-publish-all))''', - ) as ep: - pass - assert ep.returncode == 0 from check import check_org check_org(public_dir) @@ -337,32 +314,19 @@ def preprocess(args, *, skip_org_export: bool) -> None: if args.html or args.md: # TODO might want both? mode = 'html' if args.html else 'md' - prj = 'exobrain/project-org2html' if args.html else 'exobrain/project-org2md' - if args.use_new_html_export: - org_inputs = sorted(public_dir.rglob('*.org')) - ctx = Context(input_dir=public_dir, output_dir=cfg.html_dir) - with ProcessPoolExecutor() as pool: - workers = pool._max_workers - logger.debug(f'using {workers} workers') - groups = [list(group) for group in divide(workers, org_inputs)] - futures = [pool.submit(compile_org_to_html, ctx, group) for group in groups] - for group, fut in zip(groups, futures): - try: - fut.result() - except Exception as e: - raise RuntimeError(f'error while processing {group}') from e - else: - with emacs( - # ugh. such crap - *([] if args.html else ['--eval', '(setq markdown t)']), - *eargs, - '--load', src / 'publish.el', - '--eval', - f'''(let ((org-publish-project-alist `(,{prj}))) - (org-publish-all))''', - ) as ep: - pass - assert ep.returncode == 0 + + org_inputs = sorted(public_dir.rglob('*.org')) + ctx = Context(input_dir=public_dir, output_dir=cfg.html_dir) + with ProcessPoolExecutor() as pool: + workers = pool._max_workers + logger.debug(f'using {workers} workers') + groups = [list(group) for group in divide(workers, org_inputs)] + futures = [pool.submit(compile_org_to_html, ctx, group) for group in groups] + for group, fut in zip(groups, futures): + try: + fut.result() + except Exception as e: + raise RuntimeError(f'error while processing {group}') from e # for f in public_dir.rglob('*.org'): # assert not f.is_symlink(), f # just in case? @@ -381,7 +345,7 @@ def relativize(soup, *, path: Path, root: Path): return soup -def postprocess_html(*, use_new_html_export: bool) -> None: +def postprocess_html() -> None: html_dir = cfg.html_dir copy(src / 'search/search.css', html_dir / 'search.css' ) @@ -410,13 +374,12 @@ def ashtml(x): # mm, org-mode emits relative links, but we actually want abolute here so it makes sense for any page tocs = tocs.replace('href="', 'href="/') - if use_new_html_export: - with ProcessPoolExecutor() as pool: - futures = [] - for html in html_dir.rglob('*.html'): - futures.append(pool.submit(do_fixup_html, html)) - for f in futures: - f.result() + with ProcessPoolExecutor() as pool: + futures = [] + for html in html_dir.rglob('*.html'): + futures.append(pool.submit(do_fixup_html, html)) + for f in futures: + f.result() # todo eh.. implement this as an external site agnostic script diff --git a/exobrain/src/compile_org.py b/exobrain/src/compile_org.py deleted file mode 120000 index 5187785..0000000 --- a/exobrain/src/compile_org.py +++ /dev/null @@ -1 +0,0 @@ -../../src/compile_org.py \ No newline at end of file diff --git a/exobrain/src/fix-links.py b/exobrain/src/fix-links.py deleted file mode 100755 index e0bfda5..0000000 --- a/exobrain/src/fix-links.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -from pathlib import Path -import re - - -def test(): - text = ''' -* TODO [#C] [2019-08-18 Sun 20:41] How to add full text search to your website - Dev Channel - Medium :search:exobrain: - https://medium.com/dev-channel/how-to-add-full-text-search-to-your-website-4e9c80ce2bf4 misc -* TODO [#C] ok, ox-hugo generates some odd artifacts, perhaps would be easier with vanilla md export :blog:exobrain: -:PROPERTIES: -:CREATED: [2019-12-22 Sun 19:03] -:END: - -* xxx -** TODO [#B] [2019-09-02 Mon 12:41] [[https://reddit.com/r/scifi/comments/cvy78o/searching_stories_with_super_intelligence_in/eyldzyb/][Searching stories with super intelligence in humans theme]] /r/scifi :read: - Love and recommend "Brainchild" just for this. Explores multiple different angles of what we call intelligence. - - https://www.amazon.com/Brain-Child-Novel-George-Turner/dp/0688105955 - -** TODO [#C] xxx :me:blog:exobrain: -:PROPERTIES: -:CREATED: [2018-11-23 Fri 09:29] -:END: -:LOGBOOK: -- State "DONE" from "TODO" [2018-11-24 Sat 15:50] -:END: -https://github.com/karlicoss/my-awesome-list - -* TODO [#B] [2020-02-14 Fri 09:25] Building a Second Brain in Roam...And Why You Might Want To : RoamResearch :blog:exobrain: -:PROPERTIES: -:ID: 600e2919-7ba5-4dbe-85d5-4e4c642b3cc1 -:END: -https://www.reddit.com/r/RoamResearch/comments/eho7de/building_a_second_brain_in_roamand_why_you_might - -* [#B] [2020-01-20 Mon 19:56] What's wrong with computational notebooks? - Austin Z. Henley :gr: -http://web.eecs.utk.edu/~azh/blog/notebookpainpoints.html - ''' - res = process(text) - - exp = ''' -* TODO [#C] [2019-08-18 Sun 20:41] [[https://medium.com/dev-channel/how-to-add-full-text-search-to-your-website-4e9c80ce2bf4][How to add full text search to your website - Dev Channel - Medium]] :search:exobrain: - misc -* TODO [#C] ok, ox-hugo generates some odd artifacts, perhaps would be easier with vanilla md export :blog:exobrain: -:PROPERTIES: -:CREATED: [2019-12-22 Sun 19:03] -:END: - -* xxx -** TODO [#B] [2019-09-02 Mon 12:41] [[https://reddit.com/r/scifi/comments/cvy78o/searching_stories_with_super_intelligence_in/eyldzyb/][Searching stories with super intelligence in humans theme]] /r/scifi :read: - Love and recommend "Brainchild" just for this. Explores multiple different angles of what we call intelligence. - - https://www.amazon.com/Brain-Child-Novel-George-Turner/dp/0688105955 - -** TODO [#C] xxx :me:blog:exobrain: -:PROPERTIES: -:CREATED: [2018-11-23 Fri 09:29] -:END: -:LOGBOOK: -- State "DONE" from "TODO" [2018-11-24 Sat 15:50] -:END: -https://github.com/karlicoss/my-awesome-list - -* TODO [#B] [2020-02-14 Fri 09:25] Building a Second Brain in Roam...And Why You Might Want To : RoamResearch :blog:exobrain: -:PROPERTIES: -:ID: 600e2919-7ba5-4dbe-85d5-4e4c642b3cc1 -:END: -https://www.reddit.com/r/RoamResearch/comments/eho7de/building_a_second_brain_in_roamand_why_you_might - -* [#B] [2020-01-20 Mon 19:56] [[http://web.eecs.utk.edu/~azh/blog/notebookpainpoints.html][What's wrong with computational notebooks? - Austin Z. Henley]] :gr: - - ''' - print(res) - print(repr(exp)) - print(repr(res)) - - # for x, y in zip(exp, res): - # if x != y: - # print(x, y) - assert res == exp - - -import orgparse -tre = orgparse.date.gene_timestamp_regex('inactive') - -def process(text: str) -> str: - replacements = [] - pos = 0 - while True: - # eh. will handle rest of them later somehow.. - # VERBOSE because of orgparse regex - # whitespace in character class because of VERBOSE - hre = r'\*.*' + tre + r'[ ](?P[^\]\n]*?)[ ]*(:(\w+:)+)?[ ]*\n' - lre = r'\s*(?Phttp[^\s]+)' - rrr = re.compile(hre + lre, re.VERBOSE) - hm = rrr.search(text[pos:]) #, re.MULTILINE) - if hm is None: - break - - um = hm - print(um.groupdict()) - - h, hs, he = hm.group('heading'), hm.start('heading'), hm.end('heading') - u, us, ue = um.group('url') , um.start('url') , um.end('url') - - replacements.append((pos + hs, pos + he, f'[[{u}][{h}]]')) - replacements.append((pos + us, pos + ue, '')) - pos += hm.end() - - delta = 0 - for s, e, r in replacements: - # TODO ugh. a bit horrible.. - text = text[:delta + s] + r + text[delta + e:] - delta += len(r) - (e - s) - return text - - -def main(): - test() - - import argparse - p = argparse.ArgumentParser() - # p.add_argument('--dry') - p.add_argument('file', type=Path) - args = p.parse_args() - - path = args.file - text = path.read_text() - res = process(text) - import tempfile - - with tempfile.TemporaryDirectory() as td: - tdir = Path(td) - patched = tdir / 'res.org' - patched.write_text(res) - from subprocess import run, check_call - run(['git', 'diff', path, patched]) - - from kython.tui import yesno_or_fail - if yesno_or_fail('patch?'): - import shutil - shutil.copy(patched, path) - - - -if __name__ == '__main__': - main() diff --git a/exobrain/src/publish_new.el b/exobrain/src/publish_html.el similarity index 100% rename from exobrain/src/publish_new.el rename to exobrain/src/publish_html.el diff --git a/exobrain/testdata/export.el b/exobrain/src/publish_org.el similarity index 100% rename from exobrain/testdata/export.el rename to exobrain/src/publish_org.el diff --git a/exobrain/src/test.py b/exobrain/src/test.py index 90ffc45..d07346a 100644 --- a/exobrain/src/test.py +++ b/exobrain/src/test.py @@ -37,17 +37,8 @@ def tmp_data(tmp_path: Path): yield td -@pytest.mark.parametrize('use_new_org_export' , [True, False], ids=['org_new' , 'org_old']) -@pytest.mark.parametrize('use_new_html_export', [True, False], ids=['html_new', 'html_old']) -def test_build_empty(use_new_org_export: bool, use_new_html_export: bool, tmp_data: Path) -> None: - if on_ci: - if not use_new_html_export: - pytest.skip("doesn't work on ci yet") - if not use_new_org_export: - pytest.skip("doesn't work on ci yet") - oargs = ['--use-new-org-export'] if use_new_org_export else [] - hargs = ['--use-new-html-export'] if use_new_html_export else [] - check_call(build('--data-dir', tmp_data, *oargs, *hargs)) +def test_build_empty(tmp_data: Path) -> None: + check_call(build('--data-dir', tmp_data)) def _check_org(path: Path) -> None: @@ -55,23 +46,14 @@ def _check_org(path: Path) -> None: assert len(ids) > 10 -@pytest.mark.parametrize('use_new_org_export' , [True, False], ids=['org_new' , 'org_old']) -@pytest.mark.parametrize('use_new_html_export', [True, False], ids=['html_new', 'html_old']) -def test_test(use_new_org_export: bool, use_new_html_export: bool, tmp_data: Path, tmp_path: Path) -> None: - if on_ci: - if not use_new_html_export: - pytest.skip("doesn't work on ci yet") - if not use_new_org_export: - pytest.skip("doesn't work on ci yet") +def test_test(tmp_data: Path, tmp_path: Path) -> None: d = tmp_data i = d / 'input' public = d / 'public' html = d / 'html' copy(INPUT / 'test.org', i / 'test.org') - oargs = ['--use-new-org-export'] if use_new_org_export else [] - hargs = ['--use-new-html-export'] if use_new_html_export else [] - check_call(build('--data-dir', d, *oargs, *hargs)) + check_call(build('--data-dir', d)) test_org_public = (public / 'test.org').read_text() @@ -119,21 +101,11 @@ def test_test(use_new_org_export: bool, use_new_html_export: bool, tmp_data: Pat assert 'more tag inheritance tag1tag2tag_atag_btag_c' in test_html -@pytest.mark.parametrize('use_new_org_export' , [True, False], ids=['org_new' , 'org_old']) -@pytest.mark.parametrize('use_new_html_export', [True, False], ids=['html_new', 'html_old']) -def test_build_some(use_new_org_export: bool, use_new_html_export: bool, tmp_data: Path, tmp_path: Path) -> None: - if on_ci: - if not use_new_html_export: - pytest.skip("doesn't work on ci yet") - if not use_new_org_export: - pytest.skip("doesn't work on ci yet") - +def test_build_some(tmp_data: Path, tmp_path: Path) -> None: d = tmp_data def run_build(*args): - oargs = ['--use-new-org-export'] if use_new_org_export else [] - hargs = ['--use-new-html-export'] if use_new_html_export else [] - check_call(build('--data-dir', d, *oargs, *hargs, *args)) + check_call(build('--data-dir', d, *args)) i = d / 'input' diff --git a/exobrain/src/utils.py b/exobrain/src/utils.py index c1053b4..7c562f2 100644 --- a/exobrain/src/utils.py +++ b/exobrain/src/utils.py @@ -1,21 +1,8 @@ -from typing import TypeVar, Callable, Generic - -_C = TypeVar('_C') -_R = TypeVar('_R') - -# https://stackoverflow.com/a/5192374/706389 -class classproperty(Generic[_R]): - def __init__(self, f: Callable[[_C], _R]) -> None: - self.f = f - - def __get__(self, obj: None, cls: _C) -> _R: - return self.f(cls) - - from contextlib import contextmanager + @contextmanager def tmp_popen(*args, **kwargs): - import psutil # type: ignore + import psutil # type: ignore with psutil.Popen(*args, **kwargs) as p: try: yield p