diff --git a/ftmq/store/__init__.py b/ftmq/store/__init__.py index acbdc36d..04f271ae 100644 --- a/ftmq/store/__init__.py +++ b/ftmq/store/__init__.py @@ -3,6 +3,8 @@ from typing import TypeVar from urllib.parse import urlparse +import orjson +from anystore.io import smart_stream from nomenklatura import Resolver from nomenklatura.db import get_metadata @@ -13,6 +15,7 @@ from ftmq.store.memory import MemoryStore from ftmq.store.sql import SQLStore from ftmq.types import PathLike +from ftmq.util import make_proxy S = TypeVar("S", bound=Store) @@ -30,6 +33,14 @@ def get_store( resolver = get_resolver(resolver) uri = str(uri) parsed = urlparse(uri) + if parsed.scheme == "file" or uri == "-" or not parsed.scheme: + store = MemoryStore(catalog, dataset, resolver=resolver) + with store.writer() as writer: + for line in smart_stream(uri): + data = orjson.loads(line) + proxy = make_proxy(data) + writer.add_entity(proxy) + return store if parsed.scheme == "memory": return MemoryStore(catalog, dataset, resolver=resolver) if parsed.scheme == "leveldb":