Skip to content

Commit

Permalink
✨ (store) load file uri as MemoryStore
Browse files Browse the repository at this point in the history
  • Loading branch information
simonwoerpel committed Mar 13, 2024
1 parent 24d39e1 commit 8bb552e
Showing 1 changed file with 11 additions and 0 deletions.
11 changes: 11 additions & 0 deletions ftmq/store/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
from typing import TypeVar
from urllib.parse import urlparse

import orjson
from anystore.io import smart_stream
from nomenklatura import Resolver
from nomenklatura.db import get_metadata

Expand All @@ -13,6 +15,7 @@
from ftmq.store.memory import MemoryStore
from ftmq.store.sql import SQLStore
from ftmq.types import PathLike
from ftmq.util import make_proxy

S = TypeVar("S", bound=Store)

Expand All @@ -30,6 +33,14 @@ def get_store(
resolver = get_resolver(resolver)
uri = str(uri)
parsed = urlparse(uri)
if parsed.scheme == "file" or uri == "-" or not parsed.scheme:
store = MemoryStore(catalog, dataset, resolver=resolver)
with store.writer() as writer:
for line in smart_stream(uri):
data = orjson.loads(line)
proxy = make_proxy(data)
writer.add_entity(proxy)
return store
if parsed.scheme == "memory":
return MemoryStore(catalog, dataset, resolver=resolver)
if parsed.scheme == "leveldb":
Expand Down

0 comments on commit 8bb552e

Please sign in to comment.