Merge pull request #44 from plasorak/plasorak/python-codegen

python codegen

moo/jsonnet-code/opython.jsonnet

@@ -0,0 +1,36 @@
+// This holds some translation from Jsonnet schema to python syntax
+// templates.  This file provides reasonable defaults but some
+// projects may desire to interpret schema differently (eg a different
+// "Any" type or different allowed dtypes).
+
+// Suggested use is to "graft" it into a model so it is available
+// model.lang.*:
+//
+//   moo -g '/lang:opython.jsonnet' [...] some-python-template.py.j2
+{
+    types: {            // type conversion between schema and python
+        string: "str",
+        any: "dict",
+        sequence: "list",
+        boolean: "bool",
+    },
+    // fixme: there are more numpy dtypes that are supported here!
+    dtypes: {
+        i2: "np.int16",
+        i4: "np.int32",
+        i8: "np.int64",
+        u2: "np.uint16",
+        u4: "np.uint32",
+        u8: "np.uint64",
+        f4: "np.float32",
+        f8: "np.float64",
+    },
+    // imports: {          // ie, the ... in #include <...>
+    //     sequence: ["list"],
+    //     string: ["string"],
+    //     enum: ["string"],
+    //     anyOf: ["variant"],
+    //     oneOf: ["variant"],
+    //     any: ["nlohmann/json.hpp"],
+    // }
+}

moo/templates/jinjaint.py

@@ -2,8 +2,9 @@
 from jinja2 import meta, Environment, FileSystemLoader
 
 from . import cpp
+from . import python
 from . import jsonnet
-from .util import find_type, listify, relpath
+from .util import find_type, listify, relpath, debug
 from moo.util import search_path
 
 styles = dict(
@@ -35,8 +36,9 @@ def make_env(path, **kwds):
                       **kwds)
     env.filters["listify"] = listify
     env.filters["relpath"] = relpath
+    env.filters['debug'] = debug
     env.globals.update(find_type=find_type,
-                       cpp=cpp, jsonnet=jsonnet)
+                       cpp=cpp, py=python, jsonnet=jsonnet)
     return env
 
 

moo/templates/opyserdes.py.j2

@@ -0,0 +1,69 @@
+{% import 'opython.py.j2' as py_model %}
+'''
+This file is 100% generated.  Any manual edits will likely be lost.
+
+This contains functions struct and other type definitions for shema in
+{{py_model.ns(model)}} to be serialized via json
+'''
+{% set tcname = "PySerdes" %}
+
+# My structs
+{% set ctxpath = model.ctxpath or [] %}
+{% set prefix = model.path|relpath(ctxpath)|join(".") %}
+{% set all_types = model.all_types|join(', ', attribute='name') %}
+{{ all_types | debug }}
+{% if prefix %}
+from {{ prefix }}.PyStructs import {{all_types}}
+{% else %}
+from PyStructs import {{all_types}}
+{% endif %}
+
+{% if model.extrefs %}
+# {{tcname}} for externally referenced schema
+{% endif %}
+{% for ep in model.extrefs %}
+{% if ep %}
+import {{ep|listify|join(".")}}.{{tcname}}
+{% else %}
+import {{tcname}}
+{% endif %}
+{% endfor %}
+import numpy as np
+
+{% for fqn in model.byscn.record %}
+{% set r = model.byref[fqn] %}
+{% set n = fqn|listify|relpath(model.path)|join(".") %}
+#####
+##### {{r.name}} serialisation/deserialisation
+#####
+
+def {{r.name}}_to_json(obj:{{n}}) -> dict:
+    j=dict()
+    {% for f in r.fields %}
+    {% if f.item in model.byscn.record %}
+    j["{{f.name}}"] = {{f.item.split(".")[-1]}}_to_json(obj.{{f.name}})
+    {% elif f.item in model.byscn.sequence and model.byref[f.item]["items"] in model.byscn.record %}
+    j["{{f.name}}"] = [{{model.byref[f.item]["items"].split('.')[-1]}}_to_json(item) for item in obj.{{f.name}}]
+    {% else %}
+    j["{{f.name}}"] = obj.{{f.name}}
+    {% endif %}
+    {% endfor %}
+    return j
+
+def {{r.name}}_from_json(j:dict) -> {{r.name}}:
+    d = dict()
+    {% for f in r.fields %}
+    if "{{f.name}}" in j: {% if f.item in model.byscn.record %}d["{{f.name}}"] = {{f.item.split('.')[-1]}}_from_json(j["{{f.name}}"])
+    {% elif f.item in model.byscn.sequence and model.byref[f.item]["items"] in model.byscn.record %}d["{{f.name}}"] = [{{model.byref[f.item]["items"].split('.')[-1]}}_from_json(item) for item in j["{{f.name}}"]]
+    {% elif f.item in model.byscn.number %}
+
+        if not np.can_cast(j['{{f.name}}'], {{model.lang.dtypes[model.byref[f.item].dtype]}}):
+            raise RuntimeError(f"Cannot cast {{f.name}} = {% raw %}{{% endraw %}j['{{f.name}}']{% raw %}}{% endraw %} to {{model.lang.dtypes[model.byref[f.item].dtype]}}")
+        d["{{f.name}}"] = {{model.lang.dtypes[model.byref[f.item].dtype]}}(j['{{f.name}}'])
+    {% else %}d["{{f.name}}"] = j["{{f.name}}"]
+    {% endif %}
+    {% endfor %}
+    return {{r.name}}(**d)
+
+
+{% endfor %}

moo/templates/opystructs.py.j2

@@ -0,0 +1,16 @@
+{% import 'opython.py.j2' as py_model %}
+'''
+This file is 100% generated.  Any manual edits will likely be lost.
+'''
+{% set tcname = "PyStructs" %}
+
+from enum import Enum
+import numpy as np
+
+{% for ep in model.extrefs %}
+import {{ep|listify|relpath(model.extpath)|join(".")}}.{{tcname}}
+{% endfor %}
+
+{% for t in model.types %}
+{{ py_model["declare_"+t.schema](model, t)|indent}}
+{% endfor %}

moo/templates/opython.py.j2

@@ -0,0 +1,160 @@
+{# This provides some helper macros for python templates #}
+
+{% macro field_type(ft) %}
+{% if ft == "any"%}model.anytype{% else %}{{ ft }}{% endif %}
+
+{% endmacro %}
+
+{% macro declare_sequence(model, t) %}
+{{t.name}} = {{model.lang.types.sequence}}[{{t["items"]|listify|relpath(model.path)|join(".")}}]
+
+{% endmacro %}
+
+{% macro valid_string(model, field) %}
+if type({{field.name}}) != str: raise RuntimeError("{{field.name}} isn't of type string")
+{% if model["pattern"] %}
+import re
+regex = re.compile(r"{{model["pattern"]}}")
+if not regex.fullmatch({{field.name}}): raise RuntimeError(f"{{field.name}} ({ {{field.name}} }) doesn't match "+ r"{{model["pattern"]}}")
+{% endif %}
+{% endmacro %}
+
+{% macro valid_number(model, field, full_model) %}
+if not np.issubdtype(type({{field.name}}), {{full_model.lang.dtypes[model.dtype]}}): raise RuntimeError(f"{{field.name}} isn't of type {{full_model.lang.dtypes[model.dtype]}}, but of type {type({{field.name}})}")
+{% if model["constraints"] %}
+{% if "exclusiveMaximum" in model["constraints"] %}if {{field.name}} >= {{model["constraints"]["exclusiveMaximum"]}}: raise RuntimeError(f'{{field.name}} ({ {{field.name}} }) is too large (exclMax = {{model["constraints"]["exclusiveMaximum"]}})')
+{% endif %}
+{% if "exclusiveMinimum" in model["constraints"] %}if {{field.name}} <= {{model["constraints"]["exclusiveMinimum"]}}: raise RuntimeError(f'{{field.name}} ({ {{field.name}} }) is too small (exclMin = {{model["constraints"]["exclusiveMinimum"]}})')
+{% endif %}
+{% if "maximum" in model["constraints"] %}if {{field.name}} > {{model["constraints"]["maximum"]}}: raise RuntimeError(f'{{field.name}} ({ {{field.name}} }) is too large (Max = {{model["constraints"]["maximum"]}})')
+{% endif %}
+{% if "minimum" in model["constraints"] %}if {{field.name}} < {{model["constraints"]["minimum"]}}: raise RuntimeError(f'{{field.name}} ({ {{field.name}} }) is too small (Min = {{model["constraints"]["minimum"]}})')
+{% endif %}
+{% if "multipleOf" in model["constraints"] %}if {{field.name}}%{{model["constraints"]["multipleOf"]}} != 0: raise RuntimeError(f'{{field.name}} ({ {{field.name}} }) is not a multiple of {{model["constraints"]["multipleOf"]}})')
+{% endif %}
+{% endif %}
+{% endmacro %}
+
+{% macro valid_boolean(model, field) %}
+if type({{field.name}}) != {{(field.item|listify)[-1]}}: raise RuntimeError("{{field.name}} isn't of type bool")
+{% endmacro %}
+
+{% macro valid_sequence(model, field) %}
+if type({{field.name}}) != list: raise RuntimeError("{{field.name}} isn't of type list")
+if len({{field.name}}) > 0 and type({{field.name}}[0]) != {{(model['items']|listify)[-1]}}: raise RuntimeError("{{field.name}} items isn't of type {{(model['items']|listify)[-1]}}")
+{% endmacro %}
+
+{% macro valid_record(model, field) %}
+if type({{field.name}}) != {{(field.item|listify)[-1]}}: raise RuntimeError("{{field.name}} isn't of type {{(field.item|listify)[-1]}}")
+{% endmacro %}
+
+{# {% macro valid_enum(model, field) %} #}
+{# if not {{(field.item|listify)[-1]}}.parse_({{field.name}}): raise RuntimeError("Incorrect value for {{field.name}}!") #}
+{# {% endmacro %} #}
+
+{% macro declare_record(model, t) %}
+class {{t.name}}{% set comma=":" %}{% for b in t.bases %}{{comma}} {{(b.path+[b.name])|relpath(model.path)|join(".") }} {% set comma = ","%}{%endfor%}:
+{% filter indent(width=4) %}
+
+'''
+{{t.doc}}
+'''
+def __init__(self,
+{% filter indent(width=13) %}
+{% for f in t.fields %}
+{% set ext_field = py.is_external_field(f, model.extrefs, "PyStructs") %}
+{% if ext_field == [] %}{{f.name}}:{{f.item|listify|relpath(model.path)|join(".")}} = {{py.field_default(model.all_types, f)}},
+{% else %}{{f.name}}:{{ext_field|relpath(model.path)|join(".")}} = {{py.field_default(model.all_types, f)}},
+{% endif %}
+{% endfor %}
+):
+{% endfilter %} {#def __init__ args w=13#}
+{% filter indent(width=4) %}
+
+{% if t.fields == [] %}
+pass
+{% else %}
+{% for f in t.fields %}
+###################
+{% if f.doc %}# {{f.doc}}{% endif %}
+
+{% set this_model = model.byref[f.item] %}
+{% if this_model.schema == "string" %}{{valid_string(this_model, f)}}{% elif this_model.schema == "number" %}{{valid_number(this_model, f, model)}}{% elif this_model.schema == "boolean" %}{{valid_boolean(this_model, f)}}{% elif this_model.schema == "sequence" %}{{valid_sequence(this_model, f)}}{% elif this_model.schema == "record" %}{{valid_record(this_model, f)}}{% endif %}
+{# {% if this_model.schema=="enum" %} #}
+{# self.{{f.name}} = {{(f.item|listify)[-1]}}.parse_{{(f.item|listify)[-1]}}({{f.name}}) #}
+{# {% else %} #}
+self.{{f.name}} = {{f.name}}
+{# {% endif %} #}
+{% endfor %}
+{% endif %}
+{% endfilter %} {#def__init__ w=4#}
+{% endfilter %} {#class w=4#}
+
+{% endmacro %}
+
+{% macro declare_boolean(model, t) %}
+{{t.name}} = {{model.lang.types.boolean}}
+
+{% endmacro %}
+
+{% macro declare_string(model, t) %}
+{{t.name}} = {{model.lang.types.string}}
+
+{% endmacro %}
+
+{% macro declare_number(model, t) %}
+{{t.name}} = {{model.lang.dtypes[t.dtype]}}
+
+{% endmacro %}
+
+{% macro declare_any(model, t) %}
+{{t.name}} = {{model.lang.types.any}}
+
+{% endmacro %}
+
+{% macro declare_oneOf(model, t) %}
+raise RuntimeError("OneOf not implemented")
+{# {{t.name}} = std::variant<{%for one in t.types%}{{one|replace(".","::")}}{{ ", " if not loop.last }}{%endfor%}> #}
+
+{% endmacro %}
+
+{% macro declare_enum(model, t) %}
+class {{t.name}}(Enum):
+{% filter indent(width=4) %}
+
+{% for sname in t.symbols %}
+{{sname}} = {{loop.index0}}
+{% endfor %}
+
+@staticmethod
+def parse_{{t.name}}(val:str, defolt=None):
+{% filter indent(width=4) %}
+
+{% for sname in t.symbols %}
+if val == "{{sname}}": return {{t.name}}.{{sname}}
+{% endfor %}
+if defolt is not None:
+    return defolt
+else:
+    raise RuntimeError(f'Value {val} is incorrect for {{t.name}}')
+
+{% endfilter %}
+@staticmethod
+def has_index(value):
+{% filter indent(width=4) %}
+
+values = {
+{% for sname in t.symbols %}
+    {{loop.index0}}, # {{sname}}
+{% endfor %}
+}
+if value in values: return True
+return False
+{% endfilter %}
+{% endfilter %}
+{% endmacro %}
+
+{% macro ns(model) %}
+class {{ ".".join(model.path) }}
+
+{% endmacro %}

moo/templates/python.py

@@ -0,0 +1,85 @@
+'''
+This provides support for templates that produce python code.
+See also ocpp.jsonnet and ocpp.hpp.j2.
+This is added to the environment via the jinjaint module.
+'''
+import sys
+from .util import find_type
+from moo.oschema import untypify
+import numpy
+
+# fixme: this reproduces some bits that are also in otypes.
+
+def literal_value(types, fqn, val):
+    '''Convert val of type typ to a python literal syntax.'''
+    typ = find_type(types, fqn)
+    schema = typ['schema']
+
+    if schema == "boolean":
+        if not val:
+            return 'False'
+        return 'True'
+
+    if schema == "sequence":
+        if val is None:
+            return '[]'
+        seq = ', '.join([literal_value(types, typ['items'], ele) for ele in val])
+        return '[%s]' % seq
+
+    if schema == "number":
+        dtype = typ["dtype"]
+        dtype = numpy.dtype(dtype)
+        val = numpy.array(val or 0, dtype).item() # coerce
+        return f'np.{dtype}({val})'
+
+    if schema == "string":
+        if val is None:
+            return '""'
+        return f'"{val}"'
+
+    if schema == "enum":
+        if val is None:
+            val = typ.get('default', None)
+        if val is None:
+            val = typ.symbols[0]
+        nsp = [typ['name'], val]
+        return '.'.join(nsp)
+
+    if schema == "record":
+        val = val or dict()
+        seq = list()
+        for f in typ['fields']:
+            fval = val.get(f['name'], f.get('default', None))
+            if fval is None:
+                break
+            pyval = f['name']+'='+literal_value(types, f['item'], fval)
+            seq.append(pyval)
+
+        record_name = typ["name"]
+        record_args = "   "+",\n   ".join(seq)
+        s = '%s(\n%s)' % (record_name, record_args)
+        return s
+
+    if schema == "any":
+        return '{}'
+
+    if schema == "oneOf" :
+        return '{}'
+
+    sys.stderr.write(f'moo.templates.py: warning: unsupported default python record field type {schema} for {fqn} using native value')
+    return val                  # go fish
+
+
+def field_default(types, field):
+    'Return a field default as python syntax'
+    field = untypify(field)
+    types = untypify(types)
+    return literal_value(types, field['item'], field.get('default', None))
+
+def is_external_field(field, exrefs, tcname):
+    # returns the path of external field with tcname
+    for exref in exrefs:
+        if exref in field['item']:
+            item = field['item'].split('.')
+            return item[:-1]+[tcname]
+    return []