From 9c49f39cb9fbe7d686ef83c470b05e259f133670 Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Wed, 2 Nov 2022 19:01:11 +0100 Subject: [PATCH] Added support for meta data (#155) * Small fixes (and with incorrect commas) * Added support for metadata https://github.com/wc-duck/datalibrary/issues/97 * Didn't mean to change all that whitespace * More whitespace madness * Compile warning fix * Gcc fixes * Fixed memory leaks and operations on uninitialized memory * Fixed uninitialized data * PR review fixes * Review fix from PR #157 * GCC C fix * Moved metadata access to dl_reflect.h * Removed WinMerge .bak file Co-authored-by: Christian Nilsendahl --- README.md | 18 ++++- include/dl/dl.h | 2 +- include/dl/dl_defines.h | 0 include/dl/dl_reflect.h | 70 +++++++++------- src/dl.cpp | 36 ++++----- src/dl_convert.cpp | 1 + src/dl_reflect.cpp | 66 ++++++++------- src/dl_typelib_read_bin.cpp | 69 ++++++++++++++-- src/dl_typelib_read_txt.cpp | 129 +++++++++++++++++++++++++++--- src/dl_typelib_write_bin.cpp | 22 +++++ src/dl_typelib_write_c_header.cpp | 2 +- src/dl_typelib_write_txt.cpp | 123 ++++++++++++++++++++++++++-- src/dl_types.h | 31 +++++-- src/dl_util.cpp | 8 +- tests/dl_tests_error.cpp | 2 +- tests/dl_tests_metadata.cpp | 76 ++++++++++++++++++ tests/unittest.tld | 41 +++++++++- tool/dlpack/dlpack.cpp | 4 +- 18 files changed, 583 insertions(+), 117 deletions(-) mode change 100644 => 100755 include/dl/dl.h mode change 100644 => 100755 include/dl/dl_defines.h mode change 100644 => 100755 src/dl_util.cpp create mode 100644 tests/dl_tests_metadata.cpp mode change 100644 => 100755 tool/dlpack/dlpack.cpp diff --git a/README.md b/README.md index f81a7f9..c973886 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,10 @@ Type-libs in text-format follows this format. // Defaults to `uint32` if not set. "type" : "int64", - "comment": "my_enum is an externally declared enumeration and have 7 values", + "comment": "my_enum is an externally declared enumeration and have 7 values",, + + // "metadata" has a list of dl instances which are attached to the enumeration + "metadata": [ { "my_type" : { "member": 1 } } ] // "values" are the actual values of the enum-type as a dict. "values" : { @@ -161,8 +164,11 @@ Type-libs in text-format follows this format. // it was added as to shorten the text-format and still keep more descriptive names // in the generated headers. "aliases" : ["apa", "kossa"], + + "comment": "This enumerator has aliases", - "comment": "This enumerator has aliases" + // "metadata" has a list of dl instances which are attached to the enumerator + "metadata": [ { "my_type" : { "member": 1 } } ] } } } @@ -194,6 +200,9 @@ Type-libs in text-format follows this format. // If not set, no comment will be written to .h "comment" : "this type is a nice type!", + // "metadata" has a list of dl instances which are attached to the type + "metadata": [ { "my_type" : { "member": 1 } } ], + // "members" is a list of all members of this type. "members" : [ // one {} per member and the members will be ordered as they are in this list. @@ -219,7 +228,10 @@ Type-libs in text-format follows this format. "comment" : "only used in unittests to check for errors", // "const" adds the const type modifier to this member in the generated header file - "const" : true + "const" : true, + + // "metadata" has a list of dl instances which are attached to the member + "metadata": [ { "my_type" : { "member": 1 } } ] } ] } diff --git a/include/dl/dl.h b/include/dl/dl.h old mode 100644 new mode 100755 index fae87e5..2bd3b34 --- a/include/dl/dl.h +++ b/include/dl/dl.h @@ -349,7 +349,7 @@ dl_error_t DL_DLL_EXPORT dl_instance_load_inplace( dl_ctx_t dl_ctx, instance - Ptr to instance to calculate size of. out_size - Ptr where to store the amount of bytes needed to store the instances. */ -dl_error_t DL_DLL_EXPORT dl_instance_calc_size( dl_ctx_t dl_ctx, dl_typeid_t type, void* instance, size_t* out_size); +dl_error_t DL_DLL_EXPORT dl_instance_calc_size( dl_ctx_t dl_ctx, dl_typeid_t type, const void* instance, size_t* out_size); /* Function: dl_instace_store diff --git a/include/dl/dl_defines.h b/include/dl/dl_defines.h old mode 100644 new mode 100755 diff --git a/include/dl/dl_reflect.h b/include/dl/dl_reflect.h index bd97976..a239f22 100755 --- a/include/dl/dl_reflect.h +++ b/include/dl/dl_reflect.h @@ -29,15 +29,18 @@ typedef struct dl_type_context_info */ typedef struct dl_type_info { - dl_typeid_t tid; - const char* name; - const char* comment; - unsigned int size; - unsigned int alignment; - unsigned int member_count; - unsigned int is_extern : 1; - unsigned int is_union : 1; - unsigned int should_verify : 1; + dl_typeid_t tid; + const char* name; + const char* comment; + unsigned int size; + unsigned int alignment; + unsigned int member_count; + unsigned int metadata_count; + const dl_typeid_t* metadata_type_ids; + const void** metadata_instances; + unsigned int is_extern : 1; + unsigned int is_union : 1; + unsigned int should_verify : 1; } dl_type_info_t; /* @@ -46,36 +49,42 @@ typedef struct dl_type_info */ typedef struct dl_member_info { - const char* name; - const char* comment; - dl_type_atom_t atom; - dl_type_storage_t storage; - dl_typeid_t type_id; - unsigned int size; - unsigned int alignment; - unsigned int offset; - unsigned int array_count; - unsigned int bits; - unsigned int is_const : 1; - unsigned int should_verify : 1; + const char* name; + const char* comment; + dl_type_atom_t atom; + dl_type_storage_t storage; + dl_typeid_t type_id; + unsigned int size; + unsigned int alignment; + unsigned int offset; + unsigned int array_count; + unsigned int bits; + unsigned int metadata_count; + const dl_typeid_t* metadata_type_ids; + const void** metadata_instances; + unsigned int is_const : 1; + unsigned int should_verify : 1; } dl_member_info_t; /* - Struct: dl_type_info_t + Struct: dl_enum_info Struct used to retrieve information about a specific DL-enum. */ typedef struct dl_enum_info { - dl_typeid_t tid; - const char* name; - const char* comment; - dl_type_storage_t storage; - unsigned int value_count; - unsigned int is_extern : 1; + dl_typeid_t tid; + const char* name; + const char* comment; + dl_type_storage_t storage; + unsigned int value_count; + unsigned int metadata_count; + const dl_typeid_t* metadata_type_ids; + const void** metadata_instances; + unsigned int is_extern : 1; } dl_enum_info_t; /* - Struct: dl_member_info_t + Struct: dl_enum_value_info Struct used to retrieve information about a specific DL-enum-value. */ typedef struct dl_enum_value_info @@ -93,6 +102,9 @@ typedef struct dl_enum_value_info uint32_t u32; uint64_t u64; } value; + unsigned int metadata_count; + const dl_typeid_t* metadata_type_ids; + const void** metadata_instances; } dl_enum_value_info_t; #ifdef __cplusplus diff --git a/src/dl.cpp b/src/dl.cpp index dac1548..db9727e 100644 --- a/src/dl.cpp +++ b/src/dl.cpp @@ -38,6 +38,11 @@ dl_error_t dl_context_destroy(dl_ctx_t dl_ctx) dl_free( &dl_ctx->alloc, dl_ctx->typedata_strings ); dl_free( &dl_ctx->alloc, dl_ctx->default_data ); dl_free( &dl_ctx->alloc, dl_ctx->c_includes ); + for( size_t i = 0; i < dl_ctx->metadatas_count; ++i) + dl_free( &dl_ctx->alloc, dl_ctx->metadatas[i] ); + dl_free( &dl_ctx->alloc, dl_ctx->metadatas ); + dl_free( &dl_ctx->alloc, dl_ctx->metadata_infos ); + dl_free( &dl_ctx->alloc, dl_ctx->metadata_typeinfos ); dl_free( &dl_ctx->alloc, dl_ctx ); return DL_ERROR_OK; } @@ -445,28 +450,23 @@ dl_error_t dl_instance_store( dl_ctx_t dl_ctx, dl_typeid_t type_id, if( type == 0x0 ) return DL_ERROR_TYPE_NOT_FOUND; - // write header - dl_data_header header; - header.id = DL_INSTANCE_ID; - header.version = DL_INSTANCE_VERSION; - header.root_instance_type = type_id; - header.instance_size = 0; - header.is_64_bit_ptr = sizeof(void*) == 8 ? 1 : 0; - header.pad[0] = header.pad[1] = header.pad[2] = 0; - - unsigned char* store_ctx_buffer = 0x0; - size_t store_ctx_buffer_size = 0; - bool store_ctx_is_dummy = out_buffer_size == 0; + bool store_ctx_is_dummy = out_buffer_size == 0; + CDLBinStoreContext store_context(out_buffer + sizeof(dl_data_header), out_buffer_size - sizeof(dl_data_header), store_ctx_is_dummy, dl_ctx->alloc); if( out_buffer_size > 0 ) { - memcpy(out_buffer, &header, sizeof(dl_data_header)); - store_ctx_buffer = out_buffer + sizeof(dl_data_header); - store_ctx_buffer_size = out_buffer_size - sizeof(dl_data_header); + if( instance == out_buffer + sizeof(dl_data_header) ) + memset( out_buffer, 0, sizeof(dl_data_header) ); + else + memset( out_buffer, 0, out_buffer_size ); + + dl_data_header* header = (dl_data_header*)out_buffer; + header->id = DL_INSTANCE_ID; + header->version = DL_INSTANCE_VERSION; + header->root_instance_type = type_id; + header->is_64_bit_ptr = sizeof( void* ) == 8 ? 1 : 0; } - CDLBinStoreContext store_context( store_ctx_buffer, store_ctx_buffer_size, store_ctx_is_dummy, dl_ctx->alloc ); - dl_binary_writer_reserve( &store_context.writer, type->size[DL_PTR_SIZE_HOST] ); store_context.AddWrittenPtr(instance, 0); // if pointer refere to root-node, it can be found at offset 0 @@ -487,7 +487,7 @@ dl_error_t dl_instance_store( dl_ctx_t dl_ctx, dl_typeid_t type_id, return err; } -dl_error_t dl_instance_calc_size( dl_ctx_t dl_ctx, dl_typeid_t type, void* instance, size_t* out_size ) +dl_error_t dl_instance_calc_size( dl_ctx_t dl_ctx, dl_typeid_t type, const void* instance, size_t* out_size ) { return dl_instance_store( dl_ctx, type, instance, 0x0, 0, out_size ); } diff --git a/src/dl_convert.cpp b/src/dl_convert.cpp index dd6ce39..6d5eef2 100644 --- a/src/dl_convert.cpp +++ b/src/dl_convert.cpp @@ -938,6 +938,7 @@ static dl_error_t dl_internal_convert_instance( dl_ctx_t dl_ctx, if(out_instance != 0x0) { dl_data_header* new_header = (dl_data_header*)out_instance; + memset(new_header, 0, sizeof(dl_data_header)); new_header->id = DL_INSTANCE_ID; new_header->version = DL_INSTANCE_VERSION; new_header->root_instance_type = type; diff --git a/src/dl_reflect.cpp b/src/dl_reflect.cpp index b22d8d9..03e3581 100755 --- a/src/dl_reflect.cpp +++ b/src/dl_reflect.cpp @@ -33,25 +33,31 @@ dl_error_t dl_reflect_loaded_enumids( dl_ctx_t dl_ctx, dl_typeid_t* out_enums, u static void dl_reflect_copy_type_info( dl_ctx_t ctx, dl_type_info_t* typeinfo, const dl_type_desc* type ) { - typeinfo->tid = ctx->type_ids[ type - ctx->type_descs ]; - typeinfo->name = dl_internal_type_name( ctx, type ); - typeinfo->comment = dl_internal_type_comment( ctx, type ); - typeinfo->size = type->size[DL_PTR_SIZE_HOST]; - typeinfo->alignment = type->alignment[DL_PTR_SIZE_HOST]; - typeinfo->member_count = type->member_count; - typeinfo->is_extern = ( type->flags & DL_TYPE_FLAG_IS_EXTERNAL ) ? 1 : 0; - typeinfo->is_union = ( type->flags & DL_TYPE_FLAG_IS_UNION ) ? 1 : 0; - typeinfo->should_verify = ( type->flags & DL_TYPE_FLAG_VERIFY_EXTERNAL_SIZE_ALIGN ) ? 1 : 0; + typeinfo->tid = ctx->type_ids[ type - ctx->type_descs ]; + typeinfo->name = dl_internal_type_name( ctx, type ); + typeinfo->comment = dl_internal_type_comment( ctx, type ); + typeinfo->size = type->size[DL_PTR_SIZE_HOST]; + typeinfo->alignment = type->alignment[DL_PTR_SIZE_HOST]; + typeinfo->member_count = type->member_count; + typeinfo->metadata_count = type->metadata_count; + typeinfo->metadata_type_ids = &ctx->metadata_typeinfos[type->metadata_start]; + typeinfo->metadata_instances = &ctx->metadata_infos[type->metadata_start]; + typeinfo->is_extern = ( type->flags & DL_TYPE_FLAG_IS_EXTERNAL ) ? 1 : 0; + typeinfo->is_union = ( type->flags & DL_TYPE_FLAG_IS_UNION ) ? 1 : 0; + typeinfo->should_verify = ( type->flags & DL_TYPE_FLAG_VERIFY_EXTERNAL_SIZE_ALIGN ) ? 1 : 0; } static void dl_reflect_copy_enum_info( dl_ctx_t ctx, dl_enum_info_t* enuminfo, const dl_enum_desc* enum_ ) { - enuminfo->tid = ctx->enum_ids[ enum_ - ctx->enum_descs ]; - enuminfo->name = dl_internal_enum_name( ctx, enum_ ); - enuminfo->comment = dl_internal_enum_comment( ctx, enum_ ); - enuminfo->storage = enum_->storage; - enuminfo->value_count = enum_->value_count; - enuminfo->is_extern = ( enum_->flags & DL_TYPE_FLAG_IS_EXTERNAL ) ? 1 : 0; + enuminfo->tid = ctx->enum_ids[ enum_ - ctx->enum_descs ]; + enuminfo->name = dl_internal_enum_name( ctx, enum_ ); + enuminfo->comment = dl_internal_enum_comment( ctx, enum_ ); + enuminfo->storage = enum_->storage; + enuminfo->value_count = enum_->value_count; + enuminfo->metadata_count = enum_->metadata_count; + enuminfo->metadata_type_ids = &ctx->metadata_typeinfos[enum_->metadata_start]; + enuminfo->metadata_instances = &ctx->metadata_infos[enum_->metadata_start]; + enuminfo->is_extern = ( enum_->flags & DL_TYPE_FLAG_IS_EXTERNAL ) ? 1 : 0; } dl_error_t DL_DLL_EXPORT dl_reflect_loaded_types( dl_ctx_t dl_ctx, dl_type_info_t* out_types, unsigned int out_types_size ) @@ -120,18 +126,21 @@ dl_error_t DL_DLL_EXPORT dl_reflect_get_type_members( dl_ctx_t dl_ctx, dl_typeid { const dl_member_desc* member = dl_get_type_member( dl_ctx, type, member_index ); - out_members[member_index].name = dl_internal_member_name( dl_ctx, member ); - out_members[member_index].comment = dl_internal_member_comment( dl_ctx, member ); - out_members[member_index].atom = member->AtomType(); - out_members[member_index].storage = member->StorageType(); - out_members[member_index].type_id = member->type_id; - out_members[member_index].size = member->size[DL_PTR_SIZE_HOST]; - out_members[member_index].alignment = member->alignment[DL_PTR_SIZE_HOST]; - out_members[member_index].offset = member->offset[DL_PTR_SIZE_HOST]; - out_members[member_index].array_count = 0; - out_members[member_index].bits = 0; - out_members[member_index].is_const = ( member->flags & DL_MEMBER_FLAG_IS_CONST ) ? 1 : 0; - out_members[member_index].should_verify = ( member->flags & DL_MEMBER_FLAG_VERIFY_EXTERNAL_SIZE_OFFSET ) ? 1 : 0; + out_members[member_index].name = dl_internal_member_name( dl_ctx, member ); + out_members[member_index].comment = dl_internal_member_comment( dl_ctx, member ); + out_members[member_index].atom = member->AtomType(); + out_members[member_index].storage = member->StorageType(); + out_members[member_index].type_id = member->type_id; + out_members[member_index].size = member->size[DL_PTR_SIZE_HOST]; + out_members[member_index].alignment = member->alignment[DL_PTR_SIZE_HOST]; + out_members[member_index].offset = member->offset[DL_PTR_SIZE_HOST]; + out_members[member_index].array_count = 0; + out_members[member_index].bits = 0; + out_members[member_index].metadata_count = member->metadata_count; + out_members[member_index].metadata_type_ids = &dl_ctx->metadata_typeinfos[member->metadata_start]; + out_members[member_index].metadata_instances = &dl_ctx->metadata_infos[member->metadata_start]; + out_members[member_index].is_const = ( member->flags & DL_MEMBER_FLAG_IS_CONST ) ? 1 : 0; + out_members[member_index].should_verify = ( member->flags & DL_MEMBER_FLAG_VERIFY_EXTERNAL_SIZE_OFFSET ) ? 1 : 0; switch(member->AtomType()) { @@ -161,6 +170,9 @@ dl_error_t DL_DLL_EXPORT dl_reflect_get_enum_values( dl_ctx_t dl_ctx, dl_typeid_ out_values[value].name = dl_internal_enum_alias_name( dl_ctx, &dl_ctx->enum_alias_descs[v->main_alias]); out_values[value].comment = dl_internal_enum_value_comment( dl_ctx, v); out_values[value].value.u64 = v->value; + out_values[value].metadata_count = v->metadata_count; + out_values[value].metadata_type_ids = &dl_ctx->metadata_typeinfos[v->metadata_start]; + out_values[value].metadata_instances = &dl_ctx->metadata_infos[v->metadata_start]; } return DL_ERROR_OK; diff --git a/src/dl_typelib_read_bin.cpp b/src/dl_typelib_read_bin.cpp index c673b8f..1fc8143 100644 --- a/src/dl_typelib_read_bin.cpp +++ b/src/dl_typelib_read_bin.cpp @@ -33,8 +33,11 @@ static void dl_internal_read_typelibrary_header( dl_typelib_header* header, cons header->member_count = dl_swap_endian_uint32( header->member_count ); header->enum_value_count = dl_swap_endian_uint32( header->enum_value_count ); header->enum_alias_count = dl_swap_endian_uint32( header->enum_alias_count ); - - header->default_value_size = dl_swap_endian_uint32( header->default_value_size ); + + header->default_value_size = dl_swap_endian_uint32( header->default_value_size ); + header->typeinfo_strings_size = dl_swap_endian_uint32( header->typeinfo_strings_size ); + header->c_includes_size = dl_swap_endian_uint32( header->c_includes_size ); + header->metadatas_count = dl_swap_endian_uint32( header->metadatas_count ); } } @@ -45,12 +48,16 @@ static void dl_endian_swap_type_desc( dl_type_desc* desc ) desc->alignment[DL_PTR_SIZE_32BIT] = dl_swap_endian_uint32( desc->alignment[DL_PTR_SIZE_32BIT] ); desc->alignment[DL_PTR_SIZE_64BIT] = dl_swap_endian_uint32( desc->alignment[DL_PTR_SIZE_64BIT] ); desc->member_count = dl_swap_endian_uint32( desc->member_count ); + desc->metadata_count = dl_swap_endian_uint32( desc->metadata_count ); + desc->metadata_start = dl_swap_endian_uint32( desc->metadata_start ); } static void dl_endian_swap_enum_desc( dl_enum_desc* desc ) { - desc->value_count = dl_swap_endian_uint32( desc->value_count ); - desc->value_start = dl_swap_endian_uint32( desc->value_start ); + desc->value_count = dl_swap_endian_uint32( desc->value_count ); + desc->value_start = dl_swap_endian_uint32( desc->value_start ); + desc->metadata_count = dl_swap_endian_uint32( desc->metadata_count ); + desc->metadata_start = dl_swap_endian_uint32( desc->metadata_start ); } static void dl_endian_swap_member_desc( dl_member_desc* desc ) @@ -64,11 +71,15 @@ static void dl_endian_swap_member_desc( dl_member_desc* desc ) desc->alignment[DL_PTR_SIZE_32BIT] = dl_swap_endian_uint32( desc->alignment[DL_PTR_SIZE_32BIT] ); desc->alignment[DL_PTR_SIZE_64BIT] = dl_swap_endian_uint32( desc->alignment[DL_PTR_SIZE_64BIT] ); desc->default_value_offset = dl_swap_endian_uint32( desc->default_value_offset ); + desc->metadata_count = dl_swap_endian_uint32( desc->metadata_count ); + desc->metadata_start = dl_swap_endian_uint32( desc->metadata_start ); } static void dl_endian_swap_enum_value_desc( dl_enum_value_desc* desc ) { - desc->value = dl_swap_endian_uint64( desc->value ); + desc->value = dl_swap_endian_uint64( desc->value ); + desc->metadata_count = dl_swap_endian_uint32( desc->metadata_count ); + desc->metadata_start = dl_swap_endian_uint32( desc->metadata_start ); } template @@ -98,6 +109,7 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l size_t defaults_offset = enum_aliases_offset + sizeof( dl_enum_alias_desc ) * header.enum_alias_count; size_t typedata_strings_offset = defaults_offset + header.default_value_size; size_t c_includes_offset = typedata_strings_offset + header.typeinfo_strings_size; + size_t metadatas_offset = c_includes_offset + header.c_includes_size; dl_ctx->type_ids = dl_realloc_array( &dl_ctx->alloc, dl_ctx->type_ids, dl_ctx->type_count + header.type_count, dl_ctx->type_count ); dl_ctx->type_descs = dl_realloc_array( &dl_ctx->alloc, dl_ctx->type_descs, dl_ctx->type_count + header.type_count, dl_ctx->type_count ); @@ -109,6 +121,20 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l dl_ctx->typedata_strings = dl_realloc_array( &dl_ctx->alloc, dl_ctx->typedata_strings, dl_ctx->typedata_strings_size + header.typeinfo_strings_size, dl_ctx->typedata_strings_size ); if(header.c_includes_size) dl_ctx->c_includes = dl_realloc_array( &dl_ctx->alloc, dl_ctx->c_includes, dl_ctx->c_includes_size + header.c_includes_size, dl_ctx->c_includes_size ); + if(header.metadatas_count) + { + dl_ctx->metadatas = dl_realloc_array( &dl_ctx->alloc, dl_ctx->metadatas, dl_ctx->metadatas_count + header.metadatas_count, dl_ctx->metadatas_count ); + dl_ctx->metadata_infos = dl_realloc_array( &dl_ctx->alloc, dl_ctx->metadata_infos, dl_ctx->metadatas_count + header.metadatas_count, dl_ctx->metadatas_count ); + dl_ctx->metadata_typeinfos = dl_realloc_array( &dl_ctx->alloc, dl_ctx->metadata_typeinfos, dl_ctx->metadatas_count + header.metadatas_count, dl_ctx->metadatas_count ); + for( unsigned int i = 0; i < header.metadatas_count; ++i ) + { + uint32_t metadata_offset = *reinterpret_cast( lib_data + metadatas_offset + i * sizeof( uint32_t ) ); + metadata_offset = ( DL_ENDIAN_HOST == DL_ENDIAN_BIG ) ? dl_swap_endian_uint32( metadata_offset ) : metadata_offset; + const dl_data_header* metadata_header = reinterpret_cast( lib_data + metadatas_offset + header.metadatas_count * sizeof( uint32_t ) + metadata_offset ); + size_t instance_size = ( DL_ENDIAN_HOST == DL_ENDIAN_BIG ) ? dl_swap_endian_uint32( metadata_header->instance_size ) : metadata_header->instance_size; + dl_ctx->metadatas[dl_ctx->metadatas_count + i] = dl_alloc( &dl_ctx->alloc, instance_size + sizeof( dl_data_header ) ); + } + } memcpy( dl_ctx->type_ids + dl_ctx->type_count, lib_data + types_lookup_offset, sizeof( dl_typeid_t ) * header.type_count ); memcpy( dl_ctx->enum_ids + dl_ctx->enum_count, lib_data + enums_lookup_offset, sizeof( dl_typeid_t ) * header.enum_count ); @@ -138,10 +164,16 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l if(dl_ctx->type_descs[ dl_ctx->type_count + i ].comment != UINT32_MAX) dl_ctx->type_descs[ dl_ctx->type_count + i ].comment += td_str_offset; dl_ctx->type_descs[ dl_ctx->type_count + i ].member_start += dl_ctx->member_count; + if( dl_ctx->type_descs[ dl_ctx->type_count + i ].metadata_start ) + dl_ctx->type_descs[ dl_ctx->type_count + i ].metadata_start += dl_ctx->metadatas_count; } for( unsigned int i = 0; i < header.member_count; ++i ) + { dl_ctx->member_descs[ dl_ctx->member_count + i ].name += td_str_offset; + if( dl_ctx->member_descs[ dl_ctx->member_count + i ].metadata_start ) + dl_ctx->member_descs[ dl_ctx->member_count + i ].metadata_start += dl_ctx->metadatas_count; + } for( unsigned int i = 0; i < header.enum_count; ++i ) { @@ -150,6 +182,8 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l dl_ctx->enum_descs[ dl_ctx->enum_count+ i ].comment += td_str_offset; dl_ctx->enum_descs[ dl_ctx->enum_count + i ].value_start += dl_ctx->enum_value_count; dl_ctx->enum_descs[ dl_ctx->enum_count + i ].alias_start += dl_ctx->enum_alias_count; + if( dl_ctx->enum_descs[ dl_ctx->enum_count + i ].metadata_start ) + dl_ctx->enum_descs[ dl_ctx->enum_count + i ].metadata_start += dl_ctx->metadatas_count; } for( unsigned int i = 0; i < header.enum_alias_count; ++i ) @@ -161,6 +195,8 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l for( unsigned int i = 0; i < header.enum_value_count; ++i ) { dl_ctx->enum_value_descs[ dl_ctx->enum_value_count + i ].main_alias += dl_ctx->enum_alias_count; + if( dl_ctx->enum_value_descs[ dl_ctx->enum_value_count + i ].metadata_start ) + dl_ctx->enum_value_descs[ dl_ctx->enum_value_count + i ].metadata_start += dl_ctx->metadatas_count; } dl_ctx->type_count += header.type_count; @@ -169,6 +205,7 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l dl_ctx->enum_value_count += header.enum_value_count; dl_ctx->enum_alias_count += header.enum_alias_count; dl_ctx->typedata_strings_size += header.typeinfo_strings_size; + dl_ctx->c_includes_size += header.c_includes_size; // we still need to keep the capacity around here, even as they are the same as the type-counts in // the case where we were to read a typelib from text into this ctx as that would do an incremental @@ -180,6 +217,28 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l dl_ctx->enum_value_capacity = dl_ctx->enum_value_count; dl_ctx->enum_alias_capacity = dl_ctx->enum_alias_count; dl_ctx->typedata_strings_cap = dl_ctx->typedata_strings_size; + dl_ctx->c_includes_cap = dl_ctx->c_includes_size; + + // The types used for meta data must be known to be able to patch the data with dl_instance_load_inplace + for( unsigned int i = 0; i < header.metadatas_count; ++i ) + { + uint32_t metadata_offset = *reinterpret_cast( lib_data + metadatas_offset + i * sizeof( uint32_t ) ); + metadata_offset = ( DL_ENDIAN_HOST == DL_ENDIAN_BIG ) ? dl_swap_endian_uint32( metadata_offset ) : metadata_offset; + const dl_data_header* metadata_header = reinterpret_cast( lib_data + metadatas_offset + header.metadatas_count * sizeof( uint32_t ) + metadata_offset ); + size_t instance_size = ( DL_ENDIAN_HOST == DL_ENDIAN_BIG ) ? dl_swap_endian_uint32( metadata_header->instance_size ) : metadata_header->instance_size; + memcpy( dl_ctx->metadatas[dl_ctx->metadatas_count + i], metadata_header, instance_size + sizeof( dl_data_header ) ); + dl_typeid_t type_id = ( DL_ENDIAN_HOST == DL_ENDIAN_BIG ) ? dl_swap_endian_uint32( metadata_header->root_instance_type ) : metadata_header->root_instance_type; + void* loaded_instance; + size_t consumed; + dl_error_t err = dl_instance_load_inplace( dl_ctx, type_id, (uint8_t*)dl_ctx->metadatas[dl_ctx->metadatas_count + i], instance_size + sizeof( dl_data_header ), &loaded_instance, &consumed ); + DL_ASSERT( DL_ERROR_OK == err ); (void)err; + DL_ASSERT( instance_size + sizeof( dl_data_header ) == consumed ); + dl_ctx->metadata_infos[dl_ctx->metadatas_count + i] = loaded_instance; + dl_ctx->metadata_typeinfos[dl_ctx->metadatas_count + i] = type_id; + } + + dl_ctx->metadatas_count += header.metadatas_count; + dl_ctx->metadatas_cap = dl_ctx->metadatas_count; return dl_internal_load_type_library_defaults( dl_ctx, lib_data + defaults_offset, header.default_value_size ); } diff --git a/src/dl_typelib_read_txt.cpp b/src/dl_typelib_read_txt.cpp index 77504c8..7c0b0ee 100644 --- a/src/dl_typelib_read_txt.cpp +++ b/src/dl_typelib_read_txt.cpp @@ -102,6 +102,8 @@ static dl_enum_desc* dl_alloc_enum( dl_ctx_t ctx, dl_substr* name ) e->value_count = 0; e->alias_count = 0; e->alias_start = ctx->enum_alias_count; + e->metadata_count = 0; + e->metadata_start = 0; return e; } @@ -133,6 +135,20 @@ static dl_enum_alias_desc* dl_alloc_enum_alias( dl_ctx_t ctx, dl_substr* name ) return alias; } +static void** dl_alloc_metadata( dl_ctx_t ctx ) +{ + if( ctx->metadatas_cap <= ctx->metadatas_count ) + { + size_t temp_cap = ctx->metadatas_cap; + ctx->metadata_typeinfos = dl_grow_array( &ctx->alloc, ctx->metadata_typeinfos, &temp_cap, 0 ); + temp_cap = ctx->metadatas_cap; + ctx->metadata_infos = dl_grow_array( &ctx->alloc, ctx->metadata_infos, &temp_cap, 0 ); + ctx->metadatas = dl_grow_array( &ctx->alloc, ctx->metadatas, &ctx->metadatas_cap, 0 ); + } + + return ctx->metadatas + ctx->metadatas_count++; +} + static void dl_set_member_size_and_align_from_builtin( dl_type_storage_t storage, dl_member_desc* member ) { switch( storage ) @@ -509,6 +525,43 @@ static void dl_context_load_txt_type_set_flags( dl_ctx_t ctx, dl_txt_read_ctx* r type->flags |= (uint32_t)DL_TYPE_FLAG_HAS_SUBDATA; } +static void dl_context_create_metadata( dl_ctx_t ctx, dl_txt_read_ctx* read_state, void** metadata ) +{ + char** start_end = (char**)*metadata; + read_state->iter = start_end[0]; + char last_char = *start_end[1]; + *start_end[1] = '\0'; + + size_t instance_size; + dl_error_t err = dl_txt_pack_calc_size(ctx, read_state->iter, &instance_size); + if (err != DL_ERROR_OK) + dl_txt_read_failed(ctx, read_state, err, "Failed to parse metadata"); + uint8_t* instance = (uint8_t*)dl_alloc(&ctx->alloc, instance_size); + + size_t produced_bytes; + err = dl_txt_pack(ctx, read_state->iter, instance, instance_size, &produced_bytes); + if (err != DL_ERROR_OK) + dl_txt_read_failed(ctx, read_state, err, "Failed to parse metadata"); + DL_ASSERT( instance_size == produced_bytes ); + + const dl_data_header* metadata_header = reinterpret_cast( instance ); + void* loaded_instance; + size_t consumed; + err = dl_instance_load_inplace( ctx, metadata_header->root_instance_type, instance, instance_size, &loaded_instance, &consumed ); + if( err != DL_ERROR_OK ) + dl_txt_read_failed( ctx, read_state, err, "Failed to pack metadata" ); + DL_ASSERT( instance_size == consumed ); + + size_t meta_index = metadata - ctx->metadatas; + DL_ASSERT( meta_index < ctx->metadatas_count ); + ctx->metadata_infos[meta_index] = loaded_instance; + ctx->metadata_typeinfos[meta_index] = metadata_header->root_instance_type; + + *start_end[1] = last_char; + dl_free( &ctx->alloc, *metadata ); + *metadata = (void*)instance; +} + const char* dl_txt_skip_map( const char* iter, const char* end ); const char* dl_txt_skip_string( const char* str, const char* end ); @@ -555,6 +608,25 @@ static uint64_t dl_context_load_txt_type_library_read_enum_value( dl_ctx_t ctx, } } +static void dl_context_load_txt_type_library_read_metadata( dl_ctx_t ctx, + dl_txt_read_ctx* read_state, + uint32_t* meta_data_record ) +{ + meta_data_record[1] = (uint32_t) ctx->metadatas_count; + dl_txt_eat_char( ctx, read_state, '[' ); + do + { + void** metadata = dl_alloc_metadata( ctx ); + const char** range = (const char**)dl_alloc( &ctx->alloc, 2 * sizeof( const char* ) ); + *metadata = range; + range[0] = read_state->iter; + read_state->iter = dl_txt_skip_map( read_state->iter, read_state->end ); + range[1] = read_state->iter; + } while( dl_txt_try_eat_char( read_state, ',' ) ); + dl_txt_eat_char( ctx, read_state, ']' ); + meta_data_record[0] = (uint32_t) (ctx->metadatas_count - meta_data_record[1]); +} + static void dl_context_load_txt_type_library_read_enum_values( dl_ctx_t ctx, dl_type_storage_t storage, dl_txt_read_ctx* read_state, @@ -567,6 +639,8 @@ static void dl_context_load_txt_type_library_read_enum_values( dl_ctx_t ctx, alias->value_index = (uint32_t)(value - ctx->enum_value_descs); value->main_alias = (uint32_t)(alias - ctx->enum_alias_descs); value->comment = 0xFFFFFFFF; + value->metadata_start = 0; + value->metadata_count = 0; if( *read_state->iter == '{' ) { @@ -605,8 +679,12 @@ static void dl_context_load_txt_type_library_read_enum_values( dl_ctx_t ctx, value->comment = dl_alloc_string(ctx, &comment); } + else if( strncmp( "metadata", key.str, 8 ) == 0 ) + { + dl_context_load_txt_type_library_read_metadata( ctx, read_state, &value->metadata_count ); + } else - dl_txt_read_failed( ctx, read_state, DL_ERROR_MALFORMED_DATA, "unexpected key '%.*s' in type, valid keys are 'value', 'aliases' or 'comment'", key.len, key.str ); + dl_txt_read_failed( ctx, read_state, DL_ERROR_MALFORMED_DATA, "unexpected key '%.*s' in type, valid keys are 'value', 'aliases', 'metadata' or 'comment'", key.len, key.str ); } while( dl_txt_try_eat_char( read_state, ',' ) ); @@ -625,6 +703,7 @@ static void dl_context_load_txt_type_library_find_enum_keys( dl_ctx_t ctx, dl_txt_read_ctx* read_state, dl_substr* name, dl_substr* comment, + uint32_t* meta_data_record, const char** values_iter, const char** type_iter, const char** end_iter, @@ -664,8 +743,12 @@ static void dl_context_load_txt_type_library_find_enum_keys( dl_ctx_t ctx, { *comment = dl_txt_eat_and_expect_string( ctx, read_state ); } + else if( strncmp( "metadata", key.str, 8 ) == 0 ) + { + dl_context_load_txt_type_library_read_metadata( ctx, read_state, &meta_data_record[0] ); + } else - dl_txt_read_failed( ctx, read_state, DL_ERROR_MALFORMED_DATA, "unexpected key '%.*s' in type, valid keys are 'values', 'type', 'comment and 'extern'", key.len, key.str ); + dl_txt_read_failed( ctx, read_state, DL_ERROR_MALFORMED_DATA, "unexpected key '%.*s' in type, valid keys are 'values', 'type', 'comment, 'metadata' and 'extern'", key.len, key.str ); } while( dl_txt_try_eat_char( read_state, ',') ); dl_txt_eat_char( ctx, read_state, '}' ); @@ -686,7 +769,8 @@ static void dl_context_load_txt_type_library_read_enum( dl_ctx_t ctx, dl_txt_rea const char* end_iter; dl_substr comment = { 0, 0 }; bool is_extern; - dl_context_load_txt_type_library_find_enum_keys(ctx, read_state, name, &comment, &values_iter, &type_iter, &end_iter, &is_extern); + uint32_t meta_data_record[2] = {0,0}; + dl_context_load_txt_type_library_find_enum_keys(ctx, read_state, name, &comment, meta_data_record, &values_iter, &type_iter, &end_iter, &is_extern); dl_type_storage_t storage = DL_TYPE_STORAGE_CNT; if(type_iter) @@ -752,13 +836,15 @@ static void dl_context_load_txt_type_library_read_enum( dl_ctx_t ctx, dl_txt_rea // TODO: add test for missing enum value ... dl_enum_desc* edesc = dl_alloc_enum(ctx, name); - edesc->flags = is_extern ? (uint32_t)DL_TYPE_FLAG_IS_EXTERNAL : 0; - edesc->storage = storage; - edesc->value_count = ctx->enum_value_count - value_start; - edesc->value_start = value_start; - edesc->alias_count = ctx->enum_alias_count - alias_start; /// number of aliases for this enum, always at least 1. Alias 0 is consider the "main name" of the value and need to be a valid c enum name. - edesc->alias_start = alias_start; /// offset into alias list where aliases for this enum-value start. - edesc->comment = comment.len > 0 ? dl_alloc_string( ctx, &comment ) : UINT32_MAX; + edesc->flags = is_extern ? (uint32_t)DL_TYPE_FLAG_IS_EXTERNAL : 0; + edesc->storage = storage; + edesc->value_count = ctx->enum_value_count - value_start; + edesc->value_start = value_start; + edesc->alias_count = ctx->enum_alias_count - alias_start; /// number of aliases for this enum, always at least 1. Alias 0 is consider the "main name" of the value and need to be a valid c enum name. + edesc->alias_start = alias_start; /// offset into alias list where aliases for this enum-value start. + edesc->comment = comment.len > 0 ? dl_alloc_string( ctx, &comment ) : UINT32_MAX; + edesc->metadata_count = meta_data_record[0]; + edesc->metadata_start = meta_data_record[1]; } static void dl_context_load_txt_type_library_read_c_includes( dl_ctx_t ctx, dl_txt_read_ctx* read_state ) @@ -976,6 +1062,7 @@ static void dl_context_load_txt_type_library_read_member( dl_ctx_t ctx, dl_txt_r dl_substr type = {0,0}; dl_substr comment = {0,0}; dl_substr default_val = {0,0}; + uint32_t meta_data_record[2] = {0,0}; bool is_const = true; bool verify = true; @@ -1030,8 +1117,12 @@ static void dl_context_load_txt_type_library_read_member( dl_ctx_t ctx, dl_txt_r { verify = dl_txt_eat_bool( read_state ) == 1; } + else if( strncmp( "metadata", key.str, 8 ) == 0 ) + { + dl_context_load_txt_type_library_read_metadata( ctx, read_state, &meta_data_record[0] ); + } else - dl_txt_read_failed( ctx, read_state, DL_ERROR_MALFORMED_DATA, "unexpected key '%.*s' in type, valid keys are 'name', 'type', 'default', 'comment', 'const' or 'verify'", key.len, key.str ); + dl_txt_read_failed( ctx, read_state, DL_ERROR_MALFORMED_DATA, "unexpected key '%.*s' in type, valid keys are 'name', 'type', 'default', 'comment', 'const', 'metadata' or 'verify'", key.len, key.str ); } while( dl_txt_try_eat_char( read_state, ',') ); @@ -1040,6 +1131,8 @@ static void dl_context_load_txt_type_library_read_member( dl_ctx_t ctx, dl_txt_r dl_txt_read_failed(ctx, read_state, DL_ERROR_MALFORMED_DATA, "No name on member in type."); member->name = dl_alloc_string( ctx, &name ); member->comment = comment.len > 0 ? dl_alloc_string( ctx, &comment ) : UINT32_MAX; + member->metadata_count = meta_data_record[0]; + member->metadata_start = meta_data_record[1]; dl_parse_type( ctx, &type, member, read_state ); if(default_val.str) @@ -1091,6 +1184,7 @@ static void dl_context_load_txt_type_library_read_type( dl_ctx_t ctx, dl_txt_rea uint32_t member_count = 0; uint32_t member_start = ctx->member_count; dl_substr comment = {0,0}; + uint32_t meta_data_record[2] = {0,0}; do { @@ -1120,9 +1214,13 @@ static void dl_context_load_txt_type_library_read_type( dl_ctx_t ctx, dl_txt_rea { comment = dl_txt_eat_and_expect_string( ctx, read_state ); } + else if( strncmp( "metadata", key.str, 8 ) == 0 ) + { + dl_context_load_txt_type_library_read_metadata( ctx, read_state, &meta_data_record[0] ); + } else dl_txt_read_failed( ctx, read_state, DL_ERROR_MALFORMED_DATA, - "unexpected key '%.*s' in type, valid keys are 'members', 'align', 'comment', 'extern' or 'verify'", + "unexpected key '%.*s' in type, valid keys are 'members', 'align', 'comment', 'extern', 'metadata' or 'verify'", key.len, key.str ); } while( dl_txt_try_eat_char( read_state, ',') ); @@ -1143,6 +1241,9 @@ static void dl_context_load_txt_type_library_read_type( dl_ctx_t ctx, dl_txt_rea type->comment = comment.len > 0 ? dl_alloc_string( ctx, &comment ) : UINT32_MAX; + type->metadata_count = meta_data_record[0]; + type->metadata_start = meta_data_record[1]; + if( is_extern ) type->flags |= (uint32_t)DL_TYPE_FLAG_IS_EXTERNAL; if( is_union ) @@ -1198,6 +1299,7 @@ static void dl_context_load_txt_type_library_inner( dl_ctx_t ctx, dl_txt_read_ct { uint32_t type_start = ctx->type_count; uint32_t member_start = ctx->member_count; + uint32_t metadata_start = (uint32_t) ctx->metadatas_count; dl_txt_eat_char( ctx, read_state, '{' ); @@ -1269,6 +1371,9 @@ static void dl_context_load_txt_type_library_inner( dl_ctx_t ctx, dl_txt_read_ct for( unsigned int i = type_start; i < ctx->type_count; ++i ) dl_context_load_txt_type_set_flags( ctx, read_state, ctx->type_descs + i ); + + for( unsigned int i = metadata_start; i < ctx->metadatas_count; ++i ) + dl_context_create_metadata(ctx, read_state, ctx->metadatas + i); } else { diff --git a/src/dl_typelib_write_bin.cpp b/src/dl_typelib_write_bin.cpp index a2fd8e9..5b2804c 100644 --- a/src/dl_typelib_write_bin.cpp +++ b/src/dl_typelib_write_bin.cpp @@ -19,6 +19,7 @@ dl_error_t dl_context_write_type_library( dl_ctx_t dl_ctx, unsigned char* out_li header.default_value_size = (uint32_t)dl_ctx->default_data_size; header.typeinfo_strings_size = (uint32_t)dl_ctx->typedata_strings_size; header.c_includes_size = (uint32_t)dl_ctx->c_includes_size; + header.metadatas_count = (uint32_t)dl_ctx->metadatas_count; dl_binary_writer_write( &writer, &header, sizeof( dl_typelib_header ) ); if(dl_ctx->type_count) dl_binary_writer_write( &writer, dl_ctx->type_ids, sizeof( dl_typeid_t ) * dl_ctx->type_count ); @@ -31,6 +32,27 @@ dl_error_t dl_context_write_type_library( dl_ctx_t dl_ctx, unsigned char* out_li if(dl_ctx->default_data_size) dl_binary_writer_write( &writer, dl_ctx->default_data, dl_ctx->default_data_size ); if(dl_ctx->typedata_strings_size) dl_binary_writer_write( &writer, dl_ctx->typedata_strings, dl_ctx->typedata_strings_size ); if(dl_ctx->c_includes_size) dl_binary_writer_write( &writer, dl_ctx->c_includes, dl_ctx->c_includes_size ); + uint32_t metadata_offset = 0; + for( unsigned int i = 0; i < dl_ctx->metadatas_count; ++i ) + { + dl_binary_writer_write_uint32( &writer, metadata_offset ); + const dl_data_header* metadata_header = (const dl_data_header*)dl_ctx->metadatas[i]; + metadata_offset = metadata_offset + uint32_t( metadata_header->instance_size + sizeof( dl_data_header ) ); + } + for( unsigned int i = 0; i < dl_ctx->metadatas_count; ++i ) + { + const dl_data_header* metadata_header = (const dl_data_header*)dl_ctx->metadatas[i]; + size_t size; + dl_error_t err = dl_instance_calc_size( dl_ctx, metadata_header->root_instance_type, (void*)(metadata_header + 1), &size ); + DL_ASSERT( DL_ERROR_OK == err ); + uint8_t* temp_buffer = (uint8_t*) dl_alloc( &dl_ctx->alloc, size ); + size_t produced; + err = dl_instance_store( dl_ctx, metadata_header->root_instance_type, (void*)(metadata_header + 1), temp_buffer, size, &produced ); + DL_ASSERT( produced == size ); + DL_ASSERT( DL_ERROR_OK == err ); + dl_binary_writer_write( &writer, temp_buffer, produced ); + dl_free( &dl_ctx->alloc, temp_buffer ); + } // ... write default data ... diff --git a/src/dl_typelib_write_c_header.cpp b/src/dl_typelib_write_c_header.cpp index ff1329d..c5f591c 100644 --- a/src/dl_typelib_write_c_header.cpp +++ b/src/dl_typelib_write_c_header.cpp @@ -792,7 +792,7 @@ static dl_error_t dl_context_write_c_header_types( dl_binary_writer* writer, dl_ dl_binary_writer_write_string_fmt( writer, "struct %s\n{\n", type->name ); dl_binary_writer_write_string_fmt( writer, "#if defined( __cplusplus )\n" - " static const uint32_t TYPE_ID = 0x%08X;\n" + " static constexpr const uint32_t TYPE_ID = 0x%08X;\n" "#endif // defined( __cplusplus )\n\n", type->tid ); if( type->is_union ) diff --git a/src/dl_typelib_write_txt.cpp b/src/dl_typelib_write_txt.cpp index e6c801e..6e39d8a 100644 --- a/src/dl_typelib_write_txt.cpp +++ b/src/dl_typelib_write_txt.cpp @@ -1,4 +1,5 @@ #include
+#include
#include "dl_binary_writer.h" #include
@@ -40,7 +41,14 @@ static const char* dl_context_type_to_string( dl_ctx_t ctx, dl_type_storage_t st } } -static void dl_binary_writer_write_fmt( dl_binary_writer* writer, const char* fmt, ... ) // add check of fmt string +#if defined( __GNUC__ ) || defined( __clang__ ) +# ifndef _Printf_format_string_ +# define _Printf_format_string_ +# endif +static void dl_binary_writer_write_fmt( dl_binary_writer* writer, const char* fmt, ... ) __attribute__( ( __format__( __printf__, 2, 3 ) ) ); +#endif + +static void dl_binary_writer_write_fmt( dl_binary_writer* writer, _Printf_format_string_ const char* fmt, ... ) { char buffer[2048]; va_list args; @@ -66,7 +74,7 @@ static dl_error_t dl_context_write_txt_enum( dl_ctx_t ctx, dl_binary_writer* wri for( unsigned int j = 0; j < enum_info.value_count; ++j ) { - dl_binary_writer_write_fmt( writer, " \"%s\" : %u", values[j].name, values[j].value ); + dl_binary_writer_write_fmt( writer, " \"%s\" : %lu", values[j].name, values[j].value.u64 ); if( j < enum_info.value_count - 1 ) dl_binary_writer_write( writer, ",\n", 2 ); else @@ -75,7 +83,42 @@ static dl_error_t dl_context_write_txt_enum( dl_ctx_t ctx, dl_binary_writer* wri free( values ); - dl_binary_writer_write_fmt( writer, " }\n }", 1 ); + dl_binary_writer_write_fmt( writer, " }" ); + + if( enum_info.metadata_count ) + { + dl_binary_writer_write_fmt( writer, ",\n \"metadata\" : [\n" ); + for( uint32_t i = 0; i < enum_info.metadata_count; ++i ) + { + size_t produced_bytes = 0; + + // jumping through hoops, pack the data and write it as json with wrong indent + size_t packed_size; + err = dl_instance_calc_size( ctx, enum_info.metadata_type_ids[i], enum_info.metadata_instances[i], &packed_size ); + if( DL_ERROR_OK != err ) return err; + uint8_t* packed_instance = (uint8_t*)dl_alloc( &ctx->alloc, packed_size ); + err = dl_instance_store( ctx, enum_info.metadata_type_ids[i], enum_info.metadata_instances[i], packed_instance, packed_size, &produced_bytes ); + if( DL_ERROR_OK != err ) return err; + + size_t txt_instance_size; + err = dl_txt_unpack_calc_size( ctx, enum_info.metadata_type_ids[i], packed_instance, packed_size, &txt_instance_size ); + if( DL_ERROR_OK != err ) return err; + char* text_instance = (char*)dl_alloc( &ctx->alloc, txt_instance_size ); + err = dl_txt_unpack( ctx, enum_info.metadata_type_ids[i], packed_instance, packed_size, text_instance, txt_instance_size, &produced_bytes ); + if( DL_ERROR_OK != err ) return err; + dl_free( &ctx->alloc, packed_instance ); + + dl_binary_writer_write( writer, text_instance, txt_instance_size - 1 ); + dl_free( &ctx->alloc, text_instance ); + if( i < enum_info.metadata_count - 1 ) + dl_binary_writer_write( writer, ",\n", 2 ); + else + dl_binary_writer_write( writer, "\n", 1 ); + } + dl_binary_writer_write_fmt( writer, " ]" ); + } + + dl_binary_writer_write_fmt( writer, "\n }" ); return DL_ERROR_OK; } @@ -106,7 +149,7 @@ static dl_error_t dl_context_write_txt_enums( dl_ctx_t ctx, dl_binary_writer* wr return DL_ERROR_OK; } -static dl_error_t dl_context_write_txt_member( dl_ctx_t ctx, dl_binary_writer* writer, dl_member_info_t* member ) +static dl_error_t dl_context_write_txt_member( dl_ctx_t ctx, dl_binary_writer* writer, const dl_member_info_t* member ) { dl_binary_writer_write_fmt( writer, " { \"name\" : \"%s\", ", member->name ); @@ -133,8 +176,41 @@ static dl_error_t dl_context_write_txt_member( dl_ctx_t ctx, dl_binary_writer* w dl_context_type_to_string( ctx, member->storage, member->type_id ), member->array_count ); break; - default: - DL_ASSERT( false ); + default: + DL_ASSERT( false ); + } + + if( member->metadata_count ) + { + dl_binary_writer_write_fmt( writer, ",\n \"metadata\" : [\n" ); + for( uint32_t i = 0; i < member->metadata_count; ++i ) + { + size_t produced_bytes = 0; + + // jumping through hoops, pack the data and write it as json with wrong indent + size_t packed_size; + dl_error_t err = dl_instance_calc_size( ctx, member->metadata_type_ids[i], member->metadata_instances[i], &packed_size ); + if( DL_ERROR_OK != err ) return err; + uint8_t* packed_instance = (uint8_t*)dl_alloc( &ctx->alloc, packed_size ); + err = dl_instance_store( ctx, member->metadata_type_ids[i], member->metadata_instances[i], packed_instance, packed_size, &produced_bytes ); + if( DL_ERROR_OK != err ) return err; + + size_t txt_instance_size; + err = dl_txt_unpack_calc_size( ctx, member->metadata_type_ids[i], packed_instance, packed_size, &txt_instance_size ); + if( DL_ERROR_OK != err ) return err; + char* text_instance = (char*)dl_alloc( &ctx->alloc, txt_instance_size ); + err = dl_txt_unpack( ctx, member->metadata_type_ids[i], packed_instance, packed_size, text_instance, txt_instance_size, &produced_bytes ); + if( DL_ERROR_OK != err ) return err; + dl_free( &ctx->alloc, packed_instance ); + + dl_binary_writer_write( writer, text_instance, txt_instance_size - 1 ); + dl_free( &ctx->alloc, text_instance ); + if( i < member->metadata_count - 1 ) + dl_binary_writer_write( writer, ",\n", 2 ); + else + dl_binary_writer_write( writer, "\n", 1 ); + } + dl_binary_writer_write_fmt( writer, " ]" ); } dl_binary_writer_write( writer, " }", 2 ); return DL_ERROR_OK; @@ -162,8 +238,41 @@ static dl_error_t dl_context_write_txt_type( dl_ctx_t ctx, dl_binary_writer* wri else dl_binary_writer_write( writer, "\n", 1 ); } + dl_binary_writer_write_fmt( writer, " ]" ); - dl_binary_writer_write_fmt( writer, " ]\n }" ); + if( type_info.metadata_count ) + { + dl_binary_writer_write_fmt( writer, ",\n \"metadata\" : [\n" ); + for( uint32_t i = 0; i < type_info.metadata_count; ++i ) + { + size_t produced_bytes = 0; + + // jumping through hoops, pack the data and write it as json with wrong indent + size_t packed_size; + err = dl_instance_calc_size( ctx, type_info.metadata_type_ids[i], type_info.metadata_instances[i], &packed_size ); + if( DL_ERROR_OK != err ) return err; + uint8_t* packed_instance = (uint8_t*)dl_alloc( &ctx->alloc, packed_size ); + err = dl_instance_store( ctx, type_info.metadata_type_ids[i], type_info.metadata_instances[i], packed_instance, packed_size, &produced_bytes ); + if( DL_ERROR_OK != err ) return err; + + size_t txt_instance_size; + err = dl_txt_unpack_calc_size( ctx, type_info.metadata_type_ids[i], packed_instance, packed_size, &txt_instance_size ); + if( DL_ERROR_OK != err ) return err; + char* text_instance = (char*)dl_alloc( &ctx->alloc, txt_instance_size ); + err = dl_txt_unpack( ctx, type_info.metadata_type_ids[i], packed_instance, packed_size, text_instance, txt_instance_size, &produced_bytes ); + if( DL_ERROR_OK != err ) return err; + dl_free( &ctx->alloc, packed_instance ); + + dl_binary_writer_write( writer, text_instance, txt_instance_size - 1 ); + dl_free( &ctx->alloc, text_instance ); + if( i < type_info.metadata_count - 1 ) + dl_binary_writer_write( writer, ",\n", 2 ); + else + dl_binary_writer_write( writer, "\n", 1 ); + } + dl_binary_writer_write_fmt( writer, " ]" ); + } + dl_binary_writer_write_fmt( writer, "\n }" ); free( members ); return DL_ERROR_OK; } diff --git a/src/dl_types.h b/src/dl_types.h index d41fabc..35d1e57 100755 --- a/src/dl_types.h +++ b/src/dl_types.h @@ -47,7 +47,7 @@ #define DL_UNUSED #endif -static const uint32_t DL_UNUSED DL_TYPELIB_VERSION = 4; // format version for type-libraries. +static const uint32_t DL_UNUSED DL_TYPELIB_VERSION = 5; // format version for type-libraries. static const uint32_t DL_UNUSED DL_INSTANCE_VERSION = 1; // format version for instances. static const uint32_t DL_UNUSED DL_INSTANCE_VERSION_SWAPED = dl_swap_endian_uint32( DL_INSTANCE_VERSION ); static const uint32_t DL_UNUSED DL_TYPELIB_ID = ('D'<< 24) | ('L' << 16) | ('T' << 8) | 'L'; @@ -107,6 +107,7 @@ struct dl_typelib_header uint32_t default_value_size; uint32_t typeinfo_strings_size; uint32_t c_includes_size; + uint32_t metadatas_count; }; struct dl_data_header @@ -150,6 +151,8 @@ struct dl_member_desc uint32_t default_value_offset; // if M_UINT32_MAX, default value is not present, otherwise offset into default-value-data. uint32_t default_value_size; uint32_t flags; + uint32_t metadata_count; + uint32_t metadata_start; dl_type_atom_t AtomType() const { return dl_type_atom_t( (type & DL_TYPE_ATOM_MASK) >> DL_TYPE_ATOM_MIN_BIT); } dl_type_storage_t StorageType() const { return dl_type_storage_t( (type & DL_TYPE_STORAGE_MASK) >> DL_TYPE_STORAGE_MIN_BIT); } @@ -241,7 +244,8 @@ struct dl_type_desc uint32_t member_count; uint32_t member_start; uint32_t comment; - + uint32_t metadata_count; + uint32_t metadata_start; }; struct dl_enum_value_desc @@ -249,6 +253,8 @@ struct dl_enum_value_desc uint32_t main_alias; uint32_t comment; uint64_t value; + uint32_t metadata_count; + uint32_t metadata_start; }; struct dl_enum_desc @@ -261,6 +267,8 @@ struct dl_enum_desc uint32_t alias_count; /// number of aliases for this enum, always at least 1. Alias 0 is consider the "main name" of the value and need to be a valid c enum name. uint32_t alias_start; /// offset into alias list where aliases for this enum-value start. uint32_t comment; + uint32_t metadata_count; + uint32_t metadata_start; }; struct dl_enum_alias_desc @@ -281,6 +289,7 @@ struct dl_context unsigned int member_count; unsigned int enum_value_count; unsigned int enum_alias_count; + unsigned int metadatas_count; size_t type_capacity; size_t enum_capacity; @@ -305,6 +314,11 @@ struct dl_context size_t c_includes_size; size_t c_includes_cap; + void** metadatas; + size_t metadatas_cap; + const void** metadata_infos; + dl_typeid_t* metadata_typeinfos; + uint8_t* default_data; size_t default_data_size; }; @@ -315,6 +329,12 @@ struct dl_substr int len; }; +#ifdef _MSC_VER +# define DL_FORCEINLINE __forceinline +#else +# define DL_FORCEINLINE inline __attribute__( ( always_inline ) ) +#endif + // A growable array using a stack buffer while small. Use it to avoid dynamic allocations while the stack is big enough, but fall back to heap if it grows past the wanted stack size template class CArrayStatic @@ -381,12 +401,12 @@ class CArrayStatic m_nElements++; } - T& operator[](size_t _iEl) + DL_FORCEINLINE T& operator[]( size_t _iEl ) { DL_ASSERT(_iEl < m_nElements && "Index out of bound"); return m_Ptr[_iEl]; } - const T& operator[](size_t _iEl) const + DL_FORCEINLINE const T& operator[]( size_t _iEl ) const { DL_ASSERT(_iEl < m_nElements && "Index out of bound"); return m_Ptr[_iEl]; @@ -394,10 +414,11 @@ class CArrayStatic }; #if defined( __GNUC__ ) +# define _Printf_format_string_ inline void dl_log_error( dl_ctx_t dl_ctx, const char* fmt, ... ) __attribute__((format( printf, 2, 3 ))); #endif -inline void dl_log_error( dl_ctx_t dl_ctx, const char* fmt, ... ) +inline void dl_log_error( dl_ctx_t dl_ctx, _Printf_format_string_ const char* fmt, ... ) { if( dl_ctx->error_msg_func == 0x0 ) return; diff --git a/src/dl_util.cpp b/src/dl_util.cpp old mode 100644 new mode 100755 index 9c9b089..8ec0975 --- a/src/dl_util.cpp +++ b/src/dl_util.cpp @@ -9,7 +9,7 @@ #include #include -static void patch_alloc_funcs( dl_alloc_func& alloc_func, dl_realloc_func& realloc_func, dl_free_func& free_func ) +static void dl_patch_alloc_funcs( dl_alloc_func& alloc_func, dl_realloc_func& realloc_func, dl_free_func& free_func ) { if (alloc_func == nullptr) alloc_func = [](size_t size, void*) -> void* { return malloc(size); }; @@ -159,7 +159,7 @@ dl_error_t dl_util_load_from_file( dl_ctx_t dl_ctx, dl_typeid_t if( in_file != 0x0 ) { dl_realloc_func realloc_func = 0; - patch_alloc_funcs( alloc_func, realloc_func, free_func ); + dl_patch_alloc_funcs( alloc_func, realloc_func, free_func ); fseek(in_file, 0, SEEK_END); size_t size = static_cast(ftell(in_file)); @@ -183,7 +183,7 @@ dl_error_t dl_util_load_from_stream( dl_ctx_t dl_ctx, dl_typeid_t dl_alloc_func alloc_func, dl_realloc_func realloc_func, dl_free_func free_func, void* alloc_ctx ) { - patch_alloc_funcs( alloc_func, realloc_func, free_func ); + dl_patch_alloc_funcs( alloc_func, realloc_func, free_func ); unsigned char* file_content = dl_read_entire_stream( realloc_func, alloc_ctx, stream, consumed_bytes ); file_content[*consumed_bytes] = '\0'; @@ -227,7 +227,7 @@ dl_error_t dl_util_store_to_stream( dl_ctx_t dl_ctx, dl_typeid_t return error; dl_realloc_func realloc_func = 0; - patch_alloc_funcs( alloc_func, realloc_func, free_func ); + dl_patch_alloc_funcs( alloc_func, realloc_func, free_func ); // alloc memory unsigned char* packed_instance = (unsigned char*)alloc_func( packed_size, alloc_ctx ); diff --git a/tests/dl_tests_error.cpp b/tests/dl_tests_error.cpp index 116afc4..37ec6f8 100644 --- a/tests/dl_tests_error.cpp +++ b/tests/dl_tests_error.cpp @@ -78,7 +78,7 @@ TEST_F(DLError, typelib_version_mismatch_returned) // testing that errors are returned correctly by modding data. uint32_t* lib_version = modded_type_lib + 1; - EXPECT_EQ(4u, *lib_version); + EXPECT_EQ(5u, *lib_version); *lib_version = 0xFFFFFFFF; diff --git a/tests/dl_tests_metadata.cpp b/tests/dl_tests_metadata.cpp new file mode 100644 index 0000000..892eedd --- /dev/null +++ b/tests/dl_tests_metadata.cpp @@ -0,0 +1,76 @@ +#include +#include
+#include "dl_tests_base.h" + +TYPED_TEST( DLBase, metadata_type ) +{ + dl_type_info_t type_info; + EXPECT_DL_ERR_OK( dl_reflect_get_type_info( DL::Ctx, type_with_meta::TYPE_ID, &type_info ) ); + EXPECT_EQ( 2U, type_info.metadata_count ); + EXPECT_EQ( SubString_TYPE_ID, type_info.metadata_type_ids[0] ); + EXPECT_STREQ( "apa", reinterpret_cast( type_info.metadata_instances[0] )->Str ); + EXPECT_EQ( SubString_TYPE_ID, type_info.metadata_type_ids[1] ); + EXPECT_STREQ( "banan", reinterpret_cast( type_info.metadata_instances[1] )->Str ); +} + +TYPED_TEST( DLBase, metadata_member ) +{ + dl_type_info_t type_info; + EXPECT_DL_ERR_OK( dl_reflect_get_type_info( DL::Ctx, type_with_meta::TYPE_ID, &type_info ) ); + EXPECT_EQ( 1U, type_info.member_count ); + dl_member_info_t members; + EXPECT_DL_ERR_OK( dl_reflect_get_type_members( DL::Ctx, type_with_meta::TYPE_ID, &members, 1 ) ); + EXPECT_EQ( 2U, members.metadata_count ); + EXPECT_EQ( SubString_TYPE_ID, members.metadata_type_ids[0] ); + EXPECT_STREQ( "apa", reinterpret_cast( members.metadata_instances[0] )->Str ); + EXPECT_EQ( SubString_TYPE_ID, members.metadata_type_ids[1] ); + EXPECT_STREQ( "banan", reinterpret_cast( members.metadata_instances[1] )->Str ); +} + +TYPED_TEST( DLBase, metadata_enumeration ) +{ + dl_enum_info_t enum_info; + EXPECT_DL_ERR_OK( dl_reflect_get_enum_info( DL::Ctx, EnumWithMeta_TYPE_ID, &enum_info ) ); + EXPECT_EQ( 2U, enum_info.metadata_count ); + EXPECT_EQ( SubString_TYPE_ID, enum_info.metadata_type_ids[0] ); + EXPECT_STREQ( "apa", reinterpret_cast( enum_info.metadata_instances[0] )->Str ); + EXPECT_EQ( SubString_TYPE_ID, enum_info.metadata_type_ids[1] ); + EXPECT_STREQ( "banan", reinterpret_cast( enum_info.metadata_instances[1] )->Str ); +} + +TYPED_TEST( DLBase, metadata_enumerator ) +{ + dl_enum_info_t enum_info; + EXPECT_DL_ERR_OK( dl_reflect_get_enum_info( DL::Ctx, EnumWithMeta_TYPE_ID, &enum_info ) ); + EXPECT_EQ( 1U, enum_info.value_count ); + dl_enum_value_info_t values; + EXPECT_DL_ERR_OK( dl_reflect_get_enum_values( DL::Ctx, EnumWithMeta_TYPE_ID, &values, 1 ) ); + EXPECT_EQ( 2U, values.metadata_count ); + EXPECT_EQ( SubString_TYPE_ID, values.metadata_type_ids[0] ); + EXPECT_STREQ( "apa", reinterpret_cast( values.metadata_instances[0] )->Str ); + EXPECT_EQ( SubString_TYPE_ID, values.metadata_type_ids[1] ); + EXPECT_STREQ( "banan", reinterpret_cast( values.metadata_instances[1] )->Str ); +} + +TYPED_TEST( DLBase, metadata_union ) +{ + dl_type_info_t type_info; + EXPECT_DL_ERR_OK( dl_reflect_get_type_info( DL::Ctx, test_union_simple::TYPE_ID, &type_info ) ); + EXPECT_EQ( 2U, type_info.metadata_count ); + EXPECT_EQ( SubString_TYPE_ID, type_info.metadata_type_ids[0] ); + EXPECT_STREQ( "apa", reinterpret_cast( type_info.metadata_instances[0] )->Str ); + EXPECT_EQ( SubString_TYPE_ID, type_info.metadata_type_ids[1] ); + EXPECT_STREQ( "banan", reinterpret_cast( type_info.metadata_instances[1] )->Str ); +} + +TYPED_TEST( DLBase, metadata_union_member ) +{ + dl_type_info_t type_info; + EXPECT_DL_ERR_OK( dl_reflect_get_type_info( DL::Ctx, test_union_simple::TYPE_ID, &type_info ) ); + EXPECT_EQ( 3U, type_info.member_count ); + dl_member_info_t members[3]; + EXPECT_DL_ERR_OK( dl_reflect_get_type_members( DL::Ctx, test_union_simple::TYPE_ID, members, DL_ARRAY_LENGTH(members) ) ); + EXPECT_EQ( 1U, members[2].metadata_count ); + EXPECT_EQ( SubString_TYPE_ID, members[2].metadata_type_ids[0] ); + EXPECT_STREQ( "item3", reinterpret_cast( members[2].metadata_instances[0] )->Str ); +} diff --git a/tests/unittest.tld b/tests/unittest.tld index e163851..7a5ac85 100755 --- a/tests/unittest.tld +++ b/tests/unittest.tld @@ -27,6 +27,23 @@ "MULTI_ALIAS1" : { "value" : 7, "aliases" : [ "alias1", "alias2", "alias3" ] }, "MULTI_ALIAS2" : { "value" : 8, "aliases" : [ "alias4" ] } } + }, + + "EnumWithMeta" : { + "values" : { + "TESTENUMWITHMETA" : { + "value" : 1337, + "comment" : "whoppa!", + "metadata" : [ + { "SubString" : { "Str": "apa" } }, + { "SubString" : { "Str": "banan" } } + ] + } + }, + "metadata" : [ + { "SubString" : { "Str": "apa" } }, + { "SubString" : { "Str": "banan" } } + ] } }, @@ -61,7 +78,23 @@ "strArray" : { "members" : [ { "name" : "arr", "type" : "string[]" } ] }, "ptrArray" : { "members" : [ { "name" : "arr", "type" : "Pods*[]" } ] }, "enumArray" : { "members" : [ { "name" : "arr", "type" : "TestEnum1[]" } ] }, - "default_empty_type" : { "members" : [ { "name" : "empty", "type" : "uint32", "default" : 0 } ] } + "default_empty_type" : { "members" : [ { "name" : "empty", "type" : "uint32", "default" : 0 } ] }, + "type_with_meta" : { + "members" : [ + { + "name" : "i8", + "type" : "int8", + "metadata" : [ + { "SubString" : { "Str": "apa" } }, + { "SubString" : { "Str": "banan" } } + ] + } + ], + "metadata" : [ + { "SubString" : { "Str": "apa" } }, + { "SubString" : { "Str": "banan" } } + ] + } }, "unions" : { @@ -69,7 +102,11 @@ "members" : [ { "name" : "item1", "type" : "int32" }, { "name" : "item2", "type" : "fp32" }, - { "name" : "item3", "type" : "Pods" } + { "name" : "item3", "type" : "Pods", "metadata" : [ { "SubString" : { "Str": "item3" } } ] } + ], + "metadata" : [ + { "SubString" : { "Str": "apa" } }, + { "SubString" : { "Str": "banan" } } ] } }, diff --git a/tool/dlpack/dlpack.cpp b/tool/dlpack/dlpack.cpp old mode 100644 new mode 100755 index a2234a9..dd6a8df --- a/tool/dlpack/dlpack.cpp +++ b/tool/dlpack/dlpack.cpp @@ -242,8 +242,8 @@ int main( int argc, const char** argv ) out_ptr_size, instance, 0x0, - 0x0, - 0x0 ); + 0x0, + 0x0 ); if( err != DL_ERROR_OK ) M_ERROR_AND_QUIT( "DL error writing stream: %s", dl_error_to_string( err ) );