diff --git a/.gitignore b/.gitignore index 95f1422..92c976e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,7 @@ Manifest.toml coverage *.cov +*.mem + docs/build docs/Manifest.toml diff --git a/Project.toml b/Project.toml index 8824df1..11ab40c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TimeSeriesEcon" uuid = "8b6756d2-c55c-11ea-2998-5f67ea17da60" authors = ["Atai Akunov ", "Boyan Bejanov ", "Nicholas Labelle St-Pierre "] -version = "0.6.1" +version = "0.6.2" [deps] CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82" @@ -11,6 +11,7 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" @@ -19,7 +20,7 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [compat] CEnum = "0.4" -DataEcon_jll = "0.2.2" +DataEcon_jll = "0.3.1" MacroTools = "0.5" OrderedCollections = "1.4" RecipesBase = "1.0" diff --git a/src/dataecon/C.jl b/src/dataecon/C.jl index e7ad9cc..3769e02 100644 --- a/src/dataecon/C.jl +++ b/src/dataecon/C.jl @@ -5,6 +5,10 @@ export DataEcon_jll using CEnum +function de_version() + ccall((:de_version, libdaec), Ptr{Cchar}, ()) +end + function de_error(msg, len) ccall((:de_error, libdaec), Cint, (Ptr{Cchar}, Csize_t), msg, len) end @@ -23,18 +27,24 @@ end DE_BAD_AXIS_TYPE = -999 DE_BAD_CLASS = -998 DE_BAD_TYPE = -997 - DE_BAD_NAME = -996 - DE_SHORT_BUF = -995 - DE_OBJ_DNE = -994 - DE_AXIS_DNE = -993 - DE_ARG = -992 - DE_NO_OBJ = -991 - DE_EXISTS = -990 - DE_BAD_OBJ = -989 - DE_NULL = -988 - DE_DEL_ROOT = -987 - DE_MIS_ATTR = -986 - DE_INTERNAL = -985 + DE_BAD_ELTYPE = -996 + DE_BAD_ELTYPE_NONE = -995 + DE_BAD_ELTYPE_DATE = -994 + DE_BAD_NAME = -993 + DE_BAD_FREQ = -992 + DE_SHORT_BUF = -991 + DE_OBJ_DNE = -990 + DE_AXIS_DNE = -989 + DE_ARG = -988 + DE_NO_OBJ = -987 + DE_EXISTS = -986 + DE_BAD_OBJ = -985 + DE_NULL = -984 + DE_DEL_ROOT = -983 + DE_MIS_ATTR = -982 + DE_INEXACT = -981 + DE_RANGE = -980 + DE_INTERNAL = -979 end const de_file = Ptr{Cvoid} @@ -43,6 +53,14 @@ function de_open(fname, de) ccall((:de_open, libdaec), Cint, (Ptr{Cchar}, Ptr{de_file}), fname, de) end +function de_open_readonly(fname, de) + ccall((:de_open_readonly, libdaec), Cint, (Ptr{Cchar}, Ptr{de_file}), fname, de) +end + +function de_open_memory(pde) + ccall((:de_open_memory, libdaec), Cint, (Ptr{de_file},), pde) +end + function de_close(de) ccall((:de_close, libdaec), Cint, (de_file,), de) end @@ -81,70 +99,13 @@ end type_any = -1 end -@cenum frequency_t::UInt32 begin - freq_none = 0 - freq_unit = 1 - freq_daily = 4 - freq_bdaily = 5 - freq_monthly = 8 - freq_weekly = 16 - freq_weekly_sun0 = 16 - freq_weekly_mon = 17 - freq_weekly_tue = 18 - freq_weekly_wed = 19 - freq_weekly_thu = 20 - freq_weekly_fri = 21 - freq_weekly_sat = 22 - freq_weekly_sun7 = 23 - freq_weekly_sun = 23 - freq_quarterly = 32 - freq_quarterly_jan = 33 - freq_quarterly_feb = 34 - freq_quarterly_mar = 35 - freq_quarterly_apr = 33 - freq_quarterly_may = 34 - freq_quarterly_jun = 35 - freq_quarterly_jul = 33 - freq_quarterly_aug = 34 - freq_quarterly_sep = 35 - freq_quarterly_oct = 33 - freq_quarterly_nov = 34 - freq_quarterly_dec = 35 - freq_halfyearly = 64 - freq_halfyearly_jan = 65 - freq_halfyearly_feb = 66 - freq_halfyearly_mar = 67 - freq_halfyearly_apr = 68 - freq_halfyearly_may = 69 - freq_halfyearly_jun = 70 - freq_halfyearly_jul = 65 - freq_halfyearly_aug = 66 - freq_halfyearly_sep = 67 - freq_halfyearly_oct = 68 - freq_halfyearly_nov = 69 - freq_halfyearly_dec = 70 - freq_yearly = 128 - freq_yearly_jan = 129 - freq_yearly_feb = 130 - freq_yearly_mar = 131 - freq_yearly_apr = 132 - freq_yearly_may = 133 - freq_yearly_jun = 134 - freq_yearly_jul = 135 - freq_yearly_aug = 136 - freq_yearly_sep = 137 - freq_yearly_oct = 138 - freq_yearly_nov = 139 - freq_yearly_dec = 140 -end - const obj_id_t = Int64 struct object_t id::obj_id_t pid::obj_id_t - class::class_t - type::type_t + obj_class::class_t + obj_type::type_t name::Ptr{Cchar} end @@ -180,10 +141,89 @@ function de_find_fullpath(de, fullpath, id) ccall((:de_find_fullpath, libdaec), Cint, (de_file, Ptr{Cchar}, Ptr{obj_id_t}), de, fullpath, id) end +function de_catalog_size(de, pid, count) + ccall((:de_catalog_size, libdaec), Cint, (de_file, obj_id_t, Ptr{Int64}), de, pid, count) +end + function de_new_catalog(de, pid, name, id) ccall((:de_new_catalog, libdaec), Cint, (de_file, obj_id_t, Ptr{Cchar}, Ptr{obj_id_t}), de, pid, name, id) end +@cenum frequency_t::UInt32 begin + freq_none = 0 + freq_unit = 11 + freq_daily = 12 + freq_bdaily = 13 + freq_weekly = 16 + freq_weekly_sun0 = 16 + freq_weekly_mon = 17 + freq_weekly_tue = 18 + freq_weekly_wed = 19 + freq_weekly_thu = 20 + freq_weekly_fri = 21 + freq_weekly_sat = 22 + freq_weekly_sun7 = 23 + freq_weekly_sun = 23 + freq_monthly = 32 + freq_quarterly = 64 + freq_quarterly_jan = 65 + freq_quarterly_feb = 66 + freq_quarterly_mar = 67 + freq_quarterly_apr = 65 + freq_quarterly_may = 66 + freq_quarterly_jun = 67 + freq_quarterly_jul = 65 + freq_quarterly_aug = 66 + freq_quarterly_sep = 67 + freq_quarterly_oct = 65 + freq_quarterly_nov = 66 + freq_quarterly_dec = 67 + freq_halfyearly = 128 + freq_halfyearly_jan = 129 + freq_halfyearly_feb = 130 + freq_halfyearly_mar = 131 + freq_halfyearly_apr = 132 + freq_halfyearly_may = 133 + freq_halfyearly_jun = 134 + freq_halfyearly_jul = 129 + freq_halfyearly_aug = 130 + freq_halfyearly_sep = 131 + freq_halfyearly_oct = 132 + freq_halfyearly_nov = 133 + freq_halfyearly_dec = 134 + freq_yearly = 256 + freq_yearly_jan = 257 + freq_yearly_feb = 258 + freq_yearly_mar = 259 + freq_yearly_apr = 260 + freq_yearly_may = 261 + freq_yearly_jun = 262 + freq_yearly_jul = 263 + freq_yearly_aug = 264 + freq_yearly_sep = 265 + freq_yearly_oct = 266 + freq_yearly_nov = 267 + freq_yearly_dec = 268 +end + +const date_t = Int64 + +function de_pack_year_period_date(freq, year, period, date) + ccall((:de_pack_year_period_date, libdaec), Cint, (frequency_t, Int32, UInt32, Ptr{date_t}), freq, year, period, date) +end + +function de_unpack_year_period_date(freq, date, year, period) + ccall((:de_unpack_year_period_date, libdaec), Cint, (frequency_t, date_t, Ptr{Int32}, Ptr{UInt32}), freq, date, year, period) +end + +function de_pack_calendar_date(freq, year, month, day, date) + ccall((:de_pack_calendar_date, libdaec), Cint, (frequency_t, Int32, UInt32, UInt32, Ptr{date_t}), freq, year, month, day, date) +end + +function de_unpack_calendar_date(freq, date, year, month, day) + ccall((:de_unpack_calendar_date, libdaec), Cint, (frequency_t, date_t, Ptr{Int32}, Ptr{UInt32}, Ptr{UInt32}), freq, date, year, month, day) +end + struct scalar_t object::object_t frequency::frequency_t @@ -209,7 +249,7 @@ end struct axis_t id::axis_id_t - type::axis_type_t + ax_type::axis_type_t length::Int64 frequency::frequency_t first::Int64 @@ -235,6 +275,7 @@ end struct tseries_t object::object_t eltype::type_t + elfreq::frequency_t axis::axis_t nbytes::Int64 value::Ptr{Cvoid} @@ -242,8 +283,8 @@ end const vector_t = tseries_t -function de_store_tseries(de, pid, name, type, eltype, axis_id, nbytes, value, id) - ccall((:de_store_tseries, libdaec), Cint, (de_file, obj_id_t, Ptr{Cchar}, type_t, type_t, axis_id_t, Int64, Ptr{Cvoid}, Ptr{obj_id_t}), de, pid, name, type, eltype, axis_id, nbytes, value, id) +function de_store_tseries(de, pid, name, type, eltype, elfreq, axis_id, nbytes, value, id) + ccall((:de_store_tseries, libdaec), Cint, (de_file, obj_id_t, Ptr{Cchar}, type_t, type_t, frequency_t, axis_id_t, Int64, Ptr{Cvoid}, Ptr{obj_id_t}), de, pid, name, type, eltype, elfreq, axis_id, nbytes, value, id) end function de_load_tseries(de, id, tseries) @@ -253,6 +294,7 @@ end struct mvtseries_t object::object_t eltype::type_t + elfreq::frequency_t axis1::axis_t axis2::axis_t nbytes::Int64 @@ -261,8 +303,8 @@ end const matrix_t = mvtseries_t -function de_store_mvtseries(de, pid, name, type, eltype, axis1_id, axis2_id, nbytes, value, id) - ccall((:de_store_mvtseries, libdaec), Cint, (de_file, obj_id_t, Ptr{Cchar}, type_t, type_t, axis_id_t, axis_id_t, Int64, Ptr{Cvoid}, Ptr{obj_id_t}), de, pid, name, type, eltype, axis1_id, axis2_id, nbytes, value, id) +function de_store_mvtseries(de, pid, name, type, eltype, elfreq, axis1_id, axis2_id, nbytes, value, id) + ccall((:de_store_mvtseries, libdaec), Cint, (de_file, obj_id_t, Ptr{Cchar}, type_t, type_t, frequency_t, axis_id_t, axis_id_t, Int64, Ptr{Cvoid}, Ptr{obj_id_t}), de, pid, name, type, eltype, elfreq, axis1_id, axis2_id, nbytes, value, id) end function de_load_mvtseries(de, id, mvtseries) @@ -295,4 +337,16 @@ function de_finalize_search(search) ccall((:de_finalize_search, libdaec), Cint, (de_search,), search) end +const DE_VERSION = "0.3.1" + +const DE_VERNUM = 0x0310 + +const DE_VER_MAJOR = 0 + +const DE_VER_MINOR = 3 + +const DE_VER_REVISION = 1 + +const DE_VER_SUBREVISION = 0 + end # module diff --git a/src/dataecon/DataEcon.jl b/src/dataecon/DataEcon.jl index b214f63..ed55329 100644 --- a/src/dataecon/DataEcon.jl +++ b/src/dataecon/DataEcon.jl @@ -19,6 +19,17 @@ using ..TimeSeriesEcon include("C.jl") +function __init__() + # make sure the loaded library is the same version as the one that generated our C.jl + version = VersionNumber(unsafe_string(C.de_version())) + if version != VersionNumber(C.DE_VERSION) + throw(ErrorException("DataEcon library version $(version) does not match expected version $(C.DE_VERSION).")) + end + return +end + +const VERSION = VersionNumber(C.DE_VERSION) + ############################################################################# # open and close daec files @@ -70,15 +81,20 @@ Otherwise, call [`closedaec!`](@ref). """ function opendaec end -function opendaec(fname::AbstractString) +function _do_open(C_open, args...) handle = Ref{C.de_file}() + I._check(C_open(args..., handle)) + return handle +end + +function opendaec(fname::AbstractString; readonly=true, write=!readonly) fname = string(fname) - I._check(C.de_open(fname, handle)) - return DEFile(handle, fname) + open_func = _do_open(write ? C.de_open : C.de_open_readonly, fname) + return DEFile(open_func, fname) end -function opendaec(f::Function, fname::AbstractString) - de = opendaec(fname) +function opendaec(f::Function, fname::AbstractString; readonly=true, write=!readonly) + de = opendaec(fname; readonly, write) try f(de) finally @@ -86,6 +102,11 @@ function opendaec(f::Function, fname::AbstractString) end end +function opendaecmem() + handle = _do_open(C.de_open_memory) + return DEFile(handle, ":memory:") +end + """ closedaec!(de) @@ -132,7 +153,8 @@ See also [`find_object`](@ref) """ function find_fullpath(de::DEFile, fullpath::AbstractString, dne_error::Bool=true) id = Ref{C.obj_id_t}() - rc = C.de_find_fullpath(de, string(fullpath), id) + pref = startswith(fullpath, '/') ? "" : "/" + rc = C.de_find_fullpath(de, pref * string(fullpath), id) if dne_error == false && rc == C.DE_OBJ_DNE return missing end @@ -189,6 +211,31 @@ function delete_object(de::DEFile, id::C.obj_id_t) return nothing end +# import ..LittleDict + +""" + get_all_attributes(de, object_id) + +Retrieve the names and value of all attributes of the object with the given id. +They are returned in a dictionary. +""" +function get_all_attributes(de::DEFile, id::C.obj_id_t; delim="‖") + number = Ref{Int64}() + names = Ref{Ptr{Cchar}}() + values = Ref{Ptr{Cchar}}() + rc = C.de_get_all_attributes(de, id, delim, number, names, values) + I._check(rc) + if number[] == 0 + return Dict{String,String}() + end + all_names = String[split(unsafe_string(names[]), delim);] + all_values = String[split(unsafe_string(values[]), delim);] + if length(all_names) != length(all_values) + error("number of names and values don't match. Try a different delimiter.") + end + return Dict{String,String}(all_names .=> all_values) +end + """ get_attribute(de, object_id, attr_name) @@ -416,6 +463,19 @@ function new_catalog(de::DEFile, pid::C.obj_id_t, name::String) return id[] end +@inline catalog_size(de::DEFile, name::StrOrSym) = catalog_size(de, find_fullpath(de, name)) +function catalog_size(de::DEFile, pid::C.obj_id_t) + count = Ref{Int64}() + I._check(C.de_catalog_size(de, pid, count)) + return count[] +end + +@inline list_catalog(de::DEFile, name::StrOrSym; kwargs...) = list_catalog(de, find_fullpath(de, string(name)); kwargs...) +function list_catalog(de::DEFile, cid::C.obj_id_t=root_id; quiet=false, verbose=!quiet, file::IO=Base.stdout, + recursive=true, maxdepth::Int=recursive ? typemax(Int) : 1) + I._list_catalog(de, cid, maxdepth, verbose, file) +end + ############################################################################# # recursive high-level write @@ -449,7 +509,7 @@ end writedb(de::DEFile, data::Workspace) = writedb(de, root_id, data) writedb(de::DEFile, parent::AbstractString, data::Workspace) = writedb(de, find_fullpath(de, string(parent)), data) function writedb(file::AbstractString, args...) - opendaec(file) do de + opendaec(file, write=true) do de writedb(de, args...) end end @@ -478,7 +538,7 @@ function write_data(de::DEFile, pid::C.obj_id_t, name::StrOrSym, value) try I._write_data(de, pid, name, value) catch err - parent = get_fullpath(de, pid) + parent = pid == 0 ? "" : get_fullpath(de, pid) @error "Failed to write $parent/$name of type $(typeof(value))." err # rethrow() end @@ -502,12 +562,14 @@ All object contained in the specified catalog are loaded by [`read_data`](@ref). """ function readdb end -readdb(de::DEFile) = readdb(de, root_id) -readdb(de::DEFile, id::C.obj_id_t) = read_data(de, id) -readdb(de::DEFile, name::Symbol) = read_data(de, find_object(de, root_id, string(name))) -readdb(de::DEFile, catalog::AbstractString) = read_data(de, find_fullpath(de, string(catalog))) +readdb(de::DEFile, id::C.obj_id_t=root_id) = read_data(de, id) +readdb(de::DEFile, name::Symbol) = (oid = find_object(de, root_id, string(name)); read_data(de, oid)) +function readdb(de::DEFile, name::AbstractString) + oid = startswith(name, '/') ? find_fullpath(de, string(name)) : find_object(de, root_id, string(name)) + read_data(de, oid) +end function readdb(file::AbstractString, args...) - opendaec(file) do de + opendaec(file, readonly=true) do de readdb(de, args...) end end diff --git a/src/dataecon/I.jl b/src/dataecon/I.jl index 1a6d064..07d0dc6 100644 --- a/src/dataecon/I.jl +++ b/src/dataecon/I.jl @@ -42,7 +42,7 @@ function DEError() end @inline _de_error!(msg, ::Val{:debug}) = C.de_error_source(msg, sizeof(msg)) -@inline _de_error!(msg, v::Val) = (@nospecialize(v); C.de_error(msg, sizeof(msg))) +@inline _de_error!(msg, v::Val=Val(:nodebug)) = (@nospecialize(v); C.de_error(msg, sizeof(msg))) # _check() handles results from C library calls @@ -56,16 +56,41 @@ _check(rc::Cint) = rc == 0 || throw(DEError()) const StrOrSym = Union{Symbol,AbstractString} -_to_de_scalar_val(value) = throw(ArgumentError("Unable to write value of type $(typeof(value)).")) +_to_de_scalar_val(@nospecialize(value)) = throw(ArgumentError("DataEcon: unable to write scalar value of type $(typeof(value)).")) _to_de_scalar_val(value::Integer) = value _to_de_scalar_val(value::Real) = float(value) _to_de_scalar_val(value::Complex) = float(value) _to_de_scalar_val(value::StrOrSym) = string(value) _to_de_scalar_val(value::Dates.Date) = Dates.datetime2unix(convert(DateTime, value)) _to_de_scalar_val(value::Dates.DateTime) = Dates.datetime2unix(value) +_to_de_scalar_val(value::MIT)::typeof(value) = _pack_date(value) + +_pack_date(value::MIT) = value +function _pack_date(value::MIT{FR}) where {FR<:YPFrequency} + freq = _to_de_scalar_freq(FR) + year, period = mit2yp(value) + de_code = Ref{Int64}(0) + _check(C.de_pack_year_period_date(freq, year, period, de_code)) + if Int64(value) != de_code[] + @warn "MIT codes differ between TimeSeriesEcon and DataEcon for $value: $(Int(value)) vs $(de_code[])" + end + return value +end +function _pack_date(value::MIT{FR}) where {FR<:CalendarFrequency} + freq = _to_de_scalar_freq(FR) + date = Dates.Date(value) + year = Dates.year(date) + month = Dates.month(date) + day = Dates.day(date) + de_code = Ref{Int64}(0) + _check(C.de_pack_calendar_date(freq, year, month, day, de_code)) + if Int64(value) != de_code[] + @warn "MIT codes differ between TimeSeriesEcon and DataEcon for $value: $(Int(value)) vs $(de_code[])" + end + return value +end -_to_de_scalar_type(val) = _to_de_scalar_type(typeof(val)) -_to_de_scalar_type(::Type{T}) where {T} = error("Can't handle type $T") +_to_de_scalar_type(@nospecialize(val)) = _to_de_scalar_type(typeof(val)) _to_de_scalar_type(::Type{T}) where {T<:MIT} = C.type_date _to_de_scalar_type(::Type{T}) where {T<:Duration} = C.type_signed _to_de_scalar_type(::Type{T}) where {T<:Bool} = C.type_signed @@ -75,15 +100,17 @@ _to_de_scalar_type(::Type{T}) where {T<:Base.IEEEFloat} = C.type_float _to_de_scalar_type(::Type{T}) where {T<:Complex{<:Base.IEEEFloat}} = C.type_complex _to_de_scalar_type(::Type{T}) where {T<:StrOrSym} = C.type_string -_to_de_scalar_freq(val) = TimeSeriesEcon._has_frequencyof(val) ? _to_de_scalar_freq(frequencyof(val)) : C.freq_none +_to_de_scalar_freq(@nospecialize(val)) = C.freq_none +_to_de_scalar_freq(::MIT{F}) where{F} = _to_de_scalar_freq(F) +_to_de_scalar_freq(::Duration{F}) where{F} = _to_de_scalar_freq(F) _to_de_scalar_freq(::Type{Unit}) = C.freq_unit _to_de_scalar_freq(::Type{Daily}) = C.freq_daily _to_de_scalar_freq(::Type{BDaily}) = C.freq_bdaily -_to_de_scalar_freq(::Type{Weekly}) = C.freq_weekly +# _to_de_scalar_freq(::Type{Weekly}) = C.freq_weekly _to_de_scalar_freq(::Type{Monthly}) = C.freq_monthly -_to_de_scalar_freq(::Type{Quarterly}) = C.freq_quarterly -_to_de_scalar_freq(::Type{HalfYearly}) = C.freq_halfyearly -_to_de_scalar_freq(::Type{Yearly}) = C.freq_yearly +# _to_de_scalar_freq(::Type{Quarterly}) = C.freq_quarterly +# _to_de_scalar_freq(::Type{HalfYearly}) = C.freq_halfyearly +# _to_de_scalar_freq(::Type{Yearly}) = C.freq_yearly _to_de_scalar_freq(::Type{Weekly{end_day}}) where {end_day} = C.frequency_t(C.freq_weekly + mod1(end_day, 7)) _to_de_scalar_freq(::Type{Quarterly{end_month}}) where {end_month} = C.frequency_t(C.freq_quarterly + mod1(end_month, 3)) _to_de_scalar_freq(::Type{HalfYearly{end_month}}) where {end_month} = C.frequency_t(C.freq_halfyearly + mod1(end_month, 6)) @@ -98,55 +125,74 @@ _to_de_scalar_ptr_unsafe(val::String) = pointer(val) ############################################################################# # scalars read +_unpack_date(::Type{FR}, freq, de_code) where {FR<:Frequency} = convert(MIT{FR}, Int64(de_code)) +function _unpack_date(::Type{FR}, freq, de_code) where {FR<:YPFrequency} + year = Ref{Int32}() + period = Ref{UInt32}() + _check(C.de_unpack_year_period_date(freq, de_code, year, period)) + value = MIT{FR}(year[], period[]) + if Int(value) != de_code + @warn "MIT codes differ between TimeSeriesEcon and DataEcon for $value: $(Int(value)) vs $(de_code)" + end + return value +end +function _unpack_date(::Type{FR}, freq, de_code) where {FR<:CalendarFrequency} + year = Ref{Int32}() + month = Ref{UInt32}() + day = Ref{UInt32}() + _check(C.de_unpack_calendar_date(freq, de_code, year, month, day)) + value = MIT{FR}(Dates.Date(year[], month[], day[])) + if Int(value) != de_code + @warn "MIT codes differ between TimeSeriesEcon and DataEcon for $value: $(Int(value)) vs $(de_code)" + end + return value +end + function _from_de_scalar(scal::C.scalar_t) - _type = scal.object.type - if _type == C.type_string + obj_type = scal.object.obj_type + if obj_type == C.type_string return unsafe_string(Ptr{UInt8}(scal.value)) end - if _type == C.type_date - FR = _to_julia_frequency(scal.frequency) - T = _to_julia_scalar_type(Val(C.type_integer), Val(scal.nbytes)) - val = unsafe_load(Ptr{T}(scal.value)) - return convert(MIT{FR}, Int64(val)) - end - T = _to_julia_scalar_type(Val(_type), Val(scal.nbytes)) - value = unsafe_load(Ptr{T}(scal.value)) - if _type == C.type_signed && scal.frequency != C.freq_none - FR = _to_julia_frequency(scal.frequency) - value = convert(Duration{FR}, Int64(value)) + T = _to_julia_scalar_type(Val(obj_type), Val(scal.frequency), Val(scal.nbytes)) + if obj_type == C.type_date + value = unsafe_load(Ptr{Int64}(scal.value)) + value = _unpack_date(frequencyof(T), scal.frequency, value)::T + else + value = unsafe_load(Ptr{T}(scal.value)) end return value end -_to_julia_scalar_type(::Val{C.type_integer}, ::Val{0}) = Int # the default integer if size is unknown -_to_julia_scalar_type(::Val{C.type_integer}, ::Val{1}) = Int8 -_to_julia_scalar_type(::Val{C.type_integer}, ::Val{2}) = Int16 -_to_julia_scalar_type(::Val{C.type_integer}, ::Val{4}) = Int32 -_to_julia_scalar_type(::Val{C.type_integer}, ::Val{8}) = Int64 -_to_julia_scalar_type(::Val{C.type_integer}, ::Val{16}) = Int128 -_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{0}) = UInt -_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{1}) = UInt8 -_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{2}) = UInt16 -_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{4}) = UInt32 -_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{8}) = UInt64 -_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{16}) = UInt128 -_to_julia_scalar_type(::Val{C.type_float}, ::Val{0}) = Float64 -_to_julia_scalar_type(::Val{C.type_float}, ::Val{2}) = Float16 -_to_julia_scalar_type(::Val{C.type_float}, ::Val{4}) = Float32 -_to_julia_scalar_type(::Val{C.type_float}, ::Val{8}) = Float64 -_to_julia_scalar_type(::Val{C.type_complex}, ::Val{0}) = ComplexF64 -_to_julia_scalar_type(::Val{C.type_complex}, ::Val{4}) = ComplexF16 -_to_julia_scalar_type(::Val{C.type_complex}, ::Val{8}) = ComplexF32 -_to_julia_scalar_type(::Val{C.type_complex}, ::Val{16}) = ComplexF64 -_to_julia_scalar_type(::Val{C.type_string}, ::Val) = String -_to_julia_scalar_type(::Val{C.type_date}, ::Val{8}) = Int64 +_to_julia_scalar_type(::Val{C.type_integer}, ::Val{C.freq_none}, ::Val{0}) = Int # the default integer if size is unknown +_to_julia_scalar_type(::Val{C.type_integer}, ::Val{C.freq_none}, ::Val{1}) = Int8 +_to_julia_scalar_type(::Val{C.type_integer}, ::Val{C.freq_none}, ::Val{2}) = Int16 +_to_julia_scalar_type(::Val{C.type_integer}, ::Val{C.freq_none}, ::Val{4}) = Int32 +_to_julia_scalar_type(::Val{C.type_integer}, ::Val{C.freq_none}, ::Val{8}) = Int64 +_to_julia_scalar_type(::Val{C.type_integer}, ::Val{C.freq_none}, ::Val{16}) = Int128 +_to_julia_scalar_type(::Val{C.type_signed}, fr::Val, ::Val{8}) = Duration{_to_julia_frequency(fr)} +_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{C.freq_none}, ::Val{0}) = UInt +_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{C.freq_none}, ::Val{1}) = UInt8 +_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{C.freq_none}, ::Val{2}) = UInt16 +_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{C.freq_none}, ::Val{4}) = UInt32 +_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{C.freq_none}, ::Val{8}) = UInt64 +_to_julia_scalar_type(::Val{C.type_unsigned}, ::Val{C.freq_none}, ::Val{16}) = UInt128 +_to_julia_scalar_type(::Val{C.type_float}, ::Val{C.freq_none}, ::Val{0}) = Float64 +_to_julia_scalar_type(::Val{C.type_float}, ::Val{C.freq_none}, ::Val{2}) = Float16 +_to_julia_scalar_type(::Val{C.type_float}, ::Val{C.freq_none}, ::Val{4}) = Float32 +_to_julia_scalar_type(::Val{C.type_float}, ::Val{C.freq_none}, ::Val{8}) = Float64 +_to_julia_scalar_type(::Val{C.type_complex}, ::Val{C.freq_none}, ::Val{0}) = ComplexF64 +_to_julia_scalar_type(::Val{C.type_complex}, ::Val{C.freq_none}, ::Val{4}) = ComplexF16 +_to_julia_scalar_type(::Val{C.type_complex}, ::Val{C.freq_none}, ::Val{8}) = ComplexF32 +_to_julia_scalar_type(::Val{C.type_complex}, ::Val{C.freq_none}, ::Val{16}) = ComplexF64 +_to_julia_scalar_type(::Val{C.type_string}, ::Val{C.freq_none}, ::Val) = String +_to_julia_scalar_type(::Val{C.type_date}, fr::Val, ::Val{8}) = MIT{_to_julia_frequency(fr)} # _to_julia_scalar_type(::Val{C.type_other_scalar}, ::Val) = Any -_to_julia_frequency(f::C.frequency_t) = _to_julia_frequency(Val(f)) _to_julia_frequency(::Val{C.freq_unit}) = Unit _to_julia_frequency(::Val{C.freq_daily}) = Daily _to_julia_frequency(::Val{C.freq_bdaily}) = BDaily _to_julia_frequency(::Val{C.freq_monthly}) = Monthly +_to_julia_frequency(::Val{C.freq_monthly + 1}) = Monthly function _to_julia_frequency(::Val{CF}) where {CF} for (f, F) in ((C.freq_weekly => Weekly), (C.freq_quarterly => Quarterly), (C.freq_halfyearly => HalfYearly), (C.freq_yearly => Yearly)) @@ -174,7 +220,7 @@ end function _make_axis(de::DEFile, rng::AbstractUnitRange{<:MIT}) ax_id = Ref{C.axis_id_t}() - _check(C.de_axis_range(de, length(rng), _to_de_scalar_freq(rng), first(rng), ax_id)) + _check(C.de_axis_range(de, length(rng), _to_de_scalar_freq(frequencyof(rng)), _pack_date(first(rng)), ax_id)) return ax_id[] end @@ -185,31 +231,56 @@ function _make_axis(de::DEFile, rng::AbstractVector{<:Union{Symbol,AbstractStrin return ax_id[] end +# _get_axis_of(de::DEFile, vec::NTuple, dim=1) = _make_axis(de, Base.axes1(vec)) _get_axis_of(de::DEFile, vec::AbstractVector, dim=1) = _make_axis(de, Base.axes1(vec)) _get_axis_of(de::DEFile, vec::AbstractUnitRange, dim=1) = _make_axis(de, vec) _get_axis_of(de::DEFile, vec::AbstractArray, dim=1) = _make_axis(de, Base.axes(vec, dim)) +function _get_axis_range_firstdate(axis::C.axis_t) + @assert axis.ax_type == C.axis_range + FR = _to_julia_frequency(Val(axis.frequency)) + return _unpack_date(FR, axis.frequency, axis.first) +end + ############################################################################# # write tseries and mvtseries +struct _ArrayData + eltype::C.type_t + elfreq::C.frequency_t + obj_type::C.type_t + nbytes::Int + val +end + +_de_array_data(; eltype, elfreq=C.freq_none, obj_type, nbytes, val) = _ArrayData(eltype, elfreq, obj_type, nbytes, val) + _to_de_array(value) = throw(ArgumentError("Unable to write value of type $(typeof(value)).")) # handle range -_to_de_array(value::AbstractUnitRange) = - (; eltype=_to_de_scalar_type(eltype(value)), type=C.type_range, nbytes=0, val=C_NULL) +_to_de_array(::AbstractUnitRange) = _de_array_data(; eltype=C.type_none, obj_type=C.type_range, nbytes=0, val=C_NULL) # handle any vector or matrix _to_de_array(value::AbstractVecOrMat) = _to_de_array(Val(isempty(value)), value) +_eltypefreq(::Type{ET}) where {ET<:MIT} = (; eltype=C.type_date, elfreq=_to_de_scalar_freq(frequencyof(ET))) +_eltypefreq(::Type{ET}) where {ET<:Duration} = (; eltype=C.type_signed, elfreq=_to_de_scalar_freq(frequencyof(ET))) +_eltypefreq(::Type{ET}) where {ET} = (; eltype=_to_de_scalar_type(ET), elfreq=C.freq_none) + # empty array -_to_de_array(::Val{true}, value::AbstractVector{ET}) where {ET} = (; eltype=_to_de_scalar_type(ET), type=C.type_vector, nbytes=0, val=C_NULL) -_to_de_array(::Val{true}, value::AbstractMatrix{ET}) where {ET} = (; eltype=_to_de_scalar_type(ET), type=C.type_matrix, nbytes=0, val=C_NULL) +_to_de_array(::Val{true}, value::AbstractVecOrMat{ET}) where {ET} = _de_array_data(; + _eltypefreq(ET)..., + obj_type=ndims(value) == 1 ? C.type_vector : C.type_matrix, + nbytes=0, + val=C_NULL +) # vector of numbers function _to_de_array(::Val{false}, value::AbstractVecOrMat{ET}) where {ET<:Number} val = value nbytes = length(value) * _to_de_scalar_nbytes(ET) - return (; eltype=_to_de_scalar_type(ET), type=ndims(value) == 1 ? C.type_vector : C.type_matrix, nbytes, val) + return _de_array_data(; _eltypefreq(ET)..., + obj_type=ndims(value) == 1 ? C.type_vector : C.type_matrix, nbytes, val) end _to_de_array(::Val{false}, value::AbstractVecOrMat{ET}) where {ET<:StrOrSym} = _to_de_array(Val(false), map(string, value)) @@ -218,7 +289,7 @@ function _to_de_array(::Val{false}, value::VecOrMat{String}) nbytes = sum(length, value) + nel val = Vector{UInt8}(undef, nbytes) _check(C.de_pack_strings(value, nel, val, Ref(nbytes))) - return (; eltype=C.type_string, type=ndims(value) == 1 ? C.type_vector : C.type_matrix, nbytes, val) + return _de_array_data(; eltype=C.type_string, obj_type=ndims(value) == 1 ? C.type_vector : C.type_matrix, nbytes, val) end # handle bit array -- we must expand it to Array{Bool} @@ -226,22 +297,22 @@ _to_de_array(value::BitArray) = _to_de_array(Array(value)) # handle mvtseries function _to_de_array(value::MVTSeries) - (; eltype, nbytes, val) = _to_de_array(value.values) - return (; eltype, type=C.type_mvtseries, nbytes, val) + arr = _to_de_array(value.values) + return _de_array_data(; arr.eltype, arr.elfreq, obj_type=C.type_mvtseries, arr.nbytes, arr.val) end # handle tseries function _to_de_array(value::TSeries) - (; eltype, nbytes, val) = _to_de_array(value.values) - return (; eltype, type=C.type_tseries, nbytes, val) + arr = _to_de_array(value.values) + return _de_array_data(; arr.eltype, arr.elfreq, obj_type=C.type_tseries, arr.nbytes, arr.val) end function _should_store_eltype(arr, value::AbstractArray{ET}) where {ET} isempty(value) && return true - arr.type == C.type_range && return false + arr.obj_type == C.type_range && return false arr.eltype == C.type_other_scalar && return true arr.eltype == C.type_string && return ET != String - return ET != _to_julia_scalar_type(Val(arr.eltype), Val(sizeof(ET))) + return ET != _to_julia_scalar_type(Val(arr.eltype), Val(arr.elfreq), Val(sizeof(ET))) end @inline _should_store_type(v::Val, a::Any) = (@nospecialize(v, a); true) @@ -253,24 +324,24 @@ end _do_store_array(::Val{1}, args...) = C.de_store_tseries(args...) _do_store_array(::Val{2}, args...) = C.de_store_mvtseries(args...) -function _do_store_array(v::Val{N}, args...) where {N} - @nospecialize v - error("Can't store $(N)d array.") -end +# function _do_store_array(v::Val{N}, args...) where {N} +# @nospecialize v +# error("Can't store $(N)d array.") +# end import ..set_attribute import ..get_attribute -_store_array(de::DEFile, pid::C.obj_id_t, name::String, axes::NTuple{M,C.axis_id_t}, value::AbstractArray{ET,N}) where {ET,N,M} = error("Dimension mismatch: $(N)d array with $M axes.") +# _store_array(de::DEFile, pid::C.obj_id_t, name::String, axes::NTuple{M,C.axis_id_t}, value::AbstractArray{ET,N}) where {ET,N,M} = error("Dimension mismatch: $(N)d array with $M axes.") function _store_array(de::DEFile, pid::C.obj_id_t, name::String, axes::NTuple{N,C.axis_id_t}, value::AbstractArray{ET,N}) where {ET,N} id = Ref{C.obj_id_t}() arr = _to_de_array(value) GC.@preserve arr begin ptr = arr.nbytes == 0 ? C_NULL : pointer(arr.val) - _check(_do_store_array(Val(N), de, pid, name, arr.type, arr.eltype, axes..., arr.nbytes, ptr, id)) + _check(_do_store_array(Val(N), de, pid, name, arr.obj_type, arr.eltype, arr.elfreq, axes..., arr.nbytes, ptr, id)) end if _should_store_eltype(arr, value) set_attribute(de, id[], "jeltype", string(ET)) end - if _should_store_type(Val(arr.type), value) + if _should_store_type(Val(arr.obj_type), value) set_attribute(de, id[], "jtype", string(typeof(value))) end return id[] @@ -299,17 +370,16 @@ function _to_julia_array(de, id, arr) end # dispatcher -_from_de_array(arr::C.tseries_t) = _from_de_array(arr, Val(arr.object.type), Val(arr.axis.type)) -_from_de_array(arr::C.mvtseries_t) = _from_de_array(arr, Val(arr.object.type), Val(arr.axis1.type), Val(arr.axis2.type)) +_from_de_array(arr::C.tseries_t) = _from_de_array(arr, Val(arr.object.obj_type), Val(arr.axis.ax_type)) +_from_de_array(arr::C.mvtseries_t) = _from_de_array(arr, Val(arr.object.obj_type), Val(arr.axis1.ax_type), Val(arr.axis2.ax_type)) # handle type_range _from_de_array(arr::C.tseries_t, ::Val{C.type_range}, ::Val{C.axis_plain}) = 1:arr.axis.length function _from_de_array(arr::C.tseries_t, ::Val{C.type_range}, ::Val{C.axis_range}) - FR = _to_julia_frequency(arr.axis.frequency) - fd = reinterpret(MIT{FR}, arr.axis.first) + fd = _get_axis_range_firstdate(arr.axis) return fd .+ (0:arr.axis.length-1) end -_from_de_array(::C.tseries_t, ::Val{C.type_range}, ::Val{Z}) where {Z} = error("Cannot load a range of type $Z") +# _from_de_array(::C.tseries_t, ::Val{C.type_range}, ::Val{Z}) where {Z} = error("Cannot load a range of axis type $Z") # handle type_vector _from_de_array(::C.tseries_t, ::Val{C.type_vector}, ::Val{Z},) where {Z} = error("Cannot load a vector with axis of type $Z. Expected $(C.axis_plain)") @@ -327,14 +397,14 @@ function _from_de_array(arr::C.mvtseries_t, ::Val{C.type_matrix}, ::Val{C.axis_p end function _do_load_array_data(arr, ::Val{0}) - ET = _to_julia_scalar_type(Val(arr.eltype), Val(0)) + ET = _to_julia_scalar_type(Val(arr.eltype), Val(arr.elfreq), Val(0)) return ET[] end function _do_load_array_data(arr, v::Val{N}) where {N} @nospecialize v vlen = N::Int64 - ET = _to_julia_scalar_type(Val(arr.eltype), Val(arr.nbytes ÷ vlen)) + ET = _to_julia_scalar_type(Val(arr.eltype), Val(arr.elfreq), Val(arr.nbytes ÷ vlen)) vec = Vector{ET}(undef, vlen) return _my_copyto!(vec, arr) end @@ -368,8 +438,7 @@ end _from_de_array(::C.tseries_t, ::Val{C.type_tseries}, ::Val{Z}) where {Z} = error("Cannot load a tseries with axis of type $Z. Expected $(C.axis_range).") function _from_de_array(arr::C.tseries_t, ::Val{C.type_tseries}, ::Val{C.axis_range}) vlen = arr.axis.length - FR = _to_julia_frequency(arr.axis.frequency) - fd = reinterpret(MIT{FR}, arr.axis.first) + fd = _get_axis_range_firstdate(arr.axis) return TSeries(fd, _do_load_array_data(arr, Val(vlen))) end @@ -377,8 +446,7 @@ end _from_de_array(::C.mvtseries_t, ::Val{C.type_mvtseries}, ::Val{Y}, ::Val{Z}) where {Y,Z} = error("Cannot load an mvtseries with axes of type $Y and $Z. Expected $(C.axis_range) and $(C.axis_names).") function _from_de_array(arr::C.mvtseries_t, ::Val{C.type_mvtseries}, ::Val{C.axis_range}, ::Val{C.axis_names}) d1 = arr.axis1.length - FR = _to_julia_frequency(arr.axis1.frequency) - fd = reinterpret(MIT{FR}, arr.axis1.first) + fd = _get_axis_range_firstdate(arr.axis1) d2 = arr.axis2.length cols = split(Base.unsafe_string(arr.axis2.names), '\n') return MVTSeries(fd, cols, reshape(_do_load_array_data(arr, Val(d1 * d2)), d1, d2)) @@ -398,6 +466,7 @@ const StoreAsScalarType = Union{Symbol,AbstractString,Number,Dates.Date,Dates.Da _write_data(::DEFile, ::C.obj_id_t, name::StrOrSym, value) = error("Cannot determine the storage class of $name::$(typeof(value))") _write_data(de::DEFile, pid::C.obj_id_t, name::StrOrSym, value::StoreAsScalarType) = store_scalar(de, pid, string(name), value) +# _write_data(de::DEFile, pid::C.obj_id_t, name::StrOrSym, value::NTuple) = store_tseries(de, pid, string(name), value) _write_data(de::DEFile, pid::C.obj_id_t, name::StrOrSym, value::AbstractVector) = store_tseries(de, pid, string(name), value) _write_data(de::DEFile, pid::C.obj_id_t, name::StrOrSym, value::AbstractMatrix) = store_mvtseries(de, pid, string(name), value) function _write_data(de::DEFile, pid::C.obj_id_t, name::StrOrSym, data::Workspace) @@ -409,6 +478,18 @@ end ############################################################################# # helpers for readdb +function _finalize_search(rc, search) + try + if rc != C.DE_NO_OBJ + _check(rc) + end + C.de_finalize_search(search) + catch err + C.de_finalize_search(search) + rethrow(err) + end +end + import ..load_scalar import ..load_tseries import ..load_mvtseries @@ -417,7 +498,7 @@ import ..readdb function _read_data(de::DEFile, id::C.obj_id_t) obj = Ref{C.object_t}() _check(C.de_load_object(de, id, obj)) - return _read_data(de, id, Val(obj[].class)) + return _read_data(de, id, Val(obj[].obj_class)) end _read_data(de::DEFile, id::C.obj_id_t, ::Val{C.class_scalar}) = load_scalar(de, id) _read_data(de::DEFile, id::C.obj_id_t, ::Val{C.class_tseries}) = load_tseries(de, id) @@ -431,7 +512,7 @@ function _read_data(de::DEFile, id::C.obj_id_t, ::Val{C.class_catalog}) while rc == C.DE_SUCCESS name = Symbol(unsafe_string(obj[].name)) try - value = _read_data(de, obj[].id, Val(obj[].class)) + value = _read_data(de, obj[].id, Val(obj[].obj_class)) push!(data, name => value) catch err @error "Failed to load $name" err @@ -439,18 +520,49 @@ function _read_data(de::DEFile, id::C.obj_id_t, ::Val{C.class_catalog}) end rc = C.de_next_object(search[], obj) end - if rc == C.DE_NO_OBJ - C.de_finalize_search(search[]) - return data - end - try - _check(rc) - catch err - C.de_finalize_search(search[]) - rethrow(err) - end - return nothing + _finalize_search(rc, search[]) + return data end +############################################################################# +# helpers for list_catalog + +function _list_catalog(de::DEFile, cid::C.obj_id_t, maxdepth::Int, verbose::Bool, io::IO) + search = Ref{C.de_search}() + _check(C.de_list_catalog(de, cid, search)) + obj = Ref{C.object_t}() + rc = C.de_next_object(search[], obj) + list = [] + while rc == C.DE_SUCCESS + oname = unsafe_string(obj[].name) + try + fp = Ref{Ptr{Cchar}}() + depth = Ref{Int64}() + _check(C.de_get_object_info(de, obj[].id, fp, depth, C_NULL)) + if depth[] <= maxdepth + if obj[].obj_class == C.class_catalog + verbose && print(io, " "^(depth[] - 1), oname, '/') + if depth[] < maxdepth + verbose && println(io) + append!(list, _list_catalog(de, obj[].id, maxdepth, verbose, io)) + else # depth[] == maxdepth + count = Ref{Int64}() + _check(C.de_catalog_size(de, obj[].id, count)) + verbose && println(io, count[] == 0 ? "(empty)" : "($(count[]) objects)") + end + else + verbose && println(" "^(depth[] - 1), " ", oname) + push!(list, unsafe_string(fp[])) + end + end + catch err + @error "Failed to get info for $oname($(obj[].id))" err + C.de_clear_error() + end + rc = C.de_next_object(search[], obj) + end + _finalize_search(rc, search[]) + return list +end end diff --git a/src/linalg.jl b/src/linalg.jl index d0030b3..b90dfa7 100644 --- a/src/linalg.jl +++ b/src/linalg.jl @@ -7,7 +7,7 @@ import Base: *, \, / for op in (:*, :\, :/) - for AT in (AbstractMatrix, AbstractVector) + for AT in (AbstractMatrix, AbstractVector, Adjoint{<:Any, <:AbstractMatrix{T}} where {T}) for ST in (MVTSeries, TSeries) eval(quote $op(A::$AT, B::$ST) = $op(A, _vals(B)) diff --git a/src/momentintime.jl b/src/momentintime.jl index 058e212..4ad0ebf 100644 --- a/src/momentintime.jl +++ b/src/momentintime.jl @@ -188,6 +188,7 @@ MIT, Duration # -------------------------- # conversions with Int MIT{F}(x::Int) where {F<:Frequency} = reinterpret(MIT{F}, x) +MIT{F}(x::Base.BitInteger) where {F<:Frequency} = MIT{F}(Int(x)) MIT{Yearly}(x::Int) = MIT{Yearly{12}}(x) MIT{Quarterly}(x::Int) = MIT{Quarterly{3}}(x) MIT{HalfYearly}(x::Int) = MIT{HalfYearly{6}}(x) @@ -247,19 +248,19 @@ MIT{Yearly}(y::Integer, p::Integer) = MIT{Yearly{12}}(y, p) MIT{Quarterly}(y::Integer, p::Integer) = MIT{Quarterly{3}}(y, p) MIT{HalfYearly}(y::Integer, p::Integer) = MIT{HalfYearly{6}}(y, p) function _weekly_from_yp(F::Type{<:Weekly{end_day}}, y, p) where {end_day} - first_day_of_year = Dates.Date("$y-01-01") + first_day_of_year = Dates.Date(y, 1, 1) d = first_day_of_year + Day(7 * (p - 1)) return weekly(d, end_day) end function MIT{F}(y::Integer, p::Integer) where {F<:BDaily} - first_day_of_year = Dates.Date("$y-01-01") + first_day_of_year = Dates.Date(y, 1, 1) first_day = dayofweek(first_day_of_year) days_diff = first_day > 5 ? 8 - first_day : 0 d = first_day_of_year + Day(days_diff) return bdaily(d) + p - 1 end function MIT{F}(y::Integer, p::Integer) where {F<:Daily} - first_day_of_year = Dates.Date("$y-01-01") + first_day_of_year = Dates.Date(y, 1, 1) return daily(first_day_of_year) + p - 1 end @@ -341,7 +342,7 @@ Construct an `MIT{Yearly}` from an year and a period. yy(y::Integer, p::Integer=1) = MIT{Yearly{12}}(y, p) -_d0 = Date("0001-01-01") - Day(1) +_d0 = Date(1, 1, 1) - Day(1) """ daily(d::Date) @@ -387,6 +388,7 @@ is `true`, meaning that the preceding Friday is returned. """ function bdaily(d::Date; bias::Symbol=getoption(:bdaily_creation_bias)) num_weekends, rem = divrem(Dates.value(d - _d0), 7) + (rem < 0) && ((num_weekends, rem) = (num_weekends - 1, rem + 7)) adjustment = 0 if rem == 0 # Sunday if bias ∈ (:next, :nearest) @@ -424,7 +426,11 @@ macro bd_str(d) end return rng end - return bdaily(d) + return try + bdaily(d) + catch e + :(throw($e)) + end end; """ @@ -448,25 +454,27 @@ Example: second_week_of_april = bd"2022-04-04:2022-04-08" """ macro bd_str(d, bias) - if findfirst(":", d) !== nothing - throw(ArgumentError("Additional arguments are not supported when passing a range to bd\"\".")) - end - if bias ∉ ("n", "next", "p", "previous", "s", "strict", "near", "nearest") - throw(ArgumentError("""A bd\"\" string literal must terminate in one of ("", "n", "next", "p", "previous", "s", "strict", "near", "nearest").""")) + if contains(d, ":") + return :( + throw(ArgumentError("Additional arguments are not supported when passing a range to bd\"\".")) + ) end if bias == "" return bdaily(d) elseif bias == "n" || bias == "next" return bdaily(d, bias=:next) - elseif bias == "p" || bias == "p" + elseif bias == "p" || bias == "previous" return bdaily(d, bias=:previous) elseif bias == "s" || bias == "strict" return bdaily(d, bias=:strict) elseif bias == "near" || bias == "nearest" return bdaily(d, bias=:nearest) + else + return :( + throw(ArgumentError("""A bd\"\" string literal must terminate in one of ("", "n", "next", "p", "previous", "s", "strict", "near", "nearest").""")) + ) end - -end; +end """ weekly(d::Date) @@ -488,7 +496,7 @@ function weekly_from_iso(y::Int, p::Int) if p > 53 || p < 1 throw(ArgumentError("The provided period must be between 1 and 53 (inclusive).")) end - first_day_of_year = Dates.Date("$(y)-01-01") + first_day_of_year = Dates.Date(y, 1, 1) week_of_first_day = week(first_day_of_year) year_end_padding = week_of_first_day !== 1 ? 1 : 0 d2 = first_day_of_year + Day(((p - 1) + year_end_padding) * 7) @@ -563,31 +571,36 @@ end function Dates.Date(m::MIT{Monthly}, ref::Symbol=:end) year, month = divrem(Int(m), 12) if ref == :begin - return Dates.Date("$year-01-01") + Month(month) + return Dates.Date(year, 1, 1) + Month(month) end - return Dates.Date("$year-01-01") + Month(month + 1) - Day(1) + return Dates.Date(year, 1, 1) + Month(month + 1) - Day(1) end -function Dates.Date(m::MIT{Quarterly{end_month}}, ref::Symbol=:end) where end_month +function Dates.Date(m::MIT{Quarterly{end_month}}, ref::Symbol=:end) where {end_month} year, quarter = divrem(Int(m), 4) if ref == :begin - return Dates.Date("$year-01-01") + Month(quarter * 3 - (3 - end_month)) + return Dates.Date(year, 1, 1) + Month(quarter * 3 - (3 - end_month)) end - return Dates.Date("$year-01-01") + Month((quarter + 1) * 3 - (3 - end_month)) - Day(1) + return Dates.Date(year, 1, 1) + Month((quarter + 1) * 3 - (3 - end_month)) - Day(1) end -function Dates.Date(m::MIT{HalfYearly{end_month}}, ref::Symbol=:end) where end_month +function Dates.Date(m::MIT{HalfYearly{end_month}}, ref::Symbol=:end) where {end_month} year, half = divrem(Int(m), 2) if ref == :begin - return Dates.Date("$year-01-01") + Month(half * 6 - (6 - end_month)) + return Dates.Date(year, 1, 1) + Month(half * 6 - (6 - end_month)) end - return Dates.Date("$year-01-01") + Month((half + 1) * 6 - (6 - end_month)) - Day(1) + return Dates.Date(year, 1, 1) + Month((half + 1) * 6 - (6 - end_month)) - Day(1) end -function Dates.Date(m::MIT{Yearly{end_month}}, ref::Symbol=:end) where end_month +function Dates.Date(m::MIT{Yearly{end_month}}, ref::Symbol=:end) where {end_month} if ref == :begin - return Dates.Date("$(Int(m))-01-01") - Month(12 - end_month) + return Dates.Date(Int(m), 1, 1) - Month(12 - end_month) end - return Dates.Date("$(Int(m) + 1)-01-01") - Month(12 - end_month) - Day(1) + return Dates.Date(Int(m) + 1, 1, 1) - Month(12 - end_month) - Day(1) end +MIT{Daily}(d::Dates.Date) = daily(d) +MIT{BDaily}(d::Dates.Date; kwargs...) = bdaily(d; kwargs...) +MIT{Weekly}(d::Dates.Date) = weekly(d) +MIT{Weekly{end_day}}(d::Dates.Date) where {end_day} = weekly(d, end_day) + #------------------------- # pretty printing diff --git a/src/mvtseries.jl b/src/mvtseries.jl index 07c1ccb..173511d 100644 --- a/src/mvtseries.jl +++ b/src/mvtseries.jl @@ -477,12 +477,12 @@ end # single argument - list/tuple of variables - return a TSeries of the column @inline function Base.getindex(x::MVTSeries, cols::_MVTSAxes2) - inds = [_colind(x, c) for c in cols] + inds = _colind(x, cols) return MVTSeries(firstdate(x), cols, getindex(_vals(x), :, inds)) end @inline function Base.setindex!(x::MVTSeries, val, cols::_MVTSAxes2) - inds = [_colind(x, c) for c in cols] + inds = _colind(x, cols) setindex!(x.values, val, :, inds) end diff --git a/src/plotrecipes.jl b/src/plotrecipes.jl index 19490bf..71e2d80 100644 --- a/src/plotrecipes.jl +++ b/src/plotrecipes.jl @@ -41,7 +41,6 @@ end @assert y isa TSeries @assert z isa Nothing @assert frequencyof(x) == frequencyof(y) - # @info "Plotting TSeries" mit_loc = get(plotattributes, :mit_loc, :left) rng = get(plotattributes, :trange, x) rng = intersect(rng, rangeof(y)) @@ -49,6 +48,8 @@ end x := Float64.(rng) .+ mit_offset(Val(mit_loc), frequencyof(rng)) if frequencyof(x) <: YPFrequency xformatter --> mit_formatter(Val(mit_loc), frequencyof(rng)) + elseif frequencyof(x) <: Union{BDaily,Daily,<:Weekly} + x := [Date(MIT{frequencyof(x)}(Int64(i))) for i in rng] end seriestype := :path end diff --git a/src/tseries.jl b/src/tseries.jl index 3d126f8..7f3aec0 100644 --- a/src/tseries.jl +++ b/src/tseries.jl @@ -455,7 +455,10 @@ Base.copyto!(dest::TSeries, drng::AbstractRange{<:MIT}, src::TSeries) = mixed_fr function Base.copyto!(dest::TSeries{F}, drng::AbstractRange{MIT{F}}, src::TSeries{F}) where {F<:Frequency} fullindex = union(rangeof(dest), drng) resize!(dest, fullindex) - copyto!(dest.values, Int(first(drng) - firstindex(dest) + 1), src[drng].values, 1, length(drng)) + fd = Int(first(drng)) + d1 = fd - Int(firstindex(dest)) + 1 + s1 = fd - Int(firstindex(src)) + 1 + copyto!(dest.values, d1, src.values, s1, length(drng)) return dest end diff --git a/src/workspaces.jl b/src/workspaces.jl index a3910f8..e7a1567 100644 --- a/src/workspaces.jl +++ b/src/workspaces.jl @@ -52,7 +52,7 @@ Base.setproperty!(w::Workspace, sym::Symbol, val) = setindex!(w, val, sym) # MacroTools.@forward Workspace._c (Base.getindex,) Base.getindex(w::Workspace, sym) = getindex(_c(w), convert(Symbol, sym)) Base.getindex(w::Workspace, sym, syms...) = getindex(w, (sym, syms...,)) -Base.getindex(w::Workspace, syms::Vector) = getindex(w, (syms...,)) +Base.getindex(w::Workspace, syms::Vector) = Workspace(convert(Symbol, s) => w[s] for s in syms) Base.getindex(w::Workspace, syms::Tuple) = Workspace(convert(Symbol, s) => w[s] for s in syms) MacroTools.@forward Workspace._c (Base.setindex!,) @@ -60,11 +60,15 @@ MacroTools.@forward Workspace._c (Base.isempty, Base.keys, Base.haskey, Base.val MacroTools.@forward Workspace._c (Base.iterate, Base.get, Base.get!,) function Base.eltype(w::Workspace) - ET = isempty(w) ? Any : try Base.promote_typeof(values(w)...) catch; Any; end - return Pair{Symbol, ET} + ET = isempty(w) ? Any : try + Base.promote_typeof(values(w)...) + catch + Any + end + return Pair{Symbol,ET} end -Base.push!(w::Workspace, args...; kwargs...) = (push!(_c(w), args...; kwargs...); w) +Base.push!(w::Workspace, args...; kwargs...) = (push!(_c(w), args..., (k => v for (k,v) in kwargs)...); w) Base.delete!(w::Workspace, args...; kwargs...) = (delete!(_c(w), args...; kwargs...); w) @inline Base.in(name, w::Workspace) = convert(Symbol, name) ∈ keys(_c(w)) @@ -241,9 +245,19 @@ end export @weval -function Base.copyto!(x::MVTSeries, w::Workspace; range::AbstractUnitRange{<:MIT}=rangeof(x)) +Base.copyto!(x::MVTSeries, w::Workspace; verbose=false, trange=rangeof(x)) = copyto!(x, trange, w; verbose) +function Base.copyto!(x::MVTSeries, range::AbstractUnitRange{<:MIT}, w::Workspace; verbose=false) + missing = [] for (key, value) in pairs(x) - copyto!(value, range, w[key]) + w_val = get(w, key, nothing) + if !isnothing(w_val) + copyto!(value, range, w_val) + else + verbose && push!(missing, key) + end + end + if verbose + @warn "Variables not copied (missing from Workspace): $(join(missing, ", "))" end return x end diff --git a/test/runtests.jl b/test/runtests.jl index feec22d..ef03ff7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,9 +1,10 @@ -# Copyright (c) 2020-2021, Bank of Canada +# Copyright (c) 2020-2023, Bank of Canada # All rights reserved. using Test using TimeSeriesEcon using Statistics +using Suppressor include("test_mit.jl") include("test_tseries.jl") diff --git a/test/test_dataecon.jl b/test/test_dataecon.jl index 5cd44a6..f3df035 100644 --- a/test/test_dataecon.jl +++ b/test/test_dataecon.jl @@ -2,6 +2,7 @@ # All rights reserved. using Dates +using Random DE = TimeSeriesEcon.DataEcon test_file = "test.daec" @@ -9,16 +10,24 @@ rm(test_file, force=true) rm(test_file * "-journal", force=true) @testset "DE file" begin - global de = DE.opendaec(test_file) + global de + @test_throws DE.DEError DE.opendaec(test_file, readonly=true) + de = DE.opendaec(test_file, write=true) @test isopen(de) @test (DE.closedaec!(de); true) @test !isopen(de) - de = DE.opendaec(test_file) + @test (de = DE.opendaec(test_file, readonly=true); isopen(de)) + @test (DE.closedaec!(de); !isopen(de)) + de = DE.opendaec(test_file, write=true) # test find_object throws exception or returns missing @test_throws DE.DEError DE.find_object(de, DE.root_id, "nosuchobject") @test ismissing(DE.find_object(de, DE.root_id, "nosuchobject", false)) + dm = nothing + @test (dm = DE.opendaecmem(); isopen(dm)) + @test (DE.closedaec!(dm); !isopen(dm)) + # test get_fullpath works @test DE.get_fullpath(de, DE.root_id) == "/" @@ -36,6 +45,23 @@ rm(test_file * "-journal", force=true) end +@testset "DE errors" begin + @test_throws ArgumentError DE.store_scalar(de, :err, Val(0)) +end + +function _check_attributes(name, value, attr, has_jtype, has_jeltype) + sz = 0 + if (name in has_jtype) + @test get(attr, "jtype", nothing) == string(typeof(value)) + sz += 1 + end + if (name in has_jeltype) + @test get(attr, "jeltype", nothing) == string(Base.eltype(value)) + sz += 1 + end + @test length(attr) == sz +end + @testset "DE scalar" begin db = Workspace( # integers @@ -81,31 +107,22 @@ end cd2=Dates.today() ) - pid = DE.find_fullpath(de, "/scalars", false) - if ismissing(pid) - pid = DE.new_catalog(de, "scalars") - end - - # we can write them - for (name, value) in pairs(db) - @test begin - id = DE.store_scalar(de, pid, name, value) - id == DE.find_fullpath(de, "/scalars/$name") - end - end + cata = "scalars" + # Julia types that are not directly supporded by DataEcon. + has_jtype = [:ns1, :f3, :c1, :cd1, :cd2] + has_jeltype = [] - - pid1 = DE.find_fullpath(de, "/scalars1", false) - if ismissing(pid1) - pid1 = DE.new_catalog(de, "scalars1") + pid = DE.find_fullpath(de, cata, false) + if ismissing(pid) + pid = DE.new_catalog(de, cata) end # we can write them for (name, value) in pairs(db) - @test begin - id = DE.store_scalar(de, "/scalars1/$name", value) - id == DE.find_object(de, pid1, name) - end + id = DE.store_scalar(de, pid, name, value) + @test id == DE.find_fullpath(de, "/$cata/$name") + attr = DE.get_all_attributes(de, id) + _check_attributes(name, value, attr, has_jtype, has_jeltype) end ldb = Workspace() @@ -123,7 +140,23 @@ end ldb = Workspace() # we can read them for name in keys(db) - @test (push!(ldb, name => DE.load_scalar(de, "/scalars/$name")); true) + @test (push!(ldb, name => DE.load_scalar(de, "/$cata/$name")); true) + end + + # list_catalog returns what we expect + begin + @test isempty(DE.list_catalog(de; quiet=true, recursive=false)) + @test length(db) == DE.catalog_size(de, cata) + lst = DE.list_catalog(de, "/$cata"; quiet=true) + @test length(lst) == length(db) + for k in keys(db) + @test "/$cata/$k" in lst + end + lst = DE.list_catalog(de; quiet=true) + @test length(lst) == length(db) + for k in keys(db) + @test "/$cata/$k" in lst + end end # they are equal @@ -133,8 +166,8 @@ end @test @compare map(typeof, db) map(typeof, ldb) quiet # delete + recursive delete - @test (DE.delete_object(de, "scalars1"); true) - @test ismissing(DE.find_fullpath(de, "scalars1", false)) + @test (DE.delete_object(de, "scalars"); true) + @test ismissing(DE.find_fullpath(de, "scalars", false)) end @@ -155,18 +188,43 @@ end z1=TSeries(2020Q1, rand(16)), z2=TSeries(2020M7, rand(Int, 27)), z3=TSeries(w"2020-01-01"3, [1.0im .+ (1:11);]), + z4=TSeries(bd"2020-01-01", MIT{HalfYearly{1}}[rand(Int16, 20);]), b1=((1:100) .< 71) ) + cata = "series" + has_jtype = [:b1] + has_jeltype = [:vs2, :nv1, :nv5, :nv6, :nv7, :b1] + + pid = DE.find_fullpath(de, "/$cata", false) + if ismissing(pid) + pid = DE.new_catalog(de, cata) + end + # we can write them for (name, value) in pairs(db) - @test (DE.store_tseries(de, name, value); true) + id = DE.store_tseries(de, pid, name, value) + @test id == DE.find_fullpath(de, "/$cata/$name") + attr = DE.get_all_attributes(de, id) + _check_attributes(name, value, attr, has_jtype, has_jeltype) end ldb = Workspace() - # we can read them + # we can read them from (pid,name) + for name in keys(db) + @test (push!(ldb, name => DE.load_tseries(de, pid, name)); true) + end + + # they are equal + @test @compare db ldb quiet + + # their types are the same + @test @compare map(typeof, db) map(typeof, ldb) quiet + + ldb = Workspace() + # we can read them from full path for name in keys(db) - @test (push!(ldb, name => DE.load_tseries(de, name)); true) + @test (push!(ldb, name => DE.load_tseries(de, "/$cata/$name")); true) end # they are equal @@ -232,6 +290,12 @@ DE.closedaec!(de) mvts[mod1(i, length(mvts))] = i push!(b, name => copy(mvts)) end + scalar_types = Base.BitInteger_types ∪ (Float16, Float32, Float64, ComplexF16, ComplexF32, ComplexF64) + c = get!(db, :c, Workspace()) + for i = 1:100_000 + name = Symbol(:c, i) + push!(c, name => rand(rand(scalar_types))) + end tm = time() DE.writedb(test_file, "/speedtest", db) @@ -258,7 +322,7 @@ end DE.opendaec(test_file) do de @test !isempty(DE.readdb(de)) end - DE.opendaec(test_file) do de + DE.opendaec(test_file, write=true) do de @test (empty!(de); true) @test isempty(DE.readdb(de)) end @@ -272,7 +336,7 @@ rm(test_file, force=true) @testset "DE show" begin @test_throws DE.DEError DE.opendaec("/this/path/does/not/exist.daec") Core.eval(DE.I, :(debug_libdaec = :debug)) - @test_logs (:error, r".*DE\(\d+\) SQLite3: unable to open database file.*in: de_open \(.*\).*"i) begin + @test_logs (:error, r".*DE\(\d+\) SQLite3: unable to open database file.*"i) begin try DE.opendaec("/this/path/does/not/exist.daec") catch err @@ -299,3 +363,104 @@ rm(test_file, force=true) end +################################################################################################## + +# Compare the TimeSeriesEcon internal encoding of MITs to the +# encodings produced by de_pack_xyz and de_unpack_xyz. + +# Known bug: in TimeSeriesEcon, MIT{BDaily} misbehave when the year is 0 or negative. + +# Frequency \ pack/unpack || year+period | year+month+day +# =============================================================== +# YP Date || works | works as of DataEcon v0.3.1 +# Cal Date || works | works + +@testset "pack/unpack year_period" begin + # here we test the first column of the table - that is packing and unpacking + # MITs given year-period + Random.seed!(0x007) + fc = Dict{Type{<:Frequency},Base.RefValue{Int}}() + all_freqs = [Daily, BDaily, (Weekly{i} for i = 1:7)..., + Monthly, (Quarterly{i} for i = 1:3)..., + (HalfYearly{i} for i = 1:6)..., (Yearly{i} for i = 1:12)...] + for i = 1:1000 + fr = rand(all_freqs) + d1 = convert(Int, rand(Int16)) + if fr == BDaily && d1 < 1 + # make sure it's non-negative to work around known bug + d1 = 1 - d1 + end + d = MIT{fr}(d1) + y, p = isweekly(fr) ? TimeSeriesEcon._mit2yp(d) : TimeSeriesEcon.mit2yp(d) + d2 = Ref{Int64}(0) + f = DE.I._to_de_scalar_freq(fr) + @test DE.C.DE_SUCCESS == DE.C.de_pack_year_period_date(f, y, p, d2) + if d1 != d2[] + @info "Not equal" fr d1 d y p d2 + continue + end + @test d1 == d2[] + yr = Ref{Int32}(0) + pr = Ref{UInt32}(0) + @test DE.C.DE_SUCCESS == DE.C.de_unpack_year_period_date(f, d2[], yr, pr) + @test y == yr[] && p == pr[] + if !(y == yr[] && p == pr[]) + @info "Not equal" fr d1 d y p d2 yr pr + end + + get!(fc, fr, Ref(0))[] += 1 + end + # make sure we tested all frequencies + foreach(all_freqs) do fr + @test get!(fc, fr, Ref(0))[] > 10 + end +end; + + +@testset "pack/unpack year_month_day" begin + # here we test the second column of the table - that is packing and unpacking + # MITs given year-month-day. + Random.seed!(0x007) + fc = Dict{Type{<:Frequency},Base.RefValue{Int}}() + # all_freqs = [Daily, BDaily, (Weekly{i} for i = 1:7)...,] + all_freqs = [Daily, BDaily, (Weekly{i} for i = 1:7)..., + Monthly, (Quarterly{i} for i = 1:3)..., + (HalfYearly{i} for i = 1:6)..., (Yearly{i} for i = 1:12)...] + for i = 1:1000 + fr = rand(all_freqs) + d1 = convert(Int, rand(Int16)) + d = MIT{fr}(d1) + date = Date(d) + yr = Dates.year(date) + mn = Dates.month(date) + dy = Dates.day(date) + + f = DE.I._to_de_scalar_freq(fr) + + d2 = Ref{Int64}(0) + @test DE.C.DE_SUCCESS == DE.C.de_pack_calendar_date(f, yr, mn, dy, d2) + if d1 != d2[] + @info "Not equal" fr d1 d y p d2 + continue + end + @test d1 == d2[] + + Y = Ref{Int32}() + M = Ref{UInt32}() + D = Ref{UInt32}() + @test DE.C.DE_SUCCESS == DE.C.de_unpack_calendar_date(f, d1, Y, M, D) + if !(yr == Y[] && mn == M[] && dy == D[]) + @info "Not equal" fr d1 d y p d2 yr pr + end + @test yr == Y[] && mn == M[] && dy == D[] + + get!(fc, fr, Ref(0))[] += 1 + end + # make sure we tested all supported frequencies + foreach(all_freqs) do fr + @test get!(fc, fr, Ref(0))[] > 10 + end +end; + + + diff --git a/test/test_fconvert.jl b/test/test_fconvert.jl index 3768152..01fd490 100644 --- a/test/test_fconvert.jl +++ b/test/test_fconvert.jl @@ -2,7 +2,7 @@ using Suppressor using Statistics import Dates -all_frequencies = [ +all_frequencies = Type{<:Frequency}[ Daily, BDaily, Weekly, @@ -40,20 +40,19 @@ all_frequencies = [ Yearly{12} ] -function get_random_frequency(frequencies::Vector=all_frequencies) - return frequencies[rand(1:length(frequencies))] +@inline function get_random_frequency(frequencies::Vector{Type{<:Frequency}}=all_frequencies) + return rand(frequencies) end -function get_random_date() - Dates.Date("2022-01-01") + Dates.Day(floor(365*rand(1)[1])) +@inline function get_random_date() + Dates.Date(2022, 1, 1) + Dates.Day(rand(1:365)) end -function get_random_tseries(frequencies::Vector{Type}=all_frequencies) - F = get_random_frequency(frequencies) +@inline get_random_tseries(frequencies::Vector{Type{<:Frequency}}=all_frequencies) = get_random_tseries(get_random_frequency(frequencies)) +function get_random_tseries(F::Type{<:Frequency}) l = 4 * ppy(F) start_mit = fconvert(F, daily(get_random_date())) - ts = TSeries(start_mit, collect(1:l)) - ts .* rand(l) + return TSeries(start_mit .+ (0:l-1), rand) end @testset "fconvert, general" begin @@ -76,15 +75,15 @@ end @test fconvert(Yearly, q, method=:begin).values == [1.0, 5.0, 9.0] @test fconvert(Yearly, q, method=:sum).values == [10.0, 26.0] - h = TSeries(5H1, 1.0*collect(1:10)) + h = TSeries(5H1, 1.0 * collect(1:10)) yh = fconvert(Yearly, h) @test typeof(yh) == TSeries{Yearly{12},Float64,Vector{Float64}} @test fconvert(Yearly, h, method=:mean).values == [1.5, 3.5, 5.5, 7.5, 9.5] - @test fconvert(Yearly, h, method=:point, ref=:end).values == [2,4,6,8,10] - @test fconvert(Yearly, h, method=:end).values == [2,4,6,8,10] - @test fconvert(Yearly, h, method=:point, ref=:begin).values == [1,3,5,7,9] - @test fconvert(Yearly, h, method=:begin).values == [1,3,5,7,9] - @test fconvert(Yearly, h, method=:sum).values == [3,7,11,15,19] + @test fconvert(Yearly, h, method=:point, ref=:end).values == [2, 4, 6, 8, 10] + @test fconvert(Yearly, h, method=:end).values == [2, 4, 6, 8, 10] + @test fconvert(Yearly, h, method=:point, ref=:begin).values == [1, 3, 5, 7, 9] + @test fconvert(Yearly, h, method=:begin).values == [1, 3, 5, 7, 9] + @test fconvert(Yearly, h, method=:sum).values == [3, 7, 11, 15, 19] for i = 1:11 @test rangeof(fconvert(Yearly, TSeries(1M1 .+ (i:50)), method=:mean)) == 2Y:4Y @@ -92,7 +91,7 @@ end end for i = 1:3 @test rangeof(fconvert(Yearly, TSeries(1Q1 .+ (i:50)), method=:mean)) == 2Y:12Y - @test rangeof(fconvert(Yearly, TSeries(1Q1 .+ (0:47+i)))) == 1Y:12Y + @test rangeof(fconvert(Yearly, TSeries(1Q1 .+ (0:47+i)))) == 1Y:12Y end for i = 1:2 @test rangeof(fconvert(Yearly, TSeries(1H1 .+ (i:50)), method=:mean)) == 2Y:25Y @@ -217,20 +216,20 @@ end h8 = TSeries(2022H1, collect(1:5)) m8 = fconvert(Monthly, h8) @test rangeof(m8) == 2022M1:2024M6 - @test values(m8) == [1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,5,5,5,5,5,5] + @test values(m8) == [1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5] m8_begin = fconvert(Monthly, h8, method=:linear, ref=:begin) @test rangeof(m8_begin) == 2022M1:2024M6 - @test values(m8_begin)[1:13] ≈ [1,1+1/6,1+2/6,1+3/6,1+4/6,1+5/6,2,2+1/6,2+2/6,2+3/6,2+4/6,2+5/6,3] + @test values(m8_begin)[1:13] ≈ [1, 1 + 1 / 6, 1 + 2 / 6, 1 + 3 / 6, 1 + 4 / 6, 1 + 5 / 6, 2, 2 + 1 / 6, 2 + 2 / 6, 2 + 3 / 6, 2 + 4 / 6, 2 + 5 / 6, 3] m8_end = fconvert(Monthly, h8, method=:linear, ref=:end) @test rangeof(m8_end) == 2022M1:2024M6 - @test values(m8_end)[1:13] ≈ [0+1/6,0+2/6,0+3/6,0+4/6,0+5/6,1,1+1/6,1+2/6,1+3/6,1+4/6,1+5/6,2,2+1/6] + @test values(m8_end)[1:13] ≈ [0 + 1 / 6, 0 + 2 / 6, 0 + 3 / 6, 0 + 4 / 6, 0 + 5 / 6, 1, 1 + 1 / 6, 1 + 2 / 6, 1 + 3 / 6, 1 + 4 / 6, 1 + 5 / 6, 2, 2 + 1 / 6] # m8_middle = fconvert(Monthly, h8, method=:linear, ref=:middle) # @test rangeof(m8_middle) == 2022M1:2024M6 # @test values(m8_middle)[1:13] ≈ [.75-1/6, .75, .75+1/6,1.25-1/6, 1.25, 1.25+1/6, 1.75-1/6, 1.75, 1.75+1/6, 2.25-1/6, 2.25,2.25+1/6,2.75-1/6] m8_even = fconvert(Monthly, h8, method=:even) @test rangeof(m8_even) == 2022M1:2024M6 - @test values(m8_even) == [1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,5,5,5,5,5,5] ./ 6 - + @test values(m8_even) == [1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5] ./ 6 + end @testset "fconvert, YPFrequencies, to lower" begin @@ -328,18 +327,18 @@ end q9 = TSeries(2022Q1, collect(1:10)) y9_begin = fconvert(Yearly, q9, method=:point, ref=:begin) @test rangeof(y9_begin) == 2022Y:2024Y - @test y9_begin.values == [1,5,9] - @test fconvert(Yearly, q9, method=:begin) == y9_begin + @test y9_begin.values == [1, 5, 9] + @test fconvert(Yearly, q9, method=:begin) == y9_begin y9_end = fconvert(Yearly, q9, method=:point, ref=:end) @test rangeof(y9_end) == 2022Y:2023Y - @test y9_end.values == [4,8] - @test fconvert(Yearly, q9, method=:end) == y9_end + @test y9_end.values == [4, 8] + @test fconvert(Yearly, q9, method=:end) == y9_end y9_begin_m8 = fconvert(Yearly{8}, q9, method=:point, ref=:begin) @test rangeof(y9_begin_m8) == MIT{Yearly{8}}(2023):MIT{Yearly{8}}(2024) - @test y9_begin_m8.values == [3,7] + @test y9_begin_m8.values == [3, 7] y9_end_m8 = fconvert(Yearly{8}, q9, method=:point, ref=:end) @test rangeof(y9_end_m8) == MIT{Yearly{8}}(2022):MIT{Yearly{8}}(2024) - @test y9_end_m8.values == [2,6, 10] + @test y9_end_m8.values == [2, 6, 10] q10 = TSeries(2022Q2, collect(2:11)) y10_end = fconvert(Yearly{11}, q10, method=:mean, ref=:end) @@ -930,7 +929,7 @@ end """repo CONVERT(WS3, QUARTERLY(FEBRUARY), DISCRETE, AVERAGED)""" r1 = fconvert(Quarterly{2}, t3, method=:mean) - @test isapprox(r1.values, [15, 28, 41, 54], atol=1e-1) + @test isapprox(r1.values, [15, 28, 41, 54], atol=1e-1) @test rangeof(r1) == MIT{Quarterly{2}}(5):MIT{Quarterly{2}}(8) r1_range = fconvert(Quarterly{2}, rangeof(t3), trim=:both) @test r1_range == MIT{Quarterly{2}}(5):MIT{Quarterly{2}}(8) @@ -940,7 +939,7 @@ end """repo CONVERT(WS3, QUARTERLY(FEBRUARY), DISCRETE, END)""" r2 = fconvert(Quarterly{2}, t3, method=:point, ref=:end) - @test isapprox(r2.values, [8, 21, 34, 47, 60], atol=1e-2) + @test isapprox(r2.values, [8, 21, 34, 47, 60], atol=1e-2) @test rangeof(r2) == MIT{Quarterly{2}}(4):MIT{Quarterly{2}}(8) r2_range = fconvert(Quarterly{2}, rangeof(t3), trim=:end) @test r2_range == MIT{Quarterly{2}}(4):MIT{Quarterly{2}}(8) @@ -959,7 +958,7 @@ end # @test r3_MIT_start:r3_MIT_end == r3_range """repo CONVERT(WS3, QUARTERLY(FEBRUARY), DISCRETE, SUMMED)""" - r4 = fconvert(Quarterly{2}, t3, method=:sum) + r4 = fconvert(Quarterly{2}, t3, method=:sum) @test isapprox(r4.values, [195, 364, 532, 702], atol=1e-0) @test rangeof(r4) == MIT{Quarterly{2}}(5):MIT{Quarterly{2}}(8) r4_range = fconvert(Quarterly{2}, rangeof(t3), trim=:both) @@ -989,7 +988,7 @@ end """repo CONVERT(WS4, QUARTERLY(FEBRUARY), DISCRETE, AVERAGED)""" r9 = fconvert(Quarterly{2}, t4, method=:mean) - @test isapprox(r9.values, [14.50, 28.00, 41, 54], atol=1e-1) + @test isapprox(r9.values, [14.50, 28.00, 41, 54], atol=1e-1) @test rangeof(r1) == MIT{Quarterly{2}}(5):MIT{Quarterly{2}}(8) r9_range = fconvert(Quarterly{2}, rangeof(t4), trim=:both) @test r9_range == MIT{Quarterly{2}}(5):MIT{Quarterly{2}}(8) @@ -999,7 +998,7 @@ end """repo CONVERT(WS4, QUARTERLY(FEBRUARY), DISCRETE, END)""" r10 = fconvert(Quarterly{2}, t4, method=:point, ref=:end) - @test isapprox(r10.values, [7, 21, 34, 47, 60], atol=1e-2) + @test isapprox(r10.values, [7, 21, 34, 47, 60], atol=1e-2) @test rangeof(r10) == MIT{Quarterly{2}}(4):MIT{Quarterly{2}}(8) r10_range = fconvert(Quarterly{2}, rangeof(t4), trim=:end) @test r10_range == MIT{Quarterly{2}}(4):MIT{Quarterly{2}}(8) @@ -1019,7 +1018,7 @@ end """repo CONVERT(WS4, QUARTERLY(FEBRUARY), DISCRETE, SUMMED)""" r12 = fconvert(Quarterly{2}, t4, method=:sum) - @test isapprox(r12.values, [203, 364, 532, 702], atol=1e-0) + @test isapprox(r12.values, [203, 364, 532, 702], atol=1e-0) @test rangeof(r12) == MIT{Quarterly{2}}(5):MIT{Quarterly{2}}(8) r12_range = fconvert(Quarterly{2}, rangeof(t4), trim=:both) @test r12_range == MIT{Quarterly{2}}(5):MIT{Quarterly{2}}(8) @@ -1811,7 +1810,7 @@ end @test bd3.values == [ones(260)..., (2 * ones(260))...] d1_lin = fconvert(Daily, t1, method=:linear) - @test d1_lin[daily("2022-01-01"):daily("2022-01-31")].values ≈ collect(LinRange(1 - (1/28)*31, 1, 32))[2:32] + @test d1_lin[daily("2022-01-01"):daily("2022-01-31")].values ≈ collect(LinRange(1 - (1 / 28) * 31, 1, 32))[2:32] @test d1_lin[daily("2022-02-01"):daily("2022-02-28")].values == collect(LinRange(1, 2, 29))[2:29] @test d1_lin[daily("2022-04-01"):daily("2022-04-30")].values == collect(LinRange(3, 4, 31))[2:31] @test length(d1_lin) == 365 @@ -1820,7 +1819,7 @@ end @test d1_lin2[daily("2022-01-01"):daily("2022-01-31")].values == collect(LinRange(1, 2, 32))[1:31] @test d1_lin2[daily("2022-02-01"):daily("2022-02-28")].values == collect(LinRange(2, 3, 29))[1:28] @test d1_lin2[daily("2022-04-01"):daily("2022-04-30")].values == collect(LinRange(4, 5, 31))[1:30] - @test d1_lin2[daily("2022-12-01"):daily("2022-12-31")].values == collect(LinRange(12, 12+(1/30)*31, 32))[1:31] + @test d1_lin2[daily("2022-12-01"):daily("2022-12-31")].values == collect(LinRange(12, 12 + (1 / 30) * 31, 32))[1:31] @test length(d1_lin2) == 365 # d1_lin3 = fconvert(Daily, t1, method=:linear, ref=:middle) @@ -1830,7 +1829,7 @@ end # @test length(d1_lin3) == 365 d2_lin = fconvert(Daily, t2, method=:linear) - @test d2_lin[daily("2022-01-01"):daily("2022-03-31")].values == collect(LinRange(1 - 1/(30 + 31+ 30)*(31 + 28 + 31), 1, 31 + 28 + 31 + 1))[2:31+28+31+1] + @test d2_lin[daily("2022-01-01"):daily("2022-03-31")].values == collect(LinRange(1 - 1 / (30 + 31 + 30) * (31 + 28 + 31), 1, 31 + 28 + 31 + 1))[2:31+28+31+1] @test d2_lin[daily("2022-04-01"):daily("2022-06-30")].values == collect(LinRange(1, 2, 30 + 31 + 30 + 1))[2:30+31+30+1] @test d2_lin[daily("2022-07-01"):daily("2022-09-30")].values == collect(LinRange(2, 3, 31 + 31 + 30 + 1))[2:31+31+30+1] @test length(d2_lin) == 365 @@ -1839,7 +1838,7 @@ end @test d2_lin2[daily("2022-01-01"):daily("2022-03-31")].values == collect(LinRange(1, 2, 31 + 28 + 31 + 1))[1:31+28+31] @test d2_lin2[daily("2022-04-01"):daily("2022-06-30")].values == collect(LinRange(2, 3, 30 + 31 + 30 + 1))[1:30+31+30] @test d2_lin2[daily("2022-07-01"):daily("2022-09-30")].values == collect(LinRange(3, 4, 31 + 31 + 30 + 1))[1:31+31+30] - @test d2_lin2[daily("2022-10-01"):daily("2022-12-31")].values == collect(LinRange(4, 4 + 1/(31 + 31 + 30)*(31+30+31), 31 + 30 + 31 + 1))[1:31+30+31] + @test d2_lin2[daily("2022-10-01"):daily("2022-12-31")].values == collect(LinRange(4, 4 + 1 / (31 + 31 + 30) * (31 + 30 + 31), 31 + 30 + 31 + 1))[1:31+30+31] @test length(d2_lin2) == 365 d3_lin = fconvert(Daily, t3, method=:linear) @@ -1851,7 +1850,7 @@ end @test length(d3_lin2) == 365 * 2 bd1_lin = fconvert(BDaily, t1, method=:linear) - @test bd1_lin[bdaily("2022-01-01", bias=:next):bdaily("2022-01-31", bias=:previous)].values == collect(LinRange(1 - 1/20*21, 1, 22))[2:22] + @test bd1_lin[bdaily("2022-01-01", bias=:next):bdaily("2022-01-31", bias=:previous)].values == collect(LinRange(1 - 1 / 20 * 21, 1, 22))[2:22] @test bd1_lin[bdaily("2022-02-01", bias=:next):bdaily("2022-02-28", bias=:previous)].values == collect(LinRange(1, 2, 21))[2:21] @test bd1_lin[bdaily("2022-04-01", bias=:next):bdaily("2022-04-30", bias=:previous)].values == collect(LinRange(3, 4, 22))[2:22] @test length(bd1_lin) == 260 @@ -1863,7 +1862,7 @@ end @test length(bd1_lin2) == 260 bd2_lin = fconvert(BDaily, t2, method=:linear) - @test bd2_lin[bdaily("2022-01-01", bias=:next):bdaily("2022-03-31", bias=:previous)].values == collect(LinRange(1 - 1/(65)*64, 1, 64+1))[2:65] + @test bd2_lin[bdaily("2022-01-01", bias=:next):bdaily("2022-03-31", bias=:previous)].values == collect(LinRange(1 - 1 / (65) * 64, 1, 64 + 1))[2:65] @test length(bd2_lin) == 260 bd2_lin2 = fconvert(BDaily, t2, method=:linear, ref=:begin) @@ -1894,22 +1893,22 @@ end t1_2021 = TSeries(2021M1, collect(1:12)) w1 = fconvert(Weekly, t1) - @test w1[1:35] == [1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5,5,5,6,6,6,6,7,7,7,7,7,8,8,8,8] + @test w1[1:35] == [1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8] @test length(w1) == 52 - @test rangeof(w1) == weekly("2022-01-01"):weekly("2022-12-25") + @test rangeof(w1) == weekly("2022-01-01"):weekly("2022-12-25") w1_2001 = fconvert(Weekly, t1_2001) - @test w1_2001[1:35] == [1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,7,8,8,8,8,9] + @test w1_2001[1:35] == [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9] @test rangeof(w1_2001) == weekly("2001-01-01"):weekly("2001-12-30") w1_2017 = fconvert(Weekly, t1_2017) - @test w1_2017[1:35] == [1,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,7,8,8,8,8] - @test rangeof(w1_2017) == weekly("2017-01-01"):weekly("2017-12-31") + @test w1_2017[1:35] == [1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8] + @test rangeof(w1_2017) == weekly("2017-01-01"):weekly("2017-12-31") w1_2021 = fconvert(Weekly{5}, t1_2021) @test rangeof(w1_2021) == weekly("2021-01-01", 5):weekly("2021-12-31", 5) w2 = fconvert(Weekly, t2) @test w2[1:20] == [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2] @test length(w2) == 52 - @test rangeof(w2) == weekly("2022-01-01"):weekly("2022-12-25") + @test rangeof(w2) == weekly("2022-01-01"):weekly("2022-12-25") w3 = fconvert(Weekly, t3) @test w3[1:52] == ones(52) @@ -1983,29 +1982,34 @@ end end @testset "fconvert, pass function" begin - for i in 1:100 - # F_to = nothing - ts = get_random_tseries(filter(x -> x > Yearly, all_frequencies)) - F_to = get_random_frequency(filter(x -> x < frequencyof(ts), all_frequencies)) - @test fconvert(mean, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:mean, ref=:begin) - @test fconvert(mean, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:mean, ref=:end) - @test fconvert(sum, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:sum, ref=:begin) - @test fconvert(sum, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:sum, ref=:end) - @test fconvert(minimum, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:min, ref=:begin) - @test fconvert(minimum, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:min, ref=:end) - @test fconvert(maximum, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:max, ref=:begin) - @test fconvert(maximum, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:max, ref=:end) - end - for i in 1:100 - # F_to = nothing - ts = get_random_tseries(filter(x -> x < BDaily, all_frequencies)) - F_to = get_random_frequency(filter(x -> x > frequencyof(ts), all_frequencies)) - @test fconvert(TimeSeriesEcon.divide_uneven, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:even, ref=:begin) - @test fconvert(TimeSeriesEcon.divide_uneven, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:even, ref=:end) - @test fconvert(TimeSeriesEcon.repeat_uneven, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:const, ref=:begin) - @test fconvert(TimeSeriesEcon.repeat_uneven, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:const, ref=:end) - @test fconvert(TimeSeriesEcon.linear_uneven, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:linear, ref=:begin) - @test fconvert(TimeSeriesEcon.linear_uneven, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:linear, ref=:end) + for _ = 1:20 + F_from = get_random_frequency() + ts = get_random_tseries(F_from) + count = [0, 0] + while sum(count) < 20 + F_to = get_random_frequency() + if F_from > Yearly && F_from > F_to + count[1] += 1 + # convert to lower + @test fconvert(mean, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:mean, ref=:begin) + @test fconvert(mean, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:mean, ref=:end) + @test fconvert(sum, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:sum, ref=:begin) + @test fconvert(sum, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:sum, ref=:end) + @test fconvert(minimum, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:min, ref=:begin) + @test fconvert(minimum, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:min, ref=:end) + @test fconvert(maximum, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:max, ref=:begin) + @test fconvert(maximum, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:max, ref=:end) + elseif F_from < BDaily && F_from < F_to + count[2] += 1 + # convert to higher + @test fconvert(TimeSeriesEcon.divide_uneven, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:even, ref=:begin) + @test fconvert(TimeSeriesEcon.divide_uneven, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:even, ref=:end) + @test fconvert(TimeSeriesEcon.repeat_uneven, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:const, ref=:begin) + @test fconvert(TimeSeriesEcon.repeat_uneven, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:const, ref=:end) + @test fconvert(TimeSeriesEcon.linear_uneven, F_to, ts; ref=:begin) == fconvert(F_to, ts; method=:linear, ref=:begin) + @test fconvert(TimeSeriesEcon.linear_uneven, F_to, ts; ref=:end) == fconvert(F_to, ts; method=:linear, ref=:end) + end + end end end @@ -2014,7 +2018,7 @@ end second_highest(x) = length(x) == 1 ? x[1] : sort(x)[end-1] ts = TSeries(2022M1, collect(1:12)) ts_q = fconvert(second_highest, Quarterly, ts) - @test ts_q.values == [2,5,8,11] + @test ts_q.values == [2, 5, 8, 11] # to higher frequency function half_value(x, inner_lengths; kwargs...) @@ -2028,7 +2032,7 @@ end end ts2 = TSeries(2022Q1, collect(1:4)) ts_m = fconvert(half_value, Monthly, ts2) - @test ts_m.values == [0.5, 0.5, 0.5, 1,1,1,1.5,1.5,1.5,2,2,2] + @test ts_m.values == [0.5, 0.5, 0.5, 1, 1, 1, 1.5, 1.5, 1.5, 2, 2, 2] function replace_with_kwarg(x, inner_lengths; kwargs...) return kwargs[:replacement].values @@ -2041,50 +2045,16 @@ end end @testset "fconvert, all combinations" begin - frequencies = [ - Daily, - BDaily, - Weekly, - Weekly{1}, - Weekly{2}, - Weekly{3}, - Weekly{4}, - Weekly{5}, - Weekly{6}, - Weekly{7}, - Monthly, - Quarterly, - Quarterly{1}, - Quarterly{2}, - Quarterly{3}, - HalfYearly, - HalfYearly{1}, - HalfYearly{2}, - HalfYearly{3}, - HalfYearly{4}, - HalfYearly{5}, - HalfYearly{6}, - Yearly, - Yearly{1}, - Yearly{2}, - Yearly{3}, - Yearly{4}, - Yearly{5}, - Yearly{6}, - Yearly{7}, - Yearly{8}, - Yearly{9}, - Yearly{10}, - Yearly{11}, - Yearly{12} - ] + frequencies = all_frequencies # freq_short = combinations = [(F_from, F_to) for F_from in frequencies for F_to in frequencies] counter = 1 t_from = nothing last_F_from = nothing # @showprogress "combinations" for (F_from, F_to) in combinations - for (F_from, F_to) in combinations + # for (F_from, F_to) in combinations + for _ = 1:250 + (F_from, F_to) = rand(combinations) if F_from != last_F_from last_F_from = F_from t_from = TSeries(MIT{F_from}(100), collect(1:800)) @@ -2120,7 +2090,7 @@ end end end end - + # unit ranges range_to = @suppress fconvert(F_to, rangeof(t_from)) @test frequencyof(range_to) == TimeSeriesEcon.sanitize_frequency(F_to) @@ -2152,30 +2122,30 @@ end end @testset "extend series" begin - tm = TSeries(2022M2, collect(1.0:7.0)) - - ex1 = extend_series(Quarterly, tm) - @test rangeof(ex1) == 2022M1:2022M9 - @test ex1.values == [1.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 6.5] - - ex2 = extend_series(Quarterly, tm; method=:end) - @test rangeof(ex2) == 2022M1:2022M9 - @test ex2.values == [1.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 7.0] - - ex3 = extend_series(Quarterly, tm; direction=:end) - @test rangeof(ex3) == 2022M2:2022M9 - @test ex3.values == [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 6.5] - - ex4 = extend_series(Quarterly, tm; direction=:begin) - @test rangeof(ex4) == 2022M1:2022M8 - @test ex4.values == [1.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] - - ex5 = extend_series(Yearly, tm; direction=:both) - @test rangeof(ex5) == 2022M1:2022M12 - @test ex5.values == [4.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 4.0, 4.0, 4.0, 4.0] - - ex5 = extend_series(Yearly, tm; method=:end) - @test rangeof(ex5) == 2022M1:2022M12 - @test ex5.values == [1.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0] + tm = TSeries(2022M2, collect(1.0:7.0)) + + ex1 = extend_series(Quarterly, tm) + @test rangeof(ex1) == 2022M1:2022M9 + @test ex1.values == [1.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 6.5] + + ex2 = extend_series(Quarterly, tm; method=:end) + @test rangeof(ex2) == 2022M1:2022M9 + @test ex2.values == [1.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 7.0] + + ex3 = extend_series(Quarterly, tm; direction=:end) + @test rangeof(ex3) == 2022M2:2022M9 + @test ex3.values == [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 6.5] + + ex4 = extend_series(Quarterly, tm; direction=:begin) + @test rangeof(ex4) == 2022M1:2022M8 + @test ex4.values == [1.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + + ex5 = extend_series(Yearly, tm; direction=:both) + @test rangeof(ex5) == 2022M1:2022M12 + @test ex5.values == [4.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 4.0, 4.0, 4.0, 4.0] + + ex5 = extend_series(Yearly, tm; method=:end) + @test rangeof(ex5) == 2022M1:2022M12 + @test ex5.values == [1.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0] end diff --git a/test/test_mit.jl b/test/test_mit.jl index dd82484..a5f76c8 100644 --- a/test/test_mit.jl +++ b/test/test_mit.jl @@ -472,7 +472,21 @@ end @test Date(last(bd_rng)) == Date("2022-01-21") # this test does not catch the error for some reason - # @test_throws ArgumentError bd"2022-01-01:2022-01-22"p + @test_throws ArgumentError bd"2022-01-01:2022-01-22"p +end + +@testset "issue #45" begin + # errors related to divrem returning negative + # remainder for negative argument + + # Example 1: + @test_throws ArgumentError bd"0-1-1" + @test bd"0-1-1"p == bdaily(Date(-1,12,31)) + @test bd"0-1-1"n == bdaily(Date(0,1,3)) + # Example 2: + @test string(bdaily(Date(-1, 1, 1))) == "-0001-01-01" + # Example 3: + @test mit2yp(bd"1-1-1" - 1) == (0, 260) end @testset "Weekly" begin diff --git a/test/test_workspace.jl b/test/test_workspace.jl index 62a7064..04b860e 100644 --- a/test/test_workspace.jl +++ b/test/test_workspace.jl @@ -257,7 +257,7 @@ end @test compare(src, dest, quiet=true, ignoremissing=true) # dest has longer range (not that compare uses the common range) dest = MVTSeries(2020Q1 .+ (0:40), (:a, :b, :c)) - @test (copyto!(dest, src; range=2020Q1:2020Q1+19); compare(src, dest, quiet=true, ignoremissing=true)) + @test (copyto!(dest, src; trange=2020Q1:2020Q1+19); compare(src, dest, quiet=true, ignoremissing=true)) @test !compare(src, dest, quiet=true) # dest is missing some variables dest = MVTSeries(2020Q1 .+ (0:19), (:a, :c)) @@ -265,7 +265,7 @@ end @test compare(src, dest, quiet=true, ignoremissing=true) # dest is missing some variables, we initialize with NaN and we copy only a shorter range dest = MVTSeries(2020Q1 .+ (0:19), (:a, :c), NaN) - @test (copyto!(dest, src; range=2020Q1:2020Q1+10); true) + @test (copyto!(dest, src; trange=2020Q1:2020Q1+10); true) # not equal over the full range @test !compare(src, dest; quiet=true, ignoremissing=true) # not equal on the missing variable