diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..7fdc57496 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libdeflate"] + path = libdeflate + url = https://github.com/ebiggers/libdeflate diff --git a/Dockerfile b/Dockerfile index 1df89e5f1..0eef76218 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update \ RUN mkdir -p /tmp/tippecanoe-src WORKDIR /tmp/tippecanoe-src COPY . /tmp/tippecanoe-src +ENV LD_LIBRARY_PATH LD_LIBRARY_PATH:libdeflate # Build tippecanoe RUN make \ diff --git a/Dockerfile.centos7 b/Dockerfile.centos7 index c330de2f1..d1803414f 100644 --- a/Dockerfile.centos7 +++ b/Dockerfile.centos7 @@ -6,6 +6,7 @@ RUN yum install -y make sqlite-devel zlib-devel bash git gcc-c++ RUN mkdir -p /tmp/tippecanoe-src WORKDIR /tmp/tippecanoe-src COPY . /tmp/tippecanoe-src +ENV LD_LIBRARY_PATH LD_LIBRARY_PATH:libdeflate # Build tippecanoe RUN make \ diff --git a/Makefile b/Makefile index 24f150ecb..c65f9a0d6 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,11 @@ else FINAL_FLAGS := -g $(WARNING_FLAGS) $(DEBUG_FLAGS) endif -all: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit tippecanoe-json-tool +all: deflate tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit tippecanoe-json-tool docs: man/tippecanoe.1 -install: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join tippecanoe-json-tool +install: deflate tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join tippecanoe-json-tool mkdir -p $(PREFIX)/bin mkdir -p $(MANDIR) cp tippecanoe $(PREFIX)/bin/tippecanoe @@ -44,20 +44,21 @@ PG= H = $(wildcard *.h) $(wildcard *.hpp) C = $(wildcard *.c) $(wildcard *.cpp) -INCLUDES = -I/usr/local/include -I. -LIBS = -L/usr/local/lib +INCLUDES = -I/usr/local/include -I. -Ilibdeflate +LIBS = -L/usr/local/lib -L$(PWD)/libdeflate -L$(PWD) +LD_LIBRARY_PATH := /usr/local/lib:$(PWD)/libdeflate:$(PWD) tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o evaluator.o geocsv.o csv.o geojson-loop.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 libdeflate/libdeflate.a tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread libdeflate/libdeflate.a tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread @@ -73,7 +74,7 @@ unit: unit.o text.o %.o: %.cpp $(CXX) -MMD $(PG) $(INCLUDES) $(FINAL_FLAGS) $(CXXFLAGS) -c -o $@ $< -clean: +clean: deflate-clean rm -f ./tippecanoe ./tippecanoe-* ./tile-join ./unit *.o *.d */*.o */*.d tests/**/*.mbtiles tests/**/*.check indent: @@ -331,6 +332,7 @@ layer-json-test: # Use this target to regenerate the standards that the tests are compared against # after making a change that legitimately changes their output +# prep-test: $(TESTS) @@ -339,3 +341,9 @@ tests/%.json: Makefile tippecanoe tippecanoe-decode ./tippecanoe-decode -x generator $@.check.mbtiles > $@ cmp $(patsubst %.check,%,$@) $@ rm $@.check.mbtiles + +deflate: deflate-clean + cd libdeflate && make BUILDTYPE=$(BUILDTYPE) libdeflate.a + +deflate-clean: + cd libdeflate && make realclean diff --git a/libdeflate b/libdeflate new file mode 160000 index 000000000..d6d50c695 --- /dev/null +++ b/libdeflate @@ -0,0 +1 @@ +Subproject commit d6d50c69554ac25897fc988d671e8e39780bab92 diff --git a/mvt.cpp b/mvt.cpp index 27486f839..f841aa9f0 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include #include "mvt.hpp" #include "geometry.hpp" #include "protozero/varint.hpp" @@ -22,114 +22,89 @@ mvt_geometry::mvt_geometry(int nop, long long nx, long long ny) { // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp bool is_compressed(std::string const &data) { - return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); + return data.size() > 2 && (((uint8_t) data[0] & 0x60) == 0x4 || ((uint8_t) data[0] & 0x60) == 0x20); } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int decompress(std::string const &input, std::string &output) { - z_stream inflate_s; - inflate_s.zalloc = Z_NULL; - inflate_s.zfree = Z_NULL; - inflate_s.opaque = Z_NULL; - inflate_s.avail_in = 0; - inflate_s.next_in = Z_NULL; - if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { - fprintf(stderr, "Decompression error: %s\n", inflate_s.msg); + size_t avail_in = 8192; + size_t avail_out = 8192; + long unsigned int actual_output = 0; + long unsigned int actual_input = 0; + long unsigned int actual_out = 0; + long unsigned int actual_in = 0; + output.resize(avail_out); + void *current_out = (void *) output.data(); + void *current_in = (void *) input.data(); + + struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); +decompress: + int ret = libdeflate_deflate_decompress_ex(decompressor, + current_in, avail_in, + current_out, avail_out, + &actual_input, + &actual_output); + actual_in += actual_input; + actual_out += actual_output; + if (ret == LIBDEFLATE_SHORT_OUTPUT) { + output.resize(actual_out + avail_out); + current_out = (void *) ((long) output.data() + actual_out); + current_in = (void *) ((long) input.data() + actual_in); + goto decompress; } - inflate_s.next_in = (Bytef *) input.data(); - inflate_s.avail_in = input.size(); - inflate_s.next_out = (Bytef *) output.data(); - inflate_s.avail_out = output.size(); + libdeflate_free_decompressor(decompressor); - while (true) { - size_t existing_output = inflate_s.next_out - (Bytef *) output.data(); - - output.resize(existing_output + 2 * inflate_s.avail_in + 100); - inflate_s.next_out = (Bytef *) output.data() + existing_output; - inflate_s.avail_out = output.size() - existing_output; - - int ret = inflate(&inflate_s, 0); - if (ret < 0) { + if (ret != LIBDEFLATE_SUCCESS) { + if (ret == LIBDEFLATE_BAD_DATA) { + fprintf(stderr, "data not compressed"); + } else { fprintf(stderr, "Decompression error: "); - if (ret == Z_DATA_ERROR) { - fprintf(stderr, "data error"); - } - if (ret == Z_STREAM_ERROR) { - fprintf(stderr, "stream error"); - } - if (ret == Z_MEM_ERROR) { + if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) { fprintf(stderr, "out of memory"); } - if (ret == Z_BUF_ERROR) { - fprintf(stderr, "no data in buffer"); - } - fprintf(stderr, "\n"); - return 0; - } - - if (ret == Z_STREAM_END) { - break; } - - // ret must be Z_OK or Z_NEED_DICT; - // continue decompresing + fprintf(stderr, "\n"); + return -ret; } - output.resize(inflate_s.next_out - (Bytef *) output.data()); - inflateEnd(&inflate_s); - return 1; + output.resize(actual_out); + return 0; } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp int compress(std::string const &input, std::string &output) { - z_stream deflate_s; - deflate_s.zalloc = Z_NULL; - deflate_s.zfree = Z_NULL; - deflate_s.opaque = Z_NULL; - deflate_s.avail_in = 0; - deflate_s.next_in = Z_NULL; - deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); - deflate_s.next_in = (Bytef *) input.data(); - deflate_s.avail_in = input.size(); - size_t length = 0; - do { - size_t increase = input.size() / 2 + 1024; - output.resize(length + increase); - deflate_s.avail_out = increase; - deflate_s.next_out = (Bytef *) (output.data() + length); - int ret = deflate(&deflate_s, Z_FINISH); - if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { - return -1; - } - length += (increase - deflate_s.avail_out); - } while (deflate_s.avail_out == 0); - deflateEnd(&deflate_s); - output.resize(length); - return 0; + size_t avail_in = input.size(); + size_t avail_out = avail_in * 8; + output.resize(avail_out); + struct libdeflate_compressor *deflate_s = libdeflate_alloc_compressor(9); + int ret = libdeflate_deflate_compress(deflate_s, + (void *) input.data(), avail_in, + (void *) output.data(), avail_out); + libdeflate_free_compressor(deflate_s); + if (ret == 0) { + return -1; + } + output.resize(ret); + return 1; } bool mvt_tile::decode(std::string &message, bool &was_compressed) { layers.clear(); std::string src; + std::string uncompressed; - if (is_compressed(message)) { - std::string uncompressed; - if (decompress(message, uncompressed) == 0) { - exit(EXIT_FAILURE); - } + was_compressed = false; + src = message; + if (decompress(message, uncompressed) == 0) { src = uncompressed; was_compressed = true; - } else { - src = message; - was_compressed = false; } protozero::pbf_reader reader(src); while (reader.next()) { switch (reader.tag()) { - case 3: /* layer */ - { + case 3: /* layer */ { protozero::pbf_reader layer_reader(reader.get_message()); mvt_layer layer;