From e79b524e71368743ead60cdc56b5d31a19e2a01e Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 27 Apr 2017 14:03:03 -0700 Subject: [PATCH 1/2] Add an option to generate bitmap tiles as JPEG instead of PNG --- Makefile | 2 +- README.md | 1 + tile.cpp | 159 ++++++++++++++++++++++++++++++----------- tippecanoe/mbtiles.cpp | 6 +- tippecanoe/mbtiles.hpp | 2 +- 5 files changed, 124 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index e33bd87..0d6abe3 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ tile-count-decode: tippecanoe/projection.o decode.o header.o serial.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tile-count-tile: tippecanoe/projection.o tile.o header.o serial.o tippecanoe/mbtiles.o tippecanoe/mvt.o - $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -lpng + $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -lpng -ljpeg tile-count-merge: mergetool.o header.o serial.o merge.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread diff --git a/README.md b/README.md index 8642b91..37a7bdc 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ instead of merging existing tilesets. The *maxzoom* plus the *detail* always equ * `-b`: Create PNG raster tiles instead of vectors. If you are not planning to use these tiles with Mapbox GL, you will probably also want to specify `-d8` for normal 256x256 web map tile resolution. +* `-j`: Create JPEG raster tiles instead of vectors. * `-c` *rrggbb*: Specify the color to use in raster tiles as a hex color. * `-w`: Make tiles for a white background instead of a black background. diff --git a/tile.cpp b/tile.cpp index aeda0c1..36204c7 100644 --- a/tile.cpp +++ b/tile.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include "tippecanoe/projection.hpp" #include "protozero/varint.hpp" #include "protozero/pbf_reader.hpp" @@ -32,6 +33,7 @@ int first_count = 0; double count_gamma = 2.5; bool bitmap = false; +bool jpeg = false; int color = 0x888888; int white = 0; @@ -168,6 +170,23 @@ static void fail(png_structp png_ptr, png_const_charp error_msg) { exit(EXIT_FAILURE); } +void get_color(int i, png_color &out, png_byte &transparency) { + if (i < levels / 2) { + out.red = (color >> 16) & 0xFF; + out.green = (color >> 8) & 0xFF; + out.blue = (color >> 0) & 0xFF; + transparency = 255 * i / (levels / 2); + } else { + double along = 255 * (i - levels / 2) / (levels - levels / 2 - 1) / 255.0; + int fg = white ? 0x00 : 0xFF; + + out.red = ((color >> 16) & 0xFF) * (1 - along) + fg * (along); + out.green = ((color >> 8) & 0xFF) * (1 - along) + fg * (along); + out.blue = ((color >> 0) & 0xFF) * (1 - along) + fg * (along); + transparency = 255; + } +} + void make_tile(sqlite3 *outdb, tile &tile, int z, int detail, long long zoom_max, std::string const &layername) { std::string compressed; @@ -203,7 +222,7 @@ void make_tile(sqlite3 *outdb, tile &tile, int z, int detail, long long zoom_max if (bitmap) { bool anything = false; - for (size_t y = 0; y < 1U << detail; y++) { + for (size_t y = 0; y < (1U << detail); y++) { for (size_t x = 0; x < (1U << detail); x++) { long long density = normalized[y * (1 << detail) + x]; if (density > 0) { @@ -225,48 +244,101 @@ void make_tile(sqlite3 *outdb, tile &tile, int z, int detail, long long zoom_max } } - png_structp png_ptr; - png_infop info_ptr; + if (jpeg) { + struct jpeg_compress_struct cinfo; + struct jpeg_error_mgr jerr; + JSAMPROW row_pointer[1]; - png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, fail, fail); - if (png_ptr == NULL) { - fprintf(stderr, "PNG failure (write struct)\n"); - exit(EXIT_FAILURE); - } - info_ptr = png_create_info_struct(png_ptr); - if (info_ptr == NULL) { - png_destroy_write_struct(&png_ptr, NULL); - fprintf(stderr, "PNG failure (info struct)\n"); - exit(EXIT_FAILURE); + FILE *outfile = tmpfile(); // XXX write to memory instead + if (outfile == NULL) { + perror("tmpfile"); + exit(EXIT_FAILURE); + } + + cinfo.err = jpeg_std_error(&jerr); + jpeg_create_compress(&cinfo); + jpeg_stdio_dest(&cinfo, outfile); + + cinfo.image_width = 1U << detail; + cinfo.image_height = 1U << detail; + cinfo.input_components = 3; + cinfo.in_color_space = JCS_RGB; + jpeg_set_defaults(&cinfo); + jpeg_set_quality(&cinfo, 75, TRUE); + jpeg_start_compress(&cinfo, TRUE); + + for (size_t y = 0; y < (1U << detail); y++) { + unsigned char buf[3U << detail]; + int base = 0; + + if (white) { + base = 255; + } else { + base = 0; + } + + for (size_t x = 0; x < (1U << detail); x++) { + png_byte transparency; + png_color c; + + get_color(rows[y][x], c, transparency); + + c.red = c.red * transparency / 255.0 + base * (255 - transparency) / 255.0; + c.green = c.green * transparency / 255.0 + base * (255 - transparency) / 255.0; + c.blue = c.blue * transparency / 255.0 + base * (255 - transparency) / 255.0; + + buf[3 * x + 0] = c.red; + buf[3 * x + 1] = c.green; + buf[3 * x + 2] = c.blue; + } + + row_pointer[0] = buf; + jpeg_write_scanlines(&cinfo, row_pointer, 1); + } + + jpeg_finish_compress(&cinfo); + jpeg_destroy_compress(&cinfo); + + fseek(outfile, 0, SEEK_SET); + char buf[2000]; + int n; + while ((n = fread(buf, 1, 2000, outfile)) > 0) { + compressed += std::string(buf, n); + } + + fclose(outfile); } else { - png_set_IHDR(png_ptr, info_ptr, 1U << detail, 1U << detail, 8, PNG_COLOR_TYPE_PALETTE, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT); - - png_byte transparency[levels]; - png_color colors[levels]; - for (int i = 0; i < levels / 2; i++) { - colors[i].red = (color >> 16) & 0xFF; - colors[i].green = (color >> 8) & 0xFF; - colors[i].blue = (color >> 0) & 0xFF; - transparency[i] = 255 * i / (levels / 2); + png_structp png_ptr; + png_infop info_ptr; + + png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, fail, fail); + if (png_ptr == NULL) { + fprintf(stderr, "PNG failure (write struct)\n"); + exit(EXIT_FAILURE); } - for (int i = levels / 2; i < levels; i++) { - double along = 255 * (i - levels / 2) / (levels - levels / 2 - 1) / 255.0; - int fg = white ? 0x00 : 0xFF; - - colors[i].red = ((color >> 16) & 0xFF) * (1 - along) + fg * (along); - colors[i].green = ((color >> 8) & 0xFF) * (1 - along) + fg * (along); - colors[i].blue = ((color >> 0) & 0xFF) * (1 - along) + fg * (along); - transparency[i] = 255; + info_ptr = png_create_info_struct(png_ptr); + if (info_ptr == NULL) { + png_destroy_write_struct(&png_ptr, NULL); + fprintf(stderr, "PNG failure (info struct)\n"); + exit(EXIT_FAILURE); + } else { + png_set_IHDR(png_ptr, info_ptr, 1U << detail, 1U << detail, 8, PNG_COLOR_TYPE_PALETTE, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT); + + png_byte transparency[levels]; + png_color colors[levels]; + for (int i = 0; i < levels; i++) { + get_color(i, colors[i], transparency[i]); + } + png_set_tRNS(png_ptr, info_ptr, transparency, levels, NULL); + png_set_PLTE(png_ptr, info_ptr, colors, levels); + + png_set_rows(png_ptr, info_ptr, rows); + png_set_write_fn(png_ptr, &compressed, string_append, NULL); + png_write_png(png_ptr, info_ptr, 0, NULL); + png_write_end(png_ptr, info_ptr); + png_destroy_info_struct(png_ptr, &info_ptr); + png_destroy_write_struct(&png_ptr, &info_ptr); } - png_set_tRNS(png_ptr, info_ptr, transparency, levels, NULL); - png_set_PLTE(png_ptr, info_ptr, colors, levels); - - png_set_rows(png_ptr, info_ptr, rows); - png_set_write_fn(png_ptr, &compressed, string_append, NULL); - png_write_png(png_ptr, info_ptr, 0, NULL); - png_write_end(png_ptr, info_ptr); - png_destroy_info_struct(png_ptr, &info_ptr); - png_destroy_write_struct(&png_ptr, &info_ptr); } for (size_t i = 0; i < 1U << detail; i++) { @@ -1161,7 +1233,7 @@ int main(int argc, char **argv) { std::string layername = "count"; int i; - while ((i = getopt(argc, argv, "fz:Z:s:a:o:p:d:l:m:M:g:bwc:qn:y:1k")) != -1) { + while ((i = getopt(argc, argv, "fz:Z:s:a:o:p:d:l:m:M:g:bjwc:qn:y:1k")) != -1) { switch (i) { case 'f': force = true; @@ -1235,6 +1307,11 @@ int main(int argc, char **argv) { bitmap = 1; break; + case 'j': + bitmap = 1; + jpeg = 1; + break; + case 'c': color = strtoul(optarg, NULL, 16); break; @@ -1502,7 +1579,7 @@ int main(int argc, char **argv) { lm.insert(std::pair(layername, lme)); } - mbtiles_write_metadata(outdb, outfile, 0, zooms - 1, minlat, minlon, maxlat, maxlon, midlat, midlon, false, "", lm, !bitmap); + mbtiles_write_metadata(outdb, outfile, 0, zooms - 1, minlat, minlon, maxlat, maxlon, midlat, midlon, false, "", lm, jpeg ? "jpg" : bitmap ? "png" : "pbf"); write_meta(zoom_max, outdb); diff --git a/tippecanoe/mbtiles.cpp b/tippecanoe/mbtiles.cpp index d2d3480..133400c 100644 --- a/tippecanoe/mbtiles.cpp +++ b/tippecanoe/mbtiles.cpp @@ -131,7 +131,7 @@ bool type_and_string::operator<(const type_and_string &o) const { return false; } -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector) { +void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, const char *type) { char *sql, *err; sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname); @@ -217,7 +217,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); } - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", vector ? "pbf" : "png"); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", type); if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set format: %s\n", err); if (!forcetable) { @@ -226,7 +226,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int } sqlite3_free(sql); - if (vector) { + if (strcmp(type, "pbf") == 0) { std::string buf("{"); aprintf(&buf, "\"vector_layers\": [ "); diff --git a/tippecanoe/mbtiles.hpp b/tippecanoe/mbtiles.hpp index 3cb4ecf..7679fb6 100644 --- a/tippecanoe/mbtiles.hpp +++ b/tippecanoe/mbtiles.hpp @@ -20,7 +20,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector); +void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, const char *type); void mbtiles_close(sqlite3 *outdb, char **argv); From 4a8d2c976de3b8892c46f687fc697c316b109cd4 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 27 Apr 2017 14:49:55 -0700 Subject: [PATCH 2/2] Write JPEG data to memory instead of to a temporary file, for speed --- tile.cpp | 74 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/tile.cpp b/tile.cpp index 36204c7..ecde78a 100644 --- a/tile.cpp +++ b/tile.cpp @@ -187,6 +187,62 @@ void get_color(int i, png_color &out, png_byte &transparency) { } } +struct jpeg_writer { + struct jpeg_destination_mgr pub; + + unsigned char *buffer; + size_t capacity; + + std::string *s; + + jpeg_writer(std::string *out) { + capacity = 4000; + buffer = new unsigned char[capacity]; + s = out; + } + + ~jpeg_writer() { + delete[] buffer; + } +}; + +void jpeg_writer_init_destination(j_compress_ptr cinfo) { + jpeg_writer *dest = (jpeg_writer *) cinfo->dest; + + *(dest->s) = ""; + dest->pub.next_output_byte = dest->buffer; + dest->pub.free_in_buffer = dest->capacity; +} + +boolean jpeg_writer_empty_output_buffer(j_compress_ptr cinfo) { + jpeg_writer *dest = (jpeg_writer *) cinfo->dest; + + *(dest->s) += std::string((char *) dest->buffer, dest->capacity); + dest->pub.next_output_byte = dest->buffer; + dest->pub.free_in_buffer = dest->capacity; + return TRUE; +} + +void jpeg_writer_term_destination(j_compress_ptr cinfo) { + jpeg_writer *dest = (jpeg_writer *) cinfo->dest; + + size_t datacount = dest->capacity - dest->pub.free_in_buffer; + if (datacount > 0) { + *(dest->s) += std::string((char *) dest->buffer, datacount); + } +} + +jpeg_writer *jpeg_writer_dest(j_compress_ptr cinfo, std::string *out) { + cinfo->dest = (struct jpeg_destination_mgr *) new jpeg_writer(out); + + jpeg_writer *dest = (jpeg_writer *) cinfo->dest; + dest->pub.init_destination = jpeg_writer_init_destination; + dest->pub.empty_output_buffer = jpeg_writer_empty_output_buffer; + dest->pub.term_destination = jpeg_writer_term_destination; + + return dest; +} + void make_tile(sqlite3 *outdb, tile &tile, int z, int detail, long long zoom_max, std::string const &layername) { std::string compressed; @@ -249,15 +305,9 @@ void make_tile(sqlite3 *outdb, tile &tile, int z, int detail, long long zoom_max struct jpeg_error_mgr jerr; JSAMPROW row_pointer[1]; - FILE *outfile = tmpfile(); // XXX write to memory instead - if (outfile == NULL) { - perror("tmpfile"); - exit(EXIT_FAILURE); - } - cinfo.err = jpeg_std_error(&jerr); jpeg_create_compress(&cinfo); - jpeg_stdio_dest(&cinfo, outfile); + jpeg_writer *jw = jpeg_writer_dest(&cinfo, &compressed); cinfo.image_width = 1U << detail; cinfo.image_height = 1U << detail; @@ -298,15 +348,7 @@ void make_tile(sqlite3 *outdb, tile &tile, int z, int detail, long long zoom_max jpeg_finish_compress(&cinfo); jpeg_destroy_compress(&cinfo); - - fseek(outfile, 0, SEEK_SET); - char buf[2000]; - int n; - while ((n = fread(buf, 1, 2000, outfile)) > 0) { - compressed += std::string(buf, n); - } - - fclose(outfile); + delete jw; } else { png_structp png_ptr; png_infop info_ptr;