Exclude long attribute values from tilestats output, as specified

Makefile

@@ -55,7 +55,7 @@ tippecanoe-enumerate: enumerate.o
 tippecanoe-decode: decode.o projection.o mvt.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
-tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o
+tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 unit: unit.o text.o
@@ -85,7 +85,7 @@ test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test p
 %.json.check:
 	./tippecanoe -aD -f -o $@.mbtiles $(subst @,:,$(subst %,/,$(subst _, ,$(patsubst %.json.check,%,$(word 4,$(subst /, ,$@)))))) $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json) < /dev/null
 	./tippecanoe-decode $@.mbtiles > $@.out
-	cmp $(patsubst %.check,%,$@) $@.out
+	cmp $@.out $(patsubst %.check,%,$@)
 	rm $@.out $@.mbtiles
 
 parallel-test:

dirtiles.cpp

@@ -2,10 +2,11 @@
 #include <fstream>
 #include <sstream>
 #include <string>
-#include <sys/stat.h>
+#include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 #include <dirent.h>
+#include <sys/stat.h>
 #include "dirtiles.hpp"
 
 std::string dir_read_tile(std::string pbfPath) {

geojson.cpp

@@ -349,7 +349,6 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje
 
 			metakey[m] = properties->keys[i]->string;
 
-			bool track = false;
 			if (properties->values[i] != NULL) {
 				int vt = properties->values[i]->type;
 				std::string val;
@@ -412,24 +411,20 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje
 						exit(EXIT_FAILURE);
 					}
 					m++;
-					track = true;
 				} else if (vt == JSON_NUMBER) {
 					tas.type = metatype[m] = mvt_double;
 					metaval[m] = val;
 					m++;
-					track = true;
 				} else if (vt == JSON_TRUE || vt == JSON_FALSE) {
 					tas.type = metatype[m] = mvt_bool;
 					metaval[m] = val;
 					m++;
-					track = true;
 				} else if (vt == JSON_NULL) {
 					;
 				} else {
 					tas.type = metatype[m] = mvt_string;
 					metaval[m] = val;
 					m++;
-					track = true;
 				}
 			}
 

mbtiles.cpp

@@ -13,6 +13,7 @@
 #include <map>
 #include "mvt.hpp"
 #include "mbtiles.hpp"
+#include "text.hpp"
 
 sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) {
 	sqlite3 *outdb;
@@ -251,9 +252,13 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap) {
 					out.append("\t\t\t\t\t\t");
 					out.append(value.string);
 				} else {
-					out.append("\t\t\t\t\t\t\"");
-					quote(&out, value.string.c_str());
-					out.append("\"");
+					std::string trunc = truncate16(value.string, 256);
+
+					if (trunc.size() == value.string.size()) {
+						out.append("\t\t\t\t\t\t\"");
+						quote(&out, value.string.c_str());
+						out.append("\"");
+					}
 				}
 			}
 

text.cpp

@@ -50,3 +50,75 @@ std::string check_utf8(std::string s) {
 
 	return "";
 }
+
+const char *utf8_next(const char *s, long *c) {
+	if (s == NULL) {
+		*c = -1;
+		return NULL;
+	}
+
+	if (*s == '\0') {
+		*c = -1;
+		return NULL;
+	}
+
+	if ((s[0] & 0x80) == 0x80) {
+		if ((s[0] & 0xE0) == 0xC0) {
+			if ((s[1] & 0xC0) != 0x80) {
+				*c = 0xFFFD;
+				s++;
+			} else {
+				*c = ((long) (s[0] & 0x1F) << 6) | ((long) (s[1] & 0x7F));
+				s += 2;
+			}
+		} else if ((s[0] & 0xF0) == 0xE0) {
+			if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80) {
+				*c = 0xFFFD;
+				s++;
+			} else {
+				*c = ((long) (s[0] & 0x0F) << 12) | ((long) (s[1] & 0x7F) << 6) | ((long) (s[2] & 0x7F));
+				s += 3;
+			}
+		} else if ((s[0] & 0xF8) == 0xF0) {
+			if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80 || (s[3] & 0xC0) != 0x80) {
+				*c = 0xFFFD;
+				s++;
+			} else {
+				*c = ((long) (s[0] & 0x0F) << 18) | ((long) (s[1] & 0x7F) << 12) | ((long) (s[2] & 0x7F) << 6) | ((long) (s[3] & 0x7F));
+				s += 4;
+			}
+		} else {
+			*c = 0xFFFD;
+			s++;
+		}
+	} else {
+		*c = s[0];
+		s++;
+	}
+
+	return s;
+}
+
+std::string truncate16(std::string const &s, size_t runes) {
+	const char *cp = s.c_str();
+	const char *start = cp;
+	const char *lastgood = cp;
+	size_t len = 0;
+	long c;
+
+	while ((cp = utf8_next(cp, &c)) != NULL) {
+		if (c <= 0xFFFF) {
+			len++;
+		} else {
+			len += 2;
+		}
+
+		if (len <= runes) {
+			lastgood = cp;
+		} else {
+			break;
+		}
+	}
+
+	return std::string(s, 0, lastgood - start);
+}

text.hpp

@@ -4,5 +4,7 @@
 #include <string>
 
 std::string check_utf8(std::string text);
+const char *utf8_next(const char *s, long *c);
+std::string truncate16(std::string const &s, size_t runes);
 
 #endif