Factor out conversion from JSON types to vector tile attribute types

geojson.cpp

@@ -194,41 +194,20 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje
 				continue;
 			}
 
+			int type = -1;
+			std::string val;
+			stringify_value(properties->values[i], type, val, reading, line, feature);
+
+			if (type >= 0) {
+				metakey[m] = properties->keys[i]->string;
+				metatype[m] = type;
+				metaval[m] = val;
+				m++;
+
 				type_and_string tas;
 				tas.string = s;
-			tas.type = -1;
+				tas.type = type;
 
-			metakey[m] = properties->keys[i]->string;
-
-			if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) {
-				tas.type = metatype[m] = VT_STRING;
-				metaval[m] = std::string(properties->values[i]->string);
-				std::string err = check_utf8(metaval[m]);
-				if (err != "") {
-					fprintf(stderr, "%s:%d: %s\n", reading, line, err.c_str());
-					json_context(feature);
-					exit(EXIT_FAILURE);
-				}
-				m++;
-			} else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) {
-				tas.type = metatype[m] = VT_NUMBER;
-				metaval[m] = std::string(properties->values[i]->string);
-				m++;
-			} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) {
-				tas.type = metatype[m] = VT_BOOLEAN;
-				metaval[m] = std::string(properties->values[i]->type == JSON_TRUE ? "true" : "false");
-				m++;
-			} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) {
-				;
-			} else {
-				tas.type = metatype[m] = VT_STRING;
-				const char *v = json_stringify(properties->values[i]);
-				metaval[m] = std::string(v);
-				free((void *) v);  // stringify
-				m++;
-			}
-
-			if (tas.type >= 0) {
 				auto fk = layermap->find(layername);
 				fk->second.file_keys.insert(tas);
 			}

mvt.cpp

@@ -4,7 +4,9 @@
 #include <vector>
 #include <map>
 #include <zlib.h>
+#include <errno.h>
 #include "mvt.hpp"
+#include "geometry.hpp"
 #include "protozero/varint.hpp"
 #include "protozero/pbf_reader.hpp"
 #include "protozero/pbf_writer.hpp"
@@ -417,3 +419,69 @@ void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) {
 	feature.tags.push_back(ko);
 	feature.tags.push_back(vo);
 }
+
+static int is_integer(const char *s, long long *v) {
+	errno = 0;
+	char *endptr;
+
+	*v = strtoll(s, &endptr, 0);
+	if (*v == 0 && errno != 0) {
+		return 0;
+	}
+	if ((*v == LLONG_MIN || *v == LLONG_MAX) && (errno == ERANGE)) {
+		return 0;
+	}
+	if (*endptr != '\0') {
+		// Special case: If it is an integer followed by .0000 or similar,
+		// it is still an integer
+
+		if (*endptr != '.') {
+			return 0;
+		}
+		endptr++;
+		for (; *endptr != '\0'; endptr++) {
+			if (*endptr != '0') {
+				return 0;
+			}
+		}
+
+		return 1;
+	}
+
+	return 1;
+}
+
+mvt_value stringified_to_mvt_value(int type, const char *s) {
+	mvt_value tv;
+
+	if (type == VT_NUMBER) {
+		long long v;
+		if (is_integer(s, &v)) {
+			if (v >= 0) {
+				tv.type = mvt_int;
+				tv.numeric_value.int_value = v;
+			} else {
+				tv.type = mvt_sint;
+				tv.numeric_value.sint_value = v;
+			}
+		} else {
+			double d = atof(s);
+
+			if (d == (float) d) {
+				tv.type = mvt_float;
+				tv.numeric_value.float_value = d;
+			} else {
+				tv.type = mvt_double;
+				tv.numeric_value.double_value = d;
+			}
+		}
+	} else if (type == VT_BOOLEAN) {
+		tv.type = mvt_bool;
+		tv.numeric_value.bool_value = (s[0] == 't');
+	} else {
+		tv.type = mvt_string;
+		tv.string_value = s;
+	}
+
+	return tv;
+}

mvt.hpp

@@ -86,3 +86,5 @@ bool is_compressed(std::string const &data);
 int decompress(std::string const &input, std::string &output);
 int compress(std::string const &input, std::string &output);
 int dezig(unsigned n);
+
+mvt_value stringified_to_mvt_value(int type, const char *s);

plugin.cpp

@@ -182,21 +182,15 @@ mvt_layer parse_layer(int fd, unsigned z, unsigned x, unsigned y, mvt_layer cons
 			}
 
 			for (size_t i = 0; i < properties->length; i++) {
-				mvt_value v;
+				int tp = -1;
-
+				std::string s;
-				// XXX reconcile with JSON property parsing
-				if (properties->values[i]->type == JSON_STRING) {
-					v.type = mvt_string;
-					v.string_value = std::string(properties->values[i]->string);
-				} else if (properties->values[i]->type == JSON_NUMBER) {
-					v.type = mvt_double;
-					v.numeric_value.double_value = atof(properties->values[i]->string);
-				} else {
-					continue;
-				}
 
+				stringify_value(properties->values[i], tp, s, "Filter output", jp->line, j);
+				if (tp >= 0) {
+					mvt_value v = stringified_to_mvt_value(tp, s.c_str());
 					ret.tag(feature, std::string(properties->keys[i]->string), v);
 				}
+			}
 
 			ret.features.push_back(feature);
 		}

read_json.cpp

@@ -2,6 +2,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <vector>
+#include <string>
 
 extern "C" {
 #include "jsonpull/jsonpull.h"
@@ -10,6 +11,7 @@ extern "C" {
 #include "geometry.hpp"
 #include "projection.hpp"
 #include "read_json.hpp"
+#include "text.hpp"
 
 const char *geometry_names[GEOM_TYPES] = {
 	"Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon",
@@ -100,3 +102,29 @@ void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fna
 		out.push_back(draw(VT_CLOSEPATH, 0, 0));
 	}
 }
+
+void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature) {
+	if (value != NULL && value->type == JSON_STRING) {
+		type = VT_STRING;
+		stringified = std::string(value->string);
+		std::string err = check_utf8(stringified);
+		if (err != "") {
+			fprintf(stderr, "%s:%d: %s\n", reading, line, err.c_str());
+			json_context(feature);
+			exit(EXIT_FAILURE);
+		}
+	} else if (value != NULL && value->type == JSON_NUMBER) {
+		type = VT_NUMBER;
+		stringified = std::string(value->string);
+	} else if (value != NULL && (value->type == JSON_TRUE || value->type == JSON_FALSE)) {
+		type = VT_BOOLEAN;
+		stringified = std::string(value->type == JSON_TRUE ? "true" : "false");
+	} else if (value != NULL && (value->type == JSON_NULL)) {
+		;
+	} else {
+		type = VT_STRING;
+		const char *v = json_stringify(value);
+		stringified = std::string(v);
+		free((void *) v);  // stringify
+	}
+}

read_json.hpp

@@ -12,3 +12,5 @@ extern int mb_geometry[GEOM_TYPES];
 
 void json_context(json_object *j);
 void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature);
+
+void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature);

tile.cpp

@@ -62,7 +62,6 @@ bool draws_something(drawvec &geom) {
 
 int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long long> &values1, char *stringpool1, int m2, const std::vector<long long> &keys2, const std::vector<long long> &values2, char *stringpool2);
 int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
-static int is_integer(const char *s, long long *v);
 
 struct coalesce {
 	char *meta;
@@ -136,37 +135,7 @@ mvt_value retrieve_string(long long off, char *stringpool, int *otype) {
 		*otype = type;
 	}
 
-	mvt_value tv;
+	return stringified_to_mvt_value(type, s);
-	if (type == VT_NUMBER) {
-		long long v;
-		if (is_integer(s, &v)) {
-			if (v >= 0) {
-				tv.type = mvt_int;
-				tv.numeric_value.int_value = v;
-			} else {
-				tv.type = mvt_sint;
-				tv.numeric_value.sint_value = v;
-			}
-		} else {
-			double d = atof(s);
-
-			if (d == (float) d) {
-				tv.type = mvt_float;
-				tv.numeric_value.float_value = d;
-			} else {
-				tv.type = mvt_double;
-				tv.numeric_value.double_value = d;
-			}
-		}
-	} else if (type == VT_BOOLEAN) {
-		tv.type = mvt_bool;
-		tv.numeric_value.bool_value = (s[0] == 't');
-	} else {
-		tv.type = mvt_string;
-		tv.string_value = s;
-	}
-
-	return tv;
 }
 
 void decode_meta(int m, std::vector<long long> &metakeys, std::vector<long long> &metavals, char *stringpool, mvt_layer &layer, mvt_feature &feature) {
@@ -223,37 +192,6 @@ int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long
 	}
 }
 
-static int is_integer(const char *s, long long *v) {
-	errno = 0;
-	char *endptr;
-
-	*v = strtoll(s, &endptr, 0);
-	if (*v == 0 && errno != 0) {
-		return 0;
-	}
-	if ((*v == LLONG_MIN || *v == LLONG_MAX) && (errno == ERANGE)) {
-		return 0;
-	}
-	if (*endptr != '\0') {
-		// Special case: If it is an integer followed by .0000 or similar,
-		// it is still an integer
-
-		if (*endptr != '.') {
-			return 0;
-		}
-		endptr++;
-		for (; *endptr != '\0'; endptr++) {
-			if (*endptr != '0') {
-				return 0;
-			}
-		}
-
-		return 1;
-	}
-
-	return 1;
-}
-
 void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) {
 	if (geom.size() > 0 && nextzoom <= maxzoom) {
 		int xo, yo;