From 7ab94d43c6472651c3d713eeb55e807645c8178d Mon Sep 17 00:00:00 2001
From: Eric Fischer <enf@pobox.com>
Date: Tue, 6 Feb 2018 17:22:26 -0800
Subject: [PATCH] Preserve non-ASCII characters when making layernames from
 filenames

---
 main.cpp                    | 37 ++++++++++++++++++++++++++++++-------
 tests/nonascii/@@@.json     |  1 +
 tests/nonascii/out/-z0.json | 22 ++++++++++++++++++++++
 tests/nonascii/堤防.json  |  1 +
 4 files changed, 54 insertions(+), 7 deletions(-)
 create mode 100644 tests/nonascii/@@@.json
 create mode 100644 tests/nonascii/out/-z0.json
 create mode 100644 tests/nonascii/堤防.json

diff --git a/main.cpp b/main.cpp
index faa0c63..544ca32 100644
--- a/main.cpp
+++ b/main.cpp
@@ -16,7 +16,7 @@
 #include <sys/mman.h>
 #include <string.h>
 #include <fcntl.h>
-#include <ctype.h>
+#include <wctype.h>
 #include <errno.h>
 #include <limits.h>
 #include <sqlite3.h>
@@ -31,6 +31,7 @@
 #include <set>
 #include <map>
 #include <cmath>
+#include <locale.h>
 
 #ifdef __APPLE__
 #include <sys/types.h>
@@ -1169,16 +1170,35 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 			}
 
 			// Trim out characters that can't be part of selector
-			std::string out;
-			for (size_t p = 0; p < trunc.size(); p++) {
-				if (isalpha(trunc[p]) || isdigit(trunc[p]) || trunc[p] == '_') {
-					out.append(trunc, p, 1);
+			wchar_t tmp[trunc.size() + 1];
+			size_t n = mbstowcs(tmp, trunc.c_str(), trunc.size() + 1);
+			if (n == (size_t) -1) {
+				perror("charset conversion");
+				exit(EXIT_FAILURE);
+			}
+
+			size_t out = 0;
+			for (size_t p = 0; p < n; p++) {
+				if (tmp[p] == L'_' || !(iswcntrl(tmp[p]) || iswspace(tmp[p]) || iswpunct(tmp[p]))) {
+					tmp[out++] = tmp[p];
 				}
 			}
-			sources[l].layer = out;
+			tmp[out] = L'\0';
+
+			char tmp2[(out + 1) * MB_CUR_MAX];
+			n = wcstombs(tmp2, tmp, (out + 1) * MB_CUR_MAX);
+			if (n == (size_t) -1) {
+				perror("charset conversion");
+				exit(EXIT_FAILURE);
+			}
+
+			sources[l].layer = tmp2;
+			if (sources[l].layer.size() == 0) {
+				sources[l].layer = "unknown";
+			}
 
 			if (!quiet) {
-				fprintf(stderr, "For layer %d, using name \"%s\"\n", (int) l, out.c_str());
+				fprintf(stderr, "For layer %d, using name \"%s\"\n", (int) l, sources[l].layer.c_str());
 			}
 		}
 	}
@@ -2223,6 +2243,9 @@ int main(int argc, char **argv) {
 	mtrace();
 #endif
 
+	setlocale(LC_ALL, "C");
+	setlocale(LC_CTYPE, "");
+
 	init_cpus();
 
 	extern int optind;
diff --git a/tests/nonascii/@@@.json b/tests/nonascii/@@@.json
new file mode 100644
index 0000000..24d88aa
--- /dev/null
+++ b/tests/nonascii/@@@.json
@@ -0,0 +1 @@
+{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ -77.0366119, 38.9263374 ], [ 0, 0 ] ] } }
diff --git a/tests/nonascii/out/-z0.json b/tests/nonascii/out/-z0.json
new file mode 100644
index 0000000..2233d3a
--- /dev/null
+++ b/tests/nonascii/out/-z0.json
@@ -0,0 +1,22 @@
+{ "type": "FeatureCollection", "properties": {
+"bounds": "-122.238615,0.000000,0.000000,38.926337",
+"center": "0.000000,0.000000,0",
+"description": "tests/nonascii/out/-z0.json.check.mbtiles",
+"format": "pbf",
+"json": "{\"vector_layers\": [ { \"id\": \"unknown\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} }, { \"id\": \"堤防\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} } ],\"tilestats\": {\"layerCount\": 2,\"layers\": [{\"layer\": \"unknown\",\"count\": 1,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []},{\"layer\": \"堤防\",\"count\": 1,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []}]}}",
+"maxzoom": "0",
+"minzoom": "0",
+"name": "tests/nonascii/out/-z0.json.check.mbtiles",
+"type": "overlay",
+"version": "2"
+}, "features": [
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "unknown", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ -77.080078, 38.959409 ], [ 0.000000, 0.000000 ] ] } }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "layer": "堤防", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ -122.255859, 37.788081 ], [ 0.000000, 0.000000 ] ] } }
+] }
+] }
+] }
diff --git a/tests/nonascii/堤防.json b/tests/nonascii/堤防.json
new file mode 100644
index 0000000..6b6f426
--- /dev/null
+++ b/tests/nonascii/堤防.json
@@ -0,0 +1 @@
+{ "type": "Feature", "properties": { }, "geometry": { "type": "LineString", "coordinates": [ [ -122.238615, 37.783767 ], [ 0, 0 ] ] } }