From 2d625d58071c645cb401d5255c7fff2589c8476f Mon Sep 17 00:00:00 2001
From: Eric Fischer <enf@pobox.com>
Date: Wed, 7 Feb 2018 13:15:36 -0800
Subject: [PATCH] Pass through any non-ASCII characters from file names into
 layer names

The Docker build doesn't include proper locale support
---
 CHANGELOG.md                |  4 ++++
 Dockerfile                  |  2 +-
 main.cpp                    | 36 +++++++++---------------------------
 tests/nonascii/out/-z0.json |  4 ++--
 4 files changed, 16 insertions(+), 30 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08602ba..5dcaf19 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 ## 1.27.7
 
 * Add an option to produce only a single tile
+* Retain non-ASCII characters in layernames generated from filenames
+* Remember to close input files after reading them
+* Add --coalesce-fraction-as-needed and --coalesce-densest-as-needed
+* Report distances in both feet and meters
 
 ## 1.27.6
 
diff --git a/Dockerfile b/Dockerfile
index 5da22b4..1df89e5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,4 +16,4 @@ RUN make \
   && make install
 
 # Run the tests
-CMD LC_ALL=en_US.UTF-8 make test
+CMD make test
diff --git a/main.cpp b/main.cpp
index d1a2b51..4618d5d 100644
--- a/main.cpp
+++ b/main.cpp
@@ -16,7 +16,7 @@
 #include <sys/mman.h>
 #include <string.h>
 #include <fcntl.h>
-#include <wctype.h>
+#include <ctype.h>
 #include <errno.h>
 #include <limits.h>
 #include <sqlite3.h>
@@ -31,7 +31,6 @@
 #include <set>
 #include <map>
 #include <cmath>
-#include <locale.h>
 
 #ifdef __APPLE__
 #include <sys/types.h>
@@ -59,6 +58,7 @@
 #include "mvt.hpp"
 #include "dirtiles.hpp"
 #include "evaluator.hpp"
+#include "text.hpp"
 
 static int low_detail = 12;
 static int full_detail = -1;
@@ -1171,31 +1171,16 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 			}
 
 			// Trim out characters that can't be part of selector
-			wchar_t tmp[trunc.size() + 1];
-			size_t n = mbstowcs(tmp, trunc.c_str(), trunc.size() + 1);
-			if (n == (size_t) -1) {
-				perror("charset conversion");
-				exit(EXIT_FAILURE);
-			}
-
-			size_t out = 0;
-			for (size_t p = 0; p < n; p++) {
-				if (tmp[p] == L'_' || !(iswcntrl(tmp[p]) || iswspace(tmp[p]) || iswpunct(tmp[p]))) {
-					tmp[out++] = tmp[p];
+                        std::string out;
+			for (size_t p = 0; p < trunc.size(); p++) {
+				if (isalpha(trunc[p]) || isdigit(trunc[p]) || trunc[p] == '_' || (trunc[p] & 0x80) != 0) {
+					out.append(trunc, p, 1);
 				}
 			}
-			tmp[out] = L'\0';
 
-			char tmp2[(out + 1) * MB_CUR_MAX];
-			n = wcstombs(tmp2, tmp, (out + 1) * MB_CUR_MAX);
-			if (n == (size_t) -1) {
-				perror("charset conversion");
-				exit(EXIT_FAILURE);
-			}
-
-			sources[l].layer = tmp2;
-			if (sources[l].layer.size() == 0) {
-				sources[l].layer = "unknown";
+			sources[l].layer = out;
+			if (sources[l].layer.size() == 0 || check_utf8(out).size() != 0) {
+				sources[l].layer = "unknown" + std::to_string(l);
 			}
 
 			if (!quiet) {
@@ -2250,9 +2235,6 @@ int main(int argc, char **argv) {
 	mtrace();
 #endif
 
-	setlocale(LC_ALL, "C");
-	setlocale(LC_CTYPE, "");
-
 	init_cpus();
 
 	extern int optind;
diff --git a/tests/nonascii/out/-z0.json b/tests/nonascii/out/-z0.json
index 2233d3a..33774e9 100644
--- a/tests/nonascii/out/-z0.json
+++ b/tests/nonascii/out/-z0.json
@@ -3,7 +3,7 @@
 "center": "0.000000,0.000000,0",
 "description": "tests/nonascii/out/-z0.json.check.mbtiles",
 "format": "pbf",
-"json": "{\"vector_layers\": [ { \"id\": \"unknown\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} }, { \"id\": \"堤防\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} } ],\"tilestats\": {\"layerCount\": 2,\"layers\": [{\"layer\": \"unknown\",\"count\": 1,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []},{\"layer\": \"堤防\",\"count\": 1,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []}]}}",
+"json": "{\"vector_layers\": [ { \"id\": \"unknown0\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} }, { \"id\": \"堤防\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} } ],\"tilestats\": {\"layerCount\": 2,\"layers\": [{\"layer\": \"unknown0\",\"count\": 1,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []},{\"layer\": \"堤防\",\"count\": 1,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []}]}}",
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/nonascii/out/-z0.json.check.mbtiles",
@@ -11,7 +11,7 @@
 "version": "2"
 }, "features": [
 { "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
-{ "type": "FeatureCollection", "properties": { "layer": "unknown", "version": 2, "extent": 4096 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "unknown0", "version": 2, "extent": 4096 }, "features": [
 { "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ -77.080078, 38.959409 ], [ 0.000000, 0.000000 ] ] } }
 ] }
 ,