From 7154c069843d217c8cc8da7905db56e6029db54a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 12 Sep 2018 15:04:17 -0700 Subject: [PATCH] Stabilize order of features in tile-join output Order by source filename, not by unpredictable string comparison of compressed tile data --- .../just-macarthur.mbtiles.json | 8 +++---- .../merged-folder.mbtiles.json | 8 +++---- tests/join-population/merged.mbtiles.json | 8 +++---- tests/join-population/raw-merged-folder.json | 8 +++---- tests/join-population/renamed.mbtiles.json | 24 +++++++++---------- tile-join.cpp | 11 +++++++++ 6 files changed, 39 insertions(+), 28 deletions(-) diff --git a/tests/join-population/just-macarthur.mbtiles.json b/tests/join-population/just-macarthur.mbtiles.json index 345fd4c..34f4803 100644 --- a/tests/join-population/just-macarthur.mbtiles.json +++ b/tests/join-population/just-macarthur.mbtiles.json @@ -139,13 +139,13 @@ , { "type": "FeatureCollection", "properties": { "zoom": 9, "x": 82, "y": 198 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "macarthur", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "LINEARID": "1102406970094", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } -, -{ "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } -, { "type": "Feature", "properties": { "LINEARID": "1104486090991", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.138615, 37.729453 ], [ -122.137928, 37.725379 ], [ -122.137413, 37.724293 ], [ -122.134666, 37.720763 ], [ -122.133465, 37.718590 ], [ -122.131405, 37.714381 ], [ -122.129860, 37.712751 ], [ -122.127972, 37.711665 ], [ -122.125397, 37.710307 ], [ -122.121105, 37.707183 ], [ -122.117157, 37.705418 ], [ -122.109432, 37.700664 ], [ -122.104454, 37.695503 ] ] } } , { "type": "Feature", "properties": { "LINEARID": "1104474748623", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.104111, 37.695503 ], [ -122.109089, 37.700664 ], [ -122.115784, 37.704874 ], [ -122.121105, 37.707319 ], [ -122.125053, 37.710307 ], [ -122.129173, 37.712615 ], [ -122.130890, 37.714245 ], [ -122.133121, 37.718590 ], [ -122.137413, 37.724972 ], [ -122.138443, 37.729453 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1102406970094", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } ] } ] } , diff --git a/tests/join-population/merged-folder.mbtiles.json b/tests/join-population/merged-folder.mbtiles.json index 8bdb5b0..2526e9c 100644 --- a/tests/join-population/merged-folder.mbtiles.json +++ b/tests/join-population/merged-folder.mbtiles.json @@ -1090,13 +1090,13 @@ , { "type": "FeatureCollection", "properties": { "zoom": 9, "x": 82, "y": 198 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "macarthur", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "LINEARID": "1102406970094", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } -, -{ "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } -, { "type": "Feature", "properties": { "LINEARID": "1104486090991", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.138615, 37.729453 ], [ -122.137928, 37.725379 ], [ -122.137413, 37.724293 ], [ -122.134666, 37.720763 ], [ -122.133465, 37.718590 ], [ -122.131405, 37.714381 ], [ -122.129860, 37.712751 ], [ -122.127972, 37.711665 ], [ -122.125397, 37.710307 ], [ -122.121105, 37.707183 ], [ -122.117157, 37.705418 ], [ -122.109432, 37.700664 ], [ -122.104454, 37.695503 ] ] } } , { "type": "Feature", "properties": { "LINEARID": "1104474748623", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.104111, 37.695503 ], [ -122.109089, 37.700664 ], [ -122.115784, 37.704874 ], [ -122.121105, 37.707319 ], [ -122.125053, 37.710307 ], [ -122.129173, 37.712615 ], [ -122.130890, 37.714245 ], [ -122.133121, 37.718590 ], [ -122.137413, 37.724972 ], [ -122.138443, 37.729453 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1102406970094", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } ] } ] } , diff --git a/tests/join-population/merged.mbtiles.json b/tests/join-population/merged.mbtiles.json index d6aa601..2dacd04 100644 --- a/tests/join-population/merged.mbtiles.json +++ b/tests/join-population/merged.mbtiles.json @@ -1090,13 +1090,13 @@ , { "type": "FeatureCollection", "properties": { "zoom": 9, "x": 82, "y": 198 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "macarthur", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "LINEARID": "1102406970094", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } -, -{ "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } -, { "type": "Feature", "properties": { "LINEARID": "1104486090991", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.138615, 37.729453 ], [ -122.137928, 37.725379 ], [ -122.137413, 37.724293 ], [ -122.134666, 37.720763 ], [ -122.133465, 37.718590 ], [ -122.131405, 37.714381 ], [ -122.129860, 37.712751 ], [ -122.127972, 37.711665 ], [ -122.125397, 37.710307 ], [ -122.121105, 37.707183 ], [ -122.117157, 37.705418 ], [ -122.109432, 37.700664 ], [ -122.104454, 37.695503 ] ] } } , { "type": "Feature", "properties": { "LINEARID": "1104474748623", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.104111, 37.695503 ], [ -122.109089, 37.700664 ], [ -122.115784, 37.704874 ], [ -122.121105, 37.707319 ], [ -122.125053, 37.710307 ], [ -122.129173, 37.712615 ], [ -122.130890, 37.714245 ], [ -122.133121, 37.718590 ], [ -122.137413, 37.724972 ], [ -122.138443, 37.729453 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1102406970094", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } ] } ] } , diff --git a/tests/join-population/raw-merged-folder.json b/tests/join-population/raw-merged-folder.json index 6a5b00e..1de6d6d 100644 --- a/tests/join-population/raw-merged-folder.json +++ b/tests/join-population/raw-merged-folder.json @@ -1090,13 +1090,13 @@ , { "type": "FeatureCollection", "properties": { "zoom": 9, "x": 82, "y": 198, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "macarthur", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "LINEARID": "1102406970094", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } -, -{ "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } -, { "type": "Feature", "properties": { "LINEARID": "1104486090991", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.138615, 37.729453 ], [ -122.137928, 37.725379 ], [ -122.137413, 37.724293 ], [ -122.134666, 37.720763 ], [ -122.133465, 37.718590 ], [ -122.131405, 37.714381 ], [ -122.129860, 37.712751 ], [ -122.127972, 37.711665 ], [ -122.125397, 37.710307 ], [ -122.121105, 37.707183 ], [ -122.117157, 37.705418 ], [ -122.109432, 37.700664 ], [ -122.104454, 37.695503 ] ] } } , { "type": "Feature", "properties": { "LINEARID": "1104474748623", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.104111, 37.695503 ], [ -122.109089, 37.700664 ], [ -122.115784, 37.704874 ], [ -122.121105, 37.707319 ], [ -122.125053, 37.710307 ], [ -122.129173, 37.712615 ], [ -122.130890, 37.714245 ], [ -122.133121, 37.718590 ], [ -122.137413, 37.724972 ], [ -122.138443, 37.729453 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1102406970094", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.729453 ], [ -122.139130, 37.728910 ] ] } } ] } ] } , diff --git a/tests/join-population/renamed.mbtiles.json b/tests/join-population/renamed.mbtiles.json index acb0282..64b06d2 100644 --- a/tests/join-population/renamed.mbtiles.json +++ b/tests/join-population/renamed.mbtiles.json @@ -237,6 +237,14 @@ ] } , { "type": "FeatureCollection", "properties": { "zoom": 8, "x": 41, "y": 99 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "one", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { "LINEARID": "1104486090991", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.144279, 37.740313 ], [ -122.141533, 37.738141 ], [ -122.140503, 37.736784 ], [ -122.138100, 37.725379 ], [ -122.133636, 37.718590 ], [ -122.131577, 37.714517 ], [ -122.129860, 37.712887 ], [ -122.125397, 37.710443 ], [ -122.121277, 37.707183 ], [ -122.117157, 37.705553 ], [ -122.109604, 37.700664 ], [ -122.104454, 37.695503 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1104474748623", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.104111, 37.695503 ], [ -122.109261, 37.700664 ], [ -122.115784, 37.705010 ], [ -122.121277, 37.707455 ], [ -122.124023, 37.709628 ], [ -122.129860, 37.713159 ], [ -122.131577, 37.715060 ], [ -122.133293, 37.718590 ], [ -122.137413, 37.725108 ], [ -122.140503, 37.737327 ], [ -122.143936, 37.740313 ] ] } } +, +{ "type": "Feature", "properties": { "LINEARID": "1103690474250", "FULLNAME": "Macarthur", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.143250, 37.738413 ], [ -122.147369, 37.740313 ] ] } } +] } +, { "type": "FeatureCollection", "properties": { "layer": "two", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "LINEARID": "1105089465114", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.147713, 37.740313 ], [ -122.146339, 37.739770 ] ] } } , @@ -246,14 +254,6 @@ , { "type": "Feature", "properties": { "LINEARID": "1102406970095", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.139130, 37.730539 ], [ -122.138786, 37.728910 ] ] } } ] } -, -{ "type": "FeatureCollection", "properties": { "layer": "one", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "LINEARID": "1104486090991", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.144279, 37.740313 ], [ -122.141533, 37.738141 ], [ -122.140503, 37.736784 ], [ -122.138100, 37.725379 ], [ -122.133636, 37.718590 ], [ -122.131577, 37.714517 ], [ -122.129860, 37.712887 ], [ -122.125397, 37.710443 ], [ -122.121277, 37.707183 ], [ -122.117157, 37.705553 ], [ -122.109604, 37.700664 ], [ -122.104454, 37.695503 ] ] } } -, -{ "type": "Feature", "properties": { "LINEARID": "1104474748623", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.104111, 37.695503 ], [ -122.109261, 37.700664 ], [ -122.115784, 37.705010 ], [ -122.121277, 37.707455 ], [ -122.124023, 37.709628 ], [ -122.129860, 37.713159 ], [ -122.131577, 37.715060 ], [ -122.133293, 37.718590 ], [ -122.137413, 37.725108 ], [ -122.140503, 37.737327 ], [ -122.143936, 37.740313 ] ] } } -, -{ "type": "Feature", "properties": { "LINEARID": "1103690474250", "FULLNAME": "Macarthur", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.143250, 37.738413 ], [ -122.147369, 37.740313 ] ] } } -] } ] } , { "type": "FeatureCollection", "properties": { "zoom": 8, "x": 41, "y": 98 }, "features": [ @@ -449,15 +449,15 @@ ] } , { "type": "FeatureCollection", "properties": { "zoom": 10, "x": 164, "y": 396 }, "features": [ -{ "type": "FeatureCollection", "properties": { "layer": "two", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "LINEARID": "1102157509691", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.143421, 37.722324 ], [ -122.143421, 37.722257 ] ] } } -] } -, { "type": "FeatureCollection", "properties": { "layer": "one", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "LINEARID": "1104486090991", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.137156, 37.724022 ], [ -122.134666, 37.720695 ], [ -122.133808, 37.719405 ], [ -122.133465, 37.718590 ], [ -122.131319, 37.714381 ], [ -122.129774, 37.712751 ], [ -122.127972, 37.711597 ], [ -122.125311, 37.710307 ], [ -122.121019, 37.707115 ], [ -122.117157, 37.705350 ], [ -122.114067, 37.703584 ], [ -122.109346, 37.700596 ], [ -122.108402, 37.699781 ], [ -122.105570, 37.696589 ], [ -122.104454, 37.695503 ] ] } } , { "type": "Feature", "properties": { "LINEARID": "1104474748623", "FULLNAME": "Macarthur Fwy", "RTTYP": "M", "MTFCC": "S1100" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.104111, 37.695503 ], [ -122.107286, 37.699034 ], [ -122.109003, 37.700664 ], [ -122.115698, 37.704807 ], [ -122.117929, 37.705961 ], [ -122.121019, 37.707319 ], [ -122.124281, 37.709899 ], [ -122.129087, 37.712547 ], [ -122.130804, 37.714177 ], [ -122.131920, 37.716078 ], [ -122.133121, 37.718590 ], [ -122.134151, 37.720356 ], [ -122.136898, 37.724022 ] ] } } ] } +, +{ "type": "FeatureCollection", "properties": { "layer": "two", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { "LINEARID": "1102157509691", "FULLNAME": "Macarthur Blvd", "RTTYP": "M", "MTFCC": "S1400" }, "geometry": { "type": "LineString", "coordinates": [ [ -122.143421, 37.722324 ], [ -122.143421, 37.722257 ] ] } } +] } ] } , { "type": "FeatureCollection", "properties": { "zoom": 10, "x": 164, "y": 395 }, "features": [ diff --git a/tile-join.cpp b/tile-join.cpp index 1b067d6..eb9b264 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -450,6 +450,7 @@ struct reader { long long sorty = 0; long long y = 0; int z_flag = 0; + std::string fname; std::string data = ""; @@ -482,6 +483,13 @@ struct reader { return false; } + if (fname < r.fname) { + return true; + } + if (fname > r.fname) { + return false; + } + if (data < r.data) { return true; } @@ -509,6 +517,7 @@ struct reader *begin_reading(char *fname) { r->x = r->dirtiles[0].x; r->y = r->dirtiles[0].y; r->sorty = (1LL << r->zoom) - 1 - r->y; + r->fname = fname; r->data = dir_read_tile(r->dirbase, r->dirtiles[0]); r->dirtiles.erase(r->dirtiles.begin()); @@ -538,11 +547,13 @@ struct reader *begin_reading(char *fname) { r->db = db; r->stmt = stmt; r->next = NULL; + r->fname = fname; if (sqlite3_step(stmt) == SQLITE_ROW) { r->zoom = sqlite3_column_int(stmt, 0); r->x = sqlite3_column_int(stmt, 1); r->sorty = sqlite3_column_int(stmt, 2); + r->fname = fname; r->y = (1LL << r->zoom) - 1 - r->sorty; const char *data = (const char *) sqlite3_column_blob(stmt, 3);