ExternalVendorCode/tippecanoe
1
0
Fork 0
mirror of https://github.com/mapbox/tippecanoe.git synced 2025-02-01 16:58:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add an option to ignore sqlite errors on existing mbtiles files

Browse Source
This commit is contained in:
Eric Fischer 2016-01-28 14:18:31 -08:00
parent 7835f52829
commit a84061cf87
6 changed files with 72 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -64,6 +64,8 @@ Options
* -o _file_.mbtiles: Name the output file. * -o _file_.mbtiles: Name the output file.
* -f: Delete the mbtiles file if it already exists instead of giving an error * -f: Delete the mbtiles file if it already exists instead of giving an error
* -F: Proceed (without deleting existing data) if the metadata or tiles table already exists
or if metadata fields can't be set
* -t _directory_: Put the temporary files in _directory_. * -t _directory_: Put the temporary files in _directory_.
* -P: Use multiple threads to read different parts of each input file at once. * -P: Use multiple threads to read different parts of each input file at once.
This will only work if the input is line-delimited JSON with each Feature on its This will only work if the input is line-delimited JSON with each Feature on its

15
geojson.c
View File

@ -1080,7 +1080,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i
} }
} }
int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional, int read_parallel) { int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional, int read_parallel, int forcetable) {
int ret = EXIT_SUCCESS; int ret = EXIT_SUCCESS;
struct reader reader[CPUS]; struct reader reader[CPUS];
@ -2028,7 +2028,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
midlon = maxlon; midlon = maxlon;
} }
mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers); mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable);
for (i = 0; i < nlayers; i++) { for (i = 0; i < nlayers; i++) {
pool_free_strings(&file_keys1[i]); pool_free_strings(&file_keys1[i]);
@ -2069,6 +2069,7 @@ int main(int argc, char **argv) {
int basezoom = -1; int basezoom = -1;
double basezoom_marker_width = 1; double basezoom_marker_width = 1;
int force = 0; int force = 0;
int forcetable = 0;
double droprate = 2.5; double droprate = 2.5;
double gamma = 0; double gamma = 0;
int buffer = 5; int buffer = 5;
@ -2087,7 +2088,7 @@ int main(int argc, char **argv) {
additional[i] = 0; additional[i] = 0;
} }
while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fXt:g:p:vqa:B:P")) != -1) { while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fFXt:g:p:vqa:B:P")) != -1) {
switch (i) { switch (i) {
case 'n': case 'n':
name = optarg; name = optarg;
@ -2170,6 +2171,10 @@ int main(int argc, char **argv) {
force = 1; force = 1;
break; break;
case 'F':
forcetable = 1;
break;
case 't': case 't':
tmpdir = optarg; tmpdir = optarg;
break; break;
@ -2256,10 +2261,10 @@ int main(int argc, char **argv) {
unlink(outdir); unlink(outdir);
} }
sqlite3 *outdb = mbtiles_open(outdir, argv); sqlite3 *outdb = mbtiles_open(outdir, argv, forcetable);
int ret = EXIT_SUCCESS; int ret = EXIT_SUCCESS;
ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel); ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional, read_parallel, forcetable);
mbtiles_close(outdb, argv); mbtiles_close(outdb, argv);

3
man/tippecanoe.1
View File

@ -69,6 +69,9 @@ specified, the files are all merged into the single named layer.
.IP \(bu 2 .IP \(bu 2
\-f: Delete the mbtiles file if it already exists instead of giving an error \-f: Delete the mbtiles file if it already exists instead of giving an error
.IP \(bu 2 .IP \(bu 2
\-F: Proceed (without deleting existing data) if the metadata or tiles table already exists
or if metadata fields can't be set
.IP \(bu 2
\-t \fIdirectory\fP: Put the temporary files in \fIdirectory\fP\&. \-t \fIdirectory\fP: Put the temporary files in \fIdirectory\fP\&.
.IP \(bu 2 .IP \(bu 2
\-P: Use multiple threads to read different parts of each input file at once. \-P: Use multiple threads to read different parts of each input file at once.

49
mbtiles.c
View File

@ -7,8 +7,9 @@
#include <sqlite3.h> #include <sqlite3.h>
#include "pool.h" #include "pool.h"
#include "tile.h" #include "tile.h"
#include "mbtiles.h"
sqlite3 *mbtiles_open(char *dbname, char **argv) { sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) {
sqlite3 *outdb; sqlite3 *outdb;
if (sqlite3_open(dbname, &outdb) != SQLITE_OK) { if (sqlite3_open(dbname, &outdb) != SQLITE_OK) {
@ -31,20 +32,28 @@ sqlite3 *mbtiles_open(char *dbname, char **argv) {
} }
if (sqlite3_exec(outdb, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "%s: create metadata table: %s\n", argv[0], err); fprintf(stderr, "%s: create metadata table: %s\n", argv[0], err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
if (sqlite3_exec(outdb, "CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);", NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, "CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);", NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "%s: create tiles table: %s\n", argv[0], err); fprintf(stderr, "%s: create tiles table: %s\n", argv[0], err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
if (sqlite3_exec(outdb, "create unique index name on metadata (name);", NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, "create unique index name on metadata (name);", NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "%s: index metadata: %s\n", argv[0], err); fprintf(stderr, "%s: index metadata: %s\n", argv[0], err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
if (sqlite3_exec(outdb, "create unique index tile_index on tiles (zoom_level, tile_column, tile_row);", NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, "create unique index tile_index on tiles (zoom_level, tile_column, tile_row);", NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "%s: index tiles: %s\n", argv[0], err); fprintf(stderr, "%s: index tiles: %s\n", argv[0], err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
return outdb; return outdb;
} }
@ -113,70 +122,88 @@ static void aprintf(char **buf, const char *format, ...) {
free(tmp); free(tmp);
} }
void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers) { void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable) {
char *sql, *err; char *sql, *err;
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname);
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set name in metadata: %s\n", err); fprintf(stderr, "set name in metadata: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", fname); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", fname);
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set description in metadata: %s\n", err); fprintf(stderr, "set description in metadata: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", 1); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", 1);
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set metadata: %s\n", err); fprintf(stderr, "set version : %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", minzoom); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", minzoom);
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set metadata: %s\n", err); fprintf(stderr, "set minzoom: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", maxzoom); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", maxzoom);
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set metadata: %s\n", err); fprintf(stderr, "set maxzoom: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", midlon, midlat, maxzoom); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", midlon, midlat, maxzoom);
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set metadata: %s\n", err); fprintf(stderr, "set center: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat);
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set metadata: %s\n", err); fprintf(stderr, "set bounds: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", "overlay"); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", "overlay");
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set metadata: %s\n", err); fprintf(stderr, "set type: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", "pbf"); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", "pbf");
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set metadata: %s\n", err); fprintf(stderr, "set format: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
char *buf = strdup("{"); char *buf = strdup("{");
@ -217,9 +244,11 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername,
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf);
if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set metadata: %s\n", err); fprintf(stderr, "set json: %s\n", err);
if (!forcetable) {
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
sqlite3_free(sql); sqlite3_free(sql);
free(buf); free(buf);
} }

4
mbtiles.h
View File

@ -1,7 +1,7 @@
sqlite3 *mbtiles_open(char *dbname, char **argv); sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers); void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, char **layername, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, struct pool **file_keys, int nlayers, int forcetable);
void mbtiles_close(sqlite3 *outdb, char **argv); void mbtiles_close(sqlite3 *outdb, char **argv);

4
tile-join.cc
View File

@ -547,7 +547,7 @@ int main(int argc, char **argv) {
unlink(outfile); unlink(outfile);
} }
sqlite3 *outdb = mbtiles_open(outfile, argv); sqlite3 *outdb = mbtiles_open(outfile, argv, 0);
struct stats st; struct stats st;
memset(&st, 0, sizeof(st)); memset(&st, 0, sizeof(st));
st.minzoom = st.minlat = st.minlon = INT_MAX; st.minzoom = st.minlat = st.minlon = INT_MAX;
@ -566,7 +566,7 @@ int main(int argc, char **argv) {
fk[i] = &(file_keys[i]); fk[i] = &(file_keys[i]);
} }
mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers); mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers, 0);
mbtiles_close(outdb, argv); mbtiles_close(outdb, argv);
return 0; return 0;