Substantial preparatory work towards Rhizome/MeshMS store-and-forward

services in Serval DNA.

encode.c

@@ -0,0 +1 @@
+sqlite-amalgamation-3070900/encode.c

rhizome.c

@@ -0,0 +1,627 @@
+#include "mphlr.h"
+#include "sqlite-amalgamation-3070900/sqlite3.h"
+#include "sha2.h"
+
+#define MAX_MANIFEST_VARS 256
+#define MAX_MANIFEST_BYTES 8192
+typedef struct rhizome_manifest {
+  int manifest_bytes;
+  unsigned char manifestdata[MAX_MANIFEST_BYTES];
+  unsigned char manifesthash[crypto_hash_BYTES];
+
+  /* CryptoSign key pair for this manifest.
+     The filename as distributed on Rhizome will be the public key
+     of this pair, thus ensuring that noone can tamper with a bundle
+     except the creator. */
+  unsigned char cryptoSignPublic[crypto_sign_PUBLICKEYBYTES];
+  unsigned char cryptoSignSecret[crypto_sign_SECRETKEYBYTES];
+
+  /* Set non-zero after variables have been packed and
+     signature blocks appended */
+  int finalised;
+
+  int var_count;
+  char *vars[MAX_MANIFEST_VARS];
+  char *values[MAX_MANIFEST_VARS];
+
+  int sig_count;
+  unsigned char *signatureBlocks[MAX_MANIFEST_VARS];
+  unsigned char signatureTypes[MAX_MANIFEST_VARS];
+  /* 0x01 = CryptoSign signature of manifest */
+  /* 0x02 = CryptoSign signature of signatory */
+  int signature_errors; /* if non-zero, then manifest should not be trusted */
+
+  
+} rhizome_manifest;
+
+long long rhizome_space=0;
+char *rhizome_datastore_path=NULL;
+
+sqlite3 *rhizome_db=NULL;
+
+int rhizome_manifest_createid(rhizome_manifest *m);
+int rhizome_write_manifest_file(rhizome_manifest *m,char *filename);
+int rhizome_manifest_sign(rhizome_manifest *m);
+int rhizome_drop_stored_file(char *id,int maximum_priority);
+int rhizome_manifest_priority(char *id);
+rhizome_manifest *rhizome_read_manifest_file(char *filename);
+int rhizome_hash_file(char *filename,char *hash_out);
+int rhizome_manifest_get(rhizome_manifest *m,char *var,char *value_out);
+int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value);
+int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value);
+long long rhizome_file_size(char *filename);
+void rhizome_manifest_free(rhizome_manifest *m);
+int rhizome_manifest_pack_variables(rhizome_manifest *m);
+int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename);
+int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
+
+int rhizome_opendb()
+{
+  if (rhizome_db) return 0;
+  char dbname[1024];
+
+  if (!rhizome_datastore_path) {
+    fprintf(stderr,"Cannot open rhizome database -- no path specified\n");
+    exit(1);
+  }
+  if (strlen(rhizome_datastore_path)>1000) {
+    fprintf(stderr,"Cannot open rhizome database -- data store path is too long\n");
+    exit(1);
+  }
+  snprintf(dbname,1024,"%s/rhizome.db",rhizome_datastore_path);
+
+  int r=sqlite3_open(dbname,&rhizome_db);
+  if (r) {
+    fprintf(stderr,"SQLite could not open database: %s\n",sqlite3_errmsg(rhizome_db));
+    exit(1);
+  }
+
+  /* Read Rhizome configuration, and write it back out as we understand it. */
+  char conf[1024];
+  snprintf(conf,1024,"%s/rhizome.conf",rhizome_datastore_path);
+  FILE *f=fopen(conf,"r");
+  if (f) {
+    char line[1024];
+    line[0]=0; fgets(line,1024,f);
+    while (line[0]) {
+      if (sscanf(line,"space=%lld",&rhizome_space)==1) { 
+	rhizome_space*=1024; /* Units are kilobytes */
+      }
+      line[0]=0; fgets(line,1024,f);
+    }
+    fclose(f);
+  }
+  f=fopen(conf,"w");
+  if (f) {
+    fprintf(f,"space=%lld\n",rhizome_space/1024LL);
+    fclose(f);
+  }
+
+  /* Create tables if required */
+  if (sqlite3_exec(rhizome_db,"PRAGMA auto_vacuum=2;",NULL,NULL,NULL)) {
+      fprintf(stderr,"SQLite could enable incremental vacuuming: %s\n",sqlite3_errmsg(rhizome_db));
+      exit(1);
+  }
+  if (sqlite3_exec(rhizome_db,"CREATE TABLE IF NOT EXISTS GROUPS(id text not null primary key, priority integer, manifest blob, groupsecret blob);",NULL,NULL,NULL))
+    {
+      fprintf(stderr,"SQLite could not create GROUPS table: %s\n",sqlite3_errmsg(rhizome_db));
+      exit(1);
+    }
+  if (sqlite3_exec(rhizome_db,"CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, manifest blob, version integer, privatekey blob);",NULL,NULL,NULL))
+    {
+      fprintf(stderr,"SQLite could not create MANIFESTS table: %s\n",sqlite3_errmsg(rhizome_db));
+      exit(1);
+    }
+  if (sqlite3_exec(rhizome_db,"CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, data blob, length integer, highestpriority integer);",NULL,NULL,NULL))
+    {
+      fprintf(stderr,"SQLite could not create FILES table: %s\n",sqlite3_errmsg(rhizome_db));
+      exit(1);
+    }
+  if (sqlite3_exec(rhizome_db,"CREATE TABLE IF NOT EXISTS FILEMANIFESTS(fileid text not null primary key, manifestid text not null);",NULL,NULL,NULL))
+    {
+      fprintf(stderr,"SQLite could not create FILEMANIFESTS table: %s\n",sqlite3_errmsg(rhizome_db));
+      exit(1);
+    }
+  if (sqlite3_exec(rhizome_db,"CREATE TABLE IF NOT EXISTS MANIFESTGROUPS(manifestid text not null primary key, groupid text not null);",NULL,NULL,NULL))
+    {
+      fprintf(stderr,"SQLite could not create MANIFESTGROUPS table: %s\n",sqlite3_errmsg(rhizome_db));
+      exit(1);
+    }
+  
+  /* XXX Setup special groups, e.g., Serval Software and Serval Optional Data */
+
+  return 0;
+}
+
+/* 
+   Convenience wrapper for executing an SQL command that returns a single int64 value 
+ */
+long long sqlite_exec_int64(char *sqlformat,...)
+{
+  if (!rhizome_db) rhizome_opendb();
+
+  va_list ap,ap2;
+  char sqlstatement[8192];
+
+  va_start(ap,sqlformat);
+  va_copy(ap2,ap);
+
+  vsnprintf(sqlstatement,8192,sqlformat,ap2); sqlstatement[8191]=0;
+
+  va_end(ap);
+
+  sqlite3_stmt *statement;
+  if (sqlite3_prepare_v2(rhizome_db,sqlstatement,-1,&statement,NULL)!=SQLITE_OK)
+    {
+      sqlite3_close(rhizome_db);
+      rhizome_db=NULL;
+      return WHY("Could not prepare sql statement.");
+    }
+   if (sqlite3_step(statement) == SQLITE_ROW)
+     {
+       if (sqlite3_column_count(statement)!=1) {
+	 sqlite3_finalize(statement);
+	 return -1;
+       }
+       long long result= sqlite3_column_int(statement,0);
+       sqlite3_finalize(statement);
+       return result;
+     }
+   sqlite3_finalize(statement);
+   return -1;
+}
+
+long long rhizome_database_used_bytes()
+{
+  long long db_page_size=sqlite_exec_int64("PRAGMA page_size;");
+  long long db_page_count=sqlite_exec_int64("PRAGMA page_count;");
+  long long db_free_page_count=sqlite_exec_int64("PRAGMA free_count;");
+  return db_page_size*(db_page_count-db_free_page_count);
+}
+
+int rhizome_make_space(int group_priority, long long bytes)
+{
+  sqlite3_stmt *statement;
+
+  /* Asked for impossibly large amount */
+  if (bytes>=(rhizome_space-65536)) return -1;
+
+  long long db_used=rhizome_database_used_bytes(); 
+  
+  /* If there is already enough space now, then do nothing more */
+  if (db_used<=(rhizome_space-bytes-65536)) return 0;
+
+  /* Okay, not enough space, so free up some. */
+  char sql[1024];
+  snprintf(sql,1024,"select id,length from files where highestpriority<%d order by descending length",group_priority);
+  if(sqlite3_prepare_v2(rhizome_db,sql, -1, &statement, NULL) != SQLITE_OK )
+    {
+      fprintf(stderr,"SQLite error running query '%s': %s\n",sql,sqlite3_errmsg(rhizome_db));
+      sqlite3_close(rhizome_db);
+      rhizome_db=NULL;
+      exit(-1);
+    }
+
+  while ( bytes>(rhizome_space-65536-rhizome_database_used_bytes()) && sqlite3_step(statement) == SQLITE_ROW)
+    {
+      /* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */
+      char *id;
+      long long length;
+
+      /* Get values */
+      if (sqlite3_column_type(statement, 0)==SQLITE_TEXT) id=sqlite3_column_text(statement, 0);
+      else {
+	fprintf(stderr,"Incorrect type in id column of files table.\n");
+	continue; }
+      if (sqlite3_column_type(statement, 1)==SQLITE_INTEGER) length=sqlite3_column_int(statement, 1);
+      else {
+	fprintf(stderr,"Incorrect type in length column of files table.\n");
+	continue; }
+      
+      /* Try to drop this file from storage, discarding any references that do not trump the priority of this
+	 request.  The query done earlier should ensure this, but it doesn't hurt to be paranoid, and it also
+	 protects against inconsistency in the database. */
+      rhizome_drop_stored_file(id,group_priority+1);
+    }
+  sqlite3_finalize(statement);
+
+  long long equal_priority_larger_file_space_used = sqlite_exec_int64("SELECT COUNT(length) FROM FILES WHERE highestpriority=%d and length>%lld",group_priority,bytes);
+  /* XXX Get rid of any equal priority files that are larger than this one */
+
+  /* XXX Get rid of any higher priority files that are not relevant in this time or location */
+
+  /* Couldn't make space */
+  return WHY("Not implemented");
+}
+
+/* Drop the specified file from storage, and any manifests that reference it, 
+   provided that none of those manifests are being retained at a higher priority
+   than the maximum specified here. */
+int rhizome_drop_stored_file(char *id,int maximum_priority)
+{
+  char sql[1024];
+  sqlite3_stmt *statement;
+  int cannot_drop=0;
+
+  if (strlen(id)>70) return -1;
+
+  snprintf(sql,1024,"select manifests.id from manifests,filemanifests where manifests.id==filemanifests.manifestid and filemanifests.fileid='%s'",
+	   id);
+  if(sqlite3_prepare_v2(rhizome_db,sql, -1, &statement, NULL) != SQLITE_OK )
+    {
+      fprintf(stderr,"SQLite error running query '%s': %s\n",sql,sqlite3_errmsg(rhizome_db));
+      sqlite3_close(rhizome_db);
+      rhizome_db=NULL;
+      exit(-1);
+    }
+
+  while ( sqlite3_step(statement) == SQLITE_ROW)
+    {
+      /* Find manifests for this file */
+      char *id;
+      if (sqlite3_column_type(statement, 0)==SQLITE_TEXT) id=sqlite3_column_text(statement, 0);
+      else {
+	fprintf(stderr,"Incorrect type in id column of manifests table.\n");
+	continue; }
+            
+      /* Check that manifest is not part of a higher priority group.
+	 If so, we cannot drop the manifest or the file.
+         However, we will keep iterating, as we can still drop any other manifests pointing to this file
+	 that are lower priority, and thus free up a little space. */
+      if (rhizome_manifest_priority(id)>maximum_priority) {
+	cannot_drop=1;
+      } else {
+	sqlite_exec_int64("delete from filemanifests where manifestid='%s';",id);
+	sqlite_exec_int64("delete from manifests where manifestid='%s';",id);
+	sqlite_exec_int64("delete from manifestgroups where manifestid='%s';",id);	
+      }
+    }
+  sqlite3_finalize(statement);
+
+  if (!cannot_drop) {
+    sqlite_exec_int64("delete from filemanifests where fileid='%s';",id);
+    sqlite_exec_int64("delete from files where id='%s';",id);
+  }
+  return 0;
+}
+
+/* XXX Requires a messy join that might be slow. */
+int rhizome_manifest_priority(char *id)
+{
+  long long result = sqlite_exec_int64("select max(groups.priorty) from groups,manifests,manifestgroups where manifests.id='%s' and groups.id=manifestgroups.groupid and manifestgroups.manifestid=manifests.id;",id);
+  return result;
+}
+
+/* Import a bundle from the inbox folder.
+   Check that the manifest prototype is valid, and if so, complete it, and sign it if required and possible.
+   The file should be included in the specified rhizome groups, if possible.
+   (some groups may be closed groups that we do not have the private key for.)
+*/
+int rhizome_bundle_import(char *bundle,char *groups[],int verifyP, int checkFileP, int signP)
+{
+  char filename[1024];
+  char manifestname[1024];
+  char buffer[1024];
+  
+  snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,bundle); filename[1023]=0;
+  snprintf(manifestname,1024,"%s/manifest.%s",rhizome_datastore_path,bundle); manifestname[1023]=0;
+
+  /* Open files */
+  rhizome_manifest *m=rhizome_read_manifest_file(manifestname);
+  if (!m) return WHY("Could not read manifest file.");
+  char hexhash[SHA512_DIGEST_STRING_LENGTH];
+
+  if (checkFileP||signP) {
+    if (rhizome_hash_file(filename,hexhash))
+      { rhizome_manifest_free(m); return WHY("Could not hash file."); }
+  }
+
+  if (verifyP)
+    {
+      /* Make sure hashes match.
+	 Make sure that no signature verification errors were spotted on loading. */
+      int verifyErrors=0;
+      char mhexhash[1024];
+      if (checkFileP) {
+	if (rhizome_manifest_get(m,"filehash",mhexhash)==0)
+	  if (strcmp(hexhash,mhexhash)) verifyErrors++; }
+      if (m->signature_errors) verifyErrors+=m->signature_errors;
+      if (verifyErrors) {
+	rhizome_manifest_free(m);
+	unlink(manifestname);
+	unlink(filename);
+	return WHY("Errors encountered verifying bundle manifest");
+      }
+    }
+
+  if (!verifyP) {
+    if (rhizome_manifest_get(m,"id",buffer)!=0) {
+      /* No bundle id (256 bit random string being a public key in the NaCl CryptoSign crypto system),
+	 so create one, and keep the private key handy. */
+      rhizome_manifest_createid(m);
+    }
+    rhizome_manifest_set(m,"filehash",hexhash);
+    if (rhizome_manifest_get(m,"version",buffer)!=0)
+      /* Version not set, so set one */
+      rhizome_manifest_set_ll(m,"version",overlay_time_in_ms());
+    rhizome_manifest_set_ll(m,"first_byte",0);
+    rhizome_manifest_set_ll(m,"last_byte",rhizome_file_size(filename));
+  }
+   
+  /* Convert to final form for signing and writing to disk */
+  rhizome_manifest_pack_variables(m);
+    
+  /* Sign it */
+  if (signP) rhizome_manifest_sign(m);
+
+  /* Add group memberships */
+  int i;
+  for(i=0;groups[i];i++) rhizome_manifest_add_group(m,groups[i]);
+
+  /* Write manifest back to disk */
+  if (rhizome_write_manifest_file(m,manifestname)) {
+    rhizome_manifest_free(m);
+    return WHY("Could not write manifest file.");
+  }
+
+  /* Okay, it is written, and can be put directly into the rhizome database now */
+  int r=rhizome_store_bundle(m,filename);
+  if (!r) {
+    unlink(manifestname);
+    unlink(filename);
+    return 0;
+  }
+
+  return -1;
+}
+
+/* Update an existing Rhizome bundle */
+int rhizome_bundle_push_update(char *id,long long version,unsigned char *data,int appendP)
+{
+  return WHY("Not implemented");
+}
+
+rhizome_manifest *rhizome_read_manifest_file(char *filename)
+{
+  rhizome_manifest *m = calloc(sizeof(rhizome_manifest),1);
+  if (!m) return NULL;
+
+  FILE *f=fopen(filename,"r");
+  if (!f) { rhizome_manifest_free(m); return NULL; }
+  m->manifest_bytes = fread(m->manifestdata,1,MAX_MANIFEST_BYTES,f);
+  fclose(f);
+
+  /* Parse out variables, signature etc */
+  int ofs=0;
+  while(ofs<m->manifest_bytes&&m->manifestdata[ofs])
+    {
+      int i;
+      char line[1024],var[1024],value[1024];
+      while(ofs<m->manifest_bytes&&
+	    (m->manifestdata[ofs]==0x0a||
+	     m->manifestdata[ofs]==0x09||
+	     m->manifestdata[ofs]==0x20||
+	     m->manifestdata[ofs]==0x0d)) ofs++;
+      for(i=0;i<(ofs-m->manifest_bytes)
+	    &&(i<1023)
+	    &&m->manifestdata[ofs]!=0x00
+	    &&m->manifestdata[ofs]!=0x0d
+	    &&m->manifestdata[ofs]!=0x0a;i++)
+	    line[i]=m->manifestdata[ofs+i];
+      line[i]=0;
+      /* Ignore blank lines */
+      if (line[0]==0) continue;
+      if (sscanf(line,"%[^=]=%[^\n\r]",var,value)==2)
+	{
+	  if (rhizome_manifest_get(m,var,NULL)==0) {
+	    WHY("Error in manifest file (duplicate variable -- keeping first value).");
+	  }
+	  if (m->var_count<MAX_MANIFEST_VARS)
+	    {
+	      m->vars[m->var_count]=strdup(var);
+	      m->values[m->var_count]=strdup(value);
+	      m->var_count++;
+	    }
+	}
+      else
+	{
+	  /* Error in manifest file.
+	     Silently ignore for now. */
+	  WHY("Error in manifest file (badly formatted line).");
+	}
+    }
+  /* The null byte gets included in the check sum */
+  if (ofs<m->manifest_bytes) ofs++;
+
+  /* Remember where the text ends */
+  int end_of_text=ofs;
+
+  /* Calculate hash of the text part of the file, as we need to couple this with
+     each signature block to */
+  unsigned char manifest_hash[crypto_hash_BYTES];
+  crypto_hash(manifest_hash,m->manifestdata,end_of_text);
+
+  /* Read signature blocks from file.
+     XXX - What additional information/restrictions should the
+     signatures have?  start/expiry times? geo bounding box? 
+     Those elements all need to be included in the hash */
+  WHY("Signature verification not implemented");
+
+  WHY("Group membership signature reading not implemented (are we still doing it this way?)");
+  
+  m->manifest_bytes=end_of_text;
+
+  WHY("Incomplete");
+
+  rhizome_manifest_free(m);
+  return NULL;
+}
+
+int rhizome_hash_file(char *filename,char *hash_out)
+{
+  /* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one
+     go.  Trouble is, we need to run Serval DNA on filesystems that lack mmap(),
+     and may be very resource constrained. Thus we need a streamable SHA-512
+     implementation.
+  */
+  FILE *f=fopen(filename,"r");
+  if (!f) return WHY("Could not open file for reading to calculage SHA512 hash.");
+  unsigned char buffer[8192];
+  int r;
+
+  SHA512_CTX context;
+  SHA512_Init(&context);
+
+  while(!feof(f)) {
+    r=fread(buffer,1,8192,f);
+    if (r>0) SHA512_Update(&context,buffer,r);
+  }
+
+  SHA512_End(&context,(char *)hash_out);
+  return 0;
+}
+
+int rhizome_manifest_get(rhizome_manifest *m,char *var,char *out)
+{
+  int i;
+
+  if (!m) return -1;
+
+  for(i=0;i<m->var_count;i++)
+    if (!strcmp(m->vars[i],var)) {
+      if (out) strcpy(m->values[i],out);
+      return 0;
+    }
+  return -1;
+}
+
+int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value)
+{
+  int i;
+
+  if (!m) return -1;
+
+  for(i=0;i<m->var_count;i++)
+    if (!strcmp(m->vars[i],var)) {
+      free(m->values[i]); 
+      m->values[i]=strdup(value);
+      return 0;
+    }
+
+  if (m->var_count>=MAX_MANIFEST_VARS) return -1;
+  
+  m->vars[m->var_count]=strdup(var);
+  m->values[m->var_count]=strdup(value);
+  m->var_count++;
+
+  return 0;
+}
+
+int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value)
+{
+  char svalue[100];
+
+  snprintf(svalue,1024,"%lld",value);
+
+  return rhizome_manifest_set(m,var,svalue);
+}
+
+long long rhizome_file_size(char *filename)
+{
+  FILE *f;
+
+  /* XXX really should just use stat instead of opening the file */
+  f=fopen(filename,"r");
+  fseek(f,0,SEEK_END);
+  long long size=ftello(f);
+  fclose(f);
+  return size;
+}
+
+void rhizome_manifest_free(rhizome_manifest *m)
+{
+  if (!m) return;
+
+  int i;
+  for(i=0;i<m->var_count;i++)
+    { free(m->vars[i]); free(m->values[i]); }
+
+  WHY("Doesn't free signatures yet");
+
+  free(m);
+
+  return;
+}
+
+/* Convert variable list to string, complaining if it ends up
+   too long. 
+   Signatures etc will be added later. */
+int rhizome_manifest_pack_variables(rhizome_manifest *m)
+{
+  int i,ofs=0;
+
+  for(i=0;i<m->var_count;i++)
+    {
+      if ((ofs+strlen(m->vars[i])+1+strlen(m->values[i])+1+1)>MAX_MANIFEST_BYTES)
+	return WHY("Manifest variables too long in total to fit in MAX_MANIFEST_BYTES");
+      snprintf((char *)&m->manifestdata[ofs],MAX_MANIFEST_BYTES-ofs,"%s=%s\n",
+	       m->vars[i],m->values[i]);
+      ofs+=strlen((char *)&m->manifestdata[ofs]);
+    }
+  m->manifest_bytes=ofs;
+
+  return 0;
+}
+
+/* Sign this manifest using our own private CryptoSign key */
+int rhizome_manifest_sign(rhizome_manifest *m)
+{
+  return WHY("Not implemented.");
+}
+
+int rhizome_write_manifest_file(rhizome_manifest *m,char *filename)
+{
+  if (!m) return WHY("Manifest is null.");
+  if (!m->finalised) return WHY("Manifest must be finalised before it can be written.");
+  FILE *f=fopen(filename,"w");
+  int r=fwrite(m->manifestdata,m->manifest_bytes,1,f);
+  fclose(f);
+  if (r!=1) return WHY("Failed to fwrite() manifest file.");
+  return 0;
+}
+
+int rhizome_manifest_createid(rhizome_manifest *m)
+{
+  return crypto_sign_keypair(m->cryptoSignPublic,m->cryptoSignSecret);
+}
+
+/*
+  Store the specified manifest into the sqlite database.
+  We assume that sufficient space has been made for us.
+  The manifest should be finalised, and so we don't need to
+  look at the underlying manifest file, but can just write m->manifest_data
+  as a blob.
+
+  associated_filename needs to be read in and stored as a blob.  Hopefully that
+  can be done in pieces so that we don't have memory exhaustion issues on small
+  architectures.  However, we do know it's hash apriori from m, and so we can
+  skip loading the file in if it is already stored.
+
+  We need to also need to create the appropriate row(s) in the MANIFESTS, FILES, 
+  FILEMANIFESTS and MANIFESTGROUPS tables.
+ */
+int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename)
+{
+  return WHY("Not implemented.");
+}
+
+/*
+  Adds a group that this bundle should be present in.  If we have the means to sign
+  the bundle as a member of that group, then we create the appropriate signature block.
+  The group signature blocks, like all signature blocks, will be appended to the
+  manifest data during the finalisation process.
+ */
+int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid)
+{
+  return WHY("Not implemented.");
+}

sha2.h

@@ -0,0 +1,197 @@
+/*
+ * FILE:	sha2.h
+ * AUTHOR:	Aaron D. Gifford - http://www.aarongifford.com/
+ * 
+ * Copyright (c) 2000-2001, Aaron D. Gifford
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the copyright holder nor the names of contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTOR(S) ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTOR(S) BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id: sha2.h,v 1.1 2001/11/08 00:02:01 adg Exp adg $
+ */
+
+#ifndef __SHA2_H__
+#define __SHA2_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/*
+ * Import u_intXX_t size_t type definitions from system headers.  You
+ * may need to change this, or define these things yourself in this
+ * file.
+ */
+#include <sys/types.h>
+
+#ifdef SHA2_USE_INTTYPES_H
+
+#include <inttypes.h>
+
+#endif /* SHA2_USE_INTTYPES_H */
+
+
+/*** SHA-256/384/512 Various Length Definitions ***********************/
+#define SHA256_BLOCK_LENGTH		64
+#define SHA256_DIGEST_LENGTH		32
+#define SHA256_DIGEST_STRING_LENGTH	(SHA256_DIGEST_LENGTH * 2 + 1)
+#define SHA384_BLOCK_LENGTH		128
+#define SHA384_DIGEST_LENGTH		48
+#define SHA384_DIGEST_STRING_LENGTH	(SHA384_DIGEST_LENGTH * 2 + 1)
+#define SHA512_BLOCK_LENGTH		128
+#define SHA512_DIGEST_LENGTH		64
+#define SHA512_DIGEST_STRING_LENGTH	(SHA512_DIGEST_LENGTH * 2 + 1)
+
+
+/*** SHA-256/384/512 Context Structures *******************************/
+/* NOTE: If your architecture does not define either u_intXX_t types or
+ * uintXX_t (from inttypes.h), you may need to define things by hand
+ * for your system:
+ */
+#if 0
+typedef unsigned char u_int8_t;		/* 1-byte  (8-bits)  */
+typedef unsigned int u_int32_t;		/* 4-bytes (32-bits) */
+typedef unsigned long long u_int64_t;	/* 8-bytes (64-bits) */
+#endif
+/*
+ * Most BSD systems already define u_intXX_t types, as does Linux.
+ * Some systems, however, like Compaq's Tru64 Unix instead can use
+ * uintXX_t types defined by very recent ANSI C standards and included
+ * in the file:
+ *
+ *   #include <inttypes.h>
+ *
+ * If you choose to use <inttypes.h> then please define: 
+ *
+ *   #define SHA2_USE_INTTYPES_H
+ *
+ * Or on the command line during compile:
+ *
+ *   cc -DSHA2_USE_INTTYPES_H ...
+ */
+#ifdef SHA2_USE_INTTYPES_H
+
+typedef struct _SHA256_CTX {
+	uint32_t	state[8];
+	uint64_t	bitcount;
+	uint8_t	buffer[SHA256_BLOCK_LENGTH];
+} SHA256_CTX;
+typedef struct _SHA512_CTX {
+	uint64_t	state[8];
+	uint64_t	bitcount[2];
+	uint8_t	buffer[SHA512_BLOCK_LENGTH];
+} SHA512_CTX;
+
+#else /* SHA2_USE_INTTYPES_H */
+
+typedef struct _SHA256_CTX {
+	u_int32_t	state[8];
+	u_int64_t	bitcount;
+	u_int8_t	buffer[SHA256_BLOCK_LENGTH];
+} SHA256_CTX;
+typedef struct _SHA512_CTX {
+	u_int64_t	state[8];
+	u_int64_t	bitcount[2];
+	u_int8_t	buffer[SHA512_BLOCK_LENGTH];
+} SHA512_CTX;
+
+#endif /* SHA2_USE_INTTYPES_H */
+
+typedef SHA512_CTX SHA384_CTX;
+
+
+/*** SHA-256/384/512 Function Prototypes ******************************/
+#ifndef NOPROTO
+#ifdef SHA2_USE_INTTYPES_H
+
+void SHA256_Init(SHA256_CTX *);
+void SHA256_Update(SHA256_CTX*, const uint8_t*, size_t);
+void SHA256_Final(uint8_t[SHA256_DIGEST_LENGTH], SHA256_CTX*);
+char* SHA256_End(SHA256_CTX*, char[SHA256_DIGEST_STRING_LENGTH]);
+char* SHA256_Data(const uint8_t*, size_t, char[SHA256_DIGEST_STRING_LENGTH]);
+
+void SHA384_Init(SHA384_CTX*);
+void SHA384_Update(SHA384_CTX*, const uint8_t*, size_t);
+void SHA384_Final(uint8_t[SHA384_DIGEST_LENGTH], SHA384_CTX*);
+char* SHA384_End(SHA384_CTX*, char[SHA384_DIGEST_STRING_LENGTH]);
+char* SHA384_Data(const uint8_t*, size_t, char[SHA384_DIGEST_STRING_LENGTH]);
+
+void SHA512_Init(SHA512_CTX*);
+void SHA512_Update(SHA512_CTX*, const uint8_t*, size_t);
+void SHA512_Final(uint8_t[SHA512_DIGEST_LENGTH], SHA512_CTX*);
+char* SHA512_End(SHA512_CTX*, char[SHA512_DIGEST_STRING_LENGTH]);
+char* SHA512_Data(const uint8_t*, size_t, char[SHA512_DIGEST_STRING_LENGTH]);
+
+#else /* SHA2_USE_INTTYPES_H */
+
+void SHA256_Init(SHA256_CTX *);
+void SHA256_Update(SHA256_CTX*, const u_int8_t*, size_t);
+void SHA256_Final(u_int8_t[SHA256_DIGEST_LENGTH], SHA256_CTX*);
+char* SHA256_End(SHA256_CTX*, char[SHA256_DIGEST_STRING_LENGTH]);
+char* SHA256_Data(const u_int8_t*, size_t, char[SHA256_DIGEST_STRING_LENGTH]);
+
+void SHA384_Init(SHA384_CTX*);
+void SHA384_Update(SHA384_CTX*, const u_int8_t*, size_t);
+void SHA384_Final(u_int8_t[SHA384_DIGEST_LENGTH], SHA384_CTX*);
+char* SHA384_End(SHA384_CTX*, char[SHA384_DIGEST_STRING_LENGTH]);
+char* SHA384_Data(const u_int8_t*, size_t, char[SHA384_DIGEST_STRING_LENGTH]);
+
+void SHA512_Init(SHA512_CTX*);
+void SHA512_Update(SHA512_CTX*, const u_int8_t*, size_t);
+void SHA512_Final(u_int8_t[SHA512_DIGEST_LENGTH], SHA512_CTX*);
+char* SHA512_End(SHA512_CTX*, char[SHA512_DIGEST_STRING_LENGTH]);
+char* SHA512_Data(const u_int8_t*, size_t, char[SHA512_DIGEST_STRING_LENGTH]);
+
+#endif /* SHA2_USE_INTTYPES_H */
+
+#else /* NOPROTO */
+
+void SHA256_Init();
+void SHA256_Update();
+void SHA256_Final();
+char* SHA256_End();
+char* SHA256_Data();
+
+void SHA384_Init();
+void SHA384_Update();
+void SHA384_Final();
+char* SHA384_End();
+char* SHA384_Data();
+
+void SHA512_Init();
+void SHA512_Update();
+void SHA512_Final();
+char* SHA512_End();
+char* SHA512_Data();
+
+#endif /* NOPROTO */
+
+#ifdef	__cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* __SHA2_H__ */
+

sqlite-amalgamation-3070900/encode.c

@@ -0,0 +1,254 @@
+/*
+** 2002 April 25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains helper routines used to translate binary data into
+** a null-terminated string (suitable for use in SQLite) and back again.
+** These are convenience routines for use by people who want to store binary
+** data in an SQLite database.  The code in this file is not used by any other
+** part of the SQLite library.
+**
+** $Id: encode.c,v 1.1.1.1 2004/08/08 15:03:57 matt Exp $
+*/
+#include <string.h>
+#include <assert.h>
+
+/*
+** How This Encoder Works
+**
+** The output is allowed to contain any character except 0x27 (') and
+** 0x00.  This is accomplished by using an escape character to encode
+** 0x27 and 0x00 as a two-byte sequence.  The escape character is always
+** 0x01.  An 0x00 is encoded as the two byte sequence 0x01 0x01.  The
+** 0x27 character is encoded as the two byte sequence 0x01 0x28.  Finally,
+** the escape character itself is encoded as the two-character sequence
+** 0x01 0x02.
+**
+** To summarize, the encoder works by using an escape sequences as follows:
+**
+**       0x00  ->  0x01 0x01
+**       0x01  ->  0x01 0x02
+**       0x27  ->  0x01 0x28
+**
+** If that were all the encoder did, it would work, but in certain cases
+** it could double the size of the encoded string.  For example, to
+** encode a string of 100 0x27 characters would require 100 instances of
+** the 0x01 0x03 escape sequence resulting in a 200-character output.
+** We would prefer to keep the size of the encoded string smaller than
+** this.
+**
+** To minimize the encoding size, we first add a fixed offset value to each 
+** byte in the sequence.  The addition is modulo 256.  (That is to say, if
+** the sum of the original character value and the offset exceeds 256, then
+** the higher order bits are truncated.)  The offset is chosen to minimize
+** the number of characters in the string that need to be escaped.  For
+** example, in the case above where the string was composed of 100 0x27
+** characters, the offset might be 0x01.  Each of the 0x27 characters would
+** then be converted into an 0x28 character which would not need to be
+** escaped at all and so the 100 character input string would be converted
+** into just 100 characters of output.  Actually 101 characters of output - 
+** we have to record the offset used as the first byte in the sequence so
+** that the string can be decoded.  Since the offset value is stored as
+** part of the output string and the output string is not allowed to contain
+** characters 0x00 or 0x27, the offset cannot be 0x00 or 0x27.
+**
+** Here, then, are the encoding steps:
+**
+**     (1)   Choose an offset value and make it the first character of
+**           output.
+**
+**     (2)   Copy each input character into the output buffer, one by
+**           one, adding the offset value as you copy.
+**
+**     (3)   If the value of an input character plus offset is 0x00, replace
+**           that one character by the two-character sequence 0x01 0x01.
+**           If the sum is 0x01, replace it with 0x01 0x02.  If the sum
+**           is 0x27, replace it with 0x01 0x03.
+**
+**     (4)   Put a 0x00 terminator at the end of the output.
+**
+** Decoding is obvious:
+**
+**     (5)   Copy encoded characters except the first into the decode 
+**           buffer.  Set the first encoded character aside for use as
+**           the offset in step 7 below.
+**
+**     (6)   Convert each 0x01 0x01 sequence into a single character 0x00.
+**           Convert 0x01 0x02 into 0x01.  Convert 0x01 0x28 into 0x27.
+**
+**     (7)   Subtract the offset value that was the first character of
+**           the encoded buffer from all characters in the output buffer.
+**
+** The only tricky part is step (1) - how to compute an offset value to
+** minimize the size of the output buffer.  This is accomplished by testing
+** all offset values and picking the one that results in the fewest number
+** of escapes.  To do that, we first scan the entire input and count the
+** number of occurances of each character value in the input.  Suppose
+** the number of 0x00 characters is N(0), the number of occurances of 0x01
+** is N(1), and so forth up to the number of occurances of 0xff is N(255).
+** An offset of 0 is not allowed so we don't have to test it.  The number
+** of escapes required for an offset of 1 is N(1)+N(2)+N(40).  The number
+** of escapes required for an offset of 2 is N(2)+N(3)+N(41).  And so forth.
+** In this way we find the offset that gives the minimum number of escapes,
+** and thus minimizes the length of the output string.
+*/
+
+/*
+** Encode a binary buffer "in" of size n bytes so that it contains
+** no instances of characters '\'' or '\000'.  The output is 
+** null-terminated and can be used as a string value in an INSERT
+** or UPDATE statement.  Use sqlite_decode_binary() to convert the
+** string back into its original binary.
+**
+** The result is written into a preallocated output buffer "out".
+** "out" must be able to hold at least 2 +(257*n)/254 bytes.
+** In other words, the output will be expanded by as much as 3
+** bytes for every 254 bytes of input plus 2 bytes of fixed overhead.
+** (This is approximately 2 + 1.0118*n or about a 1.2% size increase.)
+**
+** The return value is the number of characters in the encoded
+** string, excluding the "\000" terminator.
+**
+** If out==NULL then no output is generated but the routine still returns
+** the number of characters that would have been generated if out had
+** not been NULL.
+*/
+int sqlite_encode_binary(const unsigned char *in, int n, unsigned char *out){
+  int i, j, e, m;
+  unsigned char x;
+  int cnt[256];
+  if( n<=0 ){
+    if( out ){
+      out[0] = 'x';
+      out[1] = 0;
+    }
+    return 1;
+  }
+  memset(cnt, 0, sizeof(cnt));
+  for(i=n-1; i>=0; i--){ cnt[in[i]]++; }
+  m = n;
+  for(i=1; i<256; i++){
+    int sum;
+    if( i=='\'' ) continue;
+    sum = cnt[i] + cnt[(i+1)&0xff] + cnt[(i+'\'')&0xff];
+    if( sum<m ){
+      m = sum;
+      e = i;
+      if( m==0 ) break;
+    }
+  }
+  if( out==0 ){
+    return n+m+1;
+  }
+  out[0] = e;
+  j = 1;
+  for(i=0; i<n; i++){
+    x = in[i] - e;
+    if( x==0 || x==1 || x=='\''){
+      out[j++] = 1;
+      x++;
+    }
+    out[j++] = x;
+  }
+  out[j] = 0;
+  assert( j==n+m+1 );
+  return j;
+}
+
+/*
+** Decode the string "in" into binary data and write it into "out".
+** This routine reverses the encoding created by sqlite_encode_binary().
+** The output will always be a few bytes less than the input.  The number
+** of bytes of output is returned.  If the input is not a well-formed
+** encoding, -1 is returned.
+**
+** The "in" and "out" parameters may point to the same buffer in order
+** to decode a string in place.
+*/
+int sqlite_decode_binary(const unsigned char *in, unsigned char *out){
+  int i, e;
+  unsigned char c;
+  e = *(in++);
+  i = 0;
+  while( (c = *(in++))!=0 ){
+    if( c==1 ){
+      c = *(in++) - 1;
+    }
+    out[i++] = c + e;
+  }
+  return i;
+}
+
+#ifdef ENCODER_TEST
+#include <stdio.h>
+/*
+** The subroutines above are not tested by the usual test suite.  To test
+** these routines, compile just this one file with a -DENCODER_TEST=1 option
+** and run the result.
+*/
+int main(int argc, char **argv){
+  int i, j, n, m, nOut, nByteIn, nByteOut;
+  unsigned char in[30000];
+  unsigned char out[33000];
+
+  nByteIn = nByteOut = 0;
+  for(i=0; i<sizeof(in); i++){
+    printf("Test %d: ", i+1);
+    n = rand() % (i+1);
+    if( i%100==0 ){
+      int k;
+      for(j=k=0; j<n; j++){
+        /* if( k==0 || k=='\'' ) k++; */
+        in[j] = k;
+        k = (k+1)&0xff;
+      }
+    }else{
+      for(j=0; j<n; j++) in[j] = rand() & 0xff;
+    }
+    nByteIn += n;
+    nOut = sqlite_encode_binary(in, n, out);
+    nByteOut += nOut;
+    if( nOut!=strlen(out) ){
+      printf(" ERROR return value is %d instead of %d\n", nOut, strlen(out));
+      exit(1);
+    }
+    if( nOut!=sqlite_encode_binary(in, n, 0) ){
+      printf(" ERROR actual output size disagrees with predicted size\n");
+      exit(1);
+    }
+    m = (256*n + 1262)/253;
+    printf("size %d->%d (max %d)", n, strlen(out)+1, m);
+    if( strlen(out)+1>m ){
+      printf(" ERROR output too big\n");
+      exit(1);
+    }
+    for(j=0; out[j]; j++){
+      if( out[j]=='\'' ){
+        printf(" ERROR contains (')\n");
+        exit(1);
+      }
+    }
+    j = sqlite_decode_binary(out, out);
+    if( j!=n ){
+      printf(" ERROR decode size %d\n", j);
+      exit(1);
+    }
+    if( memcmp(in, out, n)!=0 ){
+      printf(" ERROR decode mismatch\n");
+      exit(1);
+    }
+    printf(" OK\n");
+  }
+  fprintf(stderr,"Finished.  Total encoding: %d->%d bytes\n",
+          nByteIn, nByteOut);
+  fprintf(stderr,"Avg size increase: %.3f%%\n",
+    (nByteOut-nByteIn)*100.0/(double)nByteIn);
+}
+#endif /* ENCODER_TEST */

sqlite3.c

@@ -0,0 +1 @@
+sqlite-amalgamation-3070900/sqlite3.c