run Shutdown.shutdown on exit when using OpenJDK library

The OpenJDK library wants to track and run the shutdown hooks itself
rather than let the VM do it, so we need to tell it when we're
exiting.

Also, in machine.cpp we need to use only the modifiers specified in
the InnerClasses attribute for inner classes rather than OR them with
the flags given at the top level of the class file.

src/avian/machine.h

@@ -1592,6 +1592,9 @@ class Classpath {
   canTailCall(Thread* t, object caller, object calleeClassName,
               object calleeMethodName, object calleeMethodSpec) = 0;
 
+  virtual void
+  shutDown(Thread* t) = 0;
+
   virtual void
   dispose() = 0;
 };

src/classpath-android.cpp

@@ -469,6 +469,12 @@ class MyClasspath : public Classpath {
     return true;
   }
 
+  virtual void
+  shutDown(Thread*)
+  {
+    // ignore
+  }
+
   virtual void
   dispose()
   {

src/classpath-avian.cpp

@@ -186,6 +186,12 @@ class MyClasspath : public Classpath {
     return true;
   }
 
+  virtual void
+  shutDown(Thread*)
+  {
+    // ignore
+  }
+
   virtual void
   dispose()
   {

src/classpath-openjdk.cpp

@@ -897,6 +897,21 @@ class MyClasspath : public Classpath {
                      (&byteArrayBody(t, calleeClassName, 0))));
   }
 
+  virtual void
+  shutDown(Thread* t)
+  {
+    object c = resolveClass
+      (t, root(t, Machine::BootLoader), "java/lang/Shutdown", false);
+
+    if (c) {
+      object m = findMethodOrNull(t, c, "shutdown", "()V");
+      
+      if (m) {
+        t->m->processor->invoke(t, m, 0);
+      }
+    }
+  }
+
   virtual void
   dispose()
   { 

src/machine.cpp

@@ -2315,7 +2315,7 @@ parseAttributeTable(Thread* t, Stream& s, object class_, object pool)
             (&byteArrayBody(t, className(t, class_), 0),
              &byteArrayBody(t, innerClassReferenceInner(t, reference), 0)))
         {
-          classFlags(t, class_) |= flags;
+          classFlags(t, class_) = flags;
         }
       }
 
@@ -3321,6 +3321,8 @@ shutDown(Thread* t)
 
   setRoot(t, Machine::ShutdownHooks, 0);
 
+  t->m->classpath->shutDown(t);
+
   object h = hooks;
   PROTECT(t, h);
   for (; h; h = pairSecond(t, h)) {

test/Misc.java

@@ -259,5 +259,10 @@ public class Misc {
 
     expect((Baz.class.getModifiers() & java.lang.reflect.Modifier.STATIC)
            != 0);
+
+    expect((Protected.class.getModifiers() & java.lang.reflect.Modifier.PUBLIC)
+           == 0);
   }
+
+  protected class Protected { }
 }