Gradle API Scanner plugin (#1669)

* Skeleton plugin.
* Implement Gradle api-scanner plugin, and apply it.
* Generate API documentation for any jar without a classifier.
* Fix usage of smokeTests classifier.
* Tweak Gradle API usage.
* Upgrade to fast-classpath-scanner 2.7.0
* Include interfaces and more modifiers in the class description.
* Allow system classes to be supertypes and implemented interfaces.
* Make API Scanner plugin configuration tweakable via build.gradle.
* Add a miserable amount of unit testing.
* Sort methods and fields using their natural comparators. Way easier!
* Add README for api-scanner plugin.
* Add @OutputFiles to ScanApiTask.
* Rename ScanApiTask to ScanApi.
* Allow the ScanApi task to be disabled.
* WIP: Create a top-level GenerateApi task to collate the ScanApi output.
* Exclude package-private classes, as well as bridge/synthetic methods.
* Replace "End of Class" delimiter with '##'.
* Don't scan modules whose API is still "in flux".
* Include constructors in the API definitions.
* Finish implementation of GenerateApi task.
* Update README to include GenerateApi task.
* Filter out Kotlin's "internal" methods.
* Assign "fatjar" classifier to the fat jar artifact.
* Enhance README for GenerateApi.
* Explain effect of api-scanner plugin, and link to Corda's API strategy.
* Tweak README
* Exclude synthetic Kotlin classes by analysing @Metadata.
* Allow us to exclude some classes explicitly from the API.
This commit is contained in:
Chris Rankin 2017-09-29 16:55:26 +01:00
parent 5371b25630
commit 8f7253149a
19 changed files with 602 additions and 9 deletions

View File

@ -60,6 +60,7 @@ buildscript {
classpath "net.corda.plugins:publish-utils:$gradle_plugins_version"
classpath "net.corda.plugins:quasar-utils:$gradle_plugins_version"
classpath "net.corda.plugins:cordformation:$gradle_plugins_version"
classpath "net.corda.plugins:api-scanner:$gradle_plugins_version"
classpath 'com.github.ben-manes:gradle-versions-plugin:0.15.0'
classpath "org.jetbrains.kotlin:kotlin-noarg:$kotlin_version"
classpath "org.jetbrains.dokka:dokka-gradle-plugin:${dokka_version}"
@ -298,3 +299,7 @@ artifactory {
}
}
}
task generateApi(type: net.corda.plugins.GenerateApi){
baseName = "api-corda"
}

View File

@ -1,6 +1,7 @@
apply plugin: 'java'
apply plugin: 'kotlin'
apply plugin: 'net.corda.plugins.publish-utils'
apply plugin: 'net.corda.plugins.api-scanner'
apply plugin: 'com.jfrog.artifactory'
dependencies {

View File

@ -61,4 +61,4 @@ jar {
publish {
name jar.baseName
}
}

View File

@ -25,4 +25,4 @@ jar {
publish {
name jar.baseName
}
}

View File

@ -1,6 +1,7 @@
apply plugin: 'kotlin'
apply plugin: 'net.corda.plugins.quasar-utils'
apply plugin: 'net.corda.plugins.publish-utils'
apply plugin: 'net.corda.plugins.api-scanner'
apply plugin: 'com.jfrog.artifactory'
description 'Corda client RPC modules'

View File

@ -1,4 +1,4 @@
gradlePluginsVersion=1.0.0
gradlePluginsVersion=1.0.2
kotlinVersion=1.1.4
guavaVersion=21.0
bouncycastleVersion=1.57

View File

@ -2,6 +2,7 @@ apply plugin: 'kotlin'
apply plugin: 'kotlin-jpa'
apply plugin: 'net.corda.plugins.quasar-utils'
apply plugin: 'net.corda.plugins.publish-utils'
apply plugin: 'net.corda.plugins.api-scanner'
apply plugin: 'com.jfrog.artifactory'
description 'Corda core'
@ -94,6 +95,13 @@ jar {
baseName 'corda-core'
}
scanApi {
excludeClasses = [
// Kotlin should probably have declared this class as "synthetic".
"net.corda.core.Utils\$toFuture\$1\$subscription\$1"
]
}
publish {
name jar.baseName
}

View File

@ -0,0 +1,79 @@
# API Scanner
Generates a text summary of Corda's public API that we can check for API-breaking changes.
```bash
$ gradlew generateApi
```
See [here](../../docs/source/api-index.rst) for Corda's public API strategy. We will need to
apply this plugin to other modules in future Corda releases as those modules' APIs stabilise.
Basically, this plugin will document a module's `public` and `protected` classes/methods/fields,
excluding those from our `*.internal.*` packgages, any synthetic methods, bridge methods, or methods
identified as having Kotlin's `internal` scope. (Kotlin doesn't seem to have implemented `internal`
scope for classes or fields yet as these are currently `public` inside the `.class` file.)
## Usage
Include this line in the `build.gradle` file of every Corda module that exports public API:
```gradle
apply plugin: 'net.corda.plugins.api-scanner'
```
This will create a Gradle task called `scanApi` which will analyse that module's Jar artifacts. More precisely,
it will analyse all of the Jar artifacts that have not been assigned a Maven classifier, on the basis
that these should be the module's main artifacts.
The `scanApi` task supports the following configuration options:
```gradle
scanApi {
// Make the classpath-scanning phase more verbose.
verbose = {true|false}
// Enable / disable the task within this module.
enabled = {true|false}
// Names of classes that should be excluded from the output.
excludeClasses = [
...
]
}
```
All of the `ScanApi` tasks write their output files to their own `$buildDir/api` directory, where they
are collated into a single output file by the `GenerateApi` task. The `GenerateApi` task is declared
in the root project's `build.gradle` file:
```gradle
task generateApi(type: net.corda.plugins.GenerateApi){
baseName = "api-corda"
}
```
The final API file is written to `$buildDir/api/$baseName-$project.version.txt`
### Sample Output
```
public interface net.corda.core.contracts.Attachment extends net.corda.core.contracts.NamedByHash
public abstract void extractFile(String, java.io.OutputStream)
@org.jetbrains.annotations.NotNull public abstract List getSigners()
@org.jetbrains.annotations.NotNull public abstract java.io.InputStream open()
@org.jetbrains.annotations.NotNull public abstract jar.JarInputStream openAsJAR()
##
public interface net.corda.core.contracts.AttachmentConstraint
public abstract boolean isSatisfiedBy(net.corda.core.contracts.Attachment)
##
public final class net.corda.core.contracts.AttachmentResolutionException extends net.corda.core.flows.FlowException
public <init>(net.corda.core.crypto.SecureHash)
@org.jetbrains.annotations.NotNull public final net.corda.core.crypto.SecureHash getHash()
##
```
#### Notes
The `GenerateApi` task will collate the output of every `ScanApi` task found either in the same project,
or in any of that project's subprojects. So it is _theoretically_ possible also to collate the API output
from subtrees of modules simply by defining a new `GenerateApi` task at the root of that subtree.
## Plugin Installation
See [here](../README.rst) for full installation instructions.

View File

@ -0,0 +1,18 @@
apply plugin: 'java'
apply plugin: 'net.corda.plugins.publish-utils'
description "Generates a summary of the artifact's public API"
repositories {
mavenCentral()
}
dependencies {
compile gradleApi()
compile "io.github.lukehutch:fast-classpath-scanner:2.7.0"
testCompile "junit:junit:4.12"
}
publish {
name project.name
}

View File

@ -0,0 +1,70 @@
package net.corda.plugins;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.artifacts.ConfigurationContainer;
import org.gradle.api.file.FileCollection;
import org.gradle.api.tasks.TaskCollection;
import org.gradle.jvm.tasks.Jar;
public class ApiScanner implements Plugin<Project> {
/**
* Identify the Gradle Jar tasks creating jars
* without Maven classifiers, and generate API
* documentation for them.
* @param p Current project.
*/
@Override
public void apply(Project p) {
p.getLogger().info("Applying API scanner to {}", p.getName());
ScannerExtension extension = p.getExtensions().create("scanApi", ScannerExtension.class);
p.afterEvaluate(project -> {
TaskCollection<Jar> jarTasks = project.getTasks()
.withType(Jar.class)
.matching(jarTask -> jarTask.getClassifier().isEmpty() && jarTask.isEnabled());
if (jarTasks.isEmpty()) {
return;
}
project.getLogger().info("Adding scanApi task to {}", project.getName());
project.getTasks().create("scanApi", ScanApi.class, scanTask -> {
scanTask.setClasspath(compilationClasspath(project.getConfigurations()));
scanTask.setSources(project.files(jarTasks));
scanTask.setExcludeClasses(extension.getExcludeClasses());
scanTask.setVerbose(extension.isVerbose());
scanTask.setEnabled(extension.isEnabled());
scanTask.dependsOn(jarTasks);
// Declare this ScanApi task to be a dependency of any
// GenerateApi tasks belonging to any of our ancestors.
project.getRootProject().getTasks()
.withType(GenerateApi.class)
.matching(generateTask -> isAncestorOf(generateTask.getProject(), project))
.forEach(generateTask -> generateTask.dependsOn(scanTask));
});
});
}
/*
* Recurse through a child project's parents until we reach the root,
* and return true iff we find our target project along the way.
*/
private static boolean isAncestorOf(Project target, Project child) {
Project p = child;
while (p != null) {
if (p == target) {
return true;
}
p = p.getParent();
}
return false;
}
private static FileCollection compilationClasspath(ConfigurationContainer configurations) {
return configurations.getByName("compile")
.plus(configurations.getByName("compileOnly"));
}
}

View File

@ -0,0 +1,61 @@
package net.corda.plugins;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.FileCollection;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.TaskAction;
import java.io.*;
import java.nio.file.Files;
import static java.util.Comparator.comparing;
import static java.util.stream.Collectors.toList;
@SuppressWarnings("unused")
public class GenerateApi extends DefaultTask {
private final File outputDir;
private String baseName;
public GenerateApi() {
outputDir = new File(getProject().getBuildDir(), "api");
baseName = "api-" + getProject().getName();
}
public void setBaseName(String baseName) {
this.baseName = baseName;
}
@InputFiles
public FileCollection getSources() {
return getProject().files(getProject().getAllprojects().stream()
.flatMap(project -> project.getTasks()
.withType(ScanApi.class)
.matching(ScanApi::isEnabled)
.stream())
.flatMap(scanTask -> scanTask.getTargets().getFiles().stream())
.sorted(comparing(File::getName))
.collect(toList())
);
}
@OutputFile
public File getTarget() {
return new File(outputDir, String.format("%s-%s.txt", baseName, getProject().getVersion()));
}
@TaskAction
public void generate() {
FileCollection apiFiles = getSources();
if (!apiFiles.isEmpty() && (outputDir.isDirectory() || outputDir.mkdirs())) {
try (OutputStream output = new BufferedOutputStream(new FileOutputStream(getTarget()))) {
for (File apiFile : apiFiles) {
Files.copy(apiFile.toPath(), output);
}
} catch (IOException e) {
getLogger().error("Failed to generate API file", e);
}
}
}
}

View File

@ -0,0 +1,311 @@
package net.corda.plugins;
import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner;
import io.github.lukehutch.fastclasspathscanner.scanner.ClassInfo;
import io.github.lukehutch.fastclasspathscanner.scanner.FieldInfo;
import io.github.lukehutch.fastclasspathscanner.scanner.MethodInfo;
import io.github.lukehutch.fastclasspathscanner.scanner.ScanResult;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.ConfigurableFileCollection;
import org.gradle.api.file.FileCollection;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputFiles;
import org.gradle.api.tasks.TaskAction;
import java.io.*;
import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.*;
import java.util.stream.StreamSupport;
import static java.util.Collections.unmodifiableSet;
import static java.util.stream.Collectors.*;
@SuppressWarnings("unused")
public class ScanApi extends DefaultTask {
private static final int CLASS_MASK = Modifier.classModifiers();
private static final int INTERFACE_MASK = Modifier.interfaceModifiers() & ~Modifier.ABSTRACT;
private static final int METHOD_MASK = Modifier.methodModifiers();
private static final int FIELD_MASK = Modifier.fieldModifiers();
private static final int VISIBILITY_MASK = Modifier.PUBLIC | Modifier.PROTECTED;
/**
* This information has been lifted from:
* @link <a href="https://github.com/JetBrains/kotlin/blob/master/core/runtime.jvm/src/kotlin/Metadata.kt">Metadata.kt</a>
*/
private static final String KOTLIN_METADATA = "kotlin.Metadata";
private static final String KOTLIN_CLASSTYPE_METHOD = "k";
private static final int KOTLIN_SYNTHETIC = 3;
private final ConfigurableFileCollection sources;
private final ConfigurableFileCollection classpath;
private final Set<String> excludeClasses;
private final File outputDir;
private boolean verbose;
public ScanApi() {
sources = getProject().files();
classpath = getProject().files();
excludeClasses = new LinkedHashSet<>();
outputDir = new File(getProject().getBuildDir(), "api");
}
@Input
public FileCollection getSources() {
return sources;
}
void setSources(FileCollection sources) {
this.sources.setFrom(sources);
}
@Input
public FileCollection getClasspath() {
return classpath;
}
void setClasspath(FileCollection classpath) {
this.classpath.setFrom(classpath);
}
@Input
public Collection<String> getExcludeClasses() {
return unmodifiableSet(excludeClasses);
}
void setExcludeClasses(Collection<String> excludeClasses) {
this.excludeClasses.clear();
this.excludeClasses.addAll(excludeClasses);
}
@OutputFiles
public FileCollection getTargets() {
return getProject().files(
StreamSupport.stream(sources.spliterator(), false)
.map(this::toTarget)
.collect(toList())
);
}
public boolean isVerbose() {
return verbose;
}
void setVerbose(boolean verbose) {
this.verbose = verbose;
}
private File toTarget(File source) {
return new File(outputDir, source.getName().replaceAll(".jar$", ".txt"));
}
@TaskAction
public void scan() {
if (outputDir.isDirectory() || outputDir.mkdirs()) {
try (Scanner scanner = new Scanner(classpath)) {
for (File source : sources) {
scanner.scan(source);
}
} catch (IOException e) {
getLogger().error("Failed to write API file", e);
}
} else {
getLogger().error("Cannot create directory '{}'", outputDir.getAbsolutePath());
}
}
class Scanner implements Closeable {
private final URLClassLoader classpathLoader;
private final Class<? extends Annotation> metadataClass;
private final Method classTypeMethod;
@SuppressWarnings("unchecked")
Scanner(URLClassLoader classpathLoader) {
this.classpathLoader = classpathLoader;
Class<? extends Annotation> kClass;
Method kMethod;
try {
kClass = (Class<Annotation>) Class.forName(KOTLIN_METADATA, true, classpathLoader);
kMethod = kClass.getDeclaredMethod(KOTLIN_CLASSTYPE_METHOD);
} catch (ClassNotFoundException | NoSuchMethodException e) {
kClass = null;
kMethod = null;
}
metadataClass = kClass;
classTypeMethod = kMethod;
}
Scanner(FileCollection classpath) throws MalformedURLException {
this(new URLClassLoader(toURLs(classpath)));
}
@Override
public void close() throws IOException {
classpathLoader.close();
}
void scan(File source) {
File target = toTarget(source);
try (
URLClassLoader appLoader = new URLClassLoader(new URL[]{ toURL(source) }, classpathLoader);
PrintWriter writer = new PrintWriter(target, "UTF-8")
) {
scan(writer, appLoader);
} catch (IOException e) {
getLogger().error("API scan has failed", e);
}
}
void scan(PrintWriter writer, ClassLoader appLoader) {
ScanResult result = new FastClasspathScanner(getScanSpecification())
.overrideClassLoaders(appLoader)
.ignoreParentClassLoaders()
.ignoreMethodVisibility()
.ignoreFieldVisibility()
.enableMethodInfo()
.enableFieldInfo()
.verbose(verbose)
.scan();
writeApis(writer, result);
}
private String[] getScanSpecification() {
String[] spec = new String[2 + excludeClasses.size()];
spec[0] = "!"; // Don't blacklist system classes from the output.
spec[1] = "-dir:"; // Ignore classes on the filesystem.
int i = 2;
for (String excludeClass : excludeClasses) {
spec[i++] = '-' + excludeClass;
}
return spec;
}
private void writeApis(PrintWriter writer, ScanResult result) {
Map<String, ClassInfo> allInfo = result.getClassNameToClassInfo();
result.getNamesOfAllClasses().forEach(className -> {
if (className.contains(".internal.")) {
// These classes belong to internal Corda packages.
return;
}
ClassInfo classInfo = allInfo.get(className);
if (classInfo.getClassLoaders() == null) {
// Ignore classes that belong to one of our target ClassLoader's parents.
return;
}
Class<?> javaClass = result.classNameToClassRef(className);
if (!isVisible(javaClass.getModifiers())) {
// Excludes private and package-protected classes
return;
}
int kotlinClassType = getKotlinClassType(javaClass);
if (kotlinClassType == KOTLIN_SYNTHETIC) {
// Exclude classes synthesised by the Kotlin compiler.
return;
}
writeClass(writer, classInfo, javaClass.getModifiers());
writeMethods(writer, classInfo.getMethodAndConstructorInfo());
writeFields(writer, classInfo.getFieldInfo());
writer.println("##");
});
}
private void writeClass(PrintWriter writer, ClassInfo classInfo, int modifiers) {
if (classInfo.isAnnotation()) {
writer.append(Modifier.toString(modifiers & INTERFACE_MASK));
writer.append(" @interface ").print(classInfo);
} else if (classInfo.isStandardClass()) {
writer.append(Modifier.toString(modifiers & CLASS_MASK));
writer.append(" class ").print(classInfo);
Set<ClassInfo> superclasses = classInfo.getDirectSuperclasses();
if (!superclasses.isEmpty()) {
writer.append(" extends ").print(stringOf(superclasses));
}
Set<ClassInfo> interfaces = classInfo.getDirectlyImplementedInterfaces();
if (!interfaces.isEmpty()) {
writer.append(" implements ").print(stringOf(interfaces));
}
} else {
writer.append(Modifier.toString(modifiers & INTERFACE_MASK));
writer.append(" interface ").print(classInfo);
Set<ClassInfo> superinterfaces = classInfo.getDirectSuperinterfaces();
if (!superinterfaces.isEmpty()) {
writer.append(" extends ").print(stringOf(superinterfaces));
}
}
writer.println();
}
private void writeMethods(PrintWriter writer, List<MethodInfo> methods) {
Collections.sort(methods);
for (MethodInfo method : methods) {
if (isVisible(method.getAccessFlags()) // Only public and protected methods
&& isValid(method.getAccessFlags(), METHOD_MASK) // Excludes bridge and synthetic methods
&& !isKotlinInternalScope(method)) {
writer.append(" ").println(method);
}
}
}
private void writeFields(PrintWriter output, List<FieldInfo> fields) {
Collections.sort(fields);
for (FieldInfo field : fields) {
if (isVisible(field.getAccessFlags()) && isValid(field.getAccessFlags(), FIELD_MASK)) {
output.append(" ").println(field);
}
}
}
private int getKotlinClassType(Class<?> javaClass) {
if (metadataClass != null) {
Annotation metadata = javaClass.getAnnotation(metadataClass);
if (metadata != null) {
try {
return (int) classTypeMethod.invoke(metadata);
} catch (IllegalAccessException | InvocationTargetException e) {
getLogger().error("Failed to read Kotlin annotation", e);
}
}
}
return 0;
}
}
private static boolean isKotlinInternalScope(MethodInfo method) {
return method.getMethodName().indexOf('$') >= 0;
}
private static boolean isValid(int modifiers, int mask) {
return (modifiers & mask) == modifiers;
}
private static boolean isVisible(int accessFlags) {
return (accessFlags & VISIBILITY_MASK) != 0;
}
private static String stringOf(Collection<ClassInfo> items) {
return items.stream().map(ClassInfo::toString).collect(joining(", "));
}
private static URL toURL(File file) throws MalformedURLException {
return file.toURI().toURL();
}
private static URL[] toURLs(Iterable<File> files) throws MalformedURLException {
List<URL> urls = new LinkedList<>();
for (File file : files) {
urls.add(toURL(file));
}
return urls.toArray(new URL[urls.size()]);
}
}

View File

@ -0,0 +1,37 @@
package net.corda.plugins;
import java.util.List;
import static java.util.Collections.emptyList;
@SuppressWarnings("unused")
public class ScannerExtension {
private boolean verbose;
private boolean enabled = true;
private List<String> excludeClasses = emptyList();
public boolean isVerbose() {
return verbose;
}
public void setVerbose(boolean verbose) {
this.verbose = verbose;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public List<String> getExcludeClasses() {
return excludeClasses;
}
public void setExcludeClasses(List<String> excludeClasses) {
this.excludeClasses = excludeClasses;
}
}

View File

@ -0,0 +1 @@
implementation-class=net.corda.plugins.ApiScanner

View File

@ -25,7 +25,7 @@ buildscript {
apply plugin: 'net.corda.plugins.publish-utils'
allprojects {
version "$gradle_plugins_version"
version gradle_plugins_version
group 'net.corda.plugins'
}
@ -39,7 +39,7 @@ bintrayConfig {
projectUrl = 'https://github.com/corda/corda'
gpgSign = true
gpgPassphrase = System.getenv('CORDA_BINTRAY_GPG_PASSPHRASE')
publications = ['cordformation', 'quasar-utils', 'cordform-common']
publications = ['cordformation', 'quasar-utils', 'cordform-common', 'api-scanner']
license {
name = 'Apache-2.0'
url = 'https://www.apache.org/licenses/LICENSE-2.0'

View File

@ -51,7 +51,7 @@ task createNodeRunner(type: Jar, dependsOn: [classes]) {
manifest {
attributes('Main-Class': 'net.corda.plugins.NodeRunnerKt')
}
baseName = project.name + '-fatjar'
classifier = 'fatjar'
from { configurations.noderunner.collect { it.isDirectory() ? it : zipTree(it) } }
from sourceSets.runnodes.output
}

View File

@ -2,4 +2,5 @@ rootProject.name = 'corda-gradle-plugins'
include 'publish-utils'
include 'quasar-utils'
include 'cordformation'
include 'cordform-common'
include 'cordform-common'
include 'api-scanner'

View File

@ -192,7 +192,7 @@ task integrationTest(type: Test) {
}
task smokeTestJar(type: Jar) {
baseName = project.name + '-smoke-test'
classifier 'smokeTests'
from sourceSets.smokeTest.output
}

View File

@ -42,7 +42,7 @@ class CordappSmokeTest {
val pluginsDir = (factory.baseDirectory(aliceConfig) / "plugins").createDirectories()
// Find the jar file for the smoke tests of this module
val selfCordapp = Paths.get("build", "libs").list {
it.filter { "-smoke-test" in it.toString() }.toList().single()
it.filter { "-smokeTests" in it.toString() }.toList().single()
}
selfCordapp.copyToDirectory(pluginsDir)