From bb9c0dee073c1d11103772905dde73f75b40183e Mon Sep 17 00:00:00 2001 From: Tommy Lillehagen Date: Tue, 31 Oct 2017 17:05:44 +0000 Subject: [PATCH] Utilities for diffing the deterministic JVM with JDK8 (#86) --- sgx-jvm/tools/jvm-diff/README.md | 47 ++++++ sgx-jvm/tools/jvm-diff/compare-versions.sh | 23 +++ sgx-jvm/tools/jvm-diff/find-exclusions.sh | 36 +++++ sgx-jvm/tools/jvm-diff/flatten.sh | 5 + sgx-jvm/tools/jvm-diff/generate-report.sh | 76 +++++++++ sgx-jvm/tools/jvm-diff/report-template.html | 161 ++++++++++++++++++++ 6 files changed, 348 insertions(+) create mode 100644 sgx-jvm/tools/jvm-diff/README.md create mode 100755 sgx-jvm/tools/jvm-diff/compare-versions.sh create mode 100755 sgx-jvm/tools/jvm-diff/find-exclusions.sh create mode 100755 sgx-jvm/tools/jvm-diff/flatten.sh create mode 100755 sgx-jvm/tools/jvm-diff/generate-report.sh create mode 100644 sgx-jvm/tools/jvm-diff/report-template.html diff --git a/sgx-jvm/tools/jvm-diff/README.md b/sgx-jvm/tools/jvm-diff/README.md new file mode 100644 index 0000000000..be07c6b996 --- /dev/null +++ b/sgx-jvm/tools/jvm-diff/README.md @@ -0,0 +1,47 @@ +### Overview + +This folder has a set of utilities for generating a report outlining the +differences between two versions of the OpenJDK project, more specifically, the +`rt.jar` build artefact across versions. + +##### Files + +The following scripts and utilities are used: + + * `compare-versions.sh` - extract exclusions and binary diff between JDK8 and + Deterministic JVM + * `find-exclusions.sh` - utility used in the above script to find JavaDoc + exclusions in code base + * `generate-report.sh` - once the data has been gathered with + `compare-versions.sh`, this script can be used to generate an HTML report + * `report-template.html` - the template to use for the HTML report + * `flatten.sh` - flatten the file hierarchy of supporting diff and exclusion + files into a single directory + +##### Dependencies + + * Tweaked version of the [Package Changes Analyzer (corda/pkgdiff)](https://github.com/corda/pkgdiff) + +##### Assumptions + +The first two scripts are intended to be run from the root directory of the +OpenJDK project, and assume two built JAR files: + + * `jdk-8.0.0.jar`, which is the `rt.jar` build artefact for branch + `jdk8u/jdk8u` + * `jdk-8.0.0-determinstic.jar`, which is the `rt.jar` build artefact for + branch `deterministic-jvm8` + +### How to run? + +From the root folder of your cloned OpenJDK project: + + 1. Assuming that the files in this folder is in a subfolder `tools` of the + OpenJDK root folder + 2. Build `jdk8u/jdk8u` and copy `build//images/lib/rt.jar` to + `jdk-8.0.0.jar` + 3. Build `deterministic-jvm8` and copy `build//images/lib/rt.jar` to + `jdk-8.0.0-deterministic.jar` + 4. Run `bash tools/compare-versions.sh` + 5. Make a copy of the resulting report: + - `report/` diff --git a/sgx-jvm/tools/jvm-diff/compare-versions.sh b/sgx-jvm/tools/jvm-diff/compare-versions.sh new file mode 100755 index 0000000000..d02fd20da7 --- /dev/null +++ b/sgx-jvm/tools/jvm-diff/compare-versions.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +BASE_VERSION="jdk-8.0.0.jar" +NEW_VERSION="jdk-8.0.0-deterministic.jar" + +# Derive list of differences between the two JARs +pkgdiff -check-byte-code -extra-info pkgdiff_extra \ + "$BASE_VERSION" "$NEW_VERSION" + +# Find packages and classes marked for exclusion in JavaDoc +${SHELL} tools/find-exclusions.sh + +# Generate report +sed -n '1,/\/\* DATASET \*\//p' < ./tools/report-template.html > report.html +${SHELL} tools/generate-report.sh >> report.html +sed -n '/\/\* DATASET \*\//,$p' < ./tools/report-template.html >> report.html + +# Generate structure for upload to Azure +mkdir -p report +mv exclusions pkgdiff_extra pkgdiff_reports report/ +cd report +${SHELL} ../tools/flatten.sh +cd .. diff --git a/sgx-jvm/tools/jvm-diff/find-exclusions.sh b/sgx-jvm/tools/jvm-diff/find-exclusions.sh new file mode 100755 index 0000000000..8d134c9ca7 --- /dev/null +++ b/sgx-jvm/tools/jvm-diff/find-exclusions.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +OUTPUT_DIR=exclusions + +rm -rf "$OUTPUT_DIR" +mkdir -p "$OUTPUT_DIR" + +if [[ $(uname) = "Darwin" ]]; then + alias sed=gsed # ensure that we use GNU sed on macOS +fi + +function files_with_exclusions { + local DIRS="jaxp jaxws jdk" + find $DIRS -name '*.java' -exec grep -l '@exclude' {} \; +} + +function extract_exclusions { + echo -n "." + echo "$1" >> "$OUTPUT_DIR/files.dat" + mkdir -p "$OUTPUT_DIR/$(dirname "$1")" + cat "$1" | \ + sed 's/^\s*package/public package/' | \ + sed 's/^\s*class/public class/' | \ + sed -n '/@exclude/,/public.*[a-z]/{ + s/^.*\(@exclude.*\)$/\/\/ \1/p; + s/^\s*\(public.*[a-z].*\)$/\1/p + }' | \ + sed 's/^public package/package/' | \ + sed 's/[ ][ ]*/ /g' | \ + sed 's/[{][ ]*$//g' \ + > "$OUTPUT_DIR/$1" +} + +echo -n "Finding files with exclusions ..." +files_with_exclusions | while read f; do extract_exclusions "$f"; done +echo " [done]" diff --git a/sgx-jvm/tools/jvm-diff/flatten.sh b/sgx-jvm/tools/jvm-diff/flatten.sh new file mode 100755 index 0000000000..9aa65d42c0 --- /dev/null +++ b/sgx-jvm/tools/jvm-diff/flatten.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +find {pkgdiff_reports/jdk,exclusions} -type f \( -iname '*.html' -o -iname '*.java' \) | while read f; do + mv $f $(echo $f | tr '/' '.') +done diff --git a/sgx-jvm/tools/jvm-diff/generate-report.sh b/sgx-jvm/tools/jvm-diff/generate-report.sh new file mode 100755 index 0000000000..0d6bc82b7d --- /dev/null +++ b/sgx-jvm/tools/jvm-diff/generate-report.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +REPORT_FILE="$1" +EXCLUSIONS_FILE="$2" + +if [[ -z "$REPORT_FILE" ]]; then + REPORT_FILE=pkgdiff_extra/files.xml +fi + +if [[ -z "$EXCLUSIONS_FILE" ]]; then + EXCLUSIONS_FILE=exclusions/files.dat +fi + +function category { + CATEGORY="$1" + if [[ $CATEGORY = 'exclusions' ]]; then + if [[ ! -z "$EXCLUSIONS_FILE" ]]; then + cat $EXCLUSIONS_FILE + fi + return + fi + cat $REPORT_FILE \ + | sed '/META-INF/d' \ + | sed -n "/<$CATEGORY>/,/<\/$CATEGORY>/p" \ + | sed '1d; $d' \ + | sed 's/^[ \t]*//' +} + +function unchanged_rows { + # file + cat \ + | sed 's/^\(.*\.class\)$/["class", "\1", "unchanged", "", ""],/' \ + | sed 's/^\([^.]*\)$/["package", "\1", "unchanged", "", ""],/' +} + +function added_rows { + # file + cat \ + | sed 's/^\(.*\.class\)$/["class", "\1", "added", "", ""],/' \ + | sed 's/^\([^.]*\)$/["package", "\1", "added", "", ""],/' +} + +function removed_rows { + # file + cat \ + | sed 's/^\(.*\.class\)$/["class", "\1", "removed", "", ""],/' \ + | sed 's/^\([^.]*\)$/["package", "\1", "removed", "", ""],/' +} + +function changed_rows { + # file ([0-9]+%) + sed 's/^\([^ ]*\)[ ](\([^)]*\))$/["class", "\1", "changed", "\2", "\1"],/' +} + +function renamed_rows { + # from;to ([0-9]+%) + sed 's/^\([^;]*\);\([^ ]*\) (\([0-9%.]*\))$/["class", "\1 -> \2", "renamed", "\3", ""],/' +} + +function moved_rows { + # from;to ([0-9]+%) + sed 's/^\([^;]*\);\([^ ]*\) (\([0-9%.]*\))$/["class", "\1 -> \2", "moved", "\3", ""],/' +} + +function exclusion_rows { + # file + sed 's/^\([^ ]*\)$/["java", "\1", "exclusions", "", "\1"],/' +} + +category 'unchanged' | unchanged_rows +category 'added' | added_rows +category 'removed' | removed_rows +category 'changed' | changed_rows +category 'moved' | moved_rows +category 'renamed' | renamed_rows +category 'exclusions' | exclusion_rows diff --git a/sgx-jvm/tools/jvm-diff/report-template.html b/sgx-jvm/tools/jvm-diff/report-template.html new file mode 100644 index 0000000000..7b40fbd0a5 --- /dev/null +++ b/sgx-jvm/tools/jvm-diff/report-template.html @@ -0,0 +1,161 @@ + + + + + + JDK: 8.0.0 to 8.0.0-deterministic + + + + + + + + + + + + + + + + + + + + +
+ +