diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa924eb7..ae73c921 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,17 +13,21 @@ on: jobs: build: - name: ${{ matrix.job }} / JDK ${{ matrix.ci_java_version }} / AGP ${{ matrix.ci_agp_version }} / TraceRefs ${{ matrix.ci_tracerefs }} + name: ${{ matrix.job }} / JDK ${{ matrix.java }} / AGP ${{ matrix.agp }} / TraceRefs ${{ matrix.tracerefs }} # Use macOS for emulator hardware acceleration runs-on: macOS-latest timeout-minutes: 30 strategy: fail-fast: false # We want to see all results matrix: - ci_java_version: [1.8] - ci_agp_version: [4.0.0, 4.1.1, 4.2.0-beta02] - ci_tracerefs: [true, false] - job: [instrumentation, plugin] + java: ['8', '11'] + agp: ['4.2.1', '7.0.0-beta01'] + tracerefs: [true, false] + job: ['instrumentation', 'plugin'] + exclude: + # AGP 7.x requires JDK 11+ + - agp: 7.0.0-beta01 + java: 8 steps: - name: Checkout uses: actions/checkout@v2 @@ -37,26 +41,26 @@ jobs: - uses: actions/cache@v2 with: path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ matrix.ci_agp_version }}-${{ matrix.job }}-${{ hashFiles('checksum.txt') }} + key: ${{ runner.os }}-gradle-${{ matrix.java }}-${{ hashFiles('checksum.txt') }} restore-keys: | - ${{ runner.os }}-gradle-${{ matrix.ci_agp_version }}-${{ matrix.job }}- + ${{ runner.os }}-gradle-${{ matrix.java }}- - - name: Install JDK ${{ matrix.ci_java_version }} - uses: actions/setup-java@v1.4.3 + - name: Install JDK ${{ matrix.java }} + uses: actions/setup-java@v2 with: - java-version: ${{ matrix.ci_java_version }} + distribution: 'adopt' + java-version: ${{ matrix.java }} - name: Test plugin if: matrix.job == 'plugin' - run: ./gradlew -p keeper-gradle-plugin clean check --stacktrace -PkeeperTest.agpVersion=${{ matrix.ci_agp_version }} + run: ./gradlew -p keeper-gradle-plugin clean check --stacktrace -PkeeperTest.agpVersion=${{ matrix.agp }} - name: Run instrumentation tests uses: reactivecircus/android-emulator-runner@v2 if: matrix.job == 'instrumentation' with: - # Run connectedCheck with both Proguard and R8. # We don't want to wait for the emulator to start/stop twice, so we combine this script into the same step. - script: .github/workflows/run_instrumentation_tests.sh ${{ matrix.ci_agp_version }} ${{ matrix.ci_tracerefs }} + script: .github/workflows/run_instrumentation_tests.sh ${{ matrix.agp }} ${{ matrix.tracerefs }} api-level: 29 - name: (Fail-only) Bundle the build report @@ -72,12 +76,12 @@ jobs: - name: Reclaim memory run: ./gradlew --stop && jps|grep -E 'KotlinCompileDaemon|GradleDaemon'| awk '{print $1}'| xargs kill -9 || true - if: success() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' && matrix.ci_java_version == '1.8' && matrix.ci_agp_version == '4.0.0' && matrix.job == 'plugin' + if: success() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' && matrix.java == '8' && matrix.agp == '4.2.1' && matrix.tracerefs && matrix.job == 'plugin' - name: Upload snapshot (main only) env: - ORG_GRADLE_PROJECT_SONATYPE_NEXUS_USERNAME: ${{ secrets.SonatypeUsername }} - ORG_GRADLE_PROJECT_SONATYPE_NEXUS_PASSWORD: ${{ secrets.SonatypePassword }} + ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.SonatypeUsername }} + ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.SonatypePassword }} run: | ./publish.sh --snapshot - if: success() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' && matrix.ci_java_version == '1.8' && matrix.ci_agp_version == '4.0.0' && matrix.job == 'plugin' + if: success() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' && matrix.java == '8' && matrix.agp == '4.2.1' && matrix.tracerefs && matrix.job == 'plugin' diff --git a/.github/workflows/run_instrumentation_tests.sh b/.github/workflows/run_instrumentation_tests.sh index e9a5ca4e..996d5524 100755 --- a/.github/workflows/run_instrumentation_tests.sh +++ b/.github/workflows/run_instrumentation_tests.sh @@ -3,20 +3,11 @@ AGP_VERSION=$1 ENABLE_TRACEREFS=$2 -function printthreads { - echo "Thread dump" - jps|grep -E 'KotlinCompileDaemon|GradleDaemon'| awk '{print $1}'| xargs jstack - exit 1 -} - -trap printthreads SIGINT - -echo "Install coreutils" # For gtimeout -brew install coreutils - # We only run the sample with R8 as proguard infinite loops if we have java 8 libraries on the classpath 🙃 -./gradlew :sample:minifyExternalStagingWithR8 --stacktrace -PkeeperTest.agpVersion="${AGP_VERSION}" -PkeeperTest.enableTraceReferences="${ENABLE_TRACEREFS}" +echo "Building APK and verifying L8" +./gradlew :sample:minifyExternalStagingWithR8 --stacktrace -PkeeperTest.agpVersion="${AGP_VERSION}" -PkeeperTest.enableTraceReferences="${ENABLE_TRACEREFS}" -Pkeeper.verifyL8=true # Reclaim memory because Actions OOMs sometimes with having both an emulator and heavy gradle builds going on -./gradlew --stop && jps|grep -E 'KotlinCompileDaemon|GradleDaemon'| awk '{print $1}'| xargs kill -9 || true +./gradlew --stop || jps|grep -E 'KotlinCompileDaemon|GradleDaemon'| awk '{print $1}'| xargs kill -9 || true # Now proceed, with much of the build being cached up to this point -gtimeout --signal=SIGINT 10m ./gradlew connectedExternalStagingAndroidTest --stacktrace -PkeeperTest.agpVersion="${AGP_VERSION}" -PkeeperTest.enableTraceReferences="${ENABLE_TRACEREFS}" +echo "Running instrumentation tests" +./gradlew connectedExternalStagingAndroidTest --stacktrace -PkeeperTest.agpVersion="${AGP_VERSION}" -PkeeperTest.enableTraceReferences="${ENABLE_TRACEREFS}" diff --git a/build.gradle b/build.gradle index 4b34bc03..31e49893 100644 --- a/build.gradle +++ b/build.gradle @@ -20,13 +20,12 @@ buildscript { repositories { google() mavenCentral() - jcenter() } - String defaultAgpVersion = "4.0.0" + String defaultAgpVersion = "4.2.1" String agpVersion = findProperty("keeperTest.agpVersion")?.toString() ?: defaultAgpVersion dependencies { - classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:1.4.30" + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:1.5.0" classpath "com.android.tools.build:gradle:$agpVersion" classpath "com.slack.keeper:keeper" } @@ -37,7 +36,6 @@ allprojects { repositories { google() mavenCentral() - jcenter() } pluginManager.withPlugin("org.jetbrains.kotlin.jvm") { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 8cf6eb5a..29e41345 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/keeper-gradle-plugin/build.gradle.kts b/keeper-gradle-plugin/build.gradle.kts index 16a1b133..d8bfe341 100644 --- a/keeper-gradle-plugin/build.gradle.kts +++ b/keeper-gradle-plugin/build.gradle.kts @@ -20,17 +20,16 @@ import java.net.URL plugins { `kotlin-dsl` `java-gradle-plugin` - kotlin("jvm") version "1.4.30" - kotlin("kapt") version "1.4.30" - id("org.jetbrains.dokka") version "1.4.20" - id("com.vanniktech.maven.publish") version "0.13.0" + kotlin("jvm") version "1.5.0" + kotlin("kapt") version "1.5.0" + id("org.jetbrains.dokka") version "1.4.32" + id("com.vanniktech.maven.publish") version "0.15.1" } buildscript { repositories { - gradlePluginPortal() mavenCentral() - jcenter() + gradlePluginPortal() } } @@ -38,19 +37,16 @@ repositories { mavenCentral() google() gradlePluginPortal() - jcenter().mavenContent { - // Required for Dokka - includeModule("org.jetbrains.kotlinx", "kotlinx-html-jvm") - includeGroup("org.jetbrains.dokka") - includeModule("org.jetbrains", "markdown") - } } tasks.withType().configureEach { kotlinOptions { jvmTarget = "1.8" - @Suppress("SuspiciousCollectionReassignment") - freeCompilerArgs += listOf("-progressive") + // Because Gradle's Kotlin handling is stupid + apiVersion = "1.4" + languageVersion = "1.4" +// @Suppress("SuspiciousCollectionReassignment") +// freeCompilerArgs += listOf("-progressive") } } @@ -78,10 +74,6 @@ gradlePlugin { } } -kotlinDslPluginOptions { - experimentalWarning.set(false) -} - kotlin { explicitApi() } @@ -97,24 +89,18 @@ tasks.named("dokkaHtml") { packageListUrl.set(URL("https://developer.android.com/reference/tools/gradle-api/4.1/package-list")) url.set(URL("https://developer.android.com/reference/tools/gradle-api/4.1/classes")) } - - // Suppress Zipflinger copy - // TODO re-enable this with a proper regex -// perPackageOption { -// matchingRegex.set("com.slack.keeper.internal.zipflinger") -// suppress.set(true) -// } } } -val defaultAgpVersion = "4.0.0" +val defaultAgpVersion = "4.2.1" val agpVersion = findProperty("keeperTest.agpVersion")?.toString() ?: defaultAgpVersion // See https://github.com/slackhq/keeper/pull/11#issuecomment-579544375 for context val releaseMode = hasProperty("keeper.releaseMode") dependencies { - implementation("org.jetbrains.kotlin:kotlin-gradle-plugin-api:1.4.30") - implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:1.4.30") + implementation("org.jetbrains.kotlin:kotlin-gradle-plugin-api:1.5.0") + implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:1.5.0") + implementation("com.android:zipflinger:4.2.1") if (releaseMode) { compileOnly("com.android.tools.build:gradle:$defaultAgpVersion") @@ -122,11 +108,11 @@ dependencies { implementation("com.android.tools.build:gradle:$agpVersion") } - compileOnly("com.google.auto.service:auto-service-annotations:1.0-rc7") - kapt("com.google.auto.service:auto-service:1.0-rc7") + compileOnly("com.google.auto.service:auto-service-annotations:1.0") + kapt("com.google.auto.service:auto-service:1.0") testImplementation("com.squareup:javapoet:1.13.0") - testImplementation("com.squareup:kotlinpoet:1.7.2") + testImplementation("com.squareup:kotlinpoet:1.8.0") testImplementation("com.google.truth:truth:1.1.2") testImplementation("junit:junit:4.13.2") } diff --git a/keeper-gradle-plugin/gradle/wrapper/gradle-wrapper.properties b/keeper-gradle-plugin/gradle/wrapper/gradle-wrapper.properties index 8cf6eb5a..29e41345 100644 --- a/keeper-gradle-plugin/gradle/wrapper/gradle-wrapper.properties +++ b/keeper-gradle-plugin/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/keeper-gradle-plugin/settings.gradle.kts b/keeper-gradle-plugin/settings.gradle.kts index cdfc2076..0ce311dc 100644 --- a/keeper-gradle-plugin/settings.gradle.kts +++ b/keeper-gradle-plugin/settings.gradle.kts @@ -17,9 +17,8 @@ pluginManagement { repositories { google() - gradlePluginPortal() mavenCentral() - jcenter() + gradlePluginPortal() } } diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/InferAndroidTestKeepRules.kt b/keeper-gradle-plugin/src/main/java/com/slack/keeper/InferAndroidTestKeepRules.kt index 31a74648..888d6e91 100644 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/InferAndroidTestKeepRules.kt +++ b/keeper-gradle-plugin/src/main/java/com/slack/keeper/InferAndroidTestKeepRules.kt @@ -92,8 +92,10 @@ public abstract class InferAndroidTestKeepRules : JavaExec() { val inputJvmArgs = jvmArgsProperty.get() if (inputJvmArgs.isNotEmpty()) { logger.lifecycle( - "Starting infer exec with jvmArgs ${inputJvmArgs.joinToString(", ", prefix = "[", - postfix = "]")}. If debugging, attach the debugger now." + "Starting infer exec with jvmArgs ${ + inputJvmArgs.joinToString(", ", prefix = "[", + postfix = "]") + }. If debugging, attach the debugger now." ) jvmArgs = inputJvmArgs } @@ -109,28 +111,28 @@ public abstract class InferAndroidTestKeepRules : JavaExec() { } private fun genPrintUsesArgs(): List = - listOf( - "--keeprules", - androidJar.get().asFile.absolutePath, - appTargetJar.get().asFile.absolutePath, - androidTestSourceJar.get().asFile.absolutePath - ).also { - // print-uses is using its output to print rules - standardOutput = outputProguardRules.asFile.get().outputStream().buffered() - } + listOf( + "--keeprules", + androidJar.get().asFile.absolutePath, + appTargetJar.get().asFile.absolutePath, + androidTestSourceJar.get().asFile.absolutePath + ).also { + // print-uses is using its output to print rules + standardOutput = outputProguardRules.asFile.get().outputStream().buffered() + } private fun genTraceReferencesArgs(): List = - listOf( - "--keep-rules" to "", - "--lib" to androidJar.get().asFile.absolutePath, - "--lib" to androidTestJar.get().asFile.takeIf { it.exists() }?.absolutePath, - "--target" to appTargetJar.get().asFile.absolutePath, - "--source" to androidTestSourceJar.get().asFile.absolutePath, - "--output" to outputProguardRules.get().asFile.absolutePath - ).map { if (it.second != null) listOf(it.first, it.second) else listOf() } - .reduce { acc, any -> acc + any } - // Add user provided args coming from TraceReferences.arguments after generated ones. - .plus(traceReferencesArgs.getOrElse(listOf())) + listOf( + "--keep-rules" to "", + "--lib" to androidJar.get().asFile.absolutePath, + "--lib" to androidTestJar.get().asFile.takeIf { it.exists() }?.absolutePath, + "--target" to appTargetJar.get().asFile.absolutePath, + "--source" to androidTestSourceJar.get().asFile.absolutePath, + "--output" to outputProguardRules.get().asFile.absolutePath + ).map { if (it.second != null) listOf(it.first, it.second) else listOf() } + .reduce { acc, any -> acc + any } + // Add user provided args coming from TraceReferences.arguments after generated ones. + .plus(traceReferencesArgs.getOrElse(listOf())) public companion object { @Suppress("UNCHECKED_CAST", "UnstableApiUsage") @@ -173,8 +175,9 @@ public abstract class InferAndroidTestKeepRules : JavaExec() { this.traceReferencesArgs.set(traceReferencesArgs) outputProguardRules.set( project.layout.buildDirectory.file( - "${KeeperPlugin.INTERMEDIATES_DIR}/inferred${variantName.capitalize( - Locale.US)}KeepRules.pro")) + "${KeeperPlugin.INTERMEDIATES_DIR}/${ + variantName.capitalize(Locale.US) + }/inferredKeepRules.pro")) classpath(r8Configuration) mainClass.set(this.traceReferencesEnabled.map { enabled -> when (enabled) { diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/KeeperPlugin.kt b/keeper-gradle-plugin/src/main/java/com/slack/keeper/KeeperPlugin.kt index b7550ee0..2c52016a 100644 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/KeeperPlugin.kt +++ b/keeper-gradle-plugin/src/main/java/com/slack/keeper/KeeperPlugin.kt @@ -106,53 +106,69 @@ public class KeeperPlugin : Plugin { project.pluginManager.withPlugin("com.android.application") { val appExtension = project.extensions.getByType() val extension = project.extensions.create("keeper") - val diagnosticOutputDir = project.layout.buildDirectory.dir("$INTERMEDIATES_DIR/diagnostics") - project.configureKeepRulesGeneration(appExtension, extension, diagnosticOutputDir) - project.configureL8Rules(appExtension, extension, diagnosticOutputDir) + project.configureKeepRulesGeneration(appExtension, extension) + project.configureL8Rules(appExtension, extension) } } private fun Project.configureL8Rules( appExtension: AppExtension, - extension: KeeperExtension, - diagnosticOutputDir: Provider + extension: KeeperExtension ) { afterEvaluate { if (extension.enableL8RuleSharing.getOrElse(false)) { - val r8Enabled = !hasProperty("android.enableR8") || - property("android.enableR8")?.toString()?.toBoolean() != false - if (r8Enabled) { - appExtension.onApplicableVariants(project, extension) { testVariant, appVariant -> - val appR8Task = "minify${appVariant.name.capitalize(Locale.US)}WithR8" - val androidTestL8Task = "l8DexDesugarLib${testVariant.name.capitalize(Locale.US)}" - val inputFiles = tasks - .named(appR8Task) - .flatMap { it.projectOutputKeepRules } - - tasks - .named(androidTestL8Task) - .configure { - keepRulesFiles.from(inputFiles) - keepRulesConfigurations.set(listOf("-dontobfuscate")) - - if (extension.emitDebugInformation.getOrElse(false)) { - doFirst { - val mergedFilesContent = keepRulesFiles.files.asSequence() - .flatMap { it.walkTopDown() } - .filterNot { it.isDirectory } - .joinToString("\n") { - "# Source: ${it.absolutePath}\n${it.readText()}" + appExtension.onApplicableVariants(project, extension) { testVariant, appVariant -> + val appR8Task = "minify${appVariant.name.capitalize(Locale.US)}WithR8" + val androidTestL8Task = "l8DexDesugarLib${testVariant.name.capitalize(Locale.US)}" + val inputFiles = tasks + .named(appR8Task) + .flatMap { it.projectOutputKeepRules } + + tasks + .named(androidTestL8Task) + .configure { + val taskName = name + keepRulesFiles.from(inputFiles) + keepRulesConfigurations.set(listOf("-dontobfuscate")) + val diagnosticOutputDir = layout.buildDirectory.dir( + "$INTERMEDIATES_DIR/l8-diagnostics/$taskName") + .forUseAtConfigurationTime() + .get() + .asFile + + // We can't actually declare this because AGP's NonIncrementalTask will clear it + // during the task action +// outputs.dir(diagnosticOutputDir) +// .withPropertyName("diagnosticsDir") + + if (extension.emitDebugInformation.getOrElse(false)) { + doFirst { + val mergedFilesContent = keepRulesFiles.files.asSequence() + .flatMap { it.walkTopDown() } + .filterNot { it.isDirectory } + .joinToString("\n") { + "# Source: ${it.absolutePath}\n${it.readText()}" + } + + val configurations = keepRulesConfigurations.orNull.orEmpty() + .joinToString( + "\n", + prefix = "# Source: extra configurations\n" + ) + + + File(diagnosticOutputDir, "patchedL8Rules.pro") + .apply { + if (exists()) { + delete() } - val configurations = keepRulesConfigurations.orNull.orEmpty().joinToString("\n", prefix = "# Source: extra configurations\n") - diagnosticOutputDir.get().file("${testVariant.name}MergedL8Rules.pro") - .asFile - .writeText("$mergedFilesContent\n$configurations") - } + parentFile.mkdirs() + createNewFile() + } + .writeText("$mergedFilesContent\n$configurations") } } - } - } else { - error("enableL8RuleSharing only works if R8 is enabled!") + } } } } @@ -160,8 +176,7 @@ public class KeeperPlugin : Plugin { private fun Project.configureKeepRulesGeneration( appExtension: AppExtension, - extension: KeeperExtension, - diagnosticOutputDir: Provider + extension: KeeperExtension ) { // Set up r8 configuration val r8Configuration = configurations.create(CONFIGURATION_NAME) { @@ -180,10 +195,11 @@ public class KeeperPlugin : Plugin { } val androidJarRegularFileProvider = layout.file(provider { - resolveAndroidEmbeddedJar(appExtension, "android.jar", checkIfExisting = true) + resolveAndroidEmbeddedJar(appExtension, "android.jar", checkIfExisting = true) }) val androidTestJarRegularFileProvider = layout.file(provider { - resolveAndroidEmbeddedJar(appExtension, "optional/android.test.base.jar", checkIfExisting = false) + resolveAndroidEmbeddedJar(appExtension, "optional/android.test.base.jar", + checkIfExisting = false) }) appExtension.testVariants.configureEach { @@ -191,7 +207,7 @@ public class KeeperPlugin : Plugin { val extensionFilter = extension._variantFilter val ignoredVariant = extensionFilter?.let { logger.debug( - "$TAG Resolving ignored status for android variant ${appVariant.name}") + "$TAG Resolving ignored status for android variant ${appVariant.name}") val filter = VariantFilterImpl(appVariant) it.execute(filter) logger.debug("$TAG Variant '${appVariant.name}' ignored? ${filter._ignored}") @@ -219,11 +235,9 @@ public class KeeperPlugin : Plugin { appExtension.onApplicableVariants(project, extension) { testVariant, appVariant -> val intermediateAppJar = createIntermediateAppJar( appVariant = appVariant, - diagnosticOutputDir = diagnosticOutputDir, emitDebugInfo = extension.emitDebugInformation ) val intermediateAndroidTestJar = createIntermediateAndroidTestJar( - diagnosticOutputDir = diagnosticOutputDir, emitDebugInfo = extension.emitDebugInformation, testVariant = testVariant, appJarsProvider = intermediateAppJar.flatMap { it.appJarsFile } @@ -252,12 +266,12 @@ public class KeeperPlugin : Plugin { } private fun resolveAndroidEmbeddedJar( - appExtension: AppExtension, - path: String, - checkIfExisting: Boolean + appExtension: AppExtension, + path: String, + checkIfExisting: Boolean ): File { val compileSdkVersion = appExtension.compileSdkVersion - ?: error("No compileSdkVersion found") + ?: error("No compileSdkVersion found") val file = File("${appExtension.sdkDirectory}/platforms/${compileSdkVersion}/${path}") check(!checkIfExisting || file.exists()) { "No $path found! Expected to find it at: ${file.absolutePath}" @@ -266,9 +280,9 @@ public class KeeperPlugin : Plugin { } private fun AppExtension.onApplicableVariants( - project: Project, - extension: KeeperExtension, - body: (TestVariant, BaseVariant) -> Unit + project: Project, + extension: KeeperExtension, + body: (TestVariant, BaseVariant) -> Unit ) { testVariants.configureEach { val testVariant = this @@ -305,16 +319,7 @@ public class KeeperPlugin : Plugin { } private fun Project.applyGeneratedRules(appVariant: String, prop: Provider) { - // R8 is the default, so we'll only look to see if it's explicitly disabled - val r8Enabled = providers.gradleProperty("android.enableR8") - .forUseAtConfigurationTime() - .map { - @Suppress("PlatformExtensionReceiverOfInline") - it.toBoolean() - } - .getOrElse(true) - - val minifierTool = if (r8Enabled) "R8" else "Proguard" + val minifierTool = "R8" val targetName = interpolateTaskName(appVariant, minifierTool) @@ -332,7 +337,6 @@ public class KeeperPlugin : Plugin { * This output is used in the inferAndroidTestUsage task. */ private fun Project.createIntermediateAndroidTestJar( - diagnosticOutputDir: Provider, emitDebugInfo: Provider, testVariant: TestVariant, appJarsProvider: Provider @@ -341,7 +345,6 @@ public class KeeperPlugin : Plugin { "jar${testVariant.name.capitalize(Locale.US)}ClassesForKeeper") { group = KEEPER_TASK_GROUP this.emitDebugInfo.value(emitDebugInfo) - this.diagnosticsOutputDir.set(diagnosticOutputDir) this.appJarsFile.set(appJarsProvider) with(testVariant) { @@ -354,8 +357,12 @@ public class KeeperPlugin : Plugin { } } - archiveFile.set(layout.buildDirectory.dir(INTERMEDIATES_DIR).map { - it.file("${testVariant.name}.jar") + val outputDir = layout.buildDirectory.dir("$INTERMEDIATES_DIR/${testVariant.name}") + val diagnosticsDir = layout.buildDirectory.dir( + "$INTERMEDIATES_DIR/${testVariant.name}/diagnostics") + this.diagnosticsOutputDir.set(diagnosticsDir) + archiveFile.set(outputDir.map { + it.file("classes.jar") }) } } @@ -366,13 +373,11 @@ public class KeeperPlugin : Plugin { */ private fun Project.createIntermediateAppJar( appVariant: BaseVariant, - diagnosticOutputDir: Provider, emitDebugInfo: Provider ): TaskProvider { return tasks.register( "jar${appVariant.name.capitalize(Locale.US)}ClassesForKeeper") { group = KEEPER_TASK_GROUP - this.diagnosticsOutputDir.set(diagnosticOutputDir) this.emitDebugInfo.set(emitDebugInfo) with(appVariant) { from(layout.dir(javaCompileProvider.map { it.destinationDir })) @@ -385,9 +390,12 @@ public class KeeperPlugin : Plugin { } } - val intermediatesDir = layout.buildDirectory.dir(INTERMEDIATES_DIR) - archiveFile.set(intermediatesDir.map { it.file("${appVariant.name}.jar") }) - appJarsFile.set(intermediatesDir.map { it.file("${appVariant.name}Jars.txt") }) + val outputDir = layout.buildDirectory.dir("$INTERMEDIATES_DIR/${appVariant.name}") + val diagnosticsDir = layout.buildDirectory.dir( + "$INTERMEDIATES_DIR/${appVariant.name}/diagnostics") + diagnosticsOutputDir.set(diagnosticsDir) + archiveFile.set(outputDir.map { it.file("classes.jar") }) + appJarsFile.set(outputDir.map { it.file("jars.txt") }) } } } diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/VariantClasspathJar.kt b/keeper-gradle-plugin/src/main/java/com/slack/keeper/VariantClasspathJar.kt index 5cc3440c..31562907 100644 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/VariantClasspathJar.kt +++ b/keeper-gradle-plugin/src/main/java/com/slack/keeper/VariantClasspathJar.kt @@ -18,8 +18,8 @@ package com.slack.keeper -import com.slack.keeper.internal.zipflinger.BytesSource -import com.slack.keeper.internal.zipflinger.ZipArchive +import com.android.zipflinger.BytesSource +import com.android.zipflinger.ZipArchive import org.gradle.api.DefaultTask import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.DirectoryProperty @@ -112,7 +112,7 @@ public abstract class VariantClasspathJar : BaseKeeperJarTask() { appJarsFile.get().asFile.writeText(appJars.sorted().joinToString("\n")) - diagnostic("${archiveFile.get().asFile.nameWithoutExtension}Classes") { + diagnostic("classes") { appClasses.sorted() .joinToString("\n") } @@ -156,7 +156,7 @@ public abstract class AndroidTestVariantClasspathJar : BaseKeeperJarTask() { val appJars = appJarsFile.get().asFile.useLines { it.toSet() } val androidTestClasspath = androidTestArtifactFiles.files - diagnostic("${archiveFile.get().asFile.nameWithoutExtension}Jars") { + diagnostic("jars") { androidTestClasspath.sortedBy { it.canonicalPath } .joinToString("\n") { it.canonicalPath @@ -167,7 +167,7 @@ public abstract class AndroidTestVariantClasspathJar : BaseKeeperJarTask() { removeAll { it.canonicalPath in appJars } } - diagnostic("${archiveFile.get().asFile.nameWithoutExtension}DistinctJars") { + diagnostic("distinctJars") { distinctAndroidTestClasspath.sortedBy { it.canonicalPath } .joinToString("\n") { it.canonicalPath @@ -195,7 +195,7 @@ public abstract class AndroidTestVariantClasspathJar : BaseKeeperJarTask() { } } - diagnostic("${archiveFile.get().asFile.nameWithoutExtension}Classes") { + diagnostic("androidTestClasses") { androidTestClasses.sorted().joinToString("\n") } @@ -208,7 +208,7 @@ public abstract class AndroidTestVariantClasspathJar : BaseKeeperJarTask() { .filterTo(LinkedHashSet()) { it in androidTestClasses } if (duplicateClasses.isNotEmpty()) { - val output = diagnostic("${archiveFile.get().asFile.nameWithoutExtension}DuplicateClasses") { + val output = diagnostic("duplicateClasses") { duplicateClasses.sorted().joinToString("\n") } logger.warn("Duplicate classes found in androidTest APK and app APK! This" + diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/ZipFlingerExt.kt b/keeper-gradle-plugin/src/main/java/com/slack/keeper/ZipFlingerExt.kt index a5d6e5c0..2db7a829 100644 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/ZipFlingerExt.kt +++ b/keeper-gradle-plugin/src/main/java/com/slack/keeper/ZipFlingerExt.kt @@ -1,8 +1,9 @@ package com.slack.keeper -import com.slack.keeper.internal.zipflinger.ZipArchive -import com.slack.keeper.internal.zipflinger.ZipSource +import com.android.zipflinger.ZipArchive +import com.android.zipflinger.ZipSource import java.io.File +import java.nio.file.Path /** * Returns a sequence of pairs representing the class files and their relative names for use in a @@ -20,7 +21,7 @@ internal fun File.classesSequence(): Sequence> { * Extracts classes from the target [jar] into this archive. */ internal fun ZipArchive.extractClassesFrom(jar: File, callback: (String) -> Unit) { - val jarSource = ZipSource(jar) + val jarSource = newZipSource(jar) jarSource.entries() .filterNot { "META-INF" in it.key } .forEach { (name, entry) -> @@ -32,4 +33,18 @@ internal fun ZipArchive.extractClassesFrom(jar: File, callback: (String) -> Unit } } add(jarSource) +} + +private fun newZipSource(jar: File): ZipSource { + return try { + // AGP 4.1/4.2 + ZipSource::class.java + .getDeclaredConstructor(File::class.java) + .newInstance(jar) + } catch (e: NoSuchMethodException) { + // AGP/ZipFlinger 7+ + ZipSource::class.java + .getDeclaredConstructor(Path::class.java) + .newInstance(jar.toPath()) + } } \ No newline at end of file diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Archive.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Archive.java deleted file mode 100644 index a2f0c017..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Archive.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.Closeable; -import java.io.IOException; - -public interface Archive extends Closeable { - - /** - * Add a source to the archive. - * - * @param source The source to add to this zip archive. - * @throws IllegalStateException if the entry name already exists in the archive. - * @throws IOException if writing to the zip archive fails. - */ - void add(@NonNull BytesSource source) throws IOException; - - /** - * Add a set of selected entries from an other zip archive. - * - * @param sources A zip archive with selected entries to add to this zip archive. - * @throws IllegalStateException if the entry name already exists in the archive. - * @throws IOException if writing to the zip archive fails. - */ - void add(@NonNull ZipSource sources) throws IOException; - - /** - * Delete an entry from this archive. If the entry did not exist, this method does nothing. To - * avoid creating "holes" in the archive, it is mendatory to delete all entries first and add - * sources second. - * - * @param name The name of the entry to delete. - * @throws IllegalStateException if entries have been added. - */ - void delete(@NonNull String name) throws IOException; - - @Override - void close() throws IOException; -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/BytesSource.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/BytesSource.java deleted file mode 100644 index d2ac4f06..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/BytesSource.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.util.zip.Deflater; - -public class BytesSource extends Source { - - // Bytes to be written in the zip, after the Local File Header. - private ByteBuffer zipEntryPayload; - - // Bytes as there were provided to this class (before compression if any). - private ByteBuffer byteBuffer; - - /** - * @param bytes - * @param name - * @param compressionLevel One of java.util.zip.Deflater compression level. - */ - public BytesSource(@NonNull byte[] bytes, @NonNull String name, int compressionLevel) - throws IOException { - super(name); - build(bytes, bytes.length, compressionLevel); - } - - public BytesSource(@NonNull File file, @NonNull String name, int compressionLevel) - throws IOException { - super(name); - byte[] bytes = Files.readAllBytes(file.toPath()); - build(bytes, bytes.length, compressionLevel); - } - - /** - * @param stream BytesSource takes ownership of the InputStream and will close it after draining - * it. - * @param name - * @param compressionLevel - * @throws IOException - */ - public BytesSource(@NonNull InputStream stream, @NonNull String name, int compressionLevel) - throws IOException { - super(name); - try (NoCopyByteArrayOutputStream ncbos = new NoCopyByteArrayOutputStream(16000)) { - byte[] tmpBuffer = new byte[16000]; - int bytesRead; - while ((bytesRead = stream.read(tmpBuffer)) != -1) { - ncbos.write(tmpBuffer, 0, bytesRead); - } - stream.close(); - build(ncbos.buf(), ncbos.getCount(), compressionLevel); - } - } - - private void build(byte[] bytes, int size, int compressionLevel) throws IOException { - byteBuffer = ByteBuffer.wrap(bytes, 0, size); - crc = com.slack.keeper.internal.zipflinger.Crc32.crc32(bytes, 0, size); - uncompressedSize = size; - if (compressionLevel == Deflater.NO_COMPRESSION) { - zipEntryPayload = ByteBuffer.wrap(bytes, 0, size); - compressedSize = uncompressedSize; - compressionFlag = com.slack.keeper.internal.zipflinger.LocalFileHeader.COMPRESSION_NONE; - } else { - zipEntryPayload = Compressor.deflate(bytes, 0, size, compressionLevel); - compressedSize = zipEntryPayload.limit(); - compressionFlag = com.slack.keeper.internal.zipflinger.LocalFileHeader.COMPRESSION_DEFLATE; - } - } - - @NonNull - public ByteBuffer getBuffer() { - return byteBuffer; - } - - @Override - void prepare() {} - - @Override - long writeTo(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException { - return writer.write(zipEntryPayload); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/CentralDirectory.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/CentralDirectory.java deleted file mode 100644 index 15bf94ba..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/CentralDirectory.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import com.android.annotations.Nullable; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -class CentralDirectory { - - // The Central Directory as it was read when an archive already existed. - private final ByteBuffer buf; - - private final List deletedLocations = new ArrayList<>(); - private final Map entries; - private final Map addedEntries = new HashMap<>(); - - CentralDirectory(@NonNull ByteBuffer buf, @NonNull Map entries) { - this.buf = buf; - this.entries = entries; - } - - @NonNull - Location delete(@NonNull String name) { - if (entries.containsKey(name)) { - Entry entry = entries.get(name); - deletedLocations.add(entry.getCdLocation()); - entries.remove(name); - return entry.getLocation(); - } - if (addedEntries.containsKey(name)) { - com.slack.keeper.internal.zipflinger.CentralDirectoryRecord record = addedEntries.remove(name); - return record.getLocation(); - } - return Location.INVALID; - } - - long getNumEntries() { - return entries.size() + addedEntries.size(); - } - - void write(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException { - // Four steps operations (first write old entries then new entries): - // 1/ Sort deleted entries by location. - // 2/ Create a list of "clean" (not deleted) locations. - // 3/ Write all old (non-deleted) locations. - // 4/ Write all new entries. - - // Step 1 - Collections.sort(deletedLocations); - - // Step 2 (Build list of non-deleted locations). - List cleanCDLocations = new ArrayList<>(); - long remainingStart = 0; - long remainingSize = buf.capacity(); - - for (Location deletedLocation : deletedLocations) { - Location cleanLoc = - new Location(remainingStart, deletedLocation.first - remainingStart); - - // If cleanLoc is the left end of the remaining CD, cleanLoc size is 0. - if (cleanLoc.size() > 0) { - cleanCDLocations.add(cleanLoc); - } - remainingStart = deletedLocation.last + 1; - remainingSize -= (deletedLocation.size() + cleanLoc.size()); - } - // Add the remaining of the CD as a clear location - if (remainingSize > 0) { - cleanCDLocations.add(new Location(remainingStart, remainingSize)); - } - - // Step 3: write clean CD chunks - for (Location toWrite : cleanCDLocations) { - buf.limit(Math.toIntExact(toWrite.first + toWrite.size())); - buf.position(Math.toIntExact(toWrite.first)); - ByteBuffer view = buf.slice(); - writer.write(view); - } - - // Step 4: write new entries - - // Assess how much data the CD requires - long totalSize = 0; - for (com.slack.keeper.internal.zipflinger.CentralDirectoryRecord record : addedEntries.values()) { - totalSize += record.getSize(); - } - // Generate the CD portion of new entries - ByteBuffer cdBuffer = - ByteBuffer.allocate(Math.toIntExact(totalSize)).order(ByteOrder.LITTLE_ENDIAN); - for (com.slack.keeper.internal.zipflinger.CentralDirectoryRecord record : addedEntries.values()) { - record.write(cdBuffer); - } - - // Write new entries - cdBuffer.rewind(); - writer.write(cdBuffer); - } - - void add(@NonNull String name, @NonNull com.slack.keeper.internal.zipflinger.CentralDirectoryRecord record) { - addedEntries.put(name, record); - } - - boolean contains(@NonNull String name) { - return entries.containsKey(name) || addedEntries.containsKey(name); - } - - @NonNull - List listEntries() { - List list = new ArrayList<>(); - list.addAll(entries.keySet()); - list.addAll(addedEntries.keySet()); - return list; - } - - @Nullable - public ExtractionInfo getExtractionInfo(@NonNull String name) { - Entry entry = entries.get(name); - if (entry != null) { - return new ExtractionInfo(entry.getPayloadLocation(), entry.isCompressed()); - } - - com.slack.keeper.internal.zipflinger.CentralDirectoryRecord cd = addedEntries.get(name); - if (cd != null) { - boolean isCompressed = cd.getCompressionFlag() != com.slack.keeper.internal.zipflinger.LocalFileHeader.COMPRESSION_NONE; - return new ExtractionInfo(cd.getPayloadLocation(), isCompressed); - } - - return null; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/CentralDirectoryRecord.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/CentralDirectoryRecord.java deleted file mode 100644 index bae9037a..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/CentralDirectoryRecord.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -class CentralDirectoryRecord { - - public static final int SIGNATURE = 0x02014b50; - public static final int SIZE = 46; - public static final int DATA_DESCRIPTOR_FLAG = 0x0008; - public static final int DATA_DESCRIPTOR_SIGNATURE = 0x08074b50; - - // JDK 9 consider time&data field with value 0 as invalid. Use 1 instead. - // These are in MS-DOS 16-bit format. For actual specs, see: - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724247(v=vs.85).aspx - public static final short DEFAULT_TIME = 1 | 1 << 5 | 1 << 11; - public static final short DEFAULT_DATE = 1 | 1 << 5 | 1 << 9; - - // Zip64 extra format: - // uint16_t id (0x0001) - // uint16_t size payload (0x18) - // Payload: - // - uint64_t Uncompressed size. - // - uint64_t Compressed size. - // - uint64_t offset to LFH in archive. - private static final int ZIP64_PAYLOAD_SIZE = Long.BYTES * 3; - private static final int ZIP64_EXTRA_SIZE = Short.BYTES * 2 + ZIP64_PAYLOAD_SIZE; - - private final byte[] nameBytes; - private final int crc; - private final long compressedSize; - private final long uncompressedSize; - // Location of the Local file header to end of payload in file space. - private final Location location; - private final short compressionFlag; - private final Location payloadLocation; - private final boolean isZip64; - - CentralDirectoryRecord( - @NonNull byte[] nameBytes, - int crc, - long compressedSize, - long uncompressedSize, - Location location, - short compressionFlag, - Location payloadLocation) { - this.nameBytes = nameBytes; - this.crc = crc; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - this.location = location; - this.compressionFlag = compressionFlag; - this.payloadLocation = payloadLocation; - this.isZip64 = - compressedSize > Zip64.LONG_MAGIC - || uncompressedSize > Zip64.LONG_MAGIC - || location.first > Zip64.LONG_MAGIC; - } - - void write(@NonNull ByteBuffer buf) { - short versionNeeded = isZip64 ? Zip64.VERSION_NEEDED : 0; - int size = isZip64 ? Zip64.INT_MAGIC : com.slack.keeper.internal.zipflinger.Ints.longToUint(uncompressedSize); - int csize = isZip64 ? Zip64.INT_MAGIC : com.slack.keeper.internal.zipflinger.Ints.longToUint(compressedSize); - int offset = isZip64 ? Zip64.INT_MAGIC : com.slack.keeper.internal.zipflinger.Ints.longToUint(location.first); - - ByteBuffer extra = buildExtraField(); - - buf.putInt(SIGNATURE); - buf.putShort((short) 0); // version made by - buf.putShort(versionNeeded); - buf.putShort((short) 0); // flag - buf.putShort(compressionFlag); - buf.putShort(DEFAULT_TIME); - buf.putShort(DEFAULT_DATE); - buf.putInt(crc); - buf.putInt(csize); // compressed size - buf.putInt(size); // size - buf.putShort(com.slack.keeper.internal.zipflinger.Ints.intToUshort(nameBytes.length)); - buf.putShort(com.slack.keeper.internal.zipflinger.Ints.intToUshort(extra.capacity())); - buf.putShort((short) 0); // comment size - buf.putShort((short) 0); // disk # start - buf.putShort((short) 0); // internal att - buf.putInt(0); // external att - buf.putInt(offset); - buf.put(nameBytes); - buf.put(extra); - } - - short getCompressionFlag() { - return compressionFlag; - } - - long getSize() { - long extraSize = isZip64 ? ZIP64_EXTRA_SIZE : 0; - return SIZE + nameBytes.length + extraSize; - } - - @NonNull - Location getPayloadLocation() { - return payloadLocation; - } - - @NonNull - Location getLocation() { - return location; - } - - @NonNull - private ByteBuffer buildExtraField() { - if (!isZip64) { - return ByteBuffer.allocate(0); - } - ByteBuffer buf = ByteBuffer.allocate(ZIP64_EXTRA_SIZE).order(ByteOrder.LITTLE_ENDIAN); - buf.putShort(Zip64.EXTRA_ID); - buf.putShort(com.slack.keeper.internal.zipflinger.Ints.intToUshort(ZIP64_PAYLOAD_SIZE)); - buf.putLong(uncompressedSize); - buf.putLong(compressedSize); - buf.putLong(location.first); - buf.rewind(); - return buf; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Compressor.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Compressor.java deleted file mode 100644 index dcbc7680..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Compressor.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.zip.Deflater; -import java.util.zip.DeflaterOutputStream; -import java.util.zip.Inflater; -import java.util.zip.InflaterOutputStream; - -public class Compressor { - - @NonNull - public static ByteBuffer deflate( - @NonNull byte[] bytes, int offset, int size, int compressionLevel) throws IOException { - NoCopyByteArrayOutputStream out = new NoCopyByteArrayOutputStream(size); - - Deflater deflater = new Deflater(compressionLevel, true); - - try (DeflaterOutputStream dout = new DeflaterOutputStream(out, deflater)) { - dout.write(bytes, offset, size); - dout.flush(); - } - - return out.getByteBuffer(); - } - - @NonNull - public static ByteBuffer deflate(@NonNull byte[] bytes, int compressionLevel) - throws IOException { - return deflate(bytes, 0, bytes.length, compressionLevel); - } - - @NonNull - public static ByteBuffer inflate(@NonNull byte[] bytes) throws IOException { - NoCopyByteArrayOutputStream out = new NoCopyByteArrayOutputStream(bytes.length); - Inflater inflater = new Inflater(true); - - try (InflaterOutputStream dout = new InflaterOutputStream(out, inflater)) { - dout.write(bytes); - dout.flush(); - } - - return out.getByteBuffer(); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Crc32.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Crc32.java deleted file mode 100644 index bedc4e62..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Crc32.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.util.zip.CRC32; - -class Crc32 { - - public static int crc32(@NonNull byte[] bytes) { - return crc32(bytes, 0, bytes.length); - } - - public static int crc32(@NonNull byte[] bytes, int offset, int size) { - CRC32 crc = new CRC32(); - crc.update(bytes, offset, size); - return com.slack.keeper.internal.zipflinger.Ints.longToUint(crc.getValue()); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/EndOfCentralDirectory.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/EndOfCentralDirectory.java deleted file mode 100644 index af1e2612..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/EndOfCentralDirectory.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.FileChannel; - -class EndOfCentralDirectory { - private static final int SIGNATURE = 0x06054b50; - static final int SIZE = 22; - private static final long MAX_SIZE = com.slack.keeper.internal.zipflinger.Ints.USHRT_MAX + SIZE; - static final short DISK_NUMBER = 0; - - private int numEntries; - private Location location; - private Location cdLocation; - - private EndOfCentralDirectory() { - this.numEntries = 0; - this.location = Location.INVALID; - this.cdLocation = Location.INVALID; - } - - private void parse(@NonNull ByteBuffer buffer) { - // skip diskNumber (2) + cdDiskNumber (2) + #entries (2) - buffer.position(buffer.position() + 6); - numEntries = com.slack.keeper.internal.zipflinger.Ints.ushortToInt(buffer.getShort()); - long cdSize = com.slack.keeper.internal.zipflinger.Ints.uintToLong(buffer.getInt()); - long cdOffset = com.slack.keeper.internal.zipflinger.Ints.uintToLong(buffer.getInt()); - cdLocation = new Location(cdOffset, cdSize); - buffer.position(buffer.position() + 2); // Skip comment length - } - - @NonNull - public Location getLocation() { - return location; - } - - @NonNull - public Location getCdLocation() { - return cdLocation; - } - - public int numEntries() { - return numEntries; - } - - public void setLocation(@NonNull Location location) { - this.location = location; - } - - // Search the EOCD. If not found the returned object location will be set to Location.INVALID. - @NonNull - public static EndOfCentralDirectory find(@NonNull FileChannel channel) throws IOException { - long fileSize = channel.size(); - - EndOfCentralDirectory eocd = new EndOfCentralDirectory(); - if (fileSize < SIZE) { - return eocd; - } - - int sizeToRead = Math.toIntExact(Math.min(fileSize, MAX_SIZE)); - long offset = fileSize - sizeToRead; - - ByteBuffer buffer = ByteBuffer.allocate(sizeToRead).order(ByteOrder.LITTLE_ENDIAN); - channel.read(buffer, offset); - buffer.position(buffer.capacity() - SIZE); - while (true) { - int signature = buffer.getInt(); // Read 4 bytes. - if (signature == EndOfCentralDirectory.SIGNATURE) { - eocd.parse(buffer); - eocd.setLocation(new Location(offset + buffer.position() - SIZE, SIZE)); - break; - } - if (buffer.position() <= 4) { - break; - } - buffer.position(buffer.position() - Integer.BYTES - 1); // Backtrack 5 bytes. - } - return eocd; - } - - @NonNull - public static Location write( - @NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer, @NonNull Location cdLocation, long entriesCount) - throws IOException { - boolean isZip64 = Zip64.needZip64Footer(entriesCount, cdLocation); - - short numEntries = isZip64 ? Zip64.SHORT_MAGIC : com.slack.keeper.internal.zipflinger.Ints.longToUshort(entriesCount); - int eocdSize = isZip64 ? Zip64.INT_MAGIC : com.slack.keeper.internal.zipflinger.Ints.longToUint(cdLocation.size()); - int eocdOffset = isZip64 ? Zip64.INT_MAGIC : com.slack.keeper.internal.zipflinger.Ints.longToUint(cdLocation.first); - - ByteBuffer eocd = ByteBuffer.allocate(SIZE).order(ByteOrder.LITTLE_ENDIAN); - eocd.putInt(SIGNATURE); - eocd.putShort(DISK_NUMBER); - eocd.putShort((short) 0); // cd disk number - eocd.putShort(numEntries); - eocd.putShort(numEntries); - eocd.putInt(eocdSize); - eocd.putInt(eocdOffset); - eocd.putShort((short) 0); // comment size - - eocd.rewind(); - long position = writer.position(); - writer.write(eocd); - - return new Location(position, SIZE); - } - -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Entry.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Entry.java deleted file mode 100644 index 91b38b64..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Entry.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.nio.charset.StandardCharsets; - -public class Entry { - - // The location (in file space) of the zip entry which includes the LFH, payload and - // Data descriptor. - private Location location = Location.INVALID; - - // The location (in CD space) of this entry in the CD. - private Location cdLocation = Location.INVALID; - - // The location (in file space) of the zip entry payload (the actual file data). - private Location payloadLocation = Location.INVALID; - - private String name = ""; - private int crc; - private long compressedSize; - private long uncompressedSize; - private short compressionFlag; - - Entry() {} - - public short getCompressionFlag() { - return compressionFlag; - } - - public long getCompressedSize() { - return compressedSize; - } - - public long getUncompressedSize() { - return uncompressedSize; - } - - public String getName() { - return name; - } - - public int getCrc() { - return crc; - } - - public boolean isDirectory() { - return name.charAt(name.length() - 1) == '/'; - } - - public boolean isCompressed() { - return compressionFlag != com.slack.keeper.internal.zipflinger.LocalFileHeader.COMPRESSION_NONE; - } - - @NonNull - Location getCdLocation() { - return cdLocation; - } - - @NonNull - Location getLocation() { - return location; - } - - @NonNull - Location getPayloadLocation() { - return payloadLocation; - } - - void setCdLocation(@NonNull Location cdLocation) { - this.cdLocation = cdLocation; - } - - void setNameBytes(@NonNull byte[] nameBytes) { - this.name = new String(nameBytes, StandardCharsets.UTF_8); - } - - void setCrc(int crc) { - this.crc = crc; - } - - void setPayloadLocation(@NonNull Location payloadLocation) { - this.payloadLocation = payloadLocation; - } - - void setCompressionFlag(short compressionFlag) { - this.compressionFlag = compressionFlag; - } - - void setCompressedSize(long compressedSize) { - this.compressedSize = compressedSize; - } - - void setUncompressedSize(long ucompressedSize) { - this.uncompressedSize = ucompressedSize; - } - - void setLocation(@NonNull Location location) { - this.location = location; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ExtractionInfo.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ExtractionInfo.java deleted file mode 100644 index 73861187..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ExtractionInfo.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; - -public class ExtractionInfo { - private final Location location; - private final boolean isCompressed; - - public ExtractionInfo(@NonNull Location location, boolean isCompressed) { - this.location = location; - this.isCompressed = isCompressed; - } - - @NonNull - public Location getLocation() { - return location; - } - - public boolean isCompressed() { - return isCompressed; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/FreeStore.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/FreeStore.java deleted file mode 100644 index a995d434..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/FreeStore.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -// This works like a memory allocator except it deals with file address space instead of -// memory address space. -class FreeStore { - - static final long DEFAULT_ALIGNMENT = 4; - static final long PAGE_ALIGNMENT = 4096; - - private Zone head; - - // A zone tracks the free file address space. Two consecutive zones are never contiguous which - // mean that upon modification, if two zone "touch" each others, they are merged together into - // a bigger free zone. - // - // Used space is not tracked but inferred from each gap between free zones. - protected class Zone { - public Zone next; - public Zone prev; - public Location loc; - - public Zone() { - this.next = null; - this.prev = null; - } - - public void shrinkBy(long amount) { - assert loc.size() > amount; - loc = new Location(loc.first + amount, loc.size() - amount); - - // If the zone is empty, remove it from the list. - if (loc.size() == 0) { - prev.next = next; - if (next != null) { - next.prev = prev; - } - } - } - } - - FreeStore(@NonNull Map zipEntries) { - // Create an immutable marker of unusable space which make "insert on head" ugly code go away. - head = new Zone(); - head.loc = new Location(-1, 1); - - // Use zip entries location (used space) to build the free zones list. - List usedLocations = new ArrayList<>(); - for (Entry entry : zipEntries.values()) { - usedLocations.add(entry.getLocation()); - } - Collections.sort(usedLocations); - - Zone prevFreeZone = head; - Location prevUsedLoc = prevFreeZone.loc; - for (Location usedLoc : usedLocations) { - // If there is a gap, mark is as FREE space. - long gap = usedLoc.first - prevUsedLoc.last - 1; - if (gap > 0) { - Zone free = new Zone(); - prevFreeZone.next = free; - free.prev = prevFreeZone; - free.loc = new Location(prevUsedLoc.last + 1, gap); - prevFreeZone = free; - } - prevUsedLoc = usedLoc; - } - - // Mark everything remaining as a free zone. - Zone remainingZone = new Zone(); - remainingZone.prev = prevFreeZone; - remainingZone.next = null; - prevFreeZone.next = remainingZone; - remainingZone.loc = - new Location(prevUsedLoc.last + 1, Long.MAX_VALUE - 1 - prevUsedLoc.last); - } - - // Performs unaligned allocation. - @NonNull - Location ualloc(long requestedSize) { - Zone cursor; - for (cursor = head.next; cursor != null; cursor = cursor.next) { - // We are searching for a block big enough to contain: - // - The requested size - // - Post-padding space for potentially needed virtual entry to fill holes. - if (cursor.loc.size() >= requestedSize + com.slack.keeper.internal.zipflinger.LocalFileHeader.VIRTUAL_HEADER_SIZE) { - break; - } - } - - if (cursor == null) { - throw new IllegalStateException("Out of file address space."); - } - - Location allocated = new Location(cursor.loc.first, requestedSize); - cursor.shrinkBy(requestedSize); - return allocated; - } - - // Performs aligned allocation. The offset is necessary because what needs to be aligned is not - // the first byte in the allocation but the first byte in the zip entry payload. - // This method may return more than requested. If it does the extra space is padding that must - // be consumed by an "extra" field. - @NonNull - Location alloc(long requestedSize, long payloadOffset, long alignment) { - Zone cursor; - for (cursor = head.next; cursor != null; cursor = cursor.next) { - long padding = padFor(cursor.loc.first, payloadOffset, alignment); - // We are searching for a block big enough to contain: - // - The requested size - // - Pre-padding space for extra field ALIGNMENT - // - Post-padding space for potentially needed virtual entry to fill holes. - if (cursor.loc.size() - >= requestedSize + padding + com.slack.keeper.internal.zipflinger.LocalFileHeader.VIRTUAL_HEADER_SIZE) { - requestedSize += padding; - break; - } - } - - if (cursor == null) { - throw new IllegalStateException("Out of file address space."); - } - - Location allocated = new Location(cursor.loc.first, requestedSize); - cursor.shrinkBy(requestedSize); - return allocated; - } - - // Mark an area of the file available for allocation. This will merge up to two zones into one - // if they touch each others. - void free(@NonNull Location loc) { - Zone cursor = head.next; - while (cursor != null) { - if (loc.first > cursor.prev.loc.last && loc.last < cursor.loc.first) { - break; - } - cursor = cursor.next; - } - - if (cursor == null) { - throw new IllegalStateException("Double free"); - } - - // Insert a free zone - Zone newFreeZone = new Zone(); - newFreeZone.loc = loc; - newFreeZone.prev = cursor.prev; - newFreeZone.next = cursor; - cursor.prev.next = newFreeZone; - cursor.prev = newFreeZone; - cursor = newFreeZone; - - // If previous zone is contiguous, merge this zone into previous. - if (cursor.prev.loc.last + 1 == cursor.loc.first && cursor.prev != head) { - Zone prev = cursor.prev; - prev.next = cursor.next; - cursor.next.prev = prev; - prev.loc = new Location(prev.loc.first, prev.loc.size() + cursor.loc.size()); - cursor = prev; - } - - // If next zone is contiguous, merge this zone into next. - if (cursor.next != null && cursor.next.loc.first - 1 == cursor.loc.last) { - Zone next = cursor.next; - next.prev = cursor.prev; - cursor.prev.next = next; - next.loc = new Location(cursor.loc.first, cursor.loc.size() + next.loc.size()); - } - } - - @NonNull - Location getLastFreeLocation() { - Zone zone = head.next; - while (zone.next != null) { - zone = zone.next; - } - return zone.loc; - } - - @NonNull - List getFreeLocations() { - List locs = new ArrayList<>(); - Zone cursor = head.next; - while (cursor != null) { - locs.add(cursor.loc); - cursor = cursor.next; - } - return locs; - } - - // How much padding is needed if this address+offset is not aligned (a.k.a: An extra field will - // have to be created in order to fill this space). - static long padFor(long address, long offset, long alignment) { - long pointer = address + offset; - if ((pointer % alignment) == 0) { - return 0; - } else { - return alignment - (pointer % alignment); - } - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Ints.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Ints.java deleted file mode 100644 index 192b3530..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Ints.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -class Ints { - public static final long USHRT_MAX = 65_535L; - public static final long UINT_MAX = 0xFF_FF_FF_FFL; - - static long uintToLong(int i) { - return i & 0xFF_FF_FF_FFL; - } - - static int ushortToInt(short i) { - return i & 0xFF_FF; - } - - static int longToUint(long i) { - if ((i & 0xFF_FF_FF_FF_00_00_00_00L) != 0) { - throw new IllegalStateException("Long cannot fit in uint"); - } - return (int) i; - } - - static short intToUshort(int i) { - if ((i & 0xFF_FF_00_00) != 0) { - throw new IllegalStateException("Int cannot fit in ushort"); - } - return (short) i; - } - - static short longToUshort(long i) { - if ((i & 0xFF_FF_FF_FF_FF_FF_00_00L) != 0) { - throw new IllegalStateException("long cannot fit in ushort"); - } - return (short) i; - } - - public static long ulongToLong(long i) { - if ((i & 0x80_00_00_00_00_00_00_00L) != 0) { - throw new IllegalStateException("ulong cannot fit in long"); - } - return i; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/LocalFileHeader.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/LocalFileHeader.java deleted file mode 100644 index a1f65112..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/LocalFileHeader.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -class LocalFileHeader { - private static final int SIGNATURE = 0x04034b50; - - public static final int LOCAL_FILE_HEADER_SIZE = 30; - - // Minimum number of bytes needed to create a virtual zip entry (an entry not present in - // the Central Directory with name length = 0 and an extra field containing padding data). - public static final long VIRTUAL_HEADER_SIZE = LOCAL_FILE_HEADER_SIZE; - - public static final short COMPRESSION_NONE = 0; - public static final short COMPRESSION_DEFLATE = 8; - - static final long VIRTUAL_ENTRY_MAX_SIZE = LOCAL_FILE_HEADER_SIZE + com.slack.keeper.internal.zipflinger.Ints.USHRT_MAX; - static final long OFFSET_TO_NAME = 26; - - // Zip64 extra payload must only include uncompressed size and compressed size. It differs - // from the Central Directory Record which also features an uint64_t offset to the LFH. - private static final int ZIP64_PAYLOAD_SIZE = Long.BYTES * 2; - private static final int ZIP64_EXTRA_SIZE = Short.BYTES * 2 + ZIP64_PAYLOAD_SIZE; - - private final byte[] nameBytes; - private final short compressionFlag; - private final int crc; - private final long compressedSize; - private final long uncompressedSize; - private final boolean isZip64; - private int padding; - - LocalFileHeader(Source source) { - this.nameBytes = source.getNameBytes(); - this.compressionFlag = source.getCompressionFlag(); - this.crc = source.getCrc(); - this.compressedSize = source.getCompressedSize(); - this.uncompressedSize = source.getUncompressedSize(); - this.isZip64 = compressedSize > Zip64.LONG_MAGIC || uncompressedSize > Zip64.LONG_MAGIC; - this.padding = 0; - } - - public static void fillVirtualEntry(@NonNull ByteBuffer virtualEntry) { - int sizeToFill = virtualEntry.capacity(); - if (sizeToFill < VIRTUAL_HEADER_SIZE) { - String message = String.format("Not enough space for virtual entry (%d)", sizeToFill); - throw new IllegalStateException(message); - } - virtualEntry.order(ByteOrder.LITTLE_ENDIAN); - virtualEntry.putInt(SIGNATURE); - virtualEntry.putShort((short) 0); // Version needed - virtualEntry.putShort((short) 0); // general purpose flag - virtualEntry.putShort(COMPRESSION_NONE); - virtualEntry.putShort(com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.DEFAULT_TIME); - virtualEntry.putShort(com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.DEFAULT_DATE); - virtualEntry.putInt(0); // CRC-32 - virtualEntry.putInt(0); // compressed size - virtualEntry.putInt(0); // uncompressed size - virtualEntry.putShort((short) 0); // file name length - // -2 for the extra length ushort we have to write - virtualEntry.putShort(com.slack.keeper.internal.zipflinger.Ints.intToUshort(virtualEntry.remaining() - 2)); // extra length - virtualEntry.rewind(); - } - - public void setPadding(int padding) { - if (padding > com.slack.keeper.internal.zipflinger.Ints.USHRT_MAX) { - String err = String.format("Padding cannot be more than %s bytes", - com.slack.keeper.internal.zipflinger.Ints.USHRT_MAX); - throw new IllegalStateException(err); - } - this.padding = padding; - } - - public void write(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException { - ByteBuffer extraField = buildExtraField(); - int bytesNeeded = LOCAL_FILE_HEADER_SIZE + nameBytes.length + extraField.capacity(); - - short versionNeeded = isZip64 ? Zip64.VERSION_NEEDED : 0; - int size = isZip64 ? Zip64.INT_MAGIC : com.slack.keeper.internal.zipflinger.Ints.longToUint(uncompressedSize); - int csize = isZip64 ? Zip64.INT_MAGIC : com.slack.keeper.internal.zipflinger.Ints.longToUint(compressedSize); - - ByteBuffer buffer = ByteBuffer.allocate(bytesNeeded).order(ByteOrder.LITTLE_ENDIAN); - buffer.putInt(SIGNATURE); - buffer.putShort(versionNeeded); - - buffer.putShort((short) 0); // general purpose flag - buffer.putShort(compressionFlag); - buffer.putShort(com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.DEFAULT_TIME); - buffer.putShort(com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.DEFAULT_DATE); - buffer.putInt(crc); - buffer.putInt(csize); // compressed size - buffer.putInt(size); // size - buffer.putShort(com.slack.keeper.internal.zipflinger.Ints.intToUshort(nameBytes.length)); - buffer.putShort(com.slack.keeper.internal.zipflinger.Ints.intToUshort(extraField.capacity())); // Extra size - buffer.put(nameBytes); - buffer.put(extraField); - - buffer.rewind(); - writer.write(buffer); - } - - public long getSize() { - long extraSize = isZip64 ? ZIP64_EXTRA_SIZE : 0; - return LOCAL_FILE_HEADER_SIZE + nameBytes.length + extraSize; - } - - @NonNull - private ByteBuffer buildExtraField() { - if (!isZip64) { - return ByteBuffer.allocate(padding); - } - - ByteBuffer zip64extra = ByteBuffer.allocate(ZIP64_EXTRA_SIZE + padding); - zip64extra.order(ByteOrder.LITTLE_ENDIAN); - zip64extra.putShort(Zip64.EXTRA_ID); - zip64extra.putShort(com.slack.keeper.internal.zipflinger.Ints.intToUshort(ZIP64_PAYLOAD_SIZE)); - zip64extra.putLong(uncompressedSize); - zip64extra.putLong(compressedSize); - zip64extra.rewind(); - return zip64extra; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Location.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Location.java deleted file mode 100644 index 21f59340..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Location.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.text.NumberFormat; - -public class Location implements Comparable { - - public static final Location INVALID = new Location(Long.MAX_VALUE, Long.MAX_VALUE); - - public final long first; - public final long last; - - public Location(long first, long size) { - this.first = first; - this.last = first + size - 1; - } - - public long size() { - return last - first + 1; - } - - @NonNull - public String toString() { - return "(offset=" - + NumberFormat.getInstance().format(first) - + ", size=" - + NumberFormat.getInstance().format(size()) - + ")"; - } - - @Override - public boolean equals(@NonNull Object obj) { - if (this == obj) { - return true; - } - - if (!(obj instanceof Location)) { - return false; - } - Location other = (Location) obj; - return first == other.first && last == other.last; - } - - @Override - public int compareTo(Location o) { - return Math.toIntExact(this.first - o.first); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/NoCopyByteArrayOutputStream.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/NoCopyByteArrayOutputStream.java deleted file mode 100644 index 69ef372e..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/NoCopyByteArrayOutputStream.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; - -// A class which contrary to ByteArrayOutputStream allows to peek into the buffer -// without performing a full copy of the content. This stream does not need to be closed. -public class NoCopyByteArrayOutputStream extends ByteArrayOutputStream { - public NoCopyByteArrayOutputStream(int size) { - super(size); - } - - @NonNull - public byte[] buf() { - return buf; - } - - public int getCount() { - return count; - } - - public ByteBuffer getByteBuffer() { - return ByteBuffer.wrap(buf, 0, count); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Source.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Source.java deleted file mode 100644 index b29a8381..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Source.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -public abstract class Source { - private final String name; - private final byte[] nameBytes; - private long alignment = 0; - - protected long compressedSize; - protected long uncompressedSize; - protected int crc; - protected short compressionFlag; - - protected Source(@NonNull String name) { - this.name = name; - nameBytes = name.getBytes(StandardCharsets.UTF_8); - } - - @NonNull - public String getName() { - return name; - } - - @NonNull - byte[] getNameBytes() { - return nameBytes; - } - - boolean isAligned() { - return alignment != 0; - } - - public void align(long alignment) { - this.alignment = alignment; - } - - long getAlignment() { - return alignment; - } - - int getCrc() { - return crc; - } - - long getCompressedSize() { - return compressedSize; - } - - long getUncompressedSize() { - return uncompressedSize; - } - - short getCompressionFlag() { - return compressionFlag; - } - - // Guaranteed to be called before writeTo. After this method has been called, every fields - // in an entry must be known (csize, size, crc32, and compressionFlag). - abstract void prepare() throws IOException; - - // Return the number of bytes written. - abstract long writeTo(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException; -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/StableArchive.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/StableArchive.java deleted file mode 100644 index a24e3dad..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/StableArchive.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; - -public class StableArchive implements com.slack.keeper.internal.zipflinger.Archive { - - private final com.slack.keeper.internal.zipflinger.Archive archive; - private final ArrayList bytesSources; - private final ArrayList zipSources; - private final ArrayList deletedEntries; - - public StableArchive(com.slack.keeper.internal.zipflinger.Archive archive) { - this.archive = archive; - bytesSources = new ArrayList<>(); - zipSources = new ArrayList<>(); - deletedEntries = new ArrayList<>(); - } - - @Override - public void add(@NonNull com.slack.keeper.internal.zipflinger.BytesSource source) { - bytesSources.add(source); - } - - @Override - public void add(@NonNull ZipSource sources) { - zipSources.add(sources); - } - - @Override - public void delete(@NonNull String name) { - deletedEntries.add(name); - } - - @Override - public void close() throws IOException { - bytesSources.sort(Comparator.comparing(com.slack.keeper.internal.zipflinger.Source::getName)); - zipSources.sort(Comparator.comparing(ZipSource::getName)); - for (ZipSource zipSource : zipSources) { - zipSource.getSelectedEntries().sort(Comparator.comparing(Source::getName)); - } - deletedEntries.sort(Comparator.naturalOrder()); - - try (Archive arch = archive) { - for (String toDelete : deletedEntries) { - arch.delete(toDelete); - } - - for (BytesSource source : bytesSources) { - arch.add(source); - } - - for (ZipSource zipSource : zipSources) { - arch.add(zipSource); - } - } - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/SynchronizedArchive.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/SynchronizedArchive.java deleted file mode 100644 index 37398989..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/SynchronizedArchive.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; - -public class SynchronizedArchive implements com.slack.keeper.internal.zipflinger.Archive { - - private final com.slack.keeper.internal.zipflinger.Archive archive; - - public SynchronizedArchive(Archive archive) throws IOException { - this.archive = archive; - } - - @Override - public void add(@NonNull BytesSource source) throws IOException { - synchronized (archive) { - archive.add(source); - } - } - - @Override - public void add(@NonNull ZipSource sources) throws IOException { - synchronized (archive) { - archive.add(sources); - } - } - - @Override - public void delete(@NonNull String name) throws IOException { - synchronized (archive) { - archive.delete(name); - } - } - - @Override - public void close() throws IOException { - synchronized (archive) { - archive.close(); - } - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64.java deleted file mode 100644 index 362bbf40..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -public class Zip64 { - static final short EXTRA_ID = 0x0001; - - static final long LONG_MAGIC = 0xFF_FF_FF_FFL; - static final int INT_MAGIC = (int) LONG_MAGIC; - static final int SHORT_MAGIC = (short) LONG_MAGIC; - - static final short VERSION_NEEDED = 0x2D; - - public static boolean needZip64Footer(long numEntries, Location cdLocation) { - return numEntries > com.slack.keeper.internal.zipflinger.Ints.USHRT_MAX - || cdLocation.first > com.slack.keeper.internal.zipflinger.Ints.UINT_MAX - || cdLocation.size() > com.slack.keeper.internal.zipflinger.Ints.UINT_MAX; - } - - public enum Policy { - ALLOW, - FORBID - }; -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64Eocd.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64Eocd.java deleted file mode 100644 index b04df0ef..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64Eocd.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.FileChannel; - -public class Zip64Eocd { - private static final int SIGNATURE = 0x06064b50; - static final int SIZE = 56; - - private long numEntries; - private com.slack.keeper.internal.zipflinger.Location cdLocation; - - public Zip64Eocd(long numEntries, @NonNull com.slack.keeper.internal.zipflinger.Location cdLocation) { - this.numEntries = numEntries; - this.cdLocation = cdLocation; - } - - private Zip64Eocd() { - this(0, com.slack.keeper.internal.zipflinger.Location.INVALID); - } - - @NonNull - public com.slack.keeper.internal.zipflinger.Location write(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException { - ByteBuffer eocd = ByteBuffer.allocate(SIZE).order(ByteOrder.LITTLE_ENDIAN); - eocd.putInt(SIGNATURE); - eocd.putLong(SIZE - 12); // Peculiar specs mandate not to include 12 bytes already written. - eocd.putShort((short) 0); // Version made by - eocd.putShort(Zip64.VERSION_NEEDED); // Version needed to extract - eocd.putInt(0); // disk # - eocd.putInt(0); // total # of disks - eocd.putLong(numEntries); // # entries in cd on this disk - eocd.putLong(numEntries); // total # entries in cd - eocd.putLong(cdLocation.size()); // CD offset. - eocd.putLong(cdLocation.first); // size of CD. - eocd.rewind(); - - long position = writer.position(); - writer.write(eocd); - - return new com.slack.keeper.internal.zipflinger.Location(position, SIZE); - } - - @NonNull com.slack.keeper.internal.zipflinger.Location getCdLocation() { - return cdLocation; - } - - @NonNull - static Zip64Eocd parse(@NonNull FileChannel channel, long eocdOffset) throws IOException { - Zip64Eocd zip64Eocd = new Zip64Eocd(); - long fileSize = channel.size(); - if (eocdOffset < 0 || eocdOffset + SIZE > fileSize) { - return zip64Eocd; - } - - ByteBuffer buffer = ByteBuffer.allocate(SIZE).order(ByteOrder.LITTLE_ENDIAN); - channel.read(buffer, eocdOffset); - buffer.rewind(); - - int signature = buffer.getInt(); // signature - if (signature != SIGNATURE) { - return zip64Eocd; - } - - // Skip uninteresting fields - buffer.position(buffer.position() + 28); - // eocd.getLong(); 8 // size of zip64EOCD - // eocd.getShort(); 2 // Version made by - // eocd.getShort(); 2 // Version needed to extract - // eocd.getInt(); 4 // disk # - // eocd.getInt(); 4 // total # of disks - // eocd.getLong(); 8 // # entries in cd on this disk - long numEntries = buffer.getLong(); // total # entries in cd - long size = com.slack.keeper.internal.zipflinger.Ints.ulongToLong(buffer.getLong()); // size of CD. - long offset = com.slack.keeper.internal.zipflinger.Ints.ulongToLong(buffer.getLong()); // CD offset. - - zip64Eocd.numEntries = numEntries; - zip64Eocd.cdLocation = new Location(offset, size); - return zip64Eocd; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64Locator.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64Locator.java deleted file mode 100644 index 8aaf95b5..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/Zip64Locator.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.FileChannel; - -public class Zip64Locator { - - private static final int SIGNATURE = 0x07064b50; - public static final int SIZE = 20; - - static final int TOTAL_NUMBER_DISK = com.slack.keeper.internal.zipflinger.EndOfCentralDirectory.DISK_NUMBER + 1; - - private com.slack.keeper.internal.zipflinger.Location location; - private long offsetToEOCD64; - - private Zip64Locator() { - location = com.slack.keeper.internal.zipflinger.Location.INVALID; - offsetToEOCD64 = 0; - } - - @NonNull - public com.slack.keeper.internal.zipflinger.Location getLocation() { - return location; - } - - public long getOffsetToEOCD64() { - return offsetToEOCD64; - } - - public static com.slack.keeper.internal.zipflinger.Location write(@NonNull - com.slack.keeper.internal.zipflinger.ZipWriter writer, @NonNull - com.slack.keeper.internal.zipflinger.Location eocdLocation) - throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(SIZE).order(ByteOrder.LITTLE_ENDIAN); - buffer.putInt(SIGNATURE); - buffer.putInt(0); // CD disk number - buffer.putLong(eocdLocation.first); // offset - buffer.putInt(TOTAL_NUMBER_DISK); - buffer.rewind(); - - long position = writer.position(); - writer.write(buffer); - return new com.slack.keeper.internal.zipflinger.Location(position, SIZE); - } - - @NonNull - static Zip64Locator find(@NonNull FileChannel channel, @NonNull - com.slack.keeper.internal.zipflinger.EndOfCentralDirectory eocd) - throws IOException { - Zip64Locator locator = new Zip64Locator(); - com.slack.keeper.internal.zipflinger.Location - locatorLocation = new Location(eocd.getLocation().first - SIZE, SIZE); - long fileSize = channel.size(); - if (locatorLocation.last >= fileSize) { - return locator; - } - if (locatorLocation.first < 0) { - return locator; - } - - ByteBuffer locatorBuffer = ByteBuffer.allocate(SIZE).order(ByteOrder.LITTLE_ENDIAN); - channel.read(locatorBuffer, locatorLocation.first); - locatorBuffer.rewind(); - - if (locator.parse(locatorBuffer)) { - locator.location = locatorLocation; - } - return locator; - } - - private boolean parse(@NonNull ByteBuffer buffer) { - int signature = buffer.getInt(); - if (signature != SIGNATURE) { - return false; - } - - buffer.position(buffer.position() + 4); // skip CD disk number - offsetToEOCD64 = com.slack.keeper.internal.zipflinger.Ints.ulongToLong(buffer.getLong()); - // Don't read the rest, this is not needed - - return true; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipArchive.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipArchive.java deleted file mode 100644 index 2c94c4af..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipArchive.java +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import com.android.annotations.Nullable; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class ZipArchive implements Archive { - - private final com.slack.keeper.internal.zipflinger.FreeStore freestore; - private boolean closed; - private final File file; - private final com.slack.keeper.internal.zipflinger.CentralDirectory cd; - private final com.slack.keeper.internal.zipflinger.ZipWriter writer; - private final ZipReader reader; - private final com.slack.keeper.internal.zipflinger.Zip64.Policy policy; - private ZipInfo zipInfo; - private boolean modified; - - public ZipArchive(@NonNull File file) throws IOException { - this(file, com.slack.keeper.internal.zipflinger.Zip64.Policy.ALLOW); - } - - /** - * The object used to manipulate a zip archive. - * - * @param file the file object - * @throws IOException - */ - public ZipArchive(@NonNull File file, com.slack.keeper.internal.zipflinger.Zip64.Policy policy) throws IOException { - this.file = file; - this.policy = policy; - if (Files.exists(file.toPath())) { - ZipMap map = ZipMap.from(file, true, policy); - zipInfo = new ZipInfo(map.getPayloadLocation(), map.getCdLoc(), map.getEocdLoc()); - cd = map.getCentralDirectory(); - freestore = new com.slack.keeper.internal.zipflinger.FreeStore(map.getEntries()); - } else { - zipInfo = new ZipInfo(); - HashMap entries = new HashMap<>(); - cd = new com.slack.keeper.internal.zipflinger.CentralDirectory(ByteBuffer.allocate(0), entries); - freestore = new com.slack.keeper.internal.zipflinger.FreeStore(entries); - } - - writer = new com.slack.keeper.internal.zipflinger.ZipWriter(file); - reader = new ZipReader(file); - closed = false; - modified = false; - } - - /** - * Returns the list of zip entries found in the archive. Note that these are the entries found - * in the central directory via bottom-up parsing, not all entries present in the payload as a - * top-down parser may return. - * - * @param file the zip archive to list. - * @return the list of entries in the archive, parsed bottom-up (via the Central Directory). - * @throws IOException - */ - @NonNull - public static Map listEntries(@NonNull File file) throws IOException { - return ZipMap.from(file, false).getEntries(); - } - - @NonNull - public List listEntries() { - return cd.listEntries(); - } - - @Nullable - public ByteBuffer getContent(@NonNull String name) throws IOException { - ExtractionInfo extractInfo = cd.getExtractionInfo(name); - if (extractInfo == null) { - return null; - } - com.slack.keeper.internal.zipflinger.Location loc = extractInfo.getLocation(); - ByteBuffer payloadByteBuffer = ByteBuffer.allocate(Math.toIntExact(loc.size())); - reader.read(payloadByteBuffer, loc.first); - if (extractInfo.isCompressed()) { - return Compressor.inflate(payloadByteBuffer.array()); - } else { - return payloadByteBuffer; - } - } - - /** See Archive.add documentation */ - @Override - public void add(@NonNull BytesSource source) throws IOException { - if (closed) { - throw new IllegalStateException( - String.format("Cannot add source to closed archive %s", file)); - } - writeSource(source); - } - - /** See Archive.add documentation */ - @Override - public void add(@NonNull ZipSource sources) throws IOException { - if (closed) { - throw new IllegalStateException( - String.format("Cannot add zip source to closed archive %s", file)); - } - - try { - sources.open(); - for (com.slack.keeper.internal.zipflinger.Source source : sources.getSelectedEntries()) { - writeSource(source); - } - } finally { - sources.close(); - } - } - - /** See Archive.delete documentation */ - @Override - public void delete(@NonNull String name) { - if (closed) { - throw new IllegalStateException( - String.format("Cannot delete '%s' from closed archive %s", name, file)); - } - com.slack.keeper.internal.zipflinger.Location loc = cd.delete(name); - if (loc != com.slack.keeper.internal.zipflinger.Location.INVALID) { - freestore.free(loc); - modified = true; - } - } - - /** - * Carry all write operations to the storage system to reflect the delete/add operations - * requested via add/delete methods. - * - * @throws IOException - */ - // TODO: Zip64 -> Add boolean allowZip64 - @Override - public void close() throws IOException { - closeWithInfo(); - } - - @NonNull - public ZipInfo closeWithInfo() throws IOException { - if (closed) { - throw new IllegalStateException("Attempt to close a closed archive"); - } - closed = true; - try (com.slack.keeper.internal.zipflinger.ZipWriter w = writer; - ZipReader r = reader) { - writeArchive(w); - } - return zipInfo; - } - - @NonNull - public File getFile() { - return file; - } - - public boolean isClosed() { - return closed; - } - - private void writeArchive(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException { - // There is no need to fill space and write footers if an already existing archive - // has not been modified. - if (zipInfo.eocd != com.slack.keeper.internal.zipflinger.Location.INVALID && !modified) { - return; - } - - // Fill all empty space with virtual entry (not the last one since it represent all of - // the unused file space. - List freeLocations = freestore.getFreeLocations(); - for (int i = 0; i < freeLocations.size() - 1; i++) { - fillFreeLocation(freeLocations.get(i), writer); - } - - // Write the Central Directory - com.slack.keeper.internal.zipflinger.Location lastFreeLocation = freestore.getLastFreeLocation(); - long cdStart = lastFreeLocation.first; - writer.position(cdStart); - cd.write(writer); - com.slack.keeper.internal.zipflinger.Location - cdLocation = new com.slack.keeper.internal.zipflinger.Location(cdStart, writer.position() - cdStart); - long numEntries = cd.getNumEntries(); - - // Write zip64 EOCD and Locator (only if needed) - writeZip64Footers(writer, cdLocation, numEntries); - - // Write EOCD - com.slack.keeper.internal.zipflinger.Location eocdLocation = com.slack.keeper.internal.zipflinger.EndOfCentralDirectory.write(writer, cdLocation, numEntries); - writer.truncate(writer.position()); - - // Build and return location map - com.slack.keeper.internal.zipflinger.Location - payLoadLocation = new com.slack.keeper.internal.zipflinger.Location(0, cdStart); - - zipInfo = new ZipInfo(payLoadLocation, cdLocation, eocdLocation); - } - - private void writeZip64Footers( - @NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer, @NonNull - com.slack.keeper.internal.zipflinger.Location cdLocation, long numEntries) - throws IOException { - if (!com.slack.keeper.internal.zipflinger.Zip64.needZip64Footer(numEntries, cdLocation)) { - return; - } - - if (policy == com.slack.keeper.internal.zipflinger.Zip64.Policy.FORBID) { - String message = - String.format( - "Zip64 required but forbidden (#entries=%d, cd=%s)", - numEntries, cdLocation); - throw new IllegalStateException(message); - } - - com.slack.keeper.internal.zipflinger.Zip64Eocd eocd = new Zip64Eocd(numEntries, cdLocation); - com.slack.keeper.internal.zipflinger.Location eocdLocation = eocd.write(writer); - - Zip64Locator.write(writer, eocdLocation); - } - - // Fill archive holes with virtual entries. Use extra field to fill as much as possible. - private static void fillFreeLocation(@NonNull com.slack.keeper.internal.zipflinger.Location location, @NonNull - com.slack.keeper.internal.zipflinger.ZipWriter writer) - throws IOException { - long spaceToFill = location.size(); - - if (spaceToFill < com.slack.keeper.internal.zipflinger.LocalFileHeader.VIRTUAL_HEADER_SIZE) { - // There is not enough space to create a virtual entry here. The FreeStore - // never creates such gaps so it was already in the zip. Leave it as it is. - return; - } - - while (spaceToFill > 0) { - long entrySize; - if (spaceToFill <= com.slack.keeper.internal.zipflinger.LocalFileHeader.VIRTUAL_ENTRY_MAX_SIZE) { - // Consume all the remaining space. - entrySize = spaceToFill; - } else { - // Consume as much as possible while leaving enough for the next LFH entry. - entrySize = com.slack.keeper.internal.zipflinger.Ints.USHRT_MAX; - } - int size = Math.toIntExact(entrySize); - ByteBuffer virtualEntry = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); - com.slack.keeper.internal.zipflinger.LocalFileHeader.fillVirtualEntry(virtualEntry); - writer.write(virtualEntry, location.first + location.size() - spaceToFill); - spaceToFill -= virtualEntry.capacity(); - } - } - - private void writeSource(@NonNull com.slack.keeper.internal.zipflinger.Source source) throws IOException { - modified = true; - source.prepare(); - validateName(source); - - // Calculate the size we need (header + payload) - com.slack.keeper.internal.zipflinger.LocalFileHeader lfh = new com.slack.keeper.internal.zipflinger.LocalFileHeader(source); - long headerSize = lfh.getSize(); - long bytesNeeded = headerSize + source.getCompressedSize(); - - // Allocate file space - com.slack.keeper.internal.zipflinger.Location loc; - if (source.isAligned()) { - loc = freestore.alloc(bytesNeeded, headerSize, source.getAlignment()); - lfh.setPadding(Math.toIntExact(loc.size() - bytesNeeded)); - } else { - loc = freestore.ualloc(bytesNeeded); - } - - writer.position(loc.first); - lfh.write(writer); - - // Write payload - long payloadStart = writer.position(); - long payloadSize = source.writeTo(writer); - com.slack.keeper.internal.zipflinger.Location - payloadLocation = new com.slack.keeper.internal.zipflinger.Location(payloadStart, payloadSize); - - // Update Central Directory record - com.slack.keeper.internal.zipflinger.CentralDirectoryRecord cdRecord = - new com.slack.keeper.internal.zipflinger.CentralDirectoryRecord( - source.getNameBytes(), - source.getCrc(), - source.getCompressedSize(), - source.getUncompressedSize(), - loc, - source.getCompressionFlag(), - payloadLocation); - cd.add(source.getName(), cdRecord); - - checkPolicy(source, loc, payloadLocation); - } - - private void checkPolicy( - @NonNull com.slack.keeper.internal.zipflinger.Source source, @NonNull - com.slack.keeper.internal.zipflinger.Location cdloc, @NonNull Location payloadLoc) { - if (policy == com.slack.keeper.internal.zipflinger.Zip64.Policy.ALLOW) { - return; - } - - if (source.getUncompressedSize() >= com.slack.keeper.internal.zipflinger.Zip64.LONG_MAGIC - || source.getCompressedSize() >= com.slack.keeper.internal.zipflinger.Zip64.LONG_MAGIC - || cdloc.first >= com.slack.keeper.internal.zipflinger.Zip64.LONG_MAGIC - || payloadLoc.first >= Zip64.LONG_MAGIC) { - String message = - String.format( - "Zip64 forbidden but required in entry %s size=%d, csize=%d, cdloc=%s, loc=%s", - source.getName(), - source.getUncompressedSize(), - source.getCompressedSize(), - cdloc, - payloadLoc); - throw new IllegalStateException(message); - } - } - - private void validateName(@NonNull Source source) { - byte[] nameBytes = source.getNameBytes(); - String name = source.getName(); - if (nameBytes.length > com.slack.keeper.internal.zipflinger.Ints.USHRT_MAX) { - throw new IllegalStateException( - String.format("Name '%s' is more than %d bytes", name, com.slack.keeper.internal.zipflinger.Ints.USHRT_MAX)); - } - - if (cd.contains(name)) { - throw new IllegalStateException(String.format("Entry name '%s' collided", name)); - } - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipInfo.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipInfo.java deleted file mode 100644 index fdfa93e2..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipInfo.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -public class ZipInfo { - public final com.slack.keeper.internal.zipflinger.Location payload; - public final com.slack.keeper.internal.zipflinger.Location cd; - public final com.slack.keeper.internal.zipflinger.Location eocd; - - public ZipInfo() { - this(com.slack.keeper.internal.zipflinger.Location.INVALID, com.slack.keeper.internal.zipflinger.Location.INVALID, - com.slack.keeper.internal.zipflinger.Location.INVALID); - } - - public ZipInfo(com.slack.keeper.internal.zipflinger.Location payload, com.slack.keeper.internal.zipflinger.Location cd, Location eocd) { - this.payload = payload; - this.cd = cd; - this.eocd = eocd; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipMap.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipMap.java deleted file mode 100644 index 6343fb21..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipMap.java +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.FileChannel; -import java.nio.file.StandardOpenOption; -import java.util.HashMap; -import java.util.Map; - -public class ZipMap { - private final Map entries = new HashMap<>(); - private com.slack.keeper.internal.zipflinger.CentralDirectory cd = null; - - // To build an accurate location of entries in the zip payload, data descriptors must be read. - // This is not useful if an user only wants a list of entries in the zip but it is mandatory - // if zip entries are deleted/added. - private final boolean accountDataDescriptors; - - private File file; - private long fileSize; - - private com.slack.keeper.internal.zipflinger.Location payloadLocation; - private com.slack.keeper.internal.zipflinger.Location cdLocation; - private com.slack.keeper.internal.zipflinger.Location eocdLocation; - - private ZipMap(@NonNull File file, boolean accountDataDescriptors) { - this.file = file; - this.accountDataDescriptors = accountDataDescriptors; - } - - @NonNull - public static ZipMap from(@NonNull File zipFile, boolean accountDataDescriptors) - throws IOException { - return from(zipFile, accountDataDescriptors, com.slack.keeper.internal.zipflinger.Zip64.Policy.ALLOW); - } - - @NonNull - public static ZipMap from( - @NonNull File zipFile, boolean accountDataDescriptors, com.slack.keeper.internal.zipflinger.Zip64.Policy policy) - throws IOException { - ZipMap map = new ZipMap(zipFile, accountDataDescriptors); - map.parse(policy); - return map; - } - - @NonNull - public com.slack.keeper.internal.zipflinger.Location getPayloadLocation() { - return payloadLocation; - } - - @NonNull - public com.slack.keeper.internal.zipflinger.Location getCdLoc() { - return cdLocation; - } - - @NonNull - public com.slack.keeper.internal.zipflinger.Location getEocdLoc() { - return eocdLocation; - } - - private void parse(com.slack.keeper.internal.zipflinger.Zip64.Policy policy) throws IOException { - try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { - - fileSize = channel.size(); - - com.slack.keeper.internal.zipflinger.EndOfCentralDirectory - eocd = com.slack.keeper.internal.zipflinger.EndOfCentralDirectory.find(channel); - if (eocd.getLocation() == com.slack.keeper.internal.zipflinger.Location.INVALID) { - throw new IllegalStateException(String.format("Could not find EOCD in '%s'", file)); - } - eocdLocation = eocd.getLocation(); - cdLocation = eocd.getCdLocation(); - - // Check if this is a zip64 archive - com.slack.keeper.internal.zipflinger.Zip64Locator locator = Zip64Locator.find(channel, eocd); - if (locator.getLocation() != com.slack.keeper.internal.zipflinger.Location.INVALID) { - if (policy == com.slack.keeper.internal.zipflinger.Zip64.Policy.FORBID) { - String message = String.format("Cannot parse forbidden zip64 archive %s", file); - throw new IllegalStateException(message); - } - com.slack.keeper.internal.zipflinger.Zip64Eocd zip64EOCD = Zip64Eocd.parse(channel, locator.getOffsetToEOCD64()); - cdLocation = zip64EOCD.getCdLocation(); - if (cdLocation == com.slack.keeper.internal.zipflinger.Location.INVALID) { - String message = String.format("Zip64Locator led to bad EOCD64 in %s", file); - throw new IllegalStateException(message); - } - } - - if (cdLocation == com.slack.keeper.internal.zipflinger.Location.INVALID) { - throw new IllegalStateException(String.format("Could not find CD in '%s'", file)); - } - - parseCentralDirectory(channel, cdLocation, policy); - - payloadLocation = new com.slack.keeper.internal.zipflinger.Location(0, cdLocation.first); - } - } - - private void parseCentralDirectory( - @NonNull FileChannel channel, @NonNull - com.slack.keeper.internal.zipflinger.Location location, com.slack.keeper.internal.zipflinger.Zip64.Policy policy) - throws IOException { - if (location.size() > Integer.MAX_VALUE) { - throw new IllegalStateException("CD larger than 2GiB not supported"); - } - int size = Math.toIntExact(location.size()); - ByteBuffer buf = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); - channel.read(buf, location.first); - buf.rewind(); - - while (buf.remaining() >= 4 && buf.getInt() == com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.SIGNATURE) { - com.slack.keeper.internal.zipflinger.Entry entry = new com.slack.keeper.internal.zipflinger.Entry(); - parseCentralDirectoryRecord(buf, channel, entry); - if (!entry.getName().isEmpty()) { - entries.put(entry.getName(), entry); - } - checkPolicy(entry, policy); - } - - cd = new com.slack.keeper.internal.zipflinger.CentralDirectory(buf, entries); - - sanityCheck(location); - } - - private static void checkPolicy(@NonNull - com.slack.keeper.internal.zipflinger.Entry entry, com.slack.keeper.internal.zipflinger.Zip64.Policy policy) { - if (policy == com.slack.keeper.internal.zipflinger.Zip64.Policy.ALLOW) { - return; - } - - if (entry.getUncompressedSize() > com.slack.keeper.internal.zipflinger.Zip64.LONG_MAGIC - || entry.getCompressedSize() > com.slack.keeper.internal.zipflinger.Zip64.LONG_MAGIC - || entry.getLocation().first > com.slack.keeper.internal.zipflinger.Zip64.LONG_MAGIC) { - String message = - String.format( - "Entry %s infringes forbidden zip64 policy (size=%d, csize=%d, loc=%s)", - entry.getName(), - entry.getUncompressedSize(), - entry.getCompressedSize(), - entry.getLocation()); - throw new IllegalStateException(message); - } - } - - private void sanityCheck(com.slack.keeper.internal.zipflinger.Location cdLocation) { - //Sanity check that: - // - All payload locations are within the file (and not in the CD). - for (com.slack.keeper.internal.zipflinger.Entry e : entries.values()) { - com.slack.keeper.internal.zipflinger.Location loc = e.getLocation(); - if (loc.first < 0) { - throw new IllegalStateException("Invalid first loc '" + e.getName() + "' " + loc); - } - if (loc.last >= fileSize) { - throw new IllegalStateException( - fileSize + "Invalid last loc '" + e.getName() + "' " + loc); - } - com.slack.keeper.internal.zipflinger.Location cdLoc = e.getCdLocation(); - if (cdLoc.first < 0) { - throw new IllegalStateException( - "Invalid first cdloc '" + e.getName() + "' " + cdLoc); - } - long cdSize = cdLocation.size(); - if (cdLoc.last >= cdSize) { - throw new IllegalStateException( - cdSize + "Invalid last loc '" + e.getName() + "' " + cdLoc); - } - } - } - - @NonNull - public Map getEntries() { - return entries; - } - - @NonNull com.slack.keeper.internal.zipflinger.CentralDirectory getCentralDirectory() { - return cd; - } - - public void parseCentralDirectoryRecord( - @NonNull ByteBuffer buf, @NonNull FileChannel channel, @NonNull - com.slack.keeper.internal.zipflinger.Entry entry) - throws IOException { - long cdEntryStart = buf.position() - 4; - - buf.position(buf.position() + 4); - //short versionMadeBy = buf.getShort(); - //short versionNeededToExtract = buf.getShort(); - short flags = buf.getShort(); - short compressionFlag = buf.getShort(); - entry.setCompressionFlag(compressionFlag); - buf.position(buf.position() + 4); - //short modTime = buf.getShort(); - //short modDate = buf.getShort(); - - int crc = buf.getInt(); - entry.setCrc(crc); - - entry.setCompressedSize(com.slack.keeper.internal.zipflinger.Ints.uintToLong(buf.getInt())); - entry.setUncompressedSize(com.slack.keeper.internal.zipflinger.Ints.uintToLong(buf.getInt())); - - int pathLength = com.slack.keeper.internal.zipflinger.Ints.ushortToInt(buf.getShort()); - int extraLength = com.slack.keeper.internal.zipflinger.Ints.ushortToInt(buf.getShort()); - int commentLength = com.slack.keeper.internal.zipflinger.Ints.ushortToInt(buf.getShort()); - buf.position(buf.position() + 8); - // short diskNumber = buf.getShort(); - // short intAttributes = buf.getShort(); - // int extAttributes = bug.getInt(); - - entry.setLocation(new com.slack.keeper.internal.zipflinger.Location(com.slack.keeper.internal.zipflinger.Ints.uintToLong(buf.getInt()), 0)); - - parseName(buf, pathLength, entry); - - // Process extra field. If the entry is zip64, this may change size, csize, and offset. - if (extraLength > 0) { - int position = buf.position(); - int limit = buf.limit(); - buf.limit(position + extraLength); - parseExtra(buf.slice(), entry); - buf.limit(limit); - buf.position(position + extraLength); - } - - // Skip comment field - buf.position(buf.position() + commentLength); - - // Retrieve the local header extra size since there are no guarantee it is the same as the - // central directory size. - // Semi-paranoid mode: Also check that the local name size is the same as the cd name size. - ByteBuffer localFieldBuffer = - readLocalFields( - entry.getLocation().first + com.slack.keeper.internal.zipflinger.LocalFileHeader.OFFSET_TO_NAME, entry, channel); - int localPathLength = com.slack.keeper.internal.zipflinger.Ints.ushortToInt(localFieldBuffer.getShort()); - int localExtraLength = com.slack.keeper.internal.zipflinger.Ints.ushortToInt(localFieldBuffer.getShort()); - if (pathLength != localPathLength) { - String message = - String.format( - "Entry '%s' name differ (%d vs %d)", - entry.getName(), localPathLength, pathLength); - throw new IllegalStateException(message); - } - - // At this point we have everything we need to calculate payload location. - boolean isCompressed = compressionFlag != 0; - long payloadSize = isCompressed ? entry.getCompressedSize() : entry.getUncompressedSize(); - long start = entry.getLocation().first; - long end = - start - + com.slack.keeper.internal.zipflinger.LocalFileHeader.LOCAL_FILE_HEADER_SIZE - + pathLength - + localExtraLength - + payloadSize; - entry.setLocation(new com.slack.keeper.internal.zipflinger.Location(start, end - start)); - - com.slack.keeper.internal.zipflinger.Location payloadLocation = - new com.slack.keeper.internal.zipflinger.Location( - start - + com.slack.keeper.internal.zipflinger.LocalFileHeader.LOCAL_FILE_HEADER_SIZE - + pathLength - + localExtraLength, - payloadSize); - entry.setPayloadLocation(payloadLocation); - - // At this point we have everything we need to calculate CD location. - long cdEntrySize = com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.SIZE + pathLength + extraLength + commentLength; - entry.setCdLocation(new com.slack.keeper.internal.zipflinger.Location(cdEntryStart, cdEntrySize)); - - // Parse data descriptor to adjust crc, compressed size, and uncompressed size. - boolean hasDataDescriptor = - ((flags & com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.DATA_DESCRIPTOR_FLAG) - == com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.DATA_DESCRIPTOR_FLAG); - if (hasDataDescriptor) { - if (accountDataDescriptors) { - // This is expensive. Fortunately ZIP archive rarely use DD nowadays. - channel.position(end); - parseDataDescriptor(channel, entry); - } else { - entry.setLocation(com.slack.keeper.internal.zipflinger.Location.INVALID); - } - } - } - - private static void parseExtra(@NonNull ByteBuffer buf, @NonNull - com.slack.keeper.internal.zipflinger.Entry entry) { - buf.order(ByteOrder.LITTLE_ENDIAN); - while (buf.remaining() >= 4) { - short id = buf.getShort(); - int size = com.slack.keeper.internal.zipflinger.Ints.ushortToInt(buf.getShort()); - if (id == com.slack.keeper.internal.zipflinger.Zip64.EXTRA_ID) { - parseZip64Extra(buf, entry); - } - if (buf.remaining() >= size) { - buf.position(buf.position() + size); - } - } - } - - private static void parseZip64Extra(@NonNull ByteBuffer buf, @NonNull - com.slack.keeper.internal.zipflinger.Entry entry) { - if (entry.getUncompressedSize() == com.slack.keeper.internal.zipflinger.Zip64.LONG_MAGIC) { - if (buf.remaining() < 8) { - throw new IllegalStateException("Bad zip64 extra for entry " + entry.getName()); - } - entry.setUncompressedSize(com.slack.keeper.internal.zipflinger.Ints.ulongToLong(buf.getLong())); - } - if (entry.getCompressedSize() == com.slack.keeper.internal.zipflinger.Zip64.LONG_MAGIC) { - if (buf.remaining() < 8) { - throw new IllegalStateException("Bad zip64 extra for entry " + entry.getName()); - } - entry.setCompressedSize(com.slack.keeper.internal.zipflinger.Ints.ulongToLong(buf.getLong())); - } - if (entry.getLocation().first == Zip64.LONG_MAGIC) { - if (buf.remaining() < 8) { - throw new IllegalStateException("Bad zip64 extra for entry " + entry.getName()); - } - long offset = com.slack.keeper.internal.zipflinger.Ints.ulongToLong(buf.getLong()); - entry.setLocation(new com.slack.keeper.internal.zipflinger.Location(offset, 0)); - } - } - - private ByteBuffer readLocalFields(long offset, com.slack.keeper.internal.zipflinger.Entry entry, FileChannel channel) - throws IOException { - // The extra field is not guaranteed to be the same in the LFH and in the CDH. In practice there is - // often padding space that is not in the CD. We need to read the LFH. - ByteBuffer localFieldsBuffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); - if (offset < 0 || (offset + 4) > fileSize) { - throw new IllegalStateException( - "Entry :" + entry.getName() + " invalid offset (" + offset + ")"); - } - channel.read(localFieldsBuffer, offset); - localFieldsBuffer.rewind(); - return localFieldsBuffer; - } - - private static void parseName(@NonNull ByteBuffer buf, int length, @NonNull - com.slack.keeper.internal.zipflinger.Entry entry) { - byte[] pathBytes = new byte[length]; - buf.get(pathBytes); - entry.setNameBytes(pathBytes); - } - - private static void parseDataDescriptor(@NonNull FileChannel channel, @NonNull Entry entry) - throws IOException { - // If zip entries have data descriptor, we need to go an fetch every single entry to look if - // the "optional" marker is there. Adjust zip entry area accordingly. - - ByteBuffer dataDescriptorBuffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); - channel.read(dataDescriptorBuffer); - dataDescriptorBuffer.rewind(); - - int dataDescriptorLength = 12; - if (dataDescriptorBuffer.getInt() == com.slack.keeper.internal.zipflinger.CentralDirectoryRecord.DATA_DESCRIPTOR_SIGNATURE) { - dataDescriptorLength += 4; - } - - com.slack.keeper.internal.zipflinger.Location adjustedLocation = - new Location( - entry.getLocation().first, - entry.getLocation().size() + dataDescriptorLength); - entry.setLocation(adjustedLocation); - } - - public File getFile() { - return file; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipReader.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipReader.java deleted file mode 100644 index 2502ae5d..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipReader.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.StandardOpenOption; - -public class ZipReader implements Closeable { - - private File file; - private FileChannel channel; - private boolean isOpen; - - ZipReader(File file) { - this.file = file; - isOpen = false; - } - - @Override - public void close() throws IOException { - if (!isOpen) { - return; - } - channel.close(); - } - - void read(ByteBuffer byteBuffer, long offset) throws IOException { - ensureOpen(); - channel.read(byteBuffer, offset); - byteBuffer.rewind(); - } - - void ensureOpen() throws IOException { - if (isOpen) { - return; - } - this.channel = FileChannel.open(file.toPath(), StandardOpenOption.READ); - if (!channel.isOpen()) { - throw new IllegalStateException("Unable to open Channel to " + file.getAbsolutePath()); - } - isOpen = true; - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSource.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSource.java deleted file mode 100644 index efcdeeec..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSource.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.File; -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.zip.Deflater; - -public class ZipSource { - public static final int COMPRESSION_NO_CHANGE = -2; - private final File file; - private FileChannel channel; - private com.slack.keeper.internal.zipflinger.ZipMap map; - - private final List selectedEntries = new ArrayList<>(); - - public ZipSource(@NonNull File file) throws IOException { - this.map = ZipMap.from(file, false); - this.file = file; - } - - @NonNull - public com.slack.keeper.internal.zipflinger.Source select(@NonNull String entryName, @NonNull String newName) { - return select(entryName, newName, COMPRESSION_NO_CHANGE); - } - - /** - * Select an entry to be copied to the archive managed by zipflinger. - * - *

An entry will remain unchanged and zero-copy will happen when: - compression level is - * COMPRESSION_NO_CHANGE. - compression level is 1-9 and the entry is already compressed. - - * compression level is Deflater.NO_COMPRESSION and the entry is already uncompressed. - * - *

Otherwise, the entry is deflated/inflated accordingly via transfer to memory, crc - * calculation , and written to the target archive. - * - * @param Name of the entry in the source zip. - * @param Name of the entry in the destination zip. - * @param The desired compression level. - * @return - */ - @NonNull - public com.slack.keeper.internal.zipflinger.Source select(@NonNull String entryName, @NonNull String newName, int compressionLevel) { - com.slack.keeper.internal.zipflinger.Entry entry = map.getEntries().get(entryName); - if (entry == null) { - throw new IllegalStateException( - String.format("Cannot find '%s' in archive '%s'", entryName, map.getFile())); - } - com.slack.keeper.internal.zipflinger.Source - entrySource = newZipSourceEntryFor(newName, entry, this, compressionLevel); - selectedEntries.add(entrySource); - return entrySource; - } - - public Map entries() { - return map.getEntries(); - } - - public static ZipSource selectAll(@NonNull File file) throws IOException { - ZipSource source = new ZipSource(file); - for (com.slack.keeper.internal.zipflinger.Entry e : source.entries().values()) { - source.select(e.getName(), e.getName(), COMPRESSION_NO_CHANGE); - } - return source; - } - - void open() throws IOException { - channel = FileChannel.open(file.toPath(), StandardOpenOption.READ); - } - - void close() throws IOException { - if (channel != null) { - channel.close(); - } - } - - FileChannel getChannel() { - return channel; - } - - public List getSelectedEntries() { - return selectedEntries; - } - - Source newZipSourceEntryFor( - String newName, Entry entry, ZipSource zipSource, int compressionLevel) { - // No change case. - if (compressionLevel == COMPRESSION_NO_CHANGE - || compressionLevel == Deflater.NO_COMPRESSION && !entry.isCompressed() - || compressionLevel != Deflater.NO_COMPRESSION && entry.isCompressed()) { - return new com.slack.keeper.internal.zipflinger.ZipSourceEntry(newName, entry, this); - } - - // The entry needs to be inflated. - if (compressionLevel == Deflater.NO_COMPRESSION) { - return new com.slack.keeper.internal.zipflinger.ZipSourceEntryInflater(newName, entry, zipSource); - } - - // The entry needs to be deflated. - return new com.slack.keeper.internal.zipflinger.ZipSourceEntryDeflater(newName, entry, zipSource, compressionLevel); - } - - String getName() { - return file.getAbsolutePath(); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntry.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntry.java deleted file mode 100644 index 2a9d3843..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntry.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; - -class ZipSourceEntry extends Source { - // Location of the payload in the zipsource. - private Location payloadLoc; - private final com.slack.keeper.internal.zipflinger.ZipSource zipSource; - - protected ZipSourceEntry(@NonNull String name, @NonNull Entry entry, ZipSource zipSource) { - super(name); - this.zipSource = zipSource; - compressedSize = entry.getCompressedSize(); - uncompressedSize = entry.getUncompressedSize(); - crc = entry.getCrc(); - compressionFlag = entry.getCompressionFlag(); - payloadLoc = entry.getPayloadLocation(); - } - - @Override - void prepare() {} - - @Override - long writeTo(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException { - writer.transferFrom(zipSource.getChannel(), payloadLoc.first, payloadLoc.size()); - return payloadLoc.size(); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntryDeflater.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntryDeflater.java deleted file mode 100644 index 9d15f8fc..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntryDeflater.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.nio.ByteBuffer; - -class ZipSourceEntryDeflater extends Source { - - private final Location loc; - private final com.slack.keeper.internal.zipflinger.ZipSource zipSource; - private final int compressionLevel; - private ByteBuffer compressedByteBuffer; - - ZipSourceEntryDeflater(String newName, Entry entry, ZipSource zipSource, int compressionLevel) { - super(newName); - loc = entry.getPayloadLocation(); - this.zipSource = zipSource; - this.compressionLevel = compressionLevel; - crc = entry.getCrc(); - } - - @Override - void prepare() throws IOException { - ByteBuffer uncompressedBytes = ByteBuffer.allocate(Math.toIntExact(loc.size())); - zipSource.getChannel().read(uncompressedBytes, loc.first); - - compressedByteBuffer = Compressor.deflate(uncompressedBytes.array(), compressionLevel); - compressedSize = compressedByteBuffer.limit(); - uncompressedSize = uncompressedBytes.limit(); - compressionFlag = com.slack.keeper.internal.zipflinger.LocalFileHeader.COMPRESSION_DEFLATE; - } - - @Override - long writeTo(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException { - return writer.write(compressedByteBuffer); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntryInflater.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntryInflater.java deleted file mode 100644 index 0167f6f9..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipSourceEntryInflater.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.IOException; -import java.nio.ByteBuffer; - -class ZipSourceEntryInflater extends Source { - - private final Location loc; - private final com.slack.keeper.internal.zipflinger.ZipSource zipSource; - private ByteBuffer buffer; - - ZipSourceEntryInflater(String newName, Entry entry, ZipSource zipSource) { - super(newName); - loc = entry.getPayloadLocation(); - this.zipSource = zipSource; - crc = entry.getCrc(); - } - - @Override - void prepare() throws IOException { - ByteBuffer compressedBytes = ByteBuffer.allocate(Math.toIntExact(loc.size())); - zipSource.getChannel().read(compressedBytes, loc.first); - - buffer = Compressor.inflate(compressedBytes.array()); - compressedSize = buffer.limit(); - uncompressedSize = buffer.limit(); - compressionFlag = com.slack.keeper.internal.zipflinger.LocalFileHeader.COMPRESSION_NONE; - } - - @Override - long writeTo(@NonNull com.slack.keeper.internal.zipflinger.ZipWriter writer) throws IOException { - return writer.write(buffer); - } -} diff --git a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipWriter.java b/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipWriter.java deleted file mode 100644 index c53a37c3..00000000 --- a/keeper-gradle-plugin/src/main/java/com/slack/keeper/internal/zipflinger/ZipWriter.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.slack.keeper.internal.zipflinger; - -import com.android.annotations.NonNull; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.StandardOpenOption; - -class ZipWriter implements Closeable { - - private final File file; - private FileChannel channel; - private boolean isOpen; - - public ZipWriter(File file) { - this.file = file; - isOpen = false; - } - - @Override - public void close() throws IOException { - if (!isOpen) { - return; - } - channel.close(); - } - - public void truncate(long size) throws IOException { - ensureOpen(); - channel.truncate(size); - } - - public void position(long position) throws IOException { - ensureOpen(); - channel.position(position); - } - - public long position() throws IOException { - ensureOpen(); - return channel.position(); - } - - public int write(@NonNull ByteBuffer buffer, long position) throws IOException { - ensureOpen(); - return channel.write(buffer, position); - } - - public int write(@NonNull ByteBuffer buffer) throws IOException { - ensureOpen(); - return channel.write(buffer); - } - - public void transferFrom(@NonNull FileChannel src, long position, long count) - throws IOException { - ensureOpen(); - long copied = 0; - while (copied != count) { - copied += src.transferTo(position + copied, count - copied, channel); - } - } - - private void ensureOpen() throws IOException { - if (isOpen) { - return; - } - channel = - FileChannel.open( - file.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); - if (!channel.isOpen()) { - throw new IllegalStateException("Unable to open Channel to " + file.getAbsolutePath()); - } - isOpen = true; - } -} diff --git a/keeper-gradle-plugin/src/test/kotlin/com/slack/keeper/KeeperFunctionalTest.kt b/keeper-gradle-plugin/src/test/kotlin/com/slack/keeper/KeeperFunctionalTest.kt index a091f33e..8b824133 100644 --- a/keeper-gradle-plugin/src/test/kotlin/com/slack/keeper/KeeperFunctionalTest.kt +++ b/keeper-gradle-plugin/src/test/kotlin/com/slack/keeper/KeeperFunctionalTest.kt @@ -94,13 +94,11 @@ class KeeperFunctionalTest(private val minifierType: MinifierType) { enum class MinifierType( val taskName: String, val expectedRules: Map?>, - val isProguard: Boolean = false, val keeperExtraConfig: KeeperExtraConfig = KeeperExtraConfig.NONE ) { R8_PRINT_USES("R8", EXPECTED_PRINT_RULES_CONFIG), R8_TRACE_REFERENCES("R8", EXPECTED_TRACE_REFERENCES_CONFIG, - keeperExtraConfig = KeeperExtraConfig.TRACE_REFERENCES_ENABLED), - PROGUARD("Proguard", EXPECTED_PRINT_RULES_CONFIG, isProguard = true) + keeperExtraConfig = KeeperExtraConfig.TRACE_REFERENCES_ENABLED) } @Rule @@ -128,20 +126,20 @@ class KeeperFunctionalTest(private val minifierType: MinifierType) { .isEqualTo(TaskOutcome.SUCCESS) // Assert we correctly packaged app classes - val appJar = projectDir.generatedChild("externalStaging.jar") + val appJar = projectDir.generatedChild("externalStaging/classes.jar") val appClasses = ZipFile(appJar).readClasses() assertThat(appClasses).containsAtLeastElementsIn(EXPECTED_APP_CLASSES) assertThat(appClasses).containsNoneIn(EXPECTED_ANDROID_TEST_CLASSES) // Assert we correctly packaged androidTest classes - val androidTestJar = projectDir.generatedChild("externalStagingAndroidTest.jar") + val androidTestJar = projectDir.generatedChild("externalStagingAndroidTest/classes.jar") val androidTestClasses = ZipFile(androidTestJar).readClasses() assertThat(androidTestClasses).containsAtLeastElementsIn(EXPECTED_ANDROID_TEST_CLASSES) assertThat(androidTestClasses).containsNoneIn(EXPECTED_APP_CLASSES) // Assert we correctly generated rules val generatedRules = projectDir.generatedChild( - "inferredExternalStagingAndroidTestKeepRules.pro") + "externalStagingAndroidTest/inferredKeepRules.pro") assertThat(generatedRules.readText().trim()).isEqualTo( minifierType.expectedRules.map { indentRules(it.key, it.value) }.joinToString("\n") ) @@ -150,8 +148,7 @@ class KeeperFunctionalTest(private val minifierType: MinifierType) { // Have to compare slightly different strings because proguard's format is a little different assertThat(proguardConfigOutput.readText().trim().replace(" ", " ")).let { assertion -> minifierType.expectedRules.forEach { - val content = if (minifierType.isProguard) it.value?.reversed() else it.value - assertion.contains(indentRules(it.key, content)) + assertion.contains(indentRules(it.key, it.value)) } } } @@ -215,7 +212,7 @@ class KeeperFunctionalTest(private val minifierType: MinifierType) { // Check that we emitted a duplicate classes file val duplicateClasses = projectDir.generatedChild( - "diagnostics/externalStagingAndroidTestDuplicateClasses.txt") + "externalStagingAndroidTest/diagnostics/duplicateClasses.txt") assertThat(duplicateClasses.readText().trim()).isNotEmpty() } @@ -242,15 +239,17 @@ class KeeperFunctionalTest(private val minifierType: MinifierType) { } private fun runGradle(projectDir: File, vararg args: String): BuildResult { + val extraArgs = args.toMutableList() + extraArgs += "--stacktrace" return GradleRunner.create() .forwardStdOutput(System.out.writer()) .forwardStdError(System.err.writer()) .withProjectDir(projectDir) // TODO eventually test with configuration caching enabled // https://docs.gradle.org/nightly/userguide/configuration_cache.html#testkit - .withArguments("--stacktrace", "-Pandroid.enableR8=${!minifierType.isProguard}", *args) + .withArguments(extraArgs) .withPluginClasspath() -// .withDebug(true) + .withDebug(true) // Tests run in-process and way faster with this enabled .build() } @@ -335,14 +334,13 @@ private fun buildGradleFile( repositories { google() mavenCentral() - jcenter() } dependencies { // Note: this version doesn't really matter, the plugin's version will override it in the test - classpath "com.android.tools.build:gradle:4.0.0" + classpath "com.android.tools.build:gradle:4.2.1" //noinspection DifferentKotlinGradleVersion - classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:1.4.0" + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:1.5.0" } } @@ -355,12 +353,12 @@ private fun buildGradleFile( apply plugin: 'com.slack.keeper' android { - compileSdkVersion 29 + compileSdkVersion 30 defaultConfig { applicationId "com.slack.keeper.sample" minSdkVersion 21 - targetSdkVersion 29 + targetSdkVersion 30 } buildTypes { @@ -395,7 +393,6 @@ private fun buildGradleFile( repositories { google() mavenCentral() - jcenter() ${ if (automaticR8RepoManagement) "" else """ maven { diff --git a/sample-libraries/a/build.gradle b/sample-libraries/a/build.gradle index 5b37b8e6..5b7b27c6 100644 --- a/sample-libraries/a/build.gradle +++ b/sample-libraries/a/build.gradle @@ -39,6 +39,6 @@ android { } dependencies { - coreLibraryDesugaring "com.android.tools:desugar_jdk_libs:1.1.1" + coreLibraryDesugaring "com.android.tools:desugar_jdk_libs:1.1.5" implementation project(":sample-libraries:b") } diff --git a/sample/build.gradle b/sample/build.gradle index d84e7f35..45019a87 100644 --- a/sample/build.gradle +++ b/sample/build.gradle @@ -116,6 +116,9 @@ keeper { boolean isCi = providers.environmentVariable("CI").forUseAtConfigurationTime() .orElse("false") .get() == "true" +boolean verifyL8 = providers.environmentVariable("keeper.verifyL8").forUseAtConfigurationTime() + .orElse("false") + .get() == "true" if (isCi) { tasks.withType(InferAndroidTestKeepRules) .configureEach { @@ -129,20 +132,22 @@ if (isCi) { } } - tasks.withType(L8DexDesugarLibTask) - .matching { it.name == "l8DexDesugarLibExternalStagingAndroidTest" } - .configureEach { - doLast { - println "Checking expected input rules from diagnostics output" - String diagnosticFilePath = "build/intermediates/keeper/diagnostics/externalStagingAndroidTestMergedL8Rules.pro" - String diagnostics = file(diagnosticFilePath).text - if (!diagnostics.contains("-dontobfuscate")) { - throw new IllegalStateException("L8 diagnostic rules don't have Keeper's configured '-dontobfuscate', see ${diagnosticFilePath}") - } else if (!diagnostics.contains("-keep class j\$.time.Instant")) { - throw new IllegalStateException("L8 diagnostic rules include the main variant's R8-generated rules, see ${diagnosticFilePath}") + if (verifyL8) { + tasks.withType(L8DexDesugarLibTask) + .matching { it.name == "l8DexDesugarLibExternalStagingAndroidTest" } + .configureEach { + doLast { + println "Checking expected input rules from diagnostics output" + String diagnosticFilePath = "build/intermediates/keeper/l8-diagnostics/l8DexDesugarLibExternalStagingAndroidTest/patchedL8Rules.pro" + String diagnostics = file(diagnosticFilePath).text + if (!diagnostics.contains("-dontobfuscate")) { + throw new IllegalStateException("L8 diagnostic rules don't have Keeper's configured '-dontobfuscate', see ${diagnosticFilePath}") + } else if (!diagnostics.contains("-keep class j\$.time.Instant")) { + throw new IllegalStateException("L8 diagnostic rules include the main variant's R8-generated rules, see ${diagnosticFilePath}") + } } } - } + } } // Example demo of how to configure your own R8 repo @@ -167,15 +172,15 @@ repositories { dependencies { implementation project(":sample-libraries:a") - coreLibraryDesugaring "com.android.tools:desugar_jdk_libs:1.1.1" + coreLibraryDesugaring "com.android.tools:desugar_jdk_libs:1.1.5" androidTestImplementation project(":sample-libraries:c") androidTestImplementation "com.squareup.okio:okio:2.10.0" - androidTestImplementation "androidx.annotation:annotation:1.1.0" - androidTestImplementation "androidx.test:rules:1.3.0" - androidTestImplementation "androidx.test:runner:1.3.0" - androidTestImplementation "androidx.test:orchestrator:1.3.0" - androidTestImplementation "androidx.test.ext:junit:1.1.2" + androidTestImplementation "androidx.annotation:annotation:1.2.0" + androidTestImplementation "androidx.test:rules:1.4.0-beta01" + androidTestImplementation "androidx.test:runner:1.4.0-beta01" + androidTestUtil "androidx.test:orchestrator:1.4.0-beta01" + androidTestImplementation "androidx.test.ext:junit:1.1.3-beta01" androidTestImplementation "junit:junit:4.13.2" androidTestImplementation "com.google.truth:truth:1.1.2" }