Implementation of lambda merging for Kotlin lambda classes

.github/workflows/continuous_integration.yml

@@ -13,6 +13,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           path: proguard-main
+      - uses: actions/[email protected]
+        with:
+          repository: heckej/proguard-core
+          path: proguard-core
       - uses: actions/setup-java@v1
         with:
           java-version: 8

base/src/main/java/proguard/Configuration.java

@@ -200,6 +200,27 @@ public class Configuration
      */
     public boolean                      mergeInterfacesAggressively      = false;
 
+    /**
+     * An optional output file for listing the lambda to lambdagroup mapping.
+     * An empty file name means the standard output.
+     */
+    public File                         printLambdaGroupMapping;
+
+    /**
+     * Specifies whether Kotlin lambda classes can be merged into lambda groups.
+     */
+    public boolean                      mergeKotlinLambdaClasses;
+
+    /**
+     * Specifies whether Kotlin lambda classes have to be merged before or after the other optimisations.
+     */
+    public boolean lambdaMergingAfterOptimizing;
+
+    /**
+     * Specifies whether Kotlin lambda classes may be merged if they contain unexpected methods.
+     */
+    public boolean mergeLambdaClassesWithUnexpectedMethods;
+
     ///////////////////////////////////////////////////////////////////////////
     // Obfuscation options.
     ///////////////////////////////////////////////////////////////////////////

base/src/main/java/proguard/ConfigurationConstants.java

@@ -66,6 +66,10 @@ public class ConfigurationConstants
     public static final String ASSUME_VALUES_OPTION                    = "-assumevalues";
     public static final String ALLOW_ACCESS_MODIFICATION_OPTION        = "-allowaccessmodification";
     public static final String MERGE_INTERFACES_AGGRESSIVELY_OPTION    = "-mergeinterfacesaggressively";
+    public static final String PRINT_LAMBDAGROUP_MAPPING_OPTION        = "-printlambdagroupmapping";
+    public static final String MERGE_KOTLIN_LAMBDA_CLASSES_OPTION      = "-mergekotlinlambdaclasses";
+    public static final String LAMBDA_MERGING_AFTER_OPTIMIZE_OPTION    = "-lambdamergingafteroptimizing";
+    public static final String LAMBDA_MERGING_ALLOW_UNEXPECTED_METHODS = "-mergelambdaclasseswithunexpectedmethods";
 
     public static final String DONT_OBFUSCATE_OPTION                  = "-dontobfuscate";
     public static final String PRINT_MAPPING_OPTION                   = "-printmapping";

base/src/main/java/proguard/ConfigurationParser.java

@@ -190,6 +190,10 @@ public void parse(Configuration configuration)
             else if (ConfigurationConstants.ASSUME_VALUES_OPTION                             .startsWith(nextWord)) configuration.assumeValues                          = parseAssumeClassSpecificationArguments(configuration.assumeValues);
             else if (ConfigurationConstants.ALLOW_ACCESS_MODIFICATION_OPTION                 .startsWith(nextWord)) configuration.allowAccessModification               = parseNoArgument(true);
             else if (ConfigurationConstants.MERGE_INTERFACES_AGGRESSIVELY_OPTION             .startsWith(nextWord)) configuration.mergeInterfacesAggressively           = parseNoArgument(true);
+            else if (ConfigurationConstants.PRINT_LAMBDAGROUP_MAPPING_OPTION                 .startsWith(nextWord)) configuration.printLambdaGroupMapping               = parseOptionalFile();
+            else if (ConfigurationConstants.MERGE_KOTLIN_LAMBDA_CLASSES_OPTION               .startsWith(nextWord)) configuration.mergeKotlinLambdaClasses              = parseNoArgument(true);
+            else if (ConfigurationConstants.LAMBDA_MERGING_AFTER_OPTIMIZE_OPTION             .startsWith(nextWord)) configuration.lambdaMergingAfterOptimizing          = parseNoArgument(true);
+            else if (ConfigurationConstants.LAMBDA_MERGING_ALLOW_UNEXPECTED_METHODS          .startsWith(nextWord)) configuration.mergeLambdaClassesWithUnexpectedMethods = parseNoArgument(true);
 
             else if (ConfigurationConstants.DONT_OBFUSCATE_OPTION                            .startsWith(nextWord)) configuration.obfuscate                             = parseNoArgument(false);
             else if (ConfigurationConstants.PRINT_MAPPING_OPTION                             .startsWith(nextWord)) configuration.printMapping                          = parseOptionalFile();

base/src/main/java/proguard/ConfigurationWriter.java

@@ -124,6 +124,10 @@ public void write(Configuration configuration) throws IOException
         writeOption(ConfigurationConstants.OPTIMIZATION_PASSES,                  configuration.optimizationPasses);
         writeOption(ConfigurationConstants.ALLOW_ACCESS_MODIFICATION_OPTION,     configuration.allowAccessModification);
         writeOption(ConfigurationConstants.MERGE_INTERFACES_AGGRESSIVELY_OPTION, configuration.mergeInterfacesAggressively);
+        writeOption(ConfigurationConstants.PRINT_LAMBDAGROUP_MAPPING_OPTION,     configuration.printLambdaGroupMapping);
+        writeOption(ConfigurationConstants.MERGE_KOTLIN_LAMBDA_CLASSES_OPTION,   configuration.mergeKotlinLambdaClasses);
+        writeOption(ConfigurationConstants.LAMBDA_MERGING_AFTER_OPTIMIZE_OPTION, configuration.lambdaMergingAfterOptimizing);
+        writeOption(ConfigurationConstants.LAMBDA_MERGING_ALLOW_UNEXPECTED_METHODS, configuration.mergeLambdaClassesWithUnexpectedMethods);
 
         writeOption(ConfigurationConstants.DONT_OBFUSCATE_OPTION,                            !configuration.obfuscate);
         writeOption(ConfigurationConstants.PRINT_MAPPING_OPTION,                             configuration.printMapping);

base/src/main/java/proguard/ProGuard.java

@@ -29,7 +29,6 @@
 import proguard.configuration.ConfigurationLoggingAdder;
 import proguard.evaluation.IncompleteClassHierarchyException;
 import proguard.configuration.InitialStateInfo;
-import proguard.io.ExtraDataEntryNameMap;
 import proguard.logging.Logging;
 import proguard.mark.Marker;
 import proguard.obfuscate.NameObfuscationReferenceFixer;
@@ -39,6 +38,7 @@
 import proguard.optimize.LineNumberTrimmer;
 import proguard.optimize.Optimizer;
 import proguard.optimize.gson.GsonOptimizer;
+import proguard.optimize.kotlin.KotlinLambdaMerger;
 import proguard.optimize.peephole.LineNumberLinearizer;
 import proguard.pass.PassRunner;
 import proguard.preverify.*;
@@ -177,6 +177,11 @@ public void execute() throws Exception
                 shrink(false);
             }
 
+            if (configuration.mergeKotlinLambdaClasses && !configuration.lambdaMergingAfterOptimizing)
+            {
+                mergeKotlinLambdaClasses();
+            }
+
             // Create a matcher for filtering optimizations.
             StringMatcher filter = configuration.optimizations != null ?
                 new ListParser(new NameParser()).parse(configuration.optimizations) :
@@ -194,6 +199,11 @@ public void execute() throws Exception
                 linearizeLineNumbers();
             }
 
+            if (configuration.mergeKotlinLambdaClasses && configuration.lambdaMergingAfterOptimizing)
+            {
+                mergeKotlinLambdaClasses();
+            }
+
             if (configuration.obfuscate)
             {
                 obfuscate();
@@ -424,6 +434,14 @@ private void shrink(boolean afterOptimizer) throws Exception
         }
     }
 
+    /**
+     * Reduce the size needed to represent Kotlin lambda's.
+     * The classes that are generated for lambda's with a same structure and from the same package are merged into a group.
+     */
+    private void mergeKotlinLambdaClasses() throws Exception {
+        passRunner.run(new KotlinLambdaMerger(configuration), appView);
+    }
+
 
     /**
      * Optimizes usages of the Gson library.
@@ -452,6 +470,10 @@ private void optimize() throws Exception
             {
                 shrink(true);
             }
+            if (optimizationPass == configuration.optimizationPasses - 2)
+            {
+                mergeKotlinLambdaClasses();
+            }
         }
     }
 

base/src/main/java/proguard/optimize/Optimizer.java

@@ -38,6 +38,7 @@
 import proguard.optimize.evaluation.*;
 import proguard.optimize.evaluation.InstructionUsageMarker;
 import proguard.optimize.info.*;
+import proguard.optimize.kotlin.KotlinLambdaMerger;
 import proguard.optimize.peephole.*;
 import proguard.pass.Pass;
 import proguard.util.*;
@@ -195,11 +196,18 @@ public Optimizer(Configuration configuration)
 
 
     @Override
-    public void execute(AppView appView) throws IOException
+    public void execute(AppView appView) throws Exception
     {
         if (!moreOptimizationsPossible)
         {
-            return;
+            if (KotlinLambdaMerger.lambdaMergingDone)
+            {
+                return;
+            }
+            else
+            {
+                new KotlinLambdaMerger(configuration).execute(appView);
+            }
         }
 
         // Create a matcher for filtering optimizations.

base/src/main/java/proguard/optimize/info/ProgramClassOptimizationInfo.java

@@ -44,6 +44,8 @@ public class ProgramClassOptimizationInfo
     private volatile boolean mayBeMerged                   = true;
     private volatile Clazz   wrappedClass;
     private volatile Clazz   targetClass;
+    private volatile Clazz   lambdaGroup;
+    private volatile int     lambdaGroupClassId;
 
 
     public boolean isKept()
@@ -250,6 +252,29 @@ public Clazz getTargetClass()
     }
 
 
+    public void setLambdaGroup(Clazz lambdaGroup)
+    {
+        this.lambdaGroup = lambdaGroup;
+    }
+
+
+    public Clazz getLambdaGroup()
+    {
+        return lambdaGroup;
+    }
+
+    public void setLambdaGroupClassId(int classId)
+    {
+        this.lambdaGroupClassId = classId;
+    }
+
+
+    public int getLambdaGroupClassId()
+    {
+        return lambdaGroupClassId;
+    }
+
+
     /**
      * Merges in the given information of a class that is merged.
      */

base/src/main/java/proguard/optimize/kotlin/KotlinLambdaClassMerger.java

@@ -0,0 +1,100 @@
+package proguard.optimize.kotlin;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import proguard.Configuration;
+import proguard.classfile.*;
+import proguard.classfile.attribute.Attribute;
+import proguard.classfile.attribute.CodeAttribute;
+import proguard.classfile.attribute.ExceptionsAttribute;
+import proguard.classfile.attribute.LineNumberTableAttribute;
+import proguard.classfile.attribute.visitor.AttributeVisitor;
+import proguard.classfile.editor.ClassBuilder;
+import proguard.classfile.editor.ClassEditor;
+import proguard.classfile.editor.ConstantPoolEditor;
+import proguard.classfile.editor.InstructionSequenceBuilder;
+import proguard.classfile.instruction.*;
+import proguard.classfile.instruction.visitor.InstructionVisitor;
+import proguard.classfile.util.ClassSuperHierarchyInitializer;
+import proguard.classfile.util.ClassUtil;
+import proguard.classfile.visitor.*;
+import proguard.io.ExtraDataEntryNameMap;
+import proguard.optimize.info.ProgramClassOptimizationInfo;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class KotlinLambdaClassMerger implements ClassPoolVisitor {
+
+    public static final String NAME_LAMBDA_GROUP = "LambdaGroup";
+    private final ClassVisitor lambdaGroupVisitor;
+    private final ClassVisitor notMergedLambdaVisitor;
+    private final Configuration configuration;
+    private final ClassPool programClassPool;
+    private final ClassPool libraryClassPool;
+    private final ExtraDataEntryNameMap extraDataEntryNameMap;
+    private static final Logger logger = LogManager.getLogger(KotlinLambdaClassMerger.class);
+
+    public KotlinLambdaClassMerger(final Configuration         configuration,
+                                   final ClassPool             programClassPool,
+                                   final ClassPool             libraryClassPool,
+                                   final ClassVisitor          lambdaGroupVisitor,
+                                   final ClassVisitor          notMergedLambdaVisitor,
+                                   final ExtraDataEntryNameMap extraDataEntryNameMap)
+    {
+        this.configuration          = configuration;
+        this.programClassPool       = programClassPool;
+        this.libraryClassPool       = libraryClassPool;
+        this.lambdaGroupVisitor     = lambdaGroupVisitor;
+        this.notMergedLambdaVisitor = notMergedLambdaVisitor;
+        this.extraDataEntryNameMap  = extraDataEntryNameMap;
+    }
+
+    @Override
+    public void visitClassPool(ClassPool lambdaClassPool)
+    {
+        // don't merge lambda's if there is only one or zero
+        if (lambdaClassPool.size() < 2)
+        {
+            if (notMergedLambdaVisitor != null) {
+                lambdaClassPool.classesAccept(notMergedLambdaVisitor);
+            }
+            return;
+        }
+
+        // choose a name for the lambda group
+        // ensure that the lambda group is in the same package as the classes of the class pool
+        String lambdaGroupName = getPackagePrefixOfClasses(lambdaClassPool) + NAME_LAMBDA_GROUP;
+        logger.info("Creating lambda group with name {}", ClassUtil.externalClassName(lambdaGroupName));
+
+        // create a lambda group builder
+        KotlinLambdaGroupBuilder lambdaGroupBuilder = new KotlinLambdaGroupBuilder(lambdaGroupName,
+                                                                                   this.configuration,
+                                                                                   this.programClassPool,
+                                                                                   this.libraryClassPool,
+                                                                                   this.extraDataEntryNameMap,
+                                                                                   this.notMergedLambdaVisitor);
+
+        // visit each lambda of this package to add their implementations to the lambda group
+        lambdaClassPool.classesAccept(lambdaGroupBuilder);
+
+        ProgramClass lambdaGroup = lambdaGroupBuilder.build();
+
+        ProgramClassOptimizationInfo.setProgramClassOptimizationInfo(lambdaGroup);
+        ProgramClassOptimizationInfo optimizationInfo =
+                ProgramClassOptimizationInfo.getProgramClassOptimizationInfo(lambdaGroup);
+        optimizationInfo.setLambdaGroup(lambdaGroup);
+
+        // let the lambda group visitor visit the newly created lambda group
+        this.lambdaGroupVisitor.visitProgramClass(lambdaGroup);
+    }
+
+    private static String getPackagePrefixOfClasses(ClassPool classPool)
+    {
+        // Assume that all classes in the given class pool are in the same package.
+        String someClassName = classPool.classNames().next();
+        return ClassUtil.internalPackagePrefix(someClassName);
+    }
+}