From 598f045c4af3fefd08c9253206af2b3a50cdfb87 Mon Sep 17 00:00:00 2001 From: Alexei Nicoara Date: Thu, 9 Mar 2023 16:12:25 +0000 Subject: [PATCH 0001/1460] DO NOT MERGE Update the module versions in udc-mainline-prod Bug: 258850762 Change-Id: I8844c712bc9f94cecfe15ce343fe45f008244145 Ignore-AOSP-First: Version bump only needed on udc-mainline-prod branch not on git_master --- android/updatable_modules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/updatable_modules.go b/android/updatable_modules.go index 6d0eeb716f..db45637508 100644 --- a/android/updatable_modules.go +++ b/android/updatable_modules.go @@ -33,4 +33,4 @@ package android // * AOSP - xx9990000 // * x-mainline-prod - xx9990000 // * master - 990090000 -const DefaultUpdatableModuleVersion = "340090000" +const DefaultUpdatableModuleVersion = "349990000" From ab959882b39f43a65b9bbc63e6688bb43825263b Mon Sep 17 00:00:00 2001 From: Eric Miao Date: Thu, 1 Dec 2022 09:35:41 -0800 Subject: [PATCH 0002/1460] Enable compact resource entries for aapt linking Bug: 237583012 This enables compact resource entries by default when linking with aapt2, which helps generating resource file with smaller size. Change-Id: Ibf22f301491dcc56346a5947ee115e3059eed6aa Ignore-AOSP-First: this CL has dependency on non-AOSP-First repos --- java/aar.go | 1 + 1 file changed, 1 insertion(+) diff --git a/java/aar.go b/java/aar.go index f162a17225..324e0af2ef 100644 --- a/java/aar.go +++ b/java/aar.go @@ -176,6 +176,7 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext android.SdkConte linkFlags = append(linkFlags, a.aaptProperties.Aaptflags...) linkFlags = append(linkFlags, "--no-static-lib-packages") + linkFlags = append(linkFlags, "--enable-compact-entries") // Find implicit or explicit asset and resource dirs assetDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Asset_dirs, "assets") From 825cff4169ef07880ae4ba51b978873ed276eb73 Mon Sep 17 00:00:00 2001 From: Beatrice Marchegiani Date: Tue, 28 Mar 2023 11:30:12 +0000 Subject: [PATCH 0003/1460] Revert "Enable compact resource entries for aapt linking" This reverts commit ab959882b39f43a65b9bbc63e6688bb43825263b. Reason for revert: DroidMonitor-triggered revert due to breakage BackupFrameworksServicesRoboTests, bug b/275422786 Identified in culprit search session (https://android-build.googleplex.com/builds/culprit-assistant/run/531ea159-f8c1-4ddf-a29f-307c3a9c057e) Bug:275422786 Change-Id: I79b550ba109ce0da23074c143c65511b6f193df4 --- java/aar.go | 1 - 1 file changed, 1 deletion(-) diff --git a/java/aar.go b/java/aar.go index 324e0af2ef..f162a17225 100644 --- a/java/aar.go +++ b/java/aar.go @@ -176,7 +176,6 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext android.SdkConte linkFlags = append(linkFlags, a.aaptProperties.Aaptflags...) linkFlags = append(linkFlags, "--no-static-lib-packages") - linkFlags = append(linkFlags, "--enable-compact-entries") // Find implicit or explicit asset and resource dirs assetDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Asset_dirs, "assets") From 40eab20c34ca2084d030f0af3ef3b5a237263df3 Mon Sep 17 00:00:00 2001 From: Eric Miao Date: Thu, 30 Mar 2023 16:57:17 +0000 Subject: [PATCH 0004/1460] Revert "Revert "Enable compact resource entries for aapt linking"" This reverts commit 825cff4169ef07880ae4ba51b978873ed276eb73. Reason for revert: Failed test has been fixed - b/275572186 Change-Id: I661206f4e0e783a5c403ac15ed33a93ba3acaf40 --- java/aar.go | 1 + 1 file changed, 1 insertion(+) diff --git a/java/aar.go b/java/aar.go index 47e6efae35..4fdcb80b8b 100644 --- a/java/aar.go +++ b/java/aar.go @@ -176,6 +176,7 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext android.SdkConte linkFlags = append(linkFlags, a.aaptProperties.Aaptflags...) linkFlags = append(linkFlags, "--no-static-lib-packages") + linkFlags = append(linkFlags, "--enable-compact-entries") // Find implicit or explicit asset and resource dirs assetDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Asset_dirs, "assets") From b507831b7101cef6f1dcab6e9a9cbdd2ce6f3524 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Wed, 29 Mar 2023 23:25:49 +0000 Subject: [PATCH 0005/1460] Utilize from text core platform api surface jar in build Use JavaApiLibraryName function to redirect the usage of core platform api stubs from .txt files based on config. Test: m --build-from-text-stub Change-Id: I926a0a455fed301ba4ff9dfa509d4dbbbd076029 --- android/sdk_version.go | 9 +++++++++ java/base.go | 7 +++++-- java/config/makevars.go | 7 +++++-- java/hiddenapi_modular.go | 2 +- java/java.go | 4 +++- java/legacy_core_platform_api_usage.go | 8 ++++---- 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/android/sdk_version.go b/android/sdk_version.go index 0ae8073522..08762eff2a 100644 --- a/android/sdk_version.go +++ b/android/sdk_version.go @@ -100,6 +100,15 @@ func JavaApiLibraryName(c Config, name string) string { return name } +// JavaApiLibraryNames applies JavaApiLibraryName to the list of java_library names. +func JavaApiLibraryNames(c Config, names []string) []string { + apiLibs := make([]string, len(names)) + for i, name := range names { + apiLibs[i] = JavaApiLibraryName(c, name) + } + return apiLibs +} + func (k SdkKind) DefaultJavaLibraryName() string { switch k { case SdkPublic: diff --git a/java/base.go b/java/base.go index 991132321b..374138c781 100644 --- a/java/base.go +++ b/java/base.go @@ -1923,9 +1923,12 @@ type moduleWithSdkDep interface { func (m *Module) getSdkLinkType(ctx android.BaseModuleContext, name string) (ret sdkLinkType, stubs bool) { switch name { - case android.SdkCore.JavaLibraryName(ctx.Config()), "legacy.core.platform.api.stubs", "stable.core.platform.api.stubs", + case android.SdkCore.JavaLibraryName(ctx.Config()), + android.JavaApiLibraryName(ctx.Config(), "legacy.core.platform.api.stubs"), + android.JavaApiLibraryName(ctx.Config(), "stable.core.platform.api.stubs"), "stub-annotations", "private-stub-annotations-jar", - "core-lambda-stubs", "core-generated-annotation-stubs": + android.JavaApiLibraryName(ctx.Config(), "core-lambda-stubs"), + "core-generated-annotation-stubs": return javaCore, true case android.SdkPublic.JavaLibraryName(ctx.Config()): return javaSdk, true diff --git a/java/config/makevars.go b/java/config/makevars.go index 273aca0b4f..d383d98887 100644 --- a/java/config/makevars.go +++ b/java/config/makevars.go @@ -28,8 +28,11 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("FRAMEWORK_LIBRARIES", strings.Join(FrameworkLibraries, " ")) // These are used by make when LOCAL_PRIVATE_PLATFORM_APIS is set (equivalent to platform_apis in blueprint): - ctx.Strict("LEGACY_CORE_PLATFORM_BOOTCLASSPATH_LIBRARIES", strings.Join(LegacyCorePlatformBootclasspathLibraries, " ")) - ctx.Strict("LEGACY_CORE_PLATFORM_SYSTEM_MODULES", LegacyCorePlatformSystemModules) + ctx.Strict("LEGACY_CORE_PLATFORM_BOOTCLASSPATH_LIBRARIES", + strings.Join(android.JavaApiLibraryNames(ctx.Config(), LegacyCorePlatformBootclasspathLibraries), " ")) + ctx.Strict("LEGACY_CORE_PLATFORM_SYSTEM_MODULES", + android.JavaApiLibraryName(ctx.Config(), LegacyCorePlatformSystemModules), + ) ctx.Strict("ANDROID_JAVA_HOME", "${JavaHome}") ctx.Strict("ANDROID_JAVA8_HOME", "prebuilts/jdk/jdk8/${hostPrebuiltTag}") diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index c6176e67c6..96e084a205 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -241,7 +241,7 @@ func hiddenAPIComputeMonolithicStubLibModules(config android.Config) map[*Hidden testStubModules = append(testStubModules, android.SdkTest.JavaLibraryName(config)) } // We do not have prebuilts of the core platform api yet - corePlatformStubModules = append(corePlatformStubModules, "legacy.core.platform.api.stubs") + corePlatformStubModules = append(corePlatformStubModules, android.JavaApiLibraryName(config, "legacy.core.platform.api.stubs")) // Allow products to define their own stubs for custom product jars that apps can use. publicStubModules = append(publicStubModules, config.ProductHiddenAPIStubs()...) diff --git a/java/java.go b/java/java.go index d400b0cfb9..18d9d33742 100644 --- a/java/java.go +++ b/java/java.go @@ -456,7 +456,9 @@ func sdkDeps(ctx android.BottomUpMutatorContext, sdkContext android.SdkContext, ctx.AddVariationDependencies(nil, java9LibTag, sdkDep.java9Classpath...) ctx.AddVariationDependencies(nil, sdkLibTag, sdkDep.classpath...) if d.effectiveOptimizeEnabled() && sdkDep.hasStandardLibs() { - ctx.AddVariationDependencies(nil, proguardRaiseTag, config.LegacyCorePlatformBootclasspathLibraries...) + ctx.AddVariationDependencies(nil, proguardRaiseTag, + android.JavaApiLibraryNames(ctx.Config(), config.LegacyCorePlatformBootclasspathLibraries)..., + ) } if d.effectiveOptimizeEnabled() && sdkDep.hasFrameworkLibs() { ctx.AddVariationDependencies(nil, proguardRaiseTag, config.FrameworkLibraries...) diff --git a/java/legacy_core_platform_api_usage.go b/java/legacy_core_platform_api_usage.go index 6cb549eaf1..04c6d05b18 100644 --- a/java/legacy_core_platform_api_usage.go +++ b/java/legacy_core_platform_api_usage.go @@ -93,16 +93,16 @@ func useLegacyCorePlatformApi(ctx android.EarlyModuleContext, moduleName string) func corePlatformSystemModules(ctx android.EarlyModuleContext) string { if useLegacyCorePlatformApi(ctx, ctx.ModuleName()) { - return config.LegacyCorePlatformSystemModules + return android.JavaApiLibraryName(ctx.Config(), config.LegacyCorePlatformSystemModules) } else { - return config.StableCorePlatformSystemModules + return android.JavaApiLibraryName(ctx.Config(), config.StableCorePlatformSystemModules) } } func corePlatformBootclasspathLibraries(ctx android.EarlyModuleContext) []string { if useLegacyCorePlatformApi(ctx, ctx.ModuleName()) { - return config.LegacyCorePlatformBootclasspathLibraries + return android.JavaApiLibraryNames(ctx.Config(), config.LegacyCorePlatformBootclasspathLibraries) } else { - return config.StableCorePlatformBootclasspathLibraries + return android.JavaApiLibraryNames(ctx.Config(), config.StableCorePlatformBootclasspathLibraries) } } From 96e5370a3606478c0c20c761ce4d135755494490 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Fri, 7 Apr 2023 15:09:37 -0400 Subject: [PATCH 0006/1460] add neuralnetworks to prod allowlist Bug: 275722093 Change-Id: Ib086bdbbdd433c600efc1be4922226776983915d --- android/allowlists/allowlists.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 8227abec73..9e87a16c2a 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -1481,6 +1481,11 @@ var ( "adb_tls_connection_test", // M9: mixed builds for mainline trains launch "api_fingerprint", + // M11: neuralnetworks launch + "com.android.neuralnetworks", + "test_com.android.neuralnetworks", + "libneuralnetworks", + "libneuralnetworks_static", } // Staging-mode allowlist. Modules in this list are only built @@ -1488,11 +1493,7 @@ var ( // which will soon be added to the prod allowlist. // It is implicit that all modules in ProdMixedBuildsEnabledList will // also be built - do not add them to this list. - StagingMixedBuildsEnabledList = []string{ - "com.android.neuralnetworks", - "libneuralnetworks", - "libneuralnetworks_static", - } + StagingMixedBuildsEnabledList = []string{} // These should be the libs that are included by the apexes in the ProdMixedBuildsEnabledList ProdDclaMixedBuildsEnabledList = []string{ From f6275096eafe77cbdaaa10f2ed0336c3e44fe583 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Tue, 11 Apr 2023 11:06:07 -0400 Subject: [PATCH 0007/1460] Remove module with a cc_test_library static dep Currently we treat cc_test_library as a shared library but it really can be both a static and a shared library. This causes issues when trying to statically link a shared library. Test: bp2build.sh Change-Id: Ie3a4c0ffa607e1c9fcffe015d216be7a9bccfb4d --- android/allowlists/allowlists.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 1ece9fa7d5..ffd2aab9e9 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -1051,7 +1051,7 @@ var ( "libcfi-test", "libcfi-test-bad", "libcrash_test", - // "libcrypto_fuzz_unsafe", + "libcrypto_fuzz_unsafe", "libdl_preempt_test_1", "libdl_preempt_test_2", "libdl_test_df_1_global", @@ -1262,7 +1262,7 @@ var ( "librelocations-fat", "libsegment_gap_inner", "libsegment_gap_outer", - // "libssl_fuzz_unsafe", + "libssl_fuzz_unsafe", "libstatssocket_private", "libsysv-hash-table-library", "libtest_atexit", From f5c872f36b129d97c222df61760381c1449558f5 Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Fri, 14 Apr 2023 00:17:47 +0900 Subject: [PATCH 0008/1460] Turn on HINT_FROM_SOONG by default In most case, turning on HINT_FROM_SOONG improves build performance even though a model is not so accurate. At least, it doesn't cause performance regression (very likely) Bug: 275458825 Test: m Change-Id: Ie4ff69e68a93f6f0db3d060980657ae623816b2b --- ui/build/config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index a755d14fdd..b746cf8042 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -313,8 +313,9 @@ func UploadOnlyConfig(ctx Context, _ ...string) Config { func NewConfig(ctx Context, args ...string) Config { ret := &configImpl{ - environ: OsEnvironment(), - sandboxConfig: &SandboxConfig{}, + environ: OsEnvironment(), + sandboxConfig: &SandboxConfig{}, + ninjaWeightListSource: HINT_FROM_SOONG, } // Default matching ninja From b745e2e20590ad3dac17ec3b503dcc023c2e53bf Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Tue, 11 Apr 2023 14:28:43 +0900 Subject: [PATCH 0009/1460] Use module type, size of srcs or deps to prioritize module Instead of listing up module names, use another information from soong such as module type, or size of inputs. I focused on recall, not precision or accuracy, because false negative module can slow down whole build Bug: 273282046 Test: m --ninja_weight_source=ninja_log Change-Id: I8fcc26f1312fda36197c787d240bc3d7b5346481 --- android/allowlists/allowlists.go | 277 ++----------------------------- cmd/soong_build/main.go | 48 ++++-- 2 files changed, 52 insertions(+), 273 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 7f3dd2dbe0..eb491d8ff4 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -38,7 +38,13 @@ const ( // allows modules to opt-in. Bp2BuildDefaultFalseRecursively - DEFAULT_NINJA_WEIGHT = 1000 + // Modules with build time of more than half a minute should have high priority. + DEFAULT_PRIORITIZED_WEIGHT = 1000 + // Modules with build time of more than a few minute should have higher priority. + HIGH_PRIORITIZED_WEIGHT = 10 * DEFAULT_PRIORITIZED_WEIGHT + // Modules with inputs greater than the threshold should have high priority. + // Adjust this threshold if there are lots of wrong predictions. + INPUT_SIZE_THRESHOLD = 50 ) var ( @@ -1511,267 +1517,12 @@ var ( // "libstagefright_flacdec", // "libutils", - // TODO(b/273282046): Make this list customizable to support various targets. - // The list of modules which are expected to spend lots of build time. - // With `--ninja_weight_source=soong`, ninja builds these modules and deps first. - HugeModulesMap = map[string]int{ - "AccountManagementApp": DEFAULT_NINJA_WEIGHT, - "ActivityManagerPerfTestsStubApp1": DEFAULT_NINJA_WEIGHT, - "ActivityManagerPerfTestsStubApp2": DEFAULT_NINJA_WEIGHT, - "ActivityManagerPerfTestsStubApp3": DEFAULT_NINJA_WEIGHT, - "api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT, - "AppCompatibilityTest": DEFAULT_NINJA_WEIGHT, - "AppTransitionTests": DEFAULT_NINJA_WEIGHT, - "art_compiler_tests": DEFAULT_NINJA_WEIGHT, - "art.module.intra.core.api.stubs.source": DEFAULT_NINJA_WEIGHT, - "art.module.public.api.stubs.source": DEFAULT_NINJA_WEIGHT, - "AttestationVerificationTest": DEFAULT_NINJA_WEIGHT, - "BatteryUsageStatsProtoTests": DEFAULT_NINJA_WEIGHT, - "bluetooth_test_gd_unit": DEFAULT_NINJA_WEIGHT, - "Bluetooth": DEFAULT_NINJA_WEIGHT, - "BluetoothInstrumentationTests": DEFAULT_NINJA_WEIGHT, - "Calendar": DEFAULT_NINJA_WEIGHT, - "CalendarProvider": DEFAULT_NINJA_WEIGHT, - "Camera2": DEFAULT_NINJA_WEIGHT, - "CarRotaryControllerUnitTests": DEFAULT_NINJA_WEIGHT, - "CarSettingsForUnitTesting": DEFAULT_NINJA_WEIGHT, - "CarSettingsUnitTests": DEFAULT_NINJA_WEIGHT, - "CarSystemUI-tests": DEFAULT_NINJA_WEIGHT, - "CellBroadcastApp": DEFAULT_NINJA_WEIGHT, - "CellBroadcastLegacyApp": DEFAULT_NINJA_WEIGHT, - "CellBroadcastReceiverOemUnitTests": DEFAULT_NINJA_WEIGHT, - "CellBroadcastServiceModule": DEFAULT_NINJA_WEIGHT, - "CompanionDeviceManager": DEFAULT_NINJA_WEIGHT, - "ConnectivityChecker": DEFAULT_NINJA_WEIGHT, - "Contacts": DEFAULT_NINJA_WEIGHT, - "ContactsProvider": DEFAULT_NINJA_WEIGHT, - "ContentCapturePerfTests": DEFAULT_NINJA_WEIGHT, - "CorePerfTests": DEFAULT_NINJA_WEIGHT, - "crosvm": DEFAULT_NINJA_WEIGHT, - "CtsDomainVerificationDeviceMultiUserTestCases": DEFAULT_NINJA_WEIGHT, - "CtsLogdTestCases": DEFAULT_NINJA_WEIGHT, - "CtsMediaProviderTranscodeTests": DEFAULT_NINJA_WEIGHT, - "CtsRollbackManagerHostTestHelperApp": DEFAULT_NINJA_WEIGHT, - "CtsRollbackManagerHostTestHelperApp2": DEFAULT_NINJA_WEIGHT, - "CtsRootPackageInstallerTestCases": DEFAULT_NINJA_WEIGHT, - "CtsRootRollbackManagerHostTestHelperApp": DEFAULT_NINJA_WEIGHT, - "CtsTranscodeTestAppSupportsHevc": DEFAULT_NINJA_WEIGHT, - "CtsTranscodeTestAppSupportsSlowMotion": DEFAULT_NINJA_WEIGHT, - "CuttlefishDisplayHotplugHelperApp": DEFAULT_NINJA_WEIGHT, - "cvd-host_package": DEFAULT_NINJA_WEIGHT, - "DelegateTestApp": DEFAULT_NINJA_WEIGHT, - "DeskClock": DEFAULT_NINJA_WEIGHT, - "Development": DEFAULT_NINJA_WEIGHT, - "DeviceAdminTestApp": DEFAULT_NINJA_WEIGHT, - "DevicePolicyManagementRoleHolderTestApp": DEFAULT_NINJA_WEIGHT, - "dex2oatd": DEFAULT_NINJA_WEIGHT, - "DocumentsUI": DEFAULT_NINJA_WEIGHT, - "EasterEgg": DEFAULT_NINJA_WEIGHT, - "EffectProxyTest": DEFAULT_NINJA_WEIGHT, - "EmergencyInfo": DEFAULT_NINJA_WEIGHT, - "EmptyTestApp": DEFAULT_NINJA_WEIGHT, - "ExtServices": DEFAULT_NINJA_WEIGHT, - "FacebookAppsScenarioTests": DEFAULT_NINJA_WEIGHT, - "flickerlib-core": DEFAULT_NINJA_WEIGHT, - "flickerlib": DEFAULT_NINJA_WEIGHT, - "FlickerLibTest": DEFAULT_NINJA_WEIGHT, - "FlickerTests": DEFAULT_NINJA_WEIGHT, - "framework-minus-apex": DEFAULT_NINJA_WEIGHT, - "framework-res": DEFAULT_NINJA_WEIGHT, - "FrameworksCoreTests": DEFAULT_NINJA_WEIGHT, - "FrameworksMockingCoreTests": DEFAULT_NINJA_WEIGHT, - "FrameworksMockingServicesTests": DEFAULT_NINJA_WEIGHT, - "FrameworksNetSmokeTests": DEFAULT_NINJA_WEIGHT, - "FrameworksNetTests": DEFAULT_NINJA_WEIGHT, - "FrameworksServicesTests": DEFAULT_NINJA_WEIGHT, - "FrameworksTelephonyTests": DEFAULT_NINJA_WEIGHT, - "FrameworksUiServicesTests": DEFAULT_NINJA_WEIGHT, - "FrameworksVcnTests": DEFAULT_NINJA_WEIGHT, - "Gallery2": DEFAULT_NINJA_WEIGHT, - "GameCoreDevice": DEFAULT_NINJA_WEIGHT, - "GoogleBluetoothInstrumentationTests": DEFAULT_NINJA_WEIGHT, - "guice_munged_srcs": DEFAULT_NINJA_WEIGHT, - "HalfSheetUX": DEFAULT_NINJA_WEIGHT, - "ImePerfTests": DEFAULT_NINJA_WEIGHT, - "imgdiag": DEFAULT_NINJA_WEIGHT, - "ImsServiceEntitlement": DEFAULT_NINJA_WEIGHT, - "ImsServiceEntitlementUnitTests": DEFAULT_NINJA_WEIGHT, - "InputTests": DEFAULT_NINJA_WEIGHT, - "InstallTest": DEFAULT_NINJA_WEIGHT, - "IntentResolver": DEFAULT_NINJA_WEIGHT, - "JankBench": DEFAULT_NINJA_WEIGHT, - "jsilver": DEFAULT_NINJA_WEIGHT, - "KeyChain": DEFAULT_NINJA_WEIGHT, - "KeyChainTests": DEFAULT_NINJA_WEIGHT, - "keystore2": DEFAULT_NINJA_WEIGHT, - "LargeResourcesCompressed": DEFAULT_NINJA_WEIGHT, - "LatinIME": DEFAULT_NINJA_WEIGHT, - "Launcher3QuickStepLib": DEFAULT_NINJA_WEIGHT, - "libaom": DEFAULT_NINJA_WEIGHT, - "libart-broken": DEFAULT_NINJA_WEIGHT, - "libart-compiler": DEFAULT_NINJA_WEIGHT, - "libart-disassembler": DEFAULT_NINJA_WEIGHT, - "libart": DEFAULT_NINJA_WEIGHT, - "libartd": DEFAULT_NINJA_WEIGHT, - "libaudiohal@7.1": DEFAULT_NINJA_WEIGHT, - "libbluetooth_core_rs": DEFAULT_NINJA_WEIGHT, - "libbluetooth_gd_unit_tests": DEFAULT_NINJA_WEIGHT, - "libbluetooth_gd": DEFAULT_NINJA_WEIGHT, - "libbluetooth_rs": DEFAULT_NINJA_WEIGHT, - "libbluetooth-for-tests": DEFAULT_NINJA_WEIGHT, - "libbt_common": DEFAULT_NINJA_WEIGHT, - "libbt_packets_nonapex": DEFAULT_NINJA_WEIGHT, - "libbt_packets": DEFAULT_NINJA_WEIGHT, - "libbt_shim_ffi": DEFAULT_NINJA_WEIGHT, - "libbt_shim": DEFAULT_NINJA_WEIGHT, - "libbt-audio-hal-interface": DEFAULT_NINJA_WEIGHT, - "libbt-bta-core": DEFAULT_NINJA_WEIGHT, - "libbt-bta": DEFAULT_NINJA_WEIGHT, - "libbt-common": DEFAULT_NINJA_WEIGHT, - "libbt-hci": DEFAULT_NINJA_WEIGHT, - "libbt-platform-protos-lite": DEFAULT_NINJA_WEIGHT, - "libbt-protos-lite": DEFAULT_NINJA_WEIGHT, - "libbt-sbc-decoder": DEFAULT_NINJA_WEIGHT, - "libc": DEFAULT_NINJA_WEIGHT, - "libclap": DEFAULT_NINJA_WEIGHT, - "libcodec2_soft_av1dec_gav1": DEFAULT_NINJA_WEIGHT, - "libcompositionengine_test": DEFAULT_NINJA_WEIGHT, - "libdevices": DEFAULT_NINJA_WEIGHT, - "libfrontend_proto": DEFAULT_NINJA_WEIGHT, - "libhwtrust": DEFAULT_NINJA_WEIGHT, - "libjni": DEFAULT_NINJA_WEIGHT, - "libkeystore2": DEFAULT_NINJA_WEIGHT, - "libkmr_ta": DEFAULT_NINJA_WEIGHT, - "liblmp": DEFAULT_NINJA_WEIGHT, - "libopenjdkjvmtid": DEFAULT_NINJA_WEIGHT, - "libprotobuf_deprecated": DEFAULT_NINJA_WEIGHT, - "libprotobuf": DEFAULT_NINJA_WEIGHT, - "libregex": DEFAULT_NINJA_WEIGHT, - "LibStatsPullTests": DEFAULT_NINJA_WEIGHT, - "libstd": DEFAULT_NINJA_WEIGHT, - "libsurfaceflinger_unittest": DEFAULT_NINJA_WEIGHT, - "libsyn": DEFAULT_NINJA_WEIGHT, - "libtokio": DEFAULT_NINJA_WEIGHT, - "libuwb_core": DEFAULT_NINJA_WEIGHT, - "libuwb_uci_jni_rust": DEFAULT_NINJA_WEIGHT, - "libuwb_uci_packets": DEFAULT_NINJA_WEIGHT, - "libvpx": DEFAULT_NINJA_WEIGHT, - "libvulkan_enc": DEFAULT_NINJA_WEIGHT, - "libwebrtc": DEFAULT_NINJA_WEIGHT, - "LiveWallpapersPicker": DEFAULT_NINJA_WEIGHT, - "LockTaskApp": DEFAULT_NINJA_WEIGHT, - "LongevityPlatformLibTests": DEFAULT_NINJA_WEIGHT, - "ManagedProvisioning": DEFAULT_NINJA_WEIGHT, - "ManagedProvisioningTests": DEFAULT_NINJA_WEIGHT, - "MediaProvider": DEFAULT_NINJA_WEIGHT, - "MediaProviderClientTests": DEFAULT_NINJA_WEIGHT, - "MediaProviderLegacy": DEFAULT_NINJA_WEIGHT, - "messaging": DEFAULT_NINJA_WEIGHT, - "metalava": DEFAULT_NINJA_WEIGHT, - "MicrobenchmarkRunnerTests": DEFAULT_NINJA_WEIGHT, - "microdroid_manager": DEFAULT_NINJA_WEIGHT, - "minikin_tests": DEFAULT_NINJA_WEIGHT, - "MLCTestApp": DEFAULT_NINJA_WEIGHT, - "MmsService": DEFAULT_NINJA_WEIGHT, - "MmsServiceTests": DEFAULT_NINJA_WEIGHT, - "module-lib-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT, - "motion_tool_lib_tests": DEFAULT_NINJA_WEIGHT, - "MtpService": DEFAULT_NINJA_WEIGHT, - "MultiUserTests": DEFAULT_NINJA_WEIGHT, - "NearbyIntegrationUiTests": DEFAULT_NINJA_WEIGHT, - "net_test_bluetooth": DEFAULT_NINJA_WEIGHT, - "net_test_btif": DEFAULT_NINJA_WEIGHT, - "net_test_main_shim": DEFAULT_NINJA_WEIGHT, - "net_test_stack": DEFAULT_NINJA_WEIGHT, - "net-tests-utils": DEFAULT_NINJA_WEIGHT, - "NetworkStackCoverageTests": DEFAULT_NINJA_WEIGHT, - "NetworkStackIntegrationTests": DEFAULT_NINJA_WEIGHT, - "NetworkStackNext": DEFAULT_NINJA_WEIGHT, - "NfcNci": DEFAULT_NINJA_WEIGHT, - "NfcNciUnitTests": DEFAULT_NINJA_WEIGHT, - "NotEmptyTestApp": DEFAULT_NINJA_WEIGHT, - "NotificationFunctionalTests": DEFAULT_NINJA_WEIGHT, - "oatdumpd": DEFAULT_NINJA_WEIGHT, - "OsuLogin": DEFAULT_NINJA_WEIGHT, - "PackageInstaller": DEFAULT_NINJA_WEIGHT, - "PackageManagerComponentOverrideTests": DEFAULT_NINJA_WEIGHT, - "PackageManagerPerfTests": DEFAULT_NINJA_WEIGHT, - "PackageManagerServiceServerTests": DEFAULT_NINJA_WEIGHT, - "PackageManagerServiceUnitTests": DEFAULT_NINJA_WEIGHT, - "PackageWatchdogTest": DEFAULT_NINJA_WEIGHT, - "PandoraServerLib": DEFAULT_NINJA_WEIGHT, - "pdl": DEFAULT_NINJA_WEIGHT, - "perfetto_trace_java_protos": DEFAULT_NINJA_WEIGHT, - "perfetto_trace-full": DEFAULT_NINJA_WEIGHT, - "PermissionController": DEFAULT_NINJA_WEIGHT, - "PermissionControllerMockingTests": DEFAULT_NINJA_WEIGHT, - "PixelAppCompTests": DEFAULT_NINJA_WEIGHT, - "platform-bootclasspath": DEFAULT_NINJA_WEIGHT, - "PlatformCommonScenarioTests": DEFAULT_NINJA_WEIGHT, - "PlatformComposeCoreTests": DEFAULT_NINJA_WEIGHT, - "platformprotoslite": DEFAULT_NINJA_WEIGHT, - "PlatformRuleTests": DEFAULT_NINJA_WEIGHT, - "precompiled_sepolicy-without_apex": DEFAULT_NINJA_WEIGHT, - "PresencePolling": DEFAULT_NINJA_WEIGHT, - "PrintSpooler": DEFAULT_NINJA_WEIGHT, - "QuickSearchBox": DEFAULT_NINJA_WEIGHT, - "RemoteDPCTestApp": DEFAULT_NINJA_WEIGHT, - "RemoteProvisioningServiceTests": DEFAULT_NINJA_WEIGHT, - "RkpdAppUnitTests": DEFAULT_NINJA_WEIGHT, - "Robolectric_shadows_framework": DEFAULT_NINJA_WEIGHT, - "RoleHolderApp": DEFAULT_NINJA_WEIGHT, - "SdkSandbox": DEFAULT_NINJA_WEIGHT, - "service-appsearch": DEFAULT_NINJA_WEIGHT, - "service-connectivity": DEFAULT_NINJA_WEIGHT, - "service-uwb": DEFAULT_NINJA_WEIGHT, - "service-wifi": DEFAULT_NINJA_WEIGHT, - "services-non-updatable-stubs": DEFAULT_NINJA_WEIGHT, - "services": DEFAULT_NINJA_WEIGHT, - "Settings-core": DEFAULT_NINJA_WEIGHT, - "Settings": DEFAULT_NINJA_WEIGHT, - "SettingsIntelligence": DEFAULT_NINJA_WEIGHT, - "SettingsLibTests": DEFAULT_NINJA_WEIGHT, - "SettingsProvider": DEFAULT_NINJA_WEIGHT, - "Shell": DEFAULT_NINJA_WEIGHT, - "SimAppDialog": DEFAULT_NINJA_WEIGHT, - "sl4a": DEFAULT_NINJA_WEIGHT, - "SmsApp": DEFAULT_NINJA_WEIGHT, - "SoundPicker": DEFAULT_NINJA_WEIGHT, - "StagedInstallTest": DEFAULT_NINJA_WEIGHT, - "StatementService": DEFAULT_NINJA_WEIGHT, - "StatsdFrameworkTestApp": DEFAULT_NINJA_WEIGHT, - "StatsdFrameworkTestAppNoPermission": DEFAULT_NINJA_WEIGHT, - "statsdprotolite": DEFAULT_NINJA_WEIGHT, - "Stk": DEFAULT_NINJA_WEIGHT, - "StorageManager": DEFAULT_NINJA_WEIGHT, - "system-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT, - "SystemUI-core": DEFAULT_NINJA_WEIGHT, - "SystemUI-tests-base": DEFAULT_NINJA_WEIGHT, - "SystemUI-tests": DEFAULT_NINJA_WEIGHT, - "SystemUI": DEFAULT_NINJA_WEIGHT, - "SystemUIComposeFeatures": DEFAULT_NINJA_WEIGHT, - "SystemUIComposeFeaturesTests": DEFAULT_NINJA_WEIGHT, - "SystemUITests": DEFAULT_NINJA_WEIGHT, - "Tag": DEFAULT_NINJA_WEIGHT, - "Telecom": DEFAULT_NINJA_WEIGHT, - "TelecomUnitTests": DEFAULT_NINJA_WEIGHT, - "telephony-common": DEFAULT_NINJA_WEIGHT, - "TelephonyProvider": DEFAULT_NINJA_WEIGHT, - "TeleService": DEFAULT_NINJA_WEIGHT, - "test-api-stubs-docs-non-updatable": DEFAULT_NINJA_WEIGHT, - "TetheringIntegrationTests": DEFAULT_NINJA_WEIGHT, - "TetheringNext": DEFAULT_NINJA_WEIGHT, - "ThemePickerTests": DEFAULT_NINJA_WEIGHT, - "Traceur": DEFAULT_NINJA_WEIGHT, - "UsbManagerTests": DEFAULT_NINJA_WEIGHT, - "UsbTests": DEFAULT_NINJA_WEIGHT, - "virtmgr": DEFAULT_NINJA_WEIGHT, - "WallpaperPicker2TestLib": DEFAULT_NINJA_WEIGHT, - "WallpaperPicker2Tests": DEFAULT_NINJA_WEIGHT, - "WifiDialog": DEFAULT_NINJA_WEIGHT, - "wm-proto-parsers": DEFAULT_NINJA_WEIGHT, - "WMShellFlickerTests": DEFAULT_NINJA_WEIGHT, - "WmTests": DEFAULT_NINJA_WEIGHT, - "wpa_supplicant": DEFAULT_NINJA_WEIGHT, + // The list of module types which are expected to spend lots of build time. + // With `--ninja_weight_source=soong`, ninja builds these module types and deps first. + HugeModuleTypePrefixMap = map[string]int{ + "rust_": HIGH_PRIORITIZED_WEIGHT, + "droidstubs": DEFAULT_PRIORITIZED_WEIGHT, + "art_": DEFAULT_PRIORITIZED_WEIGHT, + "ndk_library": DEFAULT_PRIORITIZED_WEIGHT, } ) diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 79a5ce4987..c63c15c38f 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -30,6 +30,7 @@ import ( "android/soong/shared" "android/soong/ui/metrics/bp2build_metrics_proto" + "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/deptools" "github.com/google/blueprint/metrics" @@ -256,18 +257,45 @@ func apiBuildFileExcludes(ctx *android.Context) []string { } func writeNinjaHint(ctx *android.Context) error { - wantModules := make([]string, len(allowlists.HugeModulesMap)) - i := 0 - for k := range allowlists.HugeModulesMap { - wantModules[i] = k - i += 1 + // The current predictor focuses on reducing false negatives. + // If there are too many false positives (e.g., most modules are marked as positive), + // real long-running jobs cannot run early. + // Therefore, the model should be adjusted in this case. + // The model should also be adjusted if there are critical false negatives. + predicate := func(j *blueprint.JsonModule) (prioritized bool, weight int) { + prioritized = false + weight = 0 + for prefix, w := range allowlists.HugeModuleTypePrefixMap { + if strings.HasPrefix(j.Type, prefix) { + prioritized = true + weight = w + return + } + } + dep_count := len(j.Deps) + src_count := 0 + for _, a := range j.Module["Actions"].([]blueprint.JSONAction) { + src_count += len(a.Inputs) + } + input_size := dep_count + src_count + + // Current threshold is an arbitrary value which only consider recall rather than accuracy. + if input_size > allowlists.INPUT_SIZE_THRESHOLD { + prioritized = true + weight += ((input_size) / allowlists.INPUT_SIZE_THRESHOLD) * allowlists.DEFAULT_PRIORITIZED_WEIGHT + + // To prevent some modules from having too large a priority value. + if weight > allowlists.HIGH_PRIORITIZED_WEIGHT { + weight = allowlists.HIGH_PRIORITIZED_WEIGHT + } + } + return } - outputsMap := ctx.Context.GetOutputsFromModuleNames(wantModules) + + outputsMap := ctx.Context.GetWeightedOutputsFromPredicate(predicate) var outputBuilder strings.Builder - for k, v := range allowlists.HugeModulesMap { - for _, output := range outputsMap[k] { - outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, v)) - } + for output, weight := range outputsMap { + outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, weight)) } weightListFile := filepath.Join(topDir, ctx.Config().OutDir(), ".ninja_weight_list") From 3eda019e040c3ebaaa8366728bbcd5a371ec9667 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Fri, 14 Apr 2023 18:07:25 +0000 Subject: [PATCH 0010/1460] Revert "Respect Clang version override env vars." Revert submission 2478283-master-I3cf18e7814ff52d526017e4fe34b17ab3d8f1080 Reason for revert: implementing a different way Reverted changes: /q/submissionid:2478283-master-I3cf18e7814ff52d526017e4fe34b17ab3d8f1080 Change-Id: I1d690b6d322adc664e4c02d958f0183bbb78e669 --- android/config_bp2build.go | 9 --------- cc/config/global.go | 7 +++++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/android/config_bp2build.go b/android/config_bp2build.go index 830890d56d..2beeb51ec5 100644 --- a/android/config_bp2build.go +++ b/android/config_bp2build.go @@ -95,15 +95,6 @@ func (ev ExportedVariables) ExportVariableConfigMethod(name string, method inter return ev.pctx.VariableConfigMethod(name, method) } -func (ev ExportedVariables) ExportStringStaticVariableWithEnvOverride(name, envVar, defaultVal string) { - ev.ExportVariableConfigMethod(name, func(config Config) string { - if override := config.Getenv(envVar); override != "" { - return override - } - return defaultVal - }) -} - // ExportSourcePathVariable declares a static "source path" variable and exports // it to Bazel's toolchain. func (ev ExportedVariables) ExportSourcePathVariable(name string, value string) { diff --git a/cc/config/global.go b/cc/config/global.go index 0257fa3524..05dc77354e 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -417,12 +417,15 @@ func init() { exportedVars.ExportStringList("CommonGlobalIncludes", commonGlobalIncludes) pctx.PrefixedExistentPathsForSourcesVariable("CommonGlobalIncludes", "-I", commonGlobalIncludes) + exportedVars.ExportStringStaticVariable("CLANG_DEFAULT_VERSION", ClangDefaultVersion) + exportedVars.ExportStringStaticVariable("CLANG_DEFAULT_SHORT_VERSION", ClangDefaultShortVersion) + pctx.StaticVariableWithEnvOverride("ClangBase", "LLVM_PREBUILTS_BASE", ClangDefaultBase) - exportedVars.ExportStringStaticVariableWithEnvOverride("ClangVersion", "LLVM_PREBUILTS_VERSION", ClangDefaultVersion) + pctx.StaticVariableWithEnvOverride("ClangVersion", "LLVM_PREBUILTS_VERSION", ClangDefaultVersion) pctx.StaticVariable("ClangPath", "${ClangBase}/${HostPrebuiltTag}/${ClangVersion}") pctx.StaticVariable("ClangBin", "${ClangPath}/bin") - exportedVars.ExportStringStaticVariableWithEnvOverride("ClangShortVersion", "LLVM_RELEASE_VERSION", ClangDefaultShortVersion) + pctx.StaticVariableWithEnvOverride("ClangShortVersion", "LLVM_RELEASE_VERSION", ClangDefaultShortVersion) pctx.StaticVariable("ClangAsanLibDir", "${ClangBase}/linux-x86/${ClangVersion}/lib/clang/${ClangShortVersion}/lib/linux") // These are tied to the version of LLVM directly in external/llvm, so they might trail the host prebuilts From dea1d039756fc3aaf334c85c01d4a242e025da78 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 6 Dec 2022 14:50:08 -0800 Subject: [PATCH 0011/1460] Fix addrsig warnings ld -r reorders symbols and invalidates the .llvm_addrsig section, which then causes warnings if the resulting object is used with ld --icf=safe. The warning is especially common when building with musl, as the clang_rt.crt* objects have .llvm_addrsig sections, are linked into libc_musl_crt* using ld -r, and are then linked into every other binary and shared library with --icf=safe. Strip the .llvm_addrsig section after ld -r to prevent the warnings. Test: m USE_HOST_MUSL=true host-native -k Change-Id: Ia52a4756b9ebbb62115898d0de9f8641e6fea705 --- cc/builder.go | 23 +++++++++++++++++++++++ cc/object.go | 9 ++++++++- java/app_test.go | 4 ++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/cc/builder.go b/cc/builder.go index fef00d4d54..f5e0dcca55 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -125,6 +125,14 @@ var ( }, "objcopyCmd", "prefix") + // Rule to run objcopy --remove-section=.llvm_addrsig on a partially linked object + noAddrSig = pctx.AndroidStaticRule("noAddrSig", + blueprint.RuleParams{ + Command: "rm -f ${out} && $objcopyCmd --remove-section=.llvm_addrsig ${in} ${out}", + CommandDeps: []string{"$objcopyCmd"}, + }, + "objcopyCmd") + _ = pctx.SourcePathVariable("stripPath", "build/soong/scripts/strip.sh") _ = pctx.SourcePathVariable("xzCmd", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/xz") _ = pctx.SourcePathVariable("createMiniDebugInfo", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/create_minidebuginfo") @@ -1008,6 +1016,21 @@ func transformBinaryPrefixSymbols(ctx android.ModuleContext, prefix string, inpu }) } +// Generate a rule for running objcopy --remove-section=.llvm_addrsig on a partially linked object +func transformObjectNoAddrSig(ctx android.ModuleContext, inputFile android.Path, outputFile android.WritablePath) { + objcopyCmd := "${config.ClangBin}/llvm-objcopy" + + ctx.Build(pctx, android.BuildParams{ + Rule: noAddrSig, + Description: "remove addrsig " + outputFile.Base(), + Output: outputFile, + Input: inputFile, + Args: map[string]string{ + "objcopyCmd": objcopyCmd, + }, + }) +} + // Registers a build statement to invoke `strip` (to discard symbols and data from object files). func transformStrip(ctx android.ModuleContext, inputFile android.Path, outputFile android.WritablePath, flags StripFlags) { diff --git a/cc/object.go b/cc/object.go index d65cdea743..4ddce99562 100644 --- a/cc/object.go +++ b/cc/object.go @@ -309,6 +309,8 @@ func (object *objectLinker) link(ctx ModuleContext, }) } } else { + outputAddrSig := android.PathForModuleOut(ctx, "addrsig", outputName) + if String(object.Properties.Prefix_symbols) != "" { input := android.PathForModuleOut(ctx, "unprefixed", outputName) transformBinaryPrefixSymbols(ctx, String(object.Properties.Prefix_symbols), input, @@ -316,7 +318,12 @@ func (object *objectLinker) link(ctx ModuleContext, output = input } - transformObjsToObj(ctx, objs.objFiles, builderFlags, output, flags.LdFlagsDeps) + transformObjsToObj(ctx, objs.objFiles, builderFlags, outputAddrSig, flags.LdFlagsDeps) + + // ld -r reorders symbols and invalidates the .llvm_addrsig section, which then causes warnings + // if the resulting object is used with ld --icf=safe. Strip the .llvm_addrsig section to + // prevent the warnings. + transformObjectNoAddrSig(ctx, outputAddrSig, output) } ctx.CheckbuildFile(outputFile) diff --git a/java/app_test.go b/java/app_test.go index 561be684e4..02a9eac40e 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -443,9 +443,9 @@ func TestUpdatableApps_JniLibShouldBeBuiltAgainstMinSdkVersion(t *testing.T) { inputs := ctx.ModuleForTests("libjni", "android_arm64_armv8-a_sdk_shared").Description("link").Implicits var crtbeginFound, crtendFound bool expectedCrtBegin := ctx.ModuleForTests("crtbegin_so", - "android_arm64_armv8-a_sdk_29").Rule("partialLd").Output + "android_arm64_armv8-a_sdk_29").Rule("noAddrSig").Output expectedCrtEnd := ctx.ModuleForTests("crtend_so", - "android_arm64_armv8-a_sdk_29").Rule("partialLd").Output + "android_arm64_armv8-a_sdk_29").Rule("noAddrSig").Output implicits := []string{} for _, input := range inputs { implicits = append(implicits, input.String()) From c6a918aa116c958d3504b0172c86ff6067f45263 Mon Sep 17 00:00:00 2001 From: Alix Date: Fri, 7 Apr 2023 19:42:02 +0000 Subject: [PATCH 0012/1460] add support for escaping for zip2zip Change-Id: I6c2f39e24247752de065fd3116f670639322b943 Bug: 270896651 Test: zip2zip tests --- cmd/zip2zip/zip2zip.go | 6 ------ cmd/zip2zip/zip2zip_test.go | 14 +++++++------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cmd/zip2zip/zip2zip.go b/cmd/zip2zip/zip2zip.go index 491267b010..5ab96568d9 100644 --- a/cmd/zip2zip/zip2zip.go +++ b/cmd/zip2zip/zip2zip.go @@ -128,12 +128,6 @@ func zip2zip(reader *zip.Reader, writer *zip.Writer, sortOutput, sortJava, setTi } for _, arg := range args { - // Reserve escaping for future implementation, so make sure no - // one is using \ and expecting a certain behavior. - if strings.Contains(arg, "\\") { - return fmt.Errorf("\\ characters are not currently supported") - } - input, output := includeSplit(arg) var includeMatches []pair diff --git a/cmd/zip2zip/zip2zip_test.go b/cmd/zip2zip/zip2zip_test.go index 2c4e005697..c238098ec8 100644 --- a/cmd/zip2zip/zip2zip_test.go +++ b/cmd/zip2zip/zip2zip_test.go @@ -38,13 +38,6 @@ var testCases = []struct { storedFiles []string err error }{ - { - name: "unsupported \\", - - args: []string{"a\\b:b"}, - - err: fmt.Errorf("\\ characters are not currently supported"), - }, { // This is modelled after the update package build rules in build/make/core/Makefile name: "filter globs", @@ -406,6 +399,13 @@ var testCases = []struct { "b/a/b", }, }, + { + name: "escaping", + + inputFiles: []string{"a"}, + args: []string{"\\a"}, + outputFiles: []string{"a"}, + }, } func errorString(e error) string { From e8173a83cb728fb31ad6651214995bb2e763b950 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 12 Apr 2023 17:14:11 +0000 Subject: [PATCH 0013/1460] Add TestApexes to ApexInfo If any of apexes in apex_available is an apex_test, then that name will be propagated down from that apex to each apex variant. This metadata will be used to enforce that stub libraries cannot have more than one apex_available. This logic is necessary so that bp2build can select the correct stub/impl. (To avoid replicating this complexity in Bazel, we should consider dropping the test apexes in Bazel BUILD files, next CL) Bug: 277651159 Test: go build ./apex Change-Id: I63617c1dc2a2d5c9cd7758c416fec7b4db1f10a7 --- android/apex.go | 18 +++++++++++++++++ android/apex_test.go | 46 ++++++++++++++++++++++---------------------- apex/apex.go | 5 +++++ 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/android/apex.go b/android/apex.go index 5bbc02e512..823afbb111 100644 --- a/android/apex.go +++ b/android/apex.go @@ -84,6 +84,9 @@ type ApexInfo struct { // // See Prebuilt.ApexInfoMutator for more information. ForPrebuiltApex bool + + // Returns the name of the test apexes that this module is included in. + TestApexes []string } var ApexInfoProvider = blueprint.NewMutatorProvider(ApexInfo{}, "apex") @@ -287,6 +290,9 @@ type ApexProperties struct { // See ApexModule.UniqueApexVariants() UniqueApexVariationsForDeps bool `blueprint:"mutated"` + + // The test apexes that includes this apex variant + TestApexes []string `blueprint:"mutated"` } // Marker interface that identifies dependencies that are excluded from APEX contents. @@ -429,6 +435,11 @@ func (m *ApexModuleBase) TestFor() []string { return nil } +// Returns the test apexes that this module is included in. +func (m *ApexModuleBase) TestApexes() []string { + return m.ApexProperties.TestApexes +} + // Implements ApexModule func (m *ApexModuleBase) UniqueApexVariations() bool { // If needed, this will bel overridden by concrete types inheriting @@ -549,12 +560,14 @@ func mergeApexVariations(ctx PathContext, apexInfos []ApexInfo) (merged []ApexIn // Platform APIs is allowed for this module only when all APEXes containing // the module are with `use_platform_apis: true`. merged[index].UsePlatformApis = merged[index].UsePlatformApis && apexInfo.UsePlatformApis + merged[index].TestApexes = append(merged[index].TestApexes, apexInfo.TestApexes...) } else { seen[mergedName] = len(merged) apexInfo.ApexVariationName = mergedName apexInfo.InApexVariants = CopyOf(apexInfo.InApexVariants) apexInfo.InApexModules = CopyOf(apexInfo.InApexModules) apexInfo.ApexContents = append([]*ApexContents(nil), apexInfo.ApexContents...) + apexInfo.TestApexes = CopyOf(apexInfo.TestApexes) merged = append(merged, apexInfo) } aliases = append(aliases, [2]string{variantName, mergedName}) @@ -602,8 +615,10 @@ func CreateApexVariations(mctx BottomUpMutatorContext, module ApexModule) []Modu mctx.SetDefaultDependencyVariation(&defaultVariation) variations := []string{defaultVariation} + testApexes := []string{} for _, a := range apexInfos { variations = append(variations, a.ApexVariationName) + testApexes = append(testApexes, a.TestApexes...) } modules := mctx.CreateVariations(variations...) for i, mod := range modules { @@ -617,6 +632,9 @@ func CreateApexVariations(mctx BottomUpMutatorContext, module ApexModule) []Modu if !platformVariation { mctx.SetVariationProvider(mod, ApexInfoProvider, apexInfos[i-1]) } + // Set the value of TestApexes in every single apex variant. + // This allows each apex variant to be aware of the test apexes in the user provided apex_available. + mod.(ApexModule).apexModuleBase().ApexProperties.TestApexes = testApexes } for _, alias := range aliases { diff --git a/android/apex_test.go b/android/apex_test.go index 0bf4c9c367..ddc730d05a 100644 --- a/android/apex_test.go +++ b/android/apex_test.go @@ -33,10 +33,10 @@ func Test_mergeApexVariations(t *testing.T) { { name: "single", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, }, wantAliases: [][2]string{ {"foo", "apex10000"}, @@ -45,11 +45,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + {"bar", FutureApiLevel, false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, false}}, + {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, false, nil}}, wantAliases: [][2]string{ {"bar", "apex10000"}, {"foo", "apex10000"}, @@ -58,12 +58,12 @@ func Test_mergeApexVariations(t *testing.T) { { name: "don't merge version", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + {"bar", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, }, wantMerged: []ApexInfo{ - {"apex30", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, - {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"apex30", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, }, wantAliases: [][2]string{ {"bar", "apex30"}, @@ -73,11 +73,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge updatable", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -87,15 +87,15 @@ func Test_mergeApexVariations(t *testing.T) { { name: "don't merge when for prebuilt_apex", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, // This one should not be merged in with the others because it is for // a prebuilt_apex. - {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, + {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex, nil}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, - {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, + {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, + {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex, nil}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -105,11 +105,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge different UsePlatformApis but don't allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -119,11 +119,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge same UsePlatformApis and allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, true, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, true, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, true, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, true, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, diff --git a/apex/apex.go b/apex/apex.go index 5451a04008..e3f53b8212 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1064,6 +1064,10 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { apexVariationName := mctx.ModuleName() // could be com.android.foo a.properties.ApexVariationName = apexVariationName + testApexes := []string{} + if a.testApex { + testApexes = []string{apexVariationName} + } apexInfo := android.ApexInfo{ ApexVariationName: apexVariationName, MinSdkVersion: minSdkVersion, @@ -1072,6 +1076,7 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { InApexVariants: []string{apexVariationName}, InApexModules: []string{a.Name()}, // could be com.mycompany.android.foo ApexContents: []*android.ApexContents{apexContents}, + TestApexes: testApexes, } mctx.WalkDeps(func(child, parent android.Module) bool { if !continueApexDepsWalk(child, parent) { From 3b23ac0025570f850e31958e47d477fbdcf21afa Mon Sep 17 00:00:00 2001 From: Justin Yun Date: Wed, 19 Apr 2023 19:26:30 +0900 Subject: [PATCH 0014/1460] Add license visibility to prebuilt vsdk libbpf_minimal can be included as a prebuilt vsdk library. This must include the license_kinds of libbpf_minimal-license from soong. Add the visibility for the prebuilts. Bug: 271539873 Test: build against vsdk Change-Id: Ib1c3e7705e7f234b2b54c2180642d6134c6131ea --- licenses/Android.bp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/licenses/Android.bp b/licenses/Android.bp index 7267cf3743..dee72ed424 100644 --- a/licenses/Android.bp +++ b/licenses/Android.bp @@ -923,7 +923,10 @@ license_kind { license_kind { name: "SPDX-license-identifier-Linux-syscall-note", // expanding visibility requires approval from an OSPO lawyer or pcounsel - visibility: ["//external/libbpf:__subpackages__"], + visibility: [ + "//external/libbpf:__subpackages__", + "//prebuilts/vsdk:__subpackages__", + ], conditions: ["permissive"], url: "https://spdx.org/licenses/Linux-syscall-note.html", } From 0ffa1f369d2d7d4c4f7b35e2fe892434b26eb469 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Wed, 19 Apr 2023 14:18:20 -0400 Subject: [PATCH 0015/1460] add LLVM_*_VERSION envvars to allowlist Bug: 273563660 Change-Id: I7c1e6b244e137b27126abcba22684db66fae0327 --- android/bazel_handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index ade2c496ab..ace4e1a687 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -61,6 +61,8 @@ var ( "AUTO_UNINITIALIZE", "USE_CCACHE", "LLVM_NEXT", + "LLVM_PREBUILTS_VERSION", + "LLVM_RELEASE_VERSION", "ALLOW_UNKNOWN_WARNING_OPTION", // Overrides the version in the apex_manifest.json. The version is unique for From a98aab98c45aa7245cd235b8822de35aa6eb4fd6 Mon Sep 17 00:00:00 2001 From: Hang Lu Date: Fri, 17 Mar 2023 13:17:22 +0800 Subject: [PATCH 0016/1460] Enable HWASan for multiple modules in one place(Soong) Environment variables HWASAN_INCLUDE_PATHS and PRODUCT_HWASAN_INCLUDE_PATHS can be used to enable HWASan for multiple modules, by just adding the module directory to the env variable. Bug: b/271948407 Test: Set specific module directory to above env variable and check the assembly codes of output elf files after building, finding hwasan related symbols inside. Change-Id: Ic49b515830c4469ca5fa94f547b26c0fb602fc54 --- android/config.go | 7 +++++++ android/variable.go | 2 ++ cc/sanitize.go | 6 ++++++ 3 files changed, 15 insertions(+) diff --git a/android/config.go b/android/config.go index 032172d6ee..f52a48fe82 100644 --- a/android/config.go +++ b/android/config.go @@ -1557,6 +1557,13 @@ func (c *config) MemtagHeapSyncEnabledForPath(path string) bool { return HasAnyPrefix(path, c.productVariables.MemtagHeapSyncIncludePaths) && !c.MemtagHeapDisabledForPath(path) } +func (c *config) HWASanEnabledForPath(path string) bool { + if len(c.productVariables.HWASanIncludePaths) == 0 { + return false + } + return HasAnyPrefix(path, c.productVariables.HWASanIncludePaths) +} + func (c *config) VendorConfig(name string) VendorConfig { return soongconfig.Config(c.productVariables.VendorVars[name]) } diff --git a/android/variable.go b/android/variable.go index 249d53b07a..0b3111beb5 100644 --- a/android/variable.go +++ b/android/variable.go @@ -306,6 +306,8 @@ type productVariables struct { MemtagHeapAsyncIncludePaths []string `json:",omitempty"` MemtagHeapSyncIncludePaths []string `json:",omitempty"` + HWASanIncludePaths []string `json:",omitempty"` + VendorPath *string `json:",omitempty"` OdmPath *string `json:",omitempty"` ProductPath *string `json:",omitempty"` diff --git a/cc/sanitize.go b/cc/sanitize.go index 45d7fab4f5..7fddc1b836 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -593,6 +593,12 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { } } + // Enable HWASan for all components in the include paths (for Aarch64 only) + if s.Hwaddress == nil && ctx.Config().HWASanEnabledForPath(ctx.ModuleDir()) && + ctx.Arch().ArchType == android.Arm64 && ctx.toolchain().Bionic() { + s.Hwaddress = proptools.BoolPtr(true) + } + // Enable CFI for non-host components in the include paths if s.Cfi == nil && ctx.Config().CFIEnabledForPath(ctx.ModuleDir()) && !ctx.Host() { s.Cfi = proptools.BoolPtr(true) From a485d0935ce706a8058de2639056d5e3d731dd94 Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Thu, 20 Apr 2023 12:34:08 +0000 Subject: [PATCH 0017/1460] Delete mixedBuildSupportedCcTest. We can just rely on the prod/staging allowlists which name the modules directly. Test: presubmits Test: m nothing; showcommands adbd_test | grep bazel-out Fixes: 279004435 Change-Id: I666855d6bbe63c628256c3eebd7ae581c0cbdedb --- android/allowlists/allowlists.go | 5 +++++ cc/cc.go | 19 +------------------ 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index e3e4dba90e..9c1824eaf2 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -939,6 +939,11 @@ var ( "libandroidfw_tests", "aapt2_tests", // failing due to data path issues + // error: overriding commands for target + // `out/host/linux-x86/nativetest64/gmock_tests/gmock_tests__cc_runner_test', + // previously defined at out/soong/installs-aosp_arm.mk:64919` + "gmock_tests", + // cc_test with unconverted deps, or are device-only (and not verified to pass yet) "AMRWBEncTest", "AmrnbDecoderTest", // depends on unconverted modules: libaudioutils, libsndfile diff --git a/cc/cc.go b/cc/cc.go index 1997e94cd0..6054222138 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -1890,26 +1890,9 @@ func (c *Module) QueueBazelCall(ctx android.BaseModuleContext) { c.bazelHandler.QueueBazelCall(ctx, c.getBazelModuleLabel(ctx)) } -var ( - mixedBuildSupportedCcTest = []string{ - "adbd_test", - "adb_crypto_test", - "adb_pairing_auth_test", - "adb_pairing_connection_test", - "adb_tls_connection_test", - } -) - // IsMixedBuildSupported returns true if the module should be analyzed by Bazel -// in any of the --bazel-mode(s). This filters at the module level and takes -// precedence over the allowlists in allowlists/allowlists.go. +// in any of the --bazel-mode(s). func (c *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool { - _, isForTesting := ctx.Config().BazelContext.(android.MockBazelContext) - if c.testBinary() && !android.InList(c.Name(), mixedBuildSupportedCcTest) && !isForTesting { - // Per-module rollout of mixed-builds for cc_test modules. - return false - } - // TODO(b/261058727): Remove this (enable mixed builds for modules with UBSan) // Currently we can only support ubsan when minimum runtime is used. return c.bazelHandler != nil && (!isUbsanEnabled(c) || c.MinimalRuntimeNeeded()) From a8b441613af38564857d57da7998e5beabba5ae6 Mon Sep 17 00:00:00 2001 From: Trevor Radcliffe Date: Fri, 14 Apr 2023 18:25:24 +0000 Subject: [PATCH 0018/1460] convert hidden visibility flag to feature in bp2build. The context here is that `-fvisibility=default` should only be added for CFI if `-fvisibility=hidden` is not already specified. This will be achieved using toolchain features. Note that Soong itself never adds `-fvisibility=hidden`. This is only ever added in the `cflags` property of a bp file. Bug: 261733820 Test: Unit tests Change-Id: Ib821e8c30a9cd03d2929b4bd2e771bec7b33fa66 --- bp2build/cc_binary_conversion_test.go | 41 +++++++ bp2build/cc_library_conversion_test.go | 104 ++++++++++++++++++ bp2build/cc_library_shared_conversion_test.go | 41 +++++++ bp2build/cc_library_static_conversion_test.go | 41 +++++++ cc/bp2build.go | 50 ++++++++- cc/library.go | 16 ++- 6 files changed, 287 insertions(+), 6 deletions(-) diff --git a/bp2build/cc_binary_conversion_test.go b/bp2build/cc_binary_conversion_test.go index 031573274d..89eac8a000 100644 --- a/bp2build/cc_binary_conversion_test.go +++ b/bp2build/cc_binary_conversion_test.go @@ -996,3 +996,44 @@ func TestCcBinaryWithThinLtoAndWholeProgramVtables(t *testing.T) { }, }) } + +func TestCcBinaryHiddenVisibilityConvertedToFeature(t *testing.T) { + runCcBinaryTestCase(t, ccBinaryBp2buildTestCase{ + description: "cc_binary changes hidden visibility to feature", + blueprint: ` +{rule_name} { + name: "foo", + cflags: ["-fvisibility=hidden"], +}`, + targets: []testBazelTarget{ + {"cc_binary", "foo", AttrNameToString{ + "local_includes": `["."]`, + "features": `["visibility_hidden"]`, + }}, + }, + }) +} + +func TestCcBinaryHiddenVisibilityConvertedToFeatureOsSpecific(t *testing.T) { + runCcBinaryTestCase(t, ccBinaryBp2buildTestCase{ + description: "cc_binary changes hidden visibility to feature for specific os", + blueprint: ` +{rule_name} { + name: "foo", + target: { + android: { + cflags: ["-fvisibility=hidden"], + }, + }, +}`, + targets: []testBazelTarget{ + {"cc_binary", "foo", AttrNameToString{ + "local_includes": `["."]`, + "features": `select({ + "//build/bazel/platforms/os:android": ["visibility_hidden"], + "//conditions:default": [], + })`, + }}, + }, + }) +} diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index d2c463dbbd..b61b0a7cfc 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -4350,3 +4350,107 @@ cc_library { }, }) } + +func TestCcLibraryHiddenVisibilityConvertedToFeature(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library changes hidden visibility flag to feature", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: ` +cc_library { + name: "foo", + cflags: ["-fvisibility=hidden"], +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "features": `["visibility_hidden"]`, + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "features": `["visibility_hidden"]`, + "local_includes": `["."]`, + }), + }, + }) +} + +func TestCcLibraryHiddenVisibilityConvertedToFeatureSharedSpecific(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library changes hidden visibility flag to feature when specific to shared variant", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: ` +cc_library { + name: "foo", + shared: { + cflags: ["-fvisibility=hidden"], + }, +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "features": `["visibility_hidden"]`, + "local_includes": `["."]`, + }), + }, + }) +} + +func TestCcLibraryHiddenVisibilityConvertedToFeatureStaticSpecific(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library changes hidden visibility flag to feature when specific to static variant", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: ` +cc_library { + name: "foo", + static: { + cflags: ["-fvisibility=hidden"], + }, +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "features": `["visibility_hidden"]`, + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "local_includes": `["."]`, + }), + }, + }) +} + +func TestCcLibraryHiddenVisibilityConvertedToFeatureOsSpecific(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library changes hidden visibility flag to feature when specific to an os", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: ` +cc_library { + name: "foo", + target: { + android: { + cflags: ["-fvisibility=hidden"], + }, + }, +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "features": `select({ + "//build/bazel/platforms/os:android": ["visibility_hidden"], + "//conditions:default": [], + })`, + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "features": `select({ + "//build/bazel/platforms/os:android": ["visibility_hidden"], + "//conditions:default": [], + })`, + "local_includes": `["."]`, + }), + }, + }) +} diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go index cbea943e1f..29316ab1ed 100644 --- a/bp2build/cc_library_shared_conversion_test.go +++ b/bp2build/cc_library_shared_conversion_test.go @@ -1250,3 +1250,44 @@ cc_library_shared { }, }) } + +func TestCcLibrarySharedHiddenVisibilityConvertedToFeature(t *testing.T) { + runCcLibrarySharedTestCase(t, Bp2buildTestCase{ + Description: "cc_library_shared changes hidden visibility flag to feature", + Blueprint: ` +cc_library_shared{ + name: "foo", + cflags: ["-fvisibility=hidden"], +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "features": `["visibility_hidden"]`, + "local_includes": `["."]`, + }), + }, + }) +} + +func TestCcLibrarySharedHiddenVisibilityConvertedToFeatureOsSpecific(t *testing.T) { + runCcLibrarySharedTestCase(t, Bp2buildTestCase{ + Description: "cc_library_shared changes hidden visibility flag to feature for specific os", + Blueprint: ` +cc_library_shared{ + name: "foo", + target: { + android: { + cflags: ["-fvisibility=hidden"], + }, + }, +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "features": `select({ + "//build/bazel/platforms/os:android": ["visibility_hidden"], + "//conditions:default": [], + })`, + "local_includes": `["."]`, + }), + }, + }) +} diff --git a/bp2build/cc_library_static_conversion_test.go b/bp2build/cc_library_static_conversion_test.go index cd4cf51a17..f89f2dd3ff 100644 --- a/bp2build/cc_library_static_conversion_test.go +++ b/bp2build/cc_library_static_conversion_test.go @@ -2032,3 +2032,44 @@ cc_library_static { }, }) } + +func TestCcLibraryStaticHiddenVisibilityConvertedToFeature(t *testing.T) { + runCcLibraryStaticTestCase(t, Bp2buildTestCase{ + Description: "cc_library_static changes hidden visibility flag to feature", + Blueprint: ` +cc_library_static { + name: "foo", + cflags: ["-fvisibility=hidden"], +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_static", "foo", AttrNameToString{ + "features": `["visibility_hidden"]`, + "local_includes": `["."]`, + }), + }, + }) +} + +func TestCcLibraryStaticHiddenVisibilityConvertedToFeatureOsSpecific(t *testing.T) { + runCcLibraryStaticTestCase(t, Bp2buildTestCase{ + Description: "cc_library_static changes hidden visibility flag to feature for specific os", + Blueprint: ` +cc_library_static { + name: "foo", + target: { + android: { + cflags: ["-fvisibility=hidden"], + }, + }, +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_static", "foo", AttrNameToString{ + "features": `select({ + "//build/bazel/platforms/os:android": ["visibility_hidden"], + "//conditions:default": [], + })`, + "local_includes": `["."]`, + }), + }, + }) +} diff --git a/cc/bp2build.go b/cc/bp2build.go index c8f516cdc1..96c478966a 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -67,6 +67,8 @@ type staticOrSharedAttributes struct { Apex_available []string + Features bazel.StringListAttribute + sdkAttributes tidyAttributes @@ -226,7 +228,7 @@ func bp2buildParseStaticOrSharedProps(ctx android.BazelConversionPathContext, mo attrs := staticOrSharedAttributes{} setAttrs := func(axis bazel.ConfigurationAxis, config string, props StaticOrSharedProperties) { - attrs.Copts.SetSelectValue(axis, config, parseCommandLineFlags(props.Cflags, filterOutStdFlag)) + attrs.Copts.SetSelectValue(axis, config, parseCommandLineFlags(props.Cflags, filterOutStdFlag, filterOutHiddenVisibility)) attrs.Srcs.SetSelectValue(axis, config, android.BazelLabelForModuleSrc(ctx, props.Srcs)) attrs.System_dynamic_deps.SetSelectValue(axis, config, bazelLabelForSharedDeps(ctx, props.System_shared_libs)) @@ -270,6 +272,8 @@ func bp2buildParseStaticOrSharedProps(ctx android.BazelConversionPathContext, mo attrs.Apex_available = android.ConvertApexAvailableToTags(apexAvailable) + attrs.Features.Append(convertHiddenVisibilityToFeatureStaticOrShared(ctx, module, isStatic)) + if !partitionedSrcs[protoSrcPartition].IsEmpty() { // TODO(b/208815215): determine whether this is used and add support if necessary ctx.ModuleErrorf("Migrating static/shared only proto srcs is not currently supported") @@ -428,6 +432,12 @@ type compilerAttributes struct { type filterOutFn func(string) bool +// filterOutHiddenVisibility removes the flag specifying hidden visibility as +// this flag is converted to a toolchain feature +func filterOutHiddenVisibility(flag string) bool { + return flag == config.VisibilityHiddenFlag +} + func filterOutStdFlag(flag string) bool { return strings.HasPrefix(flag, "-std=") } @@ -490,7 +500,7 @@ func (ca *compilerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversi // overridden. In Bazel we always allow overriding, via flags; however, this can cause // incompatibilities, so we remove "-std=" flags from Cflag properties while leaving it in other // cases. - ca.copts.SetSelectValue(axis, config, parseCommandLineFlags(props.Cflags, filterOutStdFlag, filterOutClangUnknownCflags)) + ca.copts.SetSelectValue(axis, config, parseCommandLineFlags(props.Cflags, filterOutStdFlag, filterOutClangUnknownCflags, filterOutHiddenVisibility)) ca.asFlags.SetSelectValue(axis, config, parseCommandLineFlags(props.Asflags, nil)) ca.conlyFlags.SetSelectValue(axis, config, parseCommandLineFlags(props.Conlyflags, filterOutClangUnknownCflags)) ca.cppFlags.SetSelectValue(axis, config, parseCommandLineFlags(props.Cppflags, filterOutClangUnknownCflags)) @@ -833,6 +843,7 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) features := compilerAttrs.features.Clone().Append(linkerAttrs.features).Append(bp2buildSanitizerFeatures(ctx, module)) features = features.Append(bp2buildLtoFeatures(ctx, module)) + features = features.Append(convertHiddenVisibilityToFeatureBase(ctx, module)) features.DeduplicateAxesFromBase() addMuslSystemDynamicDeps(ctx, linkerAttrs) @@ -1547,3 +1558,38 @@ func bp2buildLtoFeatures(ctx android.BazelConversionPathContext, m *Module) baze } return ltoStringFeatures } + +func convertHiddenVisibilityToFeatureBase(ctx android.BazelConversionPathContext, m *Module) bazel.StringListAttribute { + visibilityHiddenFeature := bazel.StringListAttribute{} + bp2BuildPropParseHelper(ctx, m, &BaseCompilerProperties{}, func(axis bazel.ConfigurationAxis, configString string, props interface{}) { + if baseCompilerProps, ok := props.(*BaseCompilerProperties); ok { + convertHiddenVisibilityToFeatureHelper(&visibilityHiddenFeature, axis, configString, baseCompilerProps.Cflags) + } + }) + return visibilityHiddenFeature +} + +func convertHiddenVisibilityToFeatureStaticOrShared(ctx android.BazelConversionPathContext, m *Module, isStatic bool) bazel.StringListAttribute { + visibilityHiddenFeature := bazel.StringListAttribute{} + if isStatic { + bp2BuildPropParseHelper(ctx, m, &StaticProperties{}, func(axis bazel.ConfigurationAxis, configString string, props interface{}) { + if staticProps, ok := props.(*StaticProperties); ok { + convertHiddenVisibilityToFeatureHelper(&visibilityHiddenFeature, axis, configString, staticProps.Static.Cflags) + } + }) + } else { + bp2BuildPropParseHelper(ctx, m, &SharedProperties{}, func(axis bazel.ConfigurationAxis, configString string, props interface{}) { + if sharedProps, ok := props.(*SharedProperties); ok { + convertHiddenVisibilityToFeatureHelper(&visibilityHiddenFeature, axis, configString, sharedProps.Shared.Cflags) + } + }) + } + + return visibilityHiddenFeature +} + +func convertHiddenVisibilityToFeatureHelper(feature *bazel.StringListAttribute, axis bazel.ConfigurationAxis, configString string, cflags []string) { + if inList(config.VisibilityHiddenFlag, cflags) { + feature.SetSelectValue(axis, configString, []string{"visibility_hidden"}) + } +} diff --git a/cc/library.go b/cc/library.go index 172ca6459b..1daeeffc27 100644 --- a/cc/library.go +++ b/cc/library.go @@ -311,6 +311,11 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { asFlags = bazel.MakeStringListAttribute(nil) } + sharedFeatures := baseAttributes.features.Clone().Append(sharedAttrs.Features) + sharedFeatures.DeduplicateAxesFromBase() + staticFeatures := baseAttributes.features.Clone().Append(staticAttrs.Features) + staticFeatures.DeduplicateAxesFromBase() + staticCommonAttrs := staticOrSharedAttributes{ Srcs: *srcs.Clone().Append(staticAttrs.Srcs), Srcs_c: *compilerAttrs.cSrcs.Clone().Append(staticAttrs.Srcs_c), @@ -366,7 +371,7 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { Cpp_std: compilerAttrs.cppStd, C_std: compilerAttrs.cStd, - Features: baseAttributes.features, + Features: *staticFeatures, } sharedTargetAttrs := &bazelCcLibrarySharedAttributes{ @@ -390,7 +395,7 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { Additional_linker_inputs: linkerAttrs.additionalLinkerInputs, Strip: stripAttrsFromLinkerAttrs(&linkerAttrs), - Features: baseAttributes.features, + Features: *sharedFeatures, bazelCcHeaderAbiCheckerAttributes: bp2buildParseAbiCheckerProps(ctx, m), Fdo_profile: compilerAttrs.fdoProfile, @@ -2881,6 +2886,9 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo asFlags = bazel.MakeStringListAttribute(nil) } + features := baseAttributes.features.Clone().Append(libSharedOrStaticAttrs.Features) + features.DeduplicateAxesFromBase() + commonAttrs := staticOrSharedAttributes{ Srcs: compilerAttrs.srcs, Srcs_c: compilerAttrs.cSrcs, @@ -2922,7 +2930,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Conlyflags: compilerAttrs.conlyFlags, Asflags: asFlags, - Features: baseAttributes.features, + Features: *features, } } else { commonAttrs.Dynamic_deps.Add(baseAttributes.protoDependency) @@ -2951,7 +2959,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Strip: stripAttrsFromLinkerAttrs(&linkerAttrs), - Features: baseAttributes.features, + Features: *features, Suffix: compilerAttrs.suffix, From 5bedfee4484a8786ccd761212057f8aac9a02eb7 Mon Sep 17 00:00:00 2001 From: Inseob Kim Date: Thu, 20 Apr 2023 10:16:14 +0900 Subject: [PATCH 0019/1460] Remove deps from Soong intermediate to symbols These should be handled by Makefile dependency (LOCAL_REQUIRED_MODULES). Bug: 278832320 Test: build and see symbol files Change-Id: I80ad9d8e7e8384e2ae3537cc681bc1d1b327d04c --- apex/apex_test.go | 15 ++++++++++++--- apex/builder.go | 11 +---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 924f92344d..8a05716bbc 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -5673,6 +5673,7 @@ func TestPrebuiltSkipsSymbols(t *testing.T) { // Source module apex { name: "myapex", + binaries: ["foo"], key: "myapex.key", updatable: false, } @@ -5688,11 +5689,19 @@ func TestPrebuiltSkipsSymbols(t *testing.T) { set: "myapex.apks", `+preferProperty+` } + + cc_binary { + name: "foo", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } `) // Symbol files are installed by installing entries under ${OUT}/apex/{apex name} - android.AssertStringListContainsEquals(t, "Implicits", - ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule").Implicits.Strings(), - "out/soong/target/product/test_device/apex/myapex/apex_manifest.pb", + android.AssertStringListContainsEquals(t, "Installs", + ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().FilesToInstall().Strings(), + filepath.Join(ctx.Config().SoongOutDir(), "target/product/test_device/apex/myapex/bin/foo"), tc.installSymbolFiles) }) } diff --git a/apex/builder.go b/apex/builder.go index 2f8a4ec091..3c7671b00b 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -517,9 +517,6 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { } } implicitInputs = append(implicitInputs, fi.builtFile) - if installSymbolFiles { - implicitInputs = append(implicitInputs, installedPath) - } // Create additional symlinks pointing the file inside the APEX (if any). Note that // this is independent from the symlink optimization. @@ -527,8 +524,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { symlinkDest := imageDir.Join(ctx, symlinkPath).String() copyCommands = append(copyCommands, "ln -sfn "+filepath.Base(destPath)+" "+symlinkDest) if installSymbolFiles { - installedSymlink := ctx.InstallSymlink(apexDir.Join(ctx, filepath.Dir(symlinkPath)), filepath.Base(symlinkPath), installedPath) - implicitInputs = append(implicitInputs, installedSymlink) + ctx.InstallSymlink(apexDir.Join(ctx, filepath.Dir(symlinkPath)), filepath.Base(symlinkPath), installedPath) } } @@ -553,11 +549,6 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { installMapSet[installMapPath.String()+":"+fi.installDir+"/"+fi.builtFile.Base()] = true } implicitInputs = append(implicitInputs, a.manifestPbOut) - if installSymbolFiles { - installedManifest := ctx.InstallFile(apexDir, "apex_manifest.pb", a.manifestPbOut) - installedKey := ctx.InstallFile(apexDir, "apex_pubkey", a.publicKeyFile) - implicitInputs = append(implicitInputs, installedManifest, installedKey) - } if len(installMapSet) > 0 { var installs []string From 53506361da37db709f699cde1f55ecf29519d2a6 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Tue, 18 Apr 2023 19:33:11 +0000 Subject: [PATCH 0020/1460] Squelch output for a failure case in mixed_mode_test Currently, the expected output shows as the main text for unrelated presubmit failures. Bug: 278748024 Test: build/soong/tests/mixed_mode_test.sh Change-Id: I553a82597c7c7791a822e330eb9b2663f35567a1 --- tests/mixed_mode_test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/mixed_mode_test.sh b/tests/mixed_mode_test.sh index 05d3a664f3..a1a792de33 100755 --- a/tests/mixed_mode_test.sh +++ b/tests/mixed_mode_test.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eu +#!/bin/bash set -o pipefail @@ -88,12 +88,12 @@ EOF fail "Bazel actions not found for force-enabled module" fi - local exit_code=`run_soong --bazel-force-enabled-modules=unenabled-touch-file nothing` + unused=`run_soong --bazel-force-enabled-modules=unenabled-touch-file nothing >/dev/null` - if [[ $exit_code -ne 1 ]]; then + if [[ $? -ne 1 ]]; then fail "Expected failure due to force-enabling an unenabled module " fi } -scan_and_run_tests \ No newline at end of file +scan_and_run_tests From efc51d93964dc0b0c2601dbff34ff180532d33f0 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Fri, 21 Apr 2023 15:11:25 -0400 Subject: [PATCH 0021/1460] Correct cc_test_library migration Previously we were treating it as a shared library only, but cc_test_library produces both a static and shared library. Test: bp2build tests Change-Id: I293d4246d581212b95765fa8f084301514d00dbf --- bp2build/cc_test_conversion_test.go | 17 ++++++++++++++--- cc/cc.go | 4 ++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/bp2build/cc_test_conversion_test.go b/bp2build/cc_test_conversion_test.go index 20adddbca4..4df4d4d46f 100644 --- a/bp2build/cc_test_conversion_test.go +++ b/bp2build/cc_test_conversion_test.go @@ -76,17 +76,28 @@ cc_test { static_libs: ["hostlib"], }, }, + static_libs: ["cc_test_lib1"], + shared_libs: ["cc_test_lib2"], data: [":data_mod", "file.txt"], data_bins: [":cc_bin"], data_libs: [":cc_lib"], cflags: ["-Wall"], } + +cc_test_library { + name: "cc_test_lib1", + host_supported: true, + include_build_directory: false, +} ` + simpleModuleDoNotConvertBp2build("cc_library", "foolib") + simpleModuleDoNotConvertBp2build("cc_library_static", "hostlib") + simpleModuleDoNotConvertBp2build("genrule", "data_mod") + simpleModuleDoNotConvertBp2build("cc_binary", "cc_bin") + - simpleModuleDoNotConvertBp2build("cc_test_library", "cc_lib"), + simpleModuleDoNotConvertBp2build("cc_library", "cc_lib") + + simpleModuleDoNotConvertBp2build("cc_test_library", "cc_test_lib2"), targets: []testBazelTarget{ + {"cc_library_shared", "cc_test_lib1", AttrNameToString{}}, + {"cc_library_static", "cc_test_lib1_bp2build_cc_library_static", AttrNameToString{}}, {"cc_test", "mytest", AttrNameToString{ "copts": `["-Wall"]`, "data": `[ @@ -95,7 +106,7 @@ cc_test { ":cc_bin", ":cc_lib", ]`, - "deps": `select({ + "deps": `[":cc_test_lib1_bp2build_cc_library_static"] + select({ "//build/bazel/platforms/os:darwin": [":hostlib"], "//build/bazel/platforms/os:linux_bionic": [":hostlib"], "//build/bazel/platforms/os:linux_glibc": [":hostlib"], @@ -106,7 +117,7 @@ cc_test { "gtest": "True", "isolated": "True", "local_includes": `["."]`, - "dynamic_deps": `select({ + "dynamic_deps": `[":cc_test_lib2"] + select({ "//build/bazel/platforms/os:android": [":foolib"], "//conditions:default": [], })`, diff --git a/cc/cc.go b/cc/cc.go index 9c555a11ef..246444f16c 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -3914,8 +3914,8 @@ func (c *Module) typ() moduleType { // TODO(b/244431896) properly convert cc_test_library to its own macro. This // will let them add implicit compile deps on gtest, for example. // - // For now, treat them as regular shared libraries. - return sharedLibrary + // For now, treat them as regular libraries. + return fullLibrary } else if c.CcLibrary() { static := false shared := false From 01c4456ce21ec3aa6693c083d77c0bfe1e7cccdc Mon Sep 17 00:00:00 2001 From: Yabin Cui Date: Thu, 20 Apr 2023 14:07:29 -0700 Subject: [PATCH 0022/1460] Enable -funique-internal-linkage-names when afdo=true We used to enable unique names only when a profile is available. But A profile generated for a binary without unique names doesn't work well when building a binary with unique names. To avoid a chicken-and-egg problem, this CL enables the flag when afdo=true, whether a profile exists or not. Bug: 241523910 Test: build Change-Id: I74d834510d81d2db76e34d0488c74a60e1fcecd5 --- cc/afdo.go | 74 ++++++++++++++++++++++++++------------------ cc/afdo_test.go | 82 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 30 deletions(-) diff --git a/cc/afdo.go b/cc/afdo.go index be4f50adb8..49f69873c4 100644 --- a/cc/afdo.go +++ b/cc/afdo.go @@ -34,7 +34,7 @@ var ( var afdoProfileProjectsConfigKey = android.NewOnceKey("AfdoProfileProjects") -const afdoCFlagsFormat = "-funique-internal-linkage-names -fprofile-sample-accurate -fprofile-sample-use=%s" +const afdoCFlagsFormat = "-fprofile-sample-accurate -fprofile-sample-use=%s" func recordMissingAfdoProfileFile(ctx android.BaseModuleContext, missing string) { getNamedMapForConfig(ctx.Config(), modulesMissingProfileFileKey).Store(missing, true) @@ -66,10 +66,24 @@ func (afdo *afdo) props() []interface{} { // afdoEnabled returns true for binaries and shared libraries // that set afdo prop to True and there is a profile available func (afdo *afdo) afdoEnabled() bool { - return afdo != nil && afdo.Properties.Afdo && afdo.Properties.FdoProfilePath != nil + return afdo != nil && afdo.Properties.Afdo } func (afdo *afdo) flags(ctx ModuleContext, flags Flags) Flags { + if afdo.Properties.Afdo { + // We use `-funique-internal-linkage-names` to associate profiles to the right internal + // functions. This option should be used before generating a profile. Because a profile + // generated for a binary without unique names doesn't work well building a binary with + // unique names (they have different internal function names). + // To avoid a chicken-and-egg problem, we enable `-funique-internal-linkage-names` when + // `afdo=true`, whether a profile exists or not. + // The profile can take effect in three steps: + // 1. Add `afdo: true` in Android.bp, and build the binary. + // 2. Collect an AutoFDO profile for the binary. + // 3. Make the profile searchable by the build system. So it's used the next time the binary + // is built. + flags.Local.CFlags = append([]string{"-funique-internal-linkage-names"}, flags.Local.CFlags...) + } if path := afdo.Properties.FdoProfilePath; path != nil { // The flags are prepended to allow overriding. profileUseFlag := fmt.Sprintf(afdoCFlagsFormat, *path) @@ -129,42 +143,41 @@ var _ FdoProfileMutatorInterface = (*Module)(nil) // Propagate afdo requirements down from binaries and shared libraries func afdoDepsMutator(mctx android.TopDownMutatorContext) { if m, ok := mctx.Module().(*Module); ok && m.afdo.afdoEnabled() { - if path := m.afdo.Properties.FdoProfilePath; path != nil { - mctx.WalkDeps(func(dep android.Module, parent android.Module) bool { - tag := mctx.OtherModuleDependencyTag(dep) - libTag, isLibTag := tag.(libraryDependencyTag) - - // Do not recurse down non-static dependencies - if isLibTag { - if !libTag.static() { - return false - } - } else { - if tag != objDepTag && tag != reuseObjTag { - return false - } + path := m.afdo.Properties.FdoProfilePath + mctx.WalkDeps(func(dep android.Module, parent android.Module) bool { + tag := mctx.OtherModuleDependencyTag(dep) + libTag, isLibTag := tag.(libraryDependencyTag) + + // Do not recurse down non-static dependencies + if isLibTag { + if !libTag.static() { + return false } - - if dep, ok := dep.(*Module); ok { - dep.afdo.Properties.AfdoRDeps = append( - dep.afdo.Properties.AfdoRDeps, - afdoRdep{ - VariationName: proptools.StringPtr(encodeTarget(m.Name())), - ProfilePath: path, - }, - ) + } else { + if tag != objDepTag && tag != reuseObjTag { + return false } + } - return true - }) - } + if dep, ok := dep.(*Module); ok { + dep.afdo.Properties.AfdoRDeps = append( + dep.afdo.Properties.AfdoRDeps, + afdoRdep{ + VariationName: proptools.StringPtr(encodeTarget(m.Name())), + ProfilePath: path, + }, + ) + } + + return true + }) } } // Create afdo variants for modules that need them func afdoMutator(mctx android.BottomUpMutatorContext) { if m, ok := mctx.Module().(*Module); ok && m.afdo != nil { - if !m.static() && m.afdo.Properties.Afdo && m.afdo.Properties.FdoProfilePath != nil { + if !m.static() && m.afdo.Properties.Afdo { mctx.SetDependencyVariation(encodeTarget(m.Name())) return } @@ -182,7 +195,7 @@ func afdoMutator(mctx android.BottomUpMutatorContext) { // \ ^ // ----------------------| // We only need to create one variant per unique rdep - if variantNameToProfilePath[variantName] == nil { + if _, exists := variantNameToProfilePath[variantName]; !exists { variationNames = append(variationNames, variantName) variantNameToProfilePath[variantName] = afdoRDep.ProfilePath } @@ -197,6 +210,7 @@ func afdoMutator(mctx android.BottomUpMutatorContext) { variation := modules[i].(*Module) variation.Properties.PreventInstall = true variation.Properties.HideFromMake = true + variation.afdo.Properties.Afdo = true variation.afdo.Properties.FdoProfilePath = variantNameToProfilePath[name] } } diff --git a/cc/afdo_test.go b/cc/afdo_test.go index 1c20bfc8cb..b250ad1a1d 100644 --- a/cc/afdo_test.go +++ b/cc/afdo_test.go @@ -379,3 +379,85 @@ func TestMultipleAfdoRDeps(t *testing.T) { t.Errorf("libFoo missing dependency on non-afdo variant of libBar") } } + +func TestAfdoDepsWithoutProfile(t *testing.T) { + t.Parallel() + bp := ` + cc_library_shared { + name: "libTest", + srcs: ["test.c"], + static_libs: ["libFoo"], + afdo: true, + } + + cc_library_static { + name: "libFoo", + srcs: ["foo.c"], + static_libs: ["libBar"], + } + + cc_library_static { + name: "libBar", + srcs: ["bar.c"], + } + ` + + result := android.GroupFixturePreparers( + PrepareForTestWithFdoProfile, + prepareForCcTest, + ).RunTestWithBp(t, bp) + + // Even without a profile path, the afdo enabled libraries should be built with + // -funique-internal-linkage-names. + expectedCFlag := "-funique-internal-linkage-names" + + libTest := result.ModuleForTests("libTest", "android_arm64_armv8-a_shared") + libFooAfdoVariant := result.ModuleForTests("libFoo", "android_arm64_armv8-a_static_afdo-libTest") + libBarAfdoVariant := result.ModuleForTests("libBar", "android_arm64_armv8-a_static_afdo-libTest") + + // Check cFlags of afdo-enabled module and the afdo-variant of its static deps + cFlags := libTest.Rule("cc").Args["cFlags"] + if !strings.Contains(cFlags, expectedCFlag) { + t.Errorf("Expected 'libTest' to enable afdo, but did not find %q in cflags %q", expectedCFlag, cFlags) + } + + cFlags = libFooAfdoVariant.Rule("cc").Args["cFlags"] + if !strings.Contains(cFlags, expectedCFlag) { + t.Errorf("Expected 'libFooAfdoVariant' to enable afdo, but did not find %q in cflags %q", expectedCFlag, cFlags) + } + + cFlags = libBarAfdoVariant.Rule("cc").Args["cFlags"] + if !strings.Contains(cFlags, expectedCFlag) { + t.Errorf("Expected 'libBarAfdoVariant' to enable afdo, but did not find %q in cflags %q", expectedCFlag, cFlags) + } + // Check dependency edge from afdo-enabled module to static deps + if !hasDirectDep(result, libTest.Module(), libFooAfdoVariant.Module()) { + t.Errorf("libTest missing dependency on afdo variant of libFoo") + } + + if !hasDirectDep(result, libFooAfdoVariant.Module(), libBarAfdoVariant.Module()) { + t.Errorf("libTest missing dependency on afdo variant of libBar") + } + + // Verify non-afdo variant exists and doesn't contain afdo + libFoo := result.ModuleForTests("libFoo", "android_arm64_armv8-a_static") + libBar := result.ModuleForTests("libBar", "android_arm64_armv8-a_static") + + cFlags = libFoo.Rule("cc").Args["cFlags"] + if strings.Contains(cFlags, expectedCFlag) { + t.Errorf("Expected 'libFoo' to not enable afdo, but found %q in cflags %q", expectedCFlag, cFlags) + } + cFlags = libBar.Rule("cc").Args["cFlags"] + if strings.Contains(cFlags, expectedCFlag) { + t.Errorf("Expected 'libBar' to not enable afdo, but found %q in cflags %q", expectedCFlag, cFlags) + } + + // Check dependency edges of static deps + if hasDirectDep(result, libTest.Module(), libFoo.Module()) { + t.Errorf("libTest should not depend on non-afdo variant of libFoo") + } + + if !hasDirectDep(result, libFoo.Module(), libBar.Module()) { + t.Errorf("libFoo missing dependency on non-afdo variant of libBar") + } +} From ac693b2f815129f242d4921d23db990cba28fa58 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Mon, 24 Apr 2023 00:07:38 +0000 Subject: [PATCH 0023/1460] Use stubs for system_dynamic_deps system libraries (libc, libm, libdl) belong to the runtime apex api domain. platform and other apexes should link against stubs and not impl of these libraries. Update bp2build so that both apex/non_apex picks stubs. These axes need to be more granular in the future, but for now this works because - non_apex axis (platform) is a separate api domain than the runtime apex - com.android.runtime which matches apex axis has not been allowlisted yet. Test: TH Change-Id: I43f429a90875f30d176db17b144dd51674ec8adf --- cc/bp2build.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cc/bp2build.go b/cc/bp2build.go index c8f516cdc1..e501c6f0c7 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1107,8 +1107,11 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion // having stubs or not, so Bazel select() statement can be used to choose // source/stub variants of them. apexAvailable := module.ApexAvailable() - setStubsForDynamicDeps(ctx, axis, config, apexAvailable, sharedDeps.export, &la.dynamicDeps, 0) - setStubsForDynamicDeps(ctx, axis, config, apexAvailable, sharedDeps.implementation, &la.implementationDynamicDeps, 1) + setStubsForDynamicDeps(ctx, axis, config, apexAvailable, sharedDeps.export, &la.dynamicDeps, 0, false) + setStubsForDynamicDeps(ctx, axis, config, apexAvailable, sharedDeps.implementation, &la.implementationDynamicDeps, 1, false) + if len(systemSharedLibs) > 0 { + setStubsForDynamicDeps(ctx, axis, config, apexAvailable, bazelLabelForSharedDeps(ctx, systemSharedLibs), &la.systemDynamicDeps, 2, true) + } } if !BoolDefault(props.Pack_relocations, packRelocationsDefault) { @@ -1178,7 +1181,7 @@ func availableToSameApexes(a, b []string) bool { } func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.ConfigurationAxis, - config string, apexAvailable []string, dynamicLibs bazel.LabelList, dynamicDeps *bazel.LabelListAttribute, ind int) { + config string, apexAvailable []string, dynamicLibs bazel.LabelList, dynamicDeps *bazel.LabelListAttribute, ind int, buildNonApexWithStubs bool) { depsWithStubs := []bazel.Label{} for _, l := range dynamicLibs.Includes { @@ -1204,16 +1207,20 @@ func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.C inApexSelectValue := dynamicDeps.SelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndInApex) nonApexSelectValue := dynamicDeps.SelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex) defaultSelectValue := dynamicDeps.SelectValue(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey) + nonApexDeps := depsWithStubs + if buildNonApexWithStubs { + nonApexDeps = stubLibLabels + } if axis == bazel.NoConfigAxis { (&inApexSelectValue).Append(bazel.MakeLabelList(stubLibLabels)) - (&nonApexSelectValue).Append(bazel.MakeLabelList(depsWithStubs)) + (&nonApexSelectValue).Append(bazel.MakeLabelList(nonApexDeps)) (&defaultSelectValue).Append(bazel.MakeLabelList(depsWithStubs)) dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndInApex, bazel.FirstUniqueBazelLabelList(inApexSelectValue)) dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex, bazel.FirstUniqueBazelLabelList(nonApexSelectValue)) dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey, bazel.FirstUniqueBazelLabelList(defaultSelectValue)) } else if config == bazel.OsAndroid { (&inApexSelectValue).Append(bazel.MakeLabelList(stubLibLabels)) - (&nonApexSelectValue).Append(bazel.MakeLabelList(depsWithStubs)) + (&nonApexSelectValue).Append(bazel.MakeLabelList(nonApexDeps)) dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndInApex, bazel.FirstUniqueBazelLabelList(inApexSelectValue)) dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex, bazel.FirstUniqueBazelLabelList(nonApexSelectValue)) } From c1490ec25edd5a8d60915d2e2a5810983b2a70f0 Mon Sep 17 00:00:00 2001 From: Jeff Vander Stoep Date: Mon, 24 Apr 2023 11:28:25 +0200 Subject: [PATCH 0024/1460] Add option to use protobuf3 Users of the rust_protobuf module can use the 'use_protobuf3' option to select version 3 of the protobuf crate instead of the current default of version 2. This unblocks teams that would like to use protobuf 3 immediately and provides a mechanism for us to incrementally migrate users of protobuf 2 to protobuf 3. Test: Adds tests that are executed during every build. Bug: 270895633 Change-Id: Ib6a18b77cfa7dc3cc604cea05269004b3712bbb6 --- rust/protobuf.go | 26 ++++++++++++++++++++++- rust/protobuf_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++ rust/testing.go | 6 ++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/rust/protobuf.go b/rust/protobuf.go index e30f25d952..0cf6e8c970 100644 --- a/rust/protobuf.go +++ b/rust/protobuf.go @@ -52,6 +52,10 @@ type ProtobufProperties struct { // List of libraries which export include paths required for this module Header_libs []string `android:"arch_variant,variant_prepend"` + + // Use protobuf version 3.x. This will be deleted once we migrate all current users + // of protobuf off of 2.x. + Use_protobuf3 *bool } type protobufDecorator struct { @@ -65,6 +69,10 @@ type protobufDecorator struct { protoFlags android.ProtoFlags } +func (proto *protobufDecorator) useProtobuf3() bool { + return Bool(proto.Properties.Use_protobuf3) +} + func (proto *protobufDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) android.Path { var protoFlags android.ProtoFlags var grpcProtoFlags android.ProtoFlags @@ -73,7 +81,13 @@ func (proto *protobufDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) outDir := android.PathForModuleOut(ctx) protoFiles := android.PathsForModuleSrc(ctx, proto.Properties.Protos) grpcFiles := android.PathsForModuleSrc(ctx, proto.Properties.Grpc_protos) + + // For now protobuf2 (the deprecated version) remains the default. This will change in the + // future as we update the various users. protoPluginPath := ctx.Config().HostToolPath(ctx, "protoc-gen-rust-deprecated") + if proto.useProtobuf3() == true { + protoPluginPath = ctx.Config().HostToolPath(ctx, "protoc-gen-rust") + } commonProtoFlags = append(commonProtoFlags, defaultProtobufFlags...) commonProtoFlags = append(commonProtoFlags, proto.Properties.Proto_flags...) @@ -206,10 +220,20 @@ func (proto *protobufDecorator) SourceProviderProps() []interface{} { func (proto *protobufDecorator) SourceProviderDeps(ctx DepsContext, deps Deps) Deps { deps = proto.BaseSourceProvider.SourceProviderDeps(ctx, deps) - deps.Rustlibs = append(deps.Rustlibs, "libprotobuf_deprecated") + useProtobuf3 := proto.useProtobuf3() + if useProtobuf3 == true { + deps.Rustlibs = append(deps.Rustlibs, "libprotobuf") + } else { + deps.Rustlibs = append(deps.Rustlibs, "libprotobuf_deprecated") + } deps.HeaderLibs = append(deps.SharedLibs, proto.Properties.Header_libs...) if len(proto.Properties.Grpc_protos) > 0 { + if useProtobuf3 == true { + ctx.PropertyErrorf("protos", "rust_protobuf with grpc_protos defined must currently use "+ + "`use_protobuf3: false,` in the Android.bp file. This is temporary until the "+ + "grpcio crate is updated to use the current version of the protobuf crate.") + } deps.Rustlibs = append(deps.Rustlibs, "libgrpcio", "libfutures") deps.HeaderLibs = append(deps.HeaderLibs, "libprotobuf-cpp-full") } diff --git a/rust/protobuf_test.go b/rust/protobuf_test.go index 0aa4549751..b723f3f1a9 100644 --- a/rust/protobuf_test.go +++ b/rust/protobuf_test.go @@ -69,6 +69,55 @@ func TestRustProtobuf(t *testing.T) { } } +func TestRustProtobuf3(t *testing.T) { + ctx := testRust(t, ` + rust_protobuf { + name: "librust_proto", + protos: ["buf.proto", "proto.proto"], + crate_name: "rust_proto", + source_stem: "buf", + use_protobuf3: true, + shared_libs: ["libfoo_shared"], + static_libs: ["libfoo_static"], + } + cc_library_shared { + name: "libfoo_shared", + export_include_dirs: ["shared_include"], + } + cc_library_static { + name: "libfoo_static", + export_include_dirs: ["static_include"], + } + `) + // Check that libprotobuf is added as a dependency. + librust_proto := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_dylib").Module().(*Module) + if !android.InList("libprotobuf", librust_proto.Properties.AndroidMkDylibs) { + t.Errorf("libprotobuf dependency missing for rust_protobuf (dependency missing from AndroidMkDylibs)") + } + + // Make sure the correct plugin is being used. + librust_proto_out := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_source").Output("buf.rs") + cmd := librust_proto_out.RuleParams.Command + if w := "protoc-gen-rust"; !strings.Contains(cmd, w) { + t.Errorf("expected %q in %q", w, cmd) + } + + // Check exported include directories + if w := "-Ishared_include"; !strings.Contains(cmd, w) { + t.Errorf("expected %q in %q", w, cmd) + } + if w := "-Istatic_include"; !strings.Contains(cmd, w) { + t.Errorf("expected %q in %q", w, cmd) + } + + // Check proto.rs, the second protobuf, is listed as an output + librust_proto_outputs := ctx.ModuleForTests("librust_proto", "android_arm64_armv8-a_source").AllOutputs() + if android.InList("proto.rs", librust_proto_outputs) { + t.Errorf("rust_protobuf is not producing multiple outputs; expected 'proto.rs' in list, got: %#v ", + librust_proto_outputs) + } +} + func TestRustGrpc(t *testing.T) { ctx := testRust(t, ` rust_protobuf { diff --git a/rust/testing.go b/rust/testing.go index a33d948dba..0a6a870d8f 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -126,6 +126,12 @@ func GatherRequiredDepsForTest() string { apex_available: ["//apex_available:platform", "//apex_available:anyapex"], min_sdk_version: "29", } + rust_library { + name: "libprotobuf", + crate_name: "protobuf", + srcs: ["foo.rs"], + host_supported: true, + } rust_library { name: "libprotobuf_deprecated", crate_name: "protobuf", From ef69d47d0307b0cb8738323c2c28afe900b8867b Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Tue, 18 Apr 2023 17:32:43 -0400 Subject: [PATCH 0025/1460] disable mixed builds for sanitized cc modules The Bazel rules don't currently support any sanitizers other than ubsan, so we should disable mixed builds for modules which are sanitized. Test: go test Bug: 278772861 Bug: 253433725 Change-Id: Ia01fb8cb59154bdfb21a111b04af0350e1876b0b --- cc/cc.go | 41 ++++++-- cc/cc_test.go | 258 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+), 6 deletions(-) diff --git a/cc/cc.go b/cc/cc.go index 6054222138..82cfe9079f 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -1893,17 +1893,46 @@ func (c *Module) QueueBazelCall(ctx android.BaseModuleContext) { // IsMixedBuildSupported returns true if the module should be analyzed by Bazel // in any of the --bazel-mode(s). func (c *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool { - // TODO(b/261058727): Remove this (enable mixed builds for modules with UBSan) - // Currently we can only support ubsan when minimum runtime is used. - return c.bazelHandler != nil && (!isUbsanEnabled(c) || c.MinimalRuntimeNeeded()) + if !allEnabledSanitizersSupportedByBazel(c) { + //TODO(b/278772861) support sanitizers in Bazel rules + return false + } + return c.bazelHandler != nil } -func isUbsanEnabled(c *Module) bool { +func allEnabledSanitizersSupportedByBazel(c *Module) bool { if c.sanitize == nil { - return false + return true } sanitizeProps := &c.sanitize.Properties.SanitizeMutated - return Bool(sanitizeProps.Integer_overflow) || len(sanitizeProps.Misc_undefined) > 0 + + unsupportedSanitizers := []*bool{ + sanitizeProps.Safestack, + sanitizeProps.Cfi, + sanitizeProps.Scudo, + BoolPtr(len(c.sanitize.Properties.Sanitize.Recover) > 0), + BoolPtr(c.sanitize.Properties.Sanitize.Blocklist != nil), + } + for _, san := range unsupportedSanitizers { + if Bool(san) { + return false + } + } + + for _, san := range Sanitizers { + if san == intOverflow { + // TODO(b/261058727): enable mixed builds for all modules with UBSan + // Currently we can only support ubsan when minimum runtime is used. + ubsanEnabled := Bool(sanitizeProps.Integer_overflow) || len(sanitizeProps.Misc_undefined) > 0 + if ubsanEnabled && !c.MinimalRuntimeNeeded() { + return false + } + } else if c.sanitize.isSanitizerEnabled(san) { + return false + } + } + + return true } func GetApexConfigKey(ctx android.BaseModuleContext) *android.ApexConfigKey { diff --git a/cc/cc_test.go b/cc/cc_test.go index b986511f05..830de40c03 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -3114,6 +3114,11 @@ func TestLibDepAndroidMkExportInMixedBuilds(t *testing.T) { whole_static_libs: ["whole_static_dep"], shared_libs: ["shared_dep"], gtest: false, + sanitize: { + // cc_test modules default to memtag_heap: true, + // but this adds extra dependencies that we don't care about + never: true, + } } cc_binary { name: "binary", @@ -5101,3 +5106,256 @@ func TestDclaLibraryInApex(t *testing.T) { expectedOutputFiles := []string{"outputbase/execroot/__main__/foo.so"} android.AssertDeepEquals(t, "output files", expectedOutputFiles, outputFiles.Strings()) } + +func TestDisableSanitizerVariantsInMixedBuilds(t *testing.T) { + t.Parallel() + bp := ` + cc_library_static { + name: "foo_ubsan_minimal", + srcs: ["foo.cc"], + bazel_module: { label: "//foo_ubsan_minimal" }, + sanitize: { + all_undefined: true, + integer_overflow: true, + }, + } + cc_library_static { + name: "foo", + srcs: ["foo.cc"], + bazel_module: { label: "//foo" }, + sanitize: { + address: true, + hwaddress: true, + fuzzer: true, + integer_overflow: true, + scs: true, + }, + } + cc_library_static { + name: "foo_tsan", + srcs: ["foo.cc"], + bazel_module: { label: "//foo_tsan" }, + sanitize: { + thread: true, + }, + } + cc_library_static { + name: "foo_cfi", + srcs: ["foo.cc"], + bazel_module: { label: "//foo_cfi" }, + sanitize: { + cfi: true, + }, + } + cc_library_static { + name: "foo_memtag_stack", + srcs: ["foo.cc"], + bazel_module: { label: "//foo_memtag_stack" }, + sanitize: { + memtag_stack: true, + }, + } + cc_library_static { + name: "foo_memtag_heap", + srcs: ["foo.cc"], + bazel_module: { label: "//foo_memtag_heap" }, + sanitize: { + memtag_heap: true, + }, + } + cc_library_static { + name: "foo_safestack", + srcs: ["foo.cc"], + bazel_module: { label: "//foo_safestack" }, + sanitize: { + safestack: true, + }, + } + cc_library_static { + name: "foo_scudo", + srcs: ["foo.cc"], + bazel_module: { label: "//foo_scudo" }, + sanitize: { + scudo: true, + }, + } + ` + testcases := []struct { + name string + variant string + expectedOutputPaths []string + }{ + { + name: "foo_ubsan_minimal", + variant: "android_arm64_armv8-a_static_apex28", + expectedOutputPaths: []string{ + "outputbase/execroot/__main__/foo_ubsan_minimal.a", + }, + }, + { + name: "foo", + variant: "android_arm64_armv8-a_static_apex28", + expectedOutputPaths: []string{ + "outputbase/execroot/__main__/foo.a", + }, + }, + { + name: "foo", + variant: "android_arm_armv7-a-neon_static_asan_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo/android_arm_armv7-a-neon_static_asan_apex28/foo.a", + }, + }, + { + name: "foo", + variant: "android_arm64_armv8-a_static_hwasan_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo/android_arm64_armv8-a_static_hwasan_apex28/foo.a", + }, + }, + { + name: "foo", + variant: "android_arm64_armv8-a_static_fuzzer_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo/android_arm64_armv8-a_static_fuzzer_apex28/foo.a", + }, + }, + { + name: "foo", + variant: "android_arm_armv7-a-neon_static_asan_fuzzer_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo/android_arm_armv7-a-neon_static_asan_fuzzer_apex28/foo.a", + }, + }, + { + name: "foo", + variant: "android_arm64_armv8-a_static_hwasan_fuzzer_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo/android_arm64_armv8-a_static_hwasan_fuzzer_apex28/foo.a", + }, + }, + { + name: "foo", + variant: "android_arm64_armv8-a_static_scs_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo/android_arm64_armv8-a_static_scs_apex28/foo.a", + }, + }, + { + name: "foo", + variant: "android_arm64_armv8-a_static_hwasan_scs_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo/android_arm64_armv8-a_static_hwasan_scs_apex28/foo.a", + }, + }, + { + name: "foo", + variant: "android_arm64_armv8-a_static_hwasan_scs_fuzzer_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo/android_arm64_armv8-a_static_hwasan_scs_fuzzer_apex28/foo.a", + }, + }, + { + name: "foo_tsan", + variant: "android_arm64_armv8-a_static_apex28", + expectedOutputPaths: []string{ + "outputbase/execroot/__main__/foo_tsan.a", + }, + }, + { + name: "foo_tsan", + variant: "android_arm64_armv8-a_static_tsan_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo_tsan/android_arm64_armv8-a_static_tsan_apex28/foo_tsan.a", + }, + }, + { + name: "foo_cfi", + variant: "android_arm64_armv8-a_static_apex28", + expectedOutputPaths: []string{ + "outputbase/execroot/__main__/foo_cfi.a", + }, + }, + { + name: "foo_cfi", + variant: "android_arm64_armv8-a_static_cfi_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo_cfi/android_arm64_armv8-a_static_cfi_apex28/foo_cfi.a", + }, + }, + { + name: "foo_memtag_stack", + variant: "android_arm64_armv8-a_static_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo_memtag_stack/android_arm64_armv8-a_static_apex28/foo_memtag_stack.a", + }, + }, + { + name: "foo_memtag_heap", + variant: "android_arm64_armv8-a_static_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo_memtag_heap/android_arm64_armv8-a_static_apex28/foo_memtag_heap.a", + }, + }, + { + name: "foo_safestack", + variant: "android_arm64_armv8-a_static_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo_safestack/android_arm64_armv8-a_static_apex28/foo_safestack.a", + }, + }, + { + name: "foo_scudo", + variant: "android_arm64_armv8-a_static_apex28", + expectedOutputPaths: []string{ + "out/soong/.intermediates/foo_scudo/android_arm64_armv8-a_static_apex28/foo_scudo.a", + }, + }, + } + + ctx := android.GroupFixturePreparers( + prepareForCcTest, + prepareForAsanTest, + android.FixtureRegisterWithContext(registerTestMutators), + android.FixtureModifyConfig(func(config android.Config) { + config.BazelContext = android.MockBazelContext{ + OutputBaseDir: "outputbase", + LabelToCcInfo: map[string]cquery.CcInfo{ + "//foo_ubsan_minimal": { + RootStaticArchives: []string{"foo_ubsan_minimal.a"}, + }, + "//foo": { + RootStaticArchives: []string{"foo.a"}, + }, + "//foo_tsan": { + RootStaticArchives: []string{"foo_tsan.a"}, + }, + "//foo_cfi": { + RootStaticArchives: []string{"foo_cfi.a"}, + }, + "//foo_memtag_stack": { + RootStaticArchives: []string{"INVALID_ARCHIVE.a"}, + }, + "//foo_memtag_heap": { + RootStaticArchives: []string{"INVALID_ARCHIVE.a"}, + }, + "//foo_safestack": { + RootStaticArchives: []string{"INVALID_ARCHIVE.a"}, + }, + "//foo_scudo": { + RootStaticArchives: []string{"INVALID_ARCHIVE.a"}, + }, + }, + } + }), + ).RunTestWithBp(t, bp).TestContext + + for _, tc := range testcases { + fooMod := ctx.ModuleForTests(tc.name, tc.variant).Module() + outputFiles, err := fooMod.(android.OutputFileProducer).OutputFiles("") + if err != nil { + t.Errorf("Unexpected error getting cc_object outputfiles %s", err) + } + android.AssertPathsRelativeToTopEquals(t, "output files", tc.expectedOutputPaths, outputFiles) + } +} From 4e1bfa06e97f03fc1bb20107ba927c878b98808f Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Mon, 24 Apr 2023 22:45:14 +0000 Subject: [PATCH 0026/1460] Use copy of apex_available in AvailableToSameApexes Modifying the list in-place causes some non-determinism. Test: go test ./cc -run TestStubsForLibraryInMultipleApexes -count 1000 Bug: 275313114 Change-Id: Ia2519e146d97667ad5900cf68ab9935fcbaf08a4 --- android/apex.go | 5 +++-- cc/cc_test.go | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/android/apex.go b/android/apex.go index 5bbc02e512..5cc0584e09 100644 --- a/android/apex.go +++ b/android/apex.go @@ -502,8 +502,9 @@ func (m *ApexModuleBase) checkApexAvailableProperty(mctx BaseModuleContext) { // exactly the same set of APEXes (and platform), i.e. if their apex_available // properties have the same elements. func AvailableToSameApexes(mod1, mod2 ApexModule) bool { - mod1ApexAvail := SortedUniqueStrings(mod1.apexModuleBase().ApexProperties.Apex_available) - mod2ApexAvail := SortedUniqueStrings(mod2.apexModuleBase().ApexProperties.Apex_available) + // Use CopyOf to prevent non-determinism (b/275313114#comment1) + mod1ApexAvail := SortedUniqueStrings(CopyOf(mod1.apexModuleBase().ApexProperties.Apex_available)) + mod2ApexAvail := SortedUniqueStrings(CopyOf(mod2.apexModuleBase().ApexProperties.Apex_available)) if len(mod1ApexAvail) != len(mod2ApexAvail) { return false } diff --git a/cc/cc_test.go b/cc/cc_test.go index b986511f05..417ea9c9b4 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -3573,9 +3573,6 @@ func TestVersionedStubs(t *testing.T) { } func TestStubsForLibraryInMultipleApexes(t *testing.T) { - // TODO(b/275313114): Test exposes non-determinism which should be corrected and the test - // reenabled. - t.Skip() t.Parallel() ctx := testCc(t, ` cc_library_shared { From 75dbca2eae78bb280b50828d3c2165bda5d87ba5 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Thu, 20 Apr 2023 13:13:25 +0000 Subject: [PATCH 0027/1460] Revert "Revert "mixed builds correctly reference stubs libs"" This reverts commit 1db4348734acfd0b4a38e2ff49dca2df4bac0d84. Changes from original: - extracted function to add the current version to stub versions for bp2build. - added libc++ to mixed builds denylist Reason for revert: re-uploading with fix in topic Change-Id: Ifa0ed456bf8cb4a7f861d6826263adfedb4fdd9c --- bazel/cquery/request_type.go | 3 + bp2build/cc_library_shared_conversion_test.go | 63 +++++++++ cc/bp2build.go | 9 +- cc/cc.go | 19 ++- cc/cc_test.go | 127 ++++++++++++++++++ cc/library.go | 29 +++- 6 files changed, 235 insertions(+), 15 deletions(-) diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go index bf3a6b5c66..6a3b3c82c7 100644 --- a/bazel/cquery/request_type.go +++ b/bazel/cquery/request_type.go @@ -149,6 +149,7 @@ sharedLibraries = [] rootSharedLibraries = [] shared_info_tag = "//build/bazel/rules/cc:cc_library_shared.bzl%CcSharedLibraryOutputInfo" +stubs_tag = "//build/bazel/rules/cc:cc_stub_library.bzl%CcStubInfo" unstripped_tag = "//build/bazel/rules/cc:stripped_cc_common.bzl%CcUnstrippedInfo" unstripped = "" @@ -160,6 +161,8 @@ if shared_info_tag in p: unstripped = path if unstripped_tag in p: unstripped = p[unstripped_tag].unstripped.path +elif stubs_tag in p: + rootSharedLibraries.extend([f.path for f in target.files.to_list()]) else: for linker_input in linker_inputs: for library in linker_input.libraries: diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go index 29316ab1ed..f5d62c6926 100644 --- a/bp2build/cc_library_shared_conversion_test.go +++ b/bp2build/cc_library_shared_conversion_test.go @@ -1291,3 +1291,66 @@ cc_library_shared{ }, }) } + +func TestCcLibrarySharedStubsDessertVersionConversion(t *testing.T) { + runCcLibrarySharedTestCase(t, Bp2buildTestCase{ + Description: "cc_library_shared converts dessert codename versions to numerical versions", + Blueprint: ` +cc_library_shared { + name: "a", + include_build_directory: false, + stubs: { + symbol_file: "a.map.txt", + versions: [ + "Q", + "R", + "31", + ], + }, +} +cc_library_shared { + name: "b", + include_build_directory: false, + stubs: { + symbol_file: "b.map.txt", + versions: [ + "Q", + "R", + "31", + "current", + ], + }, +} +`, + ExpectedBazelTargets: []string{ + makeCcStubSuiteTargets("a", AttrNameToString{ + "soname": `"a.so"`, + "source_library_label": `"//:a"`, + "stubs_symbol_file": `"a.map.txt"`, + "stubs_versions": `[ + "29", + "30", + "31", + "current", + ]`, + }), + MakeBazelTarget("cc_library_shared", "a", AttrNameToString{ + "stubs_symbol_file": `"a.map.txt"`, + }), + makeCcStubSuiteTargets("b", AttrNameToString{ + "soname": `"b.so"`, + "source_library_label": `"//:b"`, + "stubs_symbol_file": `"b.map.txt"`, + "stubs_versions": `[ + "29", + "30", + "31", + "current", + ]`, + }), + MakeBazelTarget("cc_library_shared", "b", AttrNameToString{ + "stubs_symbol_file": `"b.map.txt"`, + }), + }, + }) +} diff --git a/cc/bp2build.go b/cc/bp2build.go index 96c478966a..4e39532532 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -772,8 +772,13 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) if libraryProps, ok := archVariantLibraryProperties[axis][cfg].(*LibraryProperties); ok { if axis == bazel.NoConfigAxis { - compilerAttrs.stubsSymbolFile = libraryProps.Stubs.Symbol_file - compilerAttrs.stubsVersions.SetSelectValue(axis, cfg, libraryProps.Stubs.Versions) + if libraryProps.Stubs.Symbol_file != nil { + compilerAttrs.stubsSymbolFile = libraryProps.Stubs.Symbol_file + versions := android.CopyOf(libraryProps.Stubs.Versions) + normalizeVersions(ctx, versions) + versions = addCurrentVersionIfNotPresent(versions) + compilerAttrs.stubsVersions.SetSelectValue(axis, cfg, versions) + } } if suffix := libraryProps.Suffix; suffix != nil { compilerAttrs.suffix.SetSelectValue(axis, cfg, suffix) diff --git a/cc/cc.go b/cc/cc.go index 82cfe9079f..c1a1020adf 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -1976,6 +1976,17 @@ func (c *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) { c.maybeInstall(mctx, apexInfo) } +func moduleContextFromAndroidModuleContext(actx android.ModuleContext, c *Module) ModuleContext { + ctx := &moduleContext{ + ModuleContext: actx, + moduleContextImpl: moduleContextImpl{ + mod: c, + }, + } + ctx.ctx = ctx + return ctx +} + func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { // Handle the case of a test module split by `test_per_src` mutator. // @@ -1995,13 +2006,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { c.makeLinkType = GetMakeLinkType(actx, c) - ctx := &moduleContext{ - ModuleContext: actx, - moduleContextImpl: moduleContextImpl{ - mod: c, - }, - } - ctx.ctx = ctx + ctx := moduleContextFromAndroidModuleContext(actx, c) deps := c.depsToPaths(ctx) if ctx.Failed() { diff --git a/cc/cc_test.go b/cc/cc_test.go index 830de40c03..ebe543eaf3 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -3685,6 +3685,133 @@ func TestStubsForLibraryInMultipleApexes(t *testing.T) { } } +func TestMixedBuildUsesStubs(t *testing.T) { + // TODO(b/275313114): Test exposes non-determinism which should be corrected and the test + // reenabled. + t.Skip() + t.Parallel() + bp := ` + cc_library_shared { + name: "libFoo", + bazel_module: { label: "//:libFoo" }, + srcs: ["foo.c"], + stubs: { + symbol_file: "foo.map.txt", + versions: ["current"], + }, + apex_available: ["bar", "a1"], + } + + cc_library_shared { + name: "libBar", + srcs: ["bar.c"], + shared_libs: ["libFoo"], + apex_available: ["a1"], + } + + cc_library_shared { + name: "libA1", + srcs: ["a1.c"], + shared_libs: ["libFoo"], + apex_available: ["a1"], + } + + cc_library_shared { + name: "libBarA1", + srcs: ["bara1.c"], + shared_libs: ["libFoo"], + apex_available: ["bar", "a1"], + } + + cc_library_shared { + name: "libAnyApex", + srcs: ["anyApex.c"], + shared_libs: ["libFoo"], + apex_available: ["//apex_available:anyapex"], + } + + cc_library_shared { + name: "libBaz", + srcs: ["baz.c"], + shared_libs: ["libFoo"], + apex_available: ["baz"], + } + + cc_library_shared { + name: "libQux", + srcs: ["qux.c"], + shared_libs: ["libFoo"], + apex_available: ["qux", "bar"], + }` + + result := android.GroupFixturePreparers( + prepareForCcTest, + android.FixtureModifyConfig(func(config android.Config) { + config.BazelContext = android.MockBazelContext{ + OutputBaseDir: "out/bazel", + LabelToCcInfo: map[string]cquery.CcInfo{ + "//:libFoo": { + RootDynamicLibraries: []string{"libFoo.so"}, + }, + "//:libFoo_stub_libs-current": { + RootDynamicLibraries: []string{"libFoo_stub_libs-current.so"}, + }, + }, + } + }), + ).RunTestWithBp(t, bp) + ctx := result.TestContext + + variants := ctx.ModuleVariantsForTests("libFoo") + expectedVariants := []string{ + "android_arm64_armv8-a_shared", + "android_arm64_armv8-a_shared_current", + "android_arm_armv7-a-neon_shared", + "android_arm_armv7-a-neon_shared_current", + } + variantsMismatch := false + if len(variants) != len(expectedVariants) { + variantsMismatch = true + } else { + for _, v := range expectedVariants { + if !inList(v, variants) { + variantsMismatch = false + } + } + } + if variantsMismatch { + t.Errorf("variants of libFoo expected:\n") + for _, v := range expectedVariants { + t.Errorf("%q\n", v) + } + t.Errorf(", but got:\n") + for _, v := range variants { + t.Errorf("%q\n", v) + } + } + + linkAgainstFoo := []string{"libBarA1"} + linkAgainstFooStubs := []string{"libBar", "libA1", "libBaz", "libQux", "libAnyApex"} + + libFooPath := "out/bazel/execroot/__main__/libFoo.so" + for _, lib := range linkAgainstFoo { + libLinkRule := ctx.ModuleForTests(lib, "android_arm64_armv8-a_shared").Rule("ld") + libFlags := libLinkRule.Args["libFlags"] + if !strings.Contains(libFlags, libFooPath) { + t.Errorf("%q: %q is not found in %q", lib, libFooPath, libFlags) + } + } + + libFooStubPath := "out/bazel/execroot/__main__/libFoo_stub_libs-current.so" + for _, lib := range linkAgainstFooStubs { + libLinkRule := ctx.ModuleForTests(lib, "android_arm64_armv8-a_shared").Rule("ld") + libFlags := libLinkRule.Args["libFlags"] + if !strings.Contains(libFlags, libFooStubPath) { + t.Errorf("%q: %q is not found in %q", lib, libFooStubPath, libFlags) + } + } +} + func TestVersioningMacro(t *testing.T) { t.Parallel() for _, tc := range []struct{ moduleName, expected string }{ diff --git a/cc/library.go b/cc/library.go index 1daeeffc27..ee09389cad 100644 --- a/cc/library.go +++ b/cc/library.go @@ -936,9 +936,17 @@ func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx androi func (handler *ccLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKeyApexVariant(ctx, GetApexConfigKey(ctx))) + if v := handler.module.library.stubsVersion(); v != "" { + stubsLabel := label + "_stub_libs-" + v + bazelCtx.QueueBazelRequest(stubsLabel, cquery.GetCcInfo, android.GetConfigKeyApexVariant(ctx, GetApexConfigKey(ctx))) + } } func (handler *ccLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + if v := handler.module.library.stubsVersion(); v != "" { + // if we are a stubs variant, just use the Bazel stubs target + label = label + "_stub_libs-" + v + } bazelCtx := ctx.Config().BazelContext ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKeyApexVariant(ctx, GetApexConfigKey(ctx))) if err != nil { @@ -967,6 +975,9 @@ func (handler *ccLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.Modu } handler.module.setAndroidMkVariablesFromCquery(ccInfo.CcAndroidMkInfo) + + cctx := moduleContextFromAndroidModuleContext(ctx, handler.module) + addStubDependencyProviders(cctx) } func (library *libraryDecorator) setFlagExporterInfoFromCcInfo(ctx android.ModuleContext, ccInfo cquery.CcInfo) { @@ -1792,6 +1803,12 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, Target: ctx.Target(), }) + addStubDependencyProviders(ctx) + + return unstrippedOutputFile +} + +func addStubDependencyProviders(ctx ModuleContext) { stubs := ctx.GetDirectDepsWithTag(stubImplDepTag) if len(stubs) > 0 { var stubsInfo []SharedStubLibrary @@ -1806,12 +1823,9 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, } ctx.SetProvider(SharedLibraryStubsProvider, SharedLibraryStubsInfo{ SharedStubLibraries: stubsInfo, - - IsLLNDK: ctx.IsLlndk(), + IsLLNDK: ctx.IsLlndk(), }) } - - return unstrippedOutputFile } func (library *libraryDecorator) unstrippedOutputFilePath() android.Path { @@ -2397,7 +2411,10 @@ func (library *libraryDecorator) stubsVersions(ctx android.BaseMutatorContext) [ } // Future API level is implicitly added if there isn't - vers := library.Properties.Stubs.Versions + return addCurrentVersionIfNotPresent(library.Properties.Stubs.Versions) +} + +func addCurrentVersionIfNotPresent(vers []string) []string { if inList(android.FutureApiLevel.String(), vers) { return vers } @@ -2662,7 +2679,7 @@ func LinkageMutator(mctx android.BottomUpMutatorContext) { // normalizeVersions modifies `versions` in place, so that each raw version // string becomes its normalized canonical form. // Validates that the versions in `versions` are specified in least to greatest order. -func normalizeVersions(ctx android.BaseModuleContext, versions []string) { +func normalizeVersions(ctx android.BazelConversionPathContext, versions []string) { var previous android.ApiLevel for i, v := range versions { ver, err := android.ApiLevelFromUser(ctx, v) From 912c4ab16f4c47942c66ded50f4aabba06f51d16 Mon Sep 17 00:00:00 2001 From: Jooyung Han Date: Sat, 22 Apr 2023 13:46:59 +0900 Subject: [PATCH 0028/1460] override_apex can be replaced with apex_set Bug: 279247159 Test: m nothing Change-Id: Ic4f8b2b1ee936afe2bf26de7d6cae19d93ecc831 --- android/override_module.go | 5 +-- apex/apex_test.go | 70 +++++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/android/override_module.go b/android/override_module.go index 2d30a852d0..86f582b766 100644 --- a/android/override_module.go +++ b/android/override_module.go @@ -304,8 +304,9 @@ func performOverrideMutator(ctx BottomUpMutatorContext) { for i, o := range overrides { mods[i+1].(OverridableModule).override(ctx, o) if o.getOverriddenByPrebuilt() { - // The overriding module itself, too, is overridden by a prebuilt. Skip its installation. - mods[i+1].HideFromMake() + // The overriding module itself, too, is overridden by a prebuilt. + // Copy the flag and hide it in make + mods[i+1].ReplacedByPrebuilt() } } } else if o, ok := ctx.Module().(OverrideModule); ok { diff --git a/apex/apex_test.go b/apex/apex_test.go index 440afec97a..888601f520 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -5696,7 +5696,7 @@ func TestPrebuiltSkipsSymbols(t *testing.T) { installSymbolFiles bool }{ { - name: "Source module build rule doesn't install symbol files", + name: "Source module biuld rule doesn't install symbol files", usePrebuilt: true, installSymbolFiles: false, }, @@ -5750,6 +5750,74 @@ func TestPrebuiltSkipsSymbols(t *testing.T) { } } +func TestPrebuiltSkipsSymbols_WorksWithOverrideApex(t *testing.T) { + testCases := []struct { + name string + usePrebuilt bool + installSymbolFiles bool + }{ + { + name: "Source module build rule doesn't install symbol files", + usePrebuilt: true, + installSymbolFiles: false, + }, + { + name: "Source module is installed with symbols", + usePrebuilt: false, + installSymbolFiles: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + preferProperty := "prefer: false" + if tc.usePrebuilt { + preferProperty = "prefer: true" + } + ctx := testApex(t, ` + apex { + name: "myapex", + binaries: ["foo"], + key: "myapex.key", + updatable: false, + } + + // Source module + override_apex { + name: "myapex2", + base: "myapex", + package_name: "myapex2," + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + apex_set { + name: "myapex2", + apex_name: "myapex", // needed to replace override_apex + set: "myapex2.apks", + `+preferProperty+` + } + + cc_binary { + name: "foo", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + `) + // Symbol files are installed by installing entries under ${OUT}/apex/{apex name} + android.AssertStringListContainsEquals(t, "Implicits", + ctx.ModuleForTests("myapex", "android_common_myapex2_myapex_image").Module().FilesToInstall().Strings(), + filepath.Join(ctx.Config().SoongOutDir(), "target/product/test_device/apex/myapex2/bin/foo"), + tc.installSymbolFiles) + }) + } +} + func TestApexWithTests(t *testing.T) { ctx := testApex(t, ` apex_test { From 418364213a0db1f5a159bbdec3b523d15b86274a Mon Sep 17 00:00:00 2001 From: Jooyung Han Date: Tue, 25 Apr 2023 15:04:45 +0900 Subject: [PATCH 0029/1460] Revert "apex: skip symbol files when replaced with prebuilt" This reverts commit 9551384f65eb77e43afbe3408d5437ce91804edd. 5bedfee4484a8786ccd761212057f8aac9a02eb7 is a better solution for the same problem. Bug: 279247159 Test: m checkbuild (presubmit) Change-Id: Ifd3117a4450b5275c09258d1ce896adb41a75158 --- apex/apex_test.go | 129 ---------------------------------------------- apex/builder.go | 4 -- 2 files changed, 133 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 888601f520..d08d2466e6 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -5689,135 +5689,6 @@ func TestBootDexJarsFromSourcesAndPrebuilts(t *testing.T) { }) } -func TestPrebuiltSkipsSymbols(t *testing.T) { - testCases := []struct { - name string - usePrebuilt bool - installSymbolFiles bool - }{ - { - name: "Source module biuld rule doesn't install symbol files", - usePrebuilt: true, - installSymbolFiles: false, - }, - { - name: "Source module is installed with symbols", - usePrebuilt: false, - installSymbolFiles: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - preferProperty := "prefer: false" - if tc.usePrebuilt { - preferProperty = "prefer: true" - } - ctx := testApex(t, ` - // Source module - apex { - name: "myapex", - binaries: ["foo"], - key: "myapex.key", - updatable: false, - } - - apex_key { - name: "myapex.key", - public_key: "testkey.avbpubkey", - private_key: "testkey.pem", - } - - apex_set { - name: "myapex", - set: "myapex.apks", - `+preferProperty+` - } - - cc_binary { - name: "foo", - srcs: ["mylib.cpp"], - system_shared_libs: [], - stl: "none", - apex_available: [ "myapex" ], - } - `) - // Symbol files are installed by installing entries under ${OUT}/apex/{apex name} - android.AssertStringListContainsEquals(t, "Installs", - ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().FilesToInstall().Strings(), - filepath.Join(ctx.Config().SoongOutDir(), "target/product/test_device/apex/myapex/bin/foo"), - tc.installSymbolFiles) - }) - } -} - -func TestPrebuiltSkipsSymbols_WorksWithOverrideApex(t *testing.T) { - testCases := []struct { - name string - usePrebuilt bool - installSymbolFiles bool - }{ - { - name: "Source module build rule doesn't install symbol files", - usePrebuilt: true, - installSymbolFiles: false, - }, - { - name: "Source module is installed with symbols", - usePrebuilt: false, - installSymbolFiles: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - preferProperty := "prefer: false" - if tc.usePrebuilt { - preferProperty = "prefer: true" - } - ctx := testApex(t, ` - apex { - name: "myapex", - binaries: ["foo"], - key: "myapex.key", - updatable: false, - } - - // Source module - override_apex { - name: "myapex2", - base: "myapex", - package_name: "myapex2," - } - - apex_key { - name: "myapex.key", - public_key: "testkey.avbpubkey", - private_key: "testkey.pem", - } - - apex_set { - name: "myapex2", - apex_name: "myapex", // needed to replace override_apex - set: "myapex2.apks", - `+preferProperty+` - } - - cc_binary { - name: "foo", - srcs: ["mylib.cpp"], - system_shared_libs: [], - stl: "none", - apex_available: [ "myapex" ], - } - `) - // Symbol files are installed by installing entries under ${OUT}/apex/{apex name} - android.AssertStringListContainsEquals(t, "Implicits", - ctx.ModuleForTests("myapex", "android_common_myapex2_myapex_image").Module().FilesToInstall().Strings(), - filepath.Join(ctx.Config().SoongOutDir(), "target/product/test_device/apex/myapex2/bin/foo"), - tc.installSymbolFiles) - }) - } -} - func TestApexWithTests(t *testing.T) { ctx := testApex(t, ` apex_test { diff --git a/apex/builder.go b/apex/builder.go index 3c7671b00b..2efdf1e7b2 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -468,10 +468,6 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { imageDir := android.PathForModuleOut(ctx, "image"+suffix) installSymbolFiles := (!ctx.Config().KatiEnabled() || a.ExportedToMake()) && a.installable() - // We can't install symbol files when prebuilt is used. - if a.IsReplacedByPrebuilt() { - installSymbolFiles = false - } // set of dependency module:location mappings installMapSet := make(map[string]bool) From 1db9748805147a69792b2a1a556689253c5b82f6 Mon Sep 17 00:00:00 2001 From: Justin Yun Date: Tue, 11 Apr 2023 18:20:07 +0900 Subject: [PATCH 0030/1460] Include license information for vendor and host snapshots In addition to the license text files, include license kinds information to json file. Also, use the original paths of license text files that are copied to NOTICE_FILES directory. This will be used when generating snapshots from the snapshot build. Bug: 271539873 Test: m vendor-snapshot Change-Id: I0c0427bb66f2c1fca322d5554aa66220a3b62fb3 --- cc/util.go | 11 ----------- cc/vendor_snapshot.go | 19 ++++++++----------- etc/prebuilt_etc.go | 27 ++++++++------------------- snapshot/host_snapshot.go | 18 +++++++++++------- snapshot/snapshot.go | 28 ++++++++++++++++++++++++++-- snapshot/snapshot_base.go | 15 +++++++++++++++ snapshot/util.go | 16 ++++++++++++---- 7 files changed, 80 insertions(+), 54 deletions(-) diff --git a/cc/util.go b/cc/util.go index aa0f6b5d67..6d8ac435f1 100644 --- a/cc/util.go +++ b/cc/util.go @@ -100,17 +100,6 @@ func makeSymlinkCmd(linkDirOnDevice string, linkName string, target string) stri "ln -sf " + target + " " + filepath.Join(dir, linkName) } -func combineNoticesRule(ctx android.SingletonContext, paths android.Paths, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cat, - Inputs: paths, - Output: outPath, - Description: "combine notices for " + out, - }) - return outPath -} - // Dump a map to a list file as: // // {key1} {value1} diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go index aed37753a0..e6e566025a 100644 --- a/cc/vendor_snapshot.go +++ b/cc/vendor_snapshot.go @@ -160,7 +160,7 @@ type snapshotJsonFlags struct { MinSdkVersion string `json:",omitempty"` } -var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) android.Paths { +var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) snapshot.SnapshotPaths { /* Vendor snapshot zipped artifacts directory structure for cc modules: {SNAPSHOT_ARCH}/ @@ -195,10 +195,10 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS */ var snapshotOutputs android.Paths + var snapshotNotices android.Paths includeDir := filepath.Join(snapshotArchDir, "include") configsDir := filepath.Join(snapshotArchDir, "configs") - noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES") installedNotices := make(map[string]bool) installedConfigs := make(map[string]bool) @@ -228,7 +228,7 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS prop := snapshotJsonFlags{} // Common properties among snapshots. - prop.ModuleName = ctx.ModuleName(m) + prop.InitBaseSnapshotPropsWithName(m, ctx.ModuleName(m)) if supportsVndkExt(s.Image) && m.IsVndkExt() { // vndk exts are installed to /vendor/lib(64)?/vndk(-sp)? if m.IsVndkSp() { @@ -406,13 +406,10 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS headers = append(headers, m.SnapshotHeaders()...) } - if len(m.EffectiveLicenseFiles()) > 0 { - noticeName := ctx.ModuleName(m) + ".txt" - noticeOut := filepath.Join(noticeDir, noticeName) - // skip already copied notice file - if !installedNotices[noticeOut] { - installedNotices[noticeOut] = true - snapshotOutputs = append(snapshotOutputs, combineNoticesRule(ctx, m.EffectiveLicenseFiles(), noticeOut)) + for _, notice := range m.EffectiveLicenseFiles() { + if _, ok := installedNotices[notice.String()]; !ok { + installedNotices[notice.String()] = true + snapshotNotices = append(snapshotNotices, notice) } } }) @@ -422,7 +419,7 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS snapshotOutputs = append(snapshotOutputs, copyFile(ctx, header, filepath.Join(includeDir, header.String()), s.Fake)) } - return snapshotOutputs + return snapshot.SnapshotPaths{OutputFiles: snapshotOutputs, NoticeFiles: snapshotNotices} } func init() { diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index dcd7fdcb9e..6817dcef0e 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -617,7 +617,7 @@ func isSnapshotAware(ctx android.SingletonContext, m *PrebuiltEtc, image snapsho return true } -func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) android.Paths { +func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) snapshot.SnapshotPaths { /* Snapshot zipped artifacts directory structure for etc modules: {SNAPSHOT_ARCH}/ @@ -631,7 +631,7 @@ func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.Singleto (notice files) */ var snapshotOutputs android.Paths - noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES") + var snapshotNotices android.Paths installedNotices := make(map[string]bool) ctx.VisitAllModules(func(module android.Module) { @@ -651,7 +651,7 @@ func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.Singleto prop := snapshot.SnapshotJsonFlags{} propOut := snapshotLibOut + ".json" - prop.ModuleName = m.BaseModuleName() + prop.InitBaseSnapshotProps(m) if m.subdirProperties.Relative_install_path != nil { prop.RelativeInstallPath = *m.subdirProperties.Relative_install_path } @@ -667,27 +667,16 @@ func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.Singleto } snapshotOutputs = append(snapshotOutputs, snapshot.WriteStringToFileRule(ctx, string(j), propOut)) - if len(m.EffectiveLicenseFiles()) > 0 { - noticeName := ctx.ModuleName(m) + ".txt" - noticeOut := filepath.Join(noticeDir, noticeName) - // skip already copied notice file - if !installedNotices[noticeOut] { - installedNotices[noticeOut] = true - - noticeOutPath := android.PathForOutput(ctx, noticeOut) - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cat, - Inputs: m.EffectiveLicenseFiles(), - Output: noticeOutPath, - Description: "combine notices for " + noticeOut, - }) - snapshotOutputs = append(snapshotOutputs, noticeOutPath) + for _, notice := range m.EffectiveLicenseFiles() { + if _, ok := installedNotices[notice.String()]; !ok { + installedNotices[notice.String()] = true + snapshotNotices = append(snapshotNotices, notice) } } }) - return snapshotOutputs + return snapshot.SnapshotPaths{OutputFiles: snapshotOutputs, NoticeFiles: snapshotNotices} } // For Bazel / bp2build diff --git a/snapshot/host_snapshot.go b/snapshot/host_snapshot.go index 9793218a38..edcc16348a 100644 --- a/snapshot/host_snapshot.go +++ b/snapshot/host_snapshot.go @@ -96,6 +96,7 @@ func (f *hostSnapshot) CreateMetaData(ctx android.ModuleContext, fileName string var jsonData []SnapshotJsonFlags var metaPaths android.Paths + installedNotices := make(map[string]bool) metaZipFile := android.PathForModuleOut(ctx, fileName).OutputPath // Create JSON file based on the direct dependencies @@ -104,12 +105,14 @@ func (f *hostSnapshot) CreateMetaData(ctx android.ModuleContext, fileName string if desc != nil { jsonData = append(jsonData, *desc) } - if len(dep.EffectiveLicenseFiles()) > 0 { - noticeFile := android.PathForModuleOut(ctx, "NOTICE_FILES", dep.Name()+".txt").OutputPath - android.CatFileRule(ctx, dep.EffectiveLicenseFiles(), noticeFile) - metaPaths = append(metaPaths, noticeFile) + for _, notice := range dep.EffectiveLicenseFiles() { + if _, ok := installedNotices[notice.String()]; !ok { + installedNotices[notice.String()] = true + noticeOut := android.PathForModuleOut(ctx, "NOTICE_FILES", notice.String()).OutputPath + CopyFileToOutputPathRule(pctx, ctx, notice, noticeOut) + metaPaths = append(metaPaths, noticeOut) + } } - }) // Sort notice paths and json data for repeatble build sort.Slice(jsonData, func(i, j int) bool { @@ -220,8 +223,7 @@ func hostJsonDesc(m android.Module) *SnapshotJsonFlags { } if path.Valid() && path.String() != "" { - return &SnapshotJsonFlags{ - ModuleName: m.Name(), + props := &SnapshotJsonFlags{ ModuleStemName: moduleStem, Filename: path.String(), Required: append(m.HostRequiredModuleNames(), m.RequiredModuleNames()...), @@ -229,6 +231,8 @@ func hostJsonDesc(m android.Module) *SnapshotJsonFlags { RustProcMacro: procMacro, CrateName: crateName, } + props.InitBaseSnapshotProps(m) + return props } return nil } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 206ecc947b..c95a537741 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -26,6 +26,10 @@ import ( var pctx = android.NewPackageContext("android/soong/snapshot") +func init() { + pctx.Import("android/soong/android") +} + type SnapshotSingleton struct { // Name, e.g., "vendor", "recovery", "ramdisk". name string @@ -48,8 +52,18 @@ type SnapshotSingleton struct { Fake bool } +// The output files to be included in the snapshot. +type SnapshotPaths struct { + // All files to be included in the snapshot + OutputFiles android.Paths + + // Notice files of the snapshot output files + NoticeFiles android.Paths +} + // Interface of function to capture snapshot from each module -type GenerateSnapshotAction func(snapshot SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) android.Paths +// Returns snapshot ouputs and notice files. +type GenerateSnapshotAction func(snapshot SnapshotSingleton, ctx android.SingletonContext, snapshotArchDir string) SnapshotPaths var snapshotActionList []GenerateSnapshotAction @@ -74,9 +88,19 @@ func (c *SnapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { snapshotDir = filepath.Join("fake", snapshotDir) } snapshotArchDir := filepath.Join(snapshotDir, ctx.DeviceConfig().DeviceArch()) + noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES") + installedNotices := make(map[string]bool) for _, f := range snapshotActionList { - snapshotOutputs = append(snapshotOutputs, f(*c, ctx, snapshotArchDir)...) + snapshotPaths := f(*c, ctx, snapshotArchDir) + snapshotOutputs = append(snapshotOutputs, snapshotPaths.OutputFiles...) + for _, notice := range snapshotPaths.NoticeFiles { + if _, ok := installedNotices[notice.String()]; !ok { + installedNotices[notice.String()] = true + snapshotOutputs = append(snapshotOutputs, CopyFileRule( + pctx, ctx, notice, filepath.Join(noticeDir, notice.String()))) + } + } } // All artifacts are ready. Sort them to normalize ninja and then zip. diff --git a/snapshot/snapshot_base.go b/snapshot/snapshot_base.go index 809ca3d11c..fb4ee0c839 100644 --- a/snapshot/snapshot_base.go +++ b/snapshot/snapshot_base.go @@ -120,4 +120,19 @@ type SnapshotJsonFlags struct { // dependencies Required []string `json:",omitempty"` Overrides []string `json:",omitempty"` + + // license information + LicenseKinds []string `json:",omitempty"` + LicenseTexts []string `json:",omitempty"` +} + +func (prop *SnapshotJsonFlags) InitBaseSnapshotPropsWithName(m android.Module, name string) { + prop.ModuleName = name + + prop.LicenseKinds = m.EffectiveLicenseKinds() + prop.LicenseTexts = m.EffectiveLicenseFiles().Strings() +} + +func (prop *SnapshotJsonFlags) InitBaseSnapshotProps(m android.Module) { + prop.InitBaseSnapshotPropsWithName(m, m.Name()) } diff --git a/snapshot/util.go b/snapshot/util.go index 806ac90fa0..c87c508e67 100644 --- a/snapshot/util.go +++ b/snapshot/util.go @@ -21,17 +21,25 @@ func WriteStringToFileRule(ctx android.SingletonContext, content, out string) an return outPath } -func CopyFileRule(pctx android.PackageContext, ctx android.SingletonContext, path android.Path, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) +type buildContext interface { + Build(pctx android.PackageContext, params android.BuildParams) +} + +func CopyFileToOutputPathRule(pctx android.PackageContext, ctx buildContext, path android.Path, outPath android.OutputPath) { ctx.Build(pctx, android.BuildParams{ Rule: android.Cp, Input: path, Output: outPath, - Description: "copy " + path.String() + " -> " + out, + Description: "copy " + path.String() + " -> " + outPath.String(), Args: map[string]string{ - "cpFlags": "-f -L", + "cpFlags": "-L", }, }) +} + +func CopyFileRule(pctx android.PackageContext, ctx android.SingletonContext, path android.Path, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + CopyFileToOutputPathRule(pctx, ctx, path, outPath) return outPath } From c9089dcc7b4b59dad542d0c1e6d77f466c014a86 Mon Sep 17 00:00:00 2001 From: Chris Parsons Date: Mon, 24 Apr 2023 16:21:27 +0000 Subject: [PATCH 0031/1460] Refactor and cleanup of bazel handler - Creation of a bazel command is independent of test/real implementation - Use a custom struct instead of cmd.Exec - Move mock bazel runner to the test Bug: 270989498 Test: m nothing Test: USE_PERSISTENT_BAZEL=0 m nothing Test: Treehugger Change-Id: Ieec35dad5e21aac644d5b8dc79a92397d42db861 --- android/bazel_handler.go | 102 +++++++++---------------------- android/bazel_handler_test.go | 112 ++++++++++++++++++++++++++-------- bazel/bazel_proxy.go | 38 +++++++----- 3 files changed, 136 insertions(+), 116 deletions(-) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 9c273d9a33..85d00ccef8 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -18,7 +18,6 @@ import ( "bytes" "fmt" "os" - "os/exec" "path" "path/filepath" "runtime" @@ -30,7 +29,6 @@ import ( "android/soong/bazel/cquery" "android/soong/shared" "android/soong/starlark_fmt" - "github.com/google/blueprint" "github.com/google/blueprint/metrics" @@ -220,8 +218,7 @@ type BazelContext interface { } type bazelRunner interface { - createBazelCommand(config Config, paths *bazelPaths, runName bazel.RunName, command bazelCommand, extraFlags ...string) *exec.Cmd - issueBazelCommand(bazelCmd *exec.Cmd, eventHandler *metrics.EventHandler) (output string, errorMessage string, error error) + issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (output string, errorMessage string, error error) } type bazelPaths struct { @@ -660,36 +657,6 @@ type bazelCommand struct { expression string } -type mockBazelRunner struct { - bazelCommandResults map[bazelCommand]string - // use *exec.Cmd as a key to get the bazelCommand, the map will be used in issueBazelCommand() - // Register createBazelCommand() invocations. Later, an - // issueBazelCommand() invocation can be mapped to the *exec.Cmd instance - // and then to the expected result via bazelCommandResults - tokens map[*exec.Cmd]bazelCommand - commands []bazelCommand - extraFlags []string -} - -func (r *mockBazelRunner) createBazelCommand(_ Config, _ *bazelPaths, _ bazel.RunName, - command bazelCommand, extraFlags ...string) *exec.Cmd { - r.commands = append(r.commands, command) - r.extraFlags = append(r.extraFlags, strings.Join(extraFlags, " ")) - cmd := &exec.Cmd{} - if r.tokens == nil { - r.tokens = make(map[*exec.Cmd]bazelCommand) - } - r.tokens[cmd] = command - return cmd -} - -func (r *mockBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, _ *metrics.EventHandler) (string, string, error) { - if command, ok := r.tokens[bazelCmd]; ok { - return r.bazelCommandResults[command], "", nil - } - return "", "", nil -} - type builtinBazelRunner struct { useBazelProxy bool outDir string @@ -699,17 +666,12 @@ type builtinBazelRunner struct { // Returns (stdout, stderr, error). The first and second return values are strings // containing the stdout and stderr of the run command, and an error is returned if // the invocation returned an error code. -func (r *builtinBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, eventHandler *metrics.EventHandler) (string, string, error) { +func (r *builtinBazelRunner) issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (string, string, error) { if r.useBazelProxy { eventHandler.Begin("client_proxy") defer eventHandler.End("client_proxy") proxyClient := bazel.NewProxyClient(r.outDir) - // Omit the arg containing the Bazel binary, as that is handled by the proxy - // server. - bazelFlags := bazelCmd.Args[1:] - // TODO(b/270989498): Refactor these functions to not take exec.Cmd, as its - // not actually executed for client proxying. - resp, err := proxyClient.IssueCommand(bazel.CmdRequest{bazelFlags, bazelCmd.Env}) + resp, err := proxyClient.IssueCommand(cmdRequest) if err != nil { return "", "", err @@ -721,26 +683,19 @@ func (r *builtinBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, eventHandler } else { eventHandler.Begin("bazel command") defer eventHandler.End("bazel command") - stderr := &bytes.Buffer{} - bazelCmd.Stderr = stderr - if output, err := bazelCmd.Output(); err != nil { - return "", string(stderr.Bytes()), - fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---", - err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr) - } else { - return string(output), string(stderr.Bytes()), nil - } + + stdout, stderr, err := bazel.ExecBazel(paths.bazelPath, absolutePath(paths.syntheticWorkspaceDir()), cmdRequest) + return string(stdout), string(stderr), err } } -func (r *builtinBazelRunner) createBazelCommand(config Config, paths *bazelPaths, runName bazel.RunName, command bazelCommand, - extraFlags ...string) *exec.Cmd { +func (context *mixedBuildBazelContext) createBazelCommand(config Config, runName bazel.RunName, command bazelCommand, + extraFlags ...string) bazel.CmdRequest { cmdFlags := []string{ - "--output_base=" + absolutePath(paths.outputBase), + "--output_base=" + absolutePath(context.paths.outputBase), command.command, command.expression, - // TODO(asmundak): is it needed in every build? - "--profile=" + shared.BazelMetricsFilename(paths, runName), + "--profile=" + shared.BazelMetricsFilename(context.paths, runName), // We don't need to set --host_platforms because it's set in bazelrc files // that the bazel shell script wrapper passes @@ -755,15 +710,13 @@ func (r *builtinBazelRunner) createBazelCommand(config Config, paths *bazelPaths } cmdFlags = append(cmdFlags, extraFlags...) - bazelCmd := exec.Command(paths.bazelPath, cmdFlags...) - bazelCmd.Dir = absolutePath(paths.syntheticWorkspaceDir()) extraEnv := []string{ - "HOME=" + paths.homeDir, + "HOME=" + context.paths.homeDir, pwdPrefix(), - "BUILD_DIR=" + absolutePath(paths.soongOutDir), + "BUILD_DIR=" + absolutePath(context.paths.soongOutDir), // Make OUT_DIR absolute here so build/bazel/bin/bazel uses the correct // OUT_DIR at /out, instead of /out/soong/workspace/out. - "OUT_DIR=" + absolutePath(paths.outDir()), + "OUT_DIR=" + absolutePath(context.paths.outDir()), // Disables local host detection of gcc; toolchain information is defined // explicitly in BUILD files. "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1", @@ -775,15 +728,15 @@ func (r *builtinBazelRunner) createBazelCommand(config Config, paths *bazelPaths } extraEnv = append(extraEnv, fmt.Sprintf("%s=%s", envvar, val)) } - bazelCmd.Env = append(os.Environ(), extraEnv...) + envVars := append(os.Environ(), extraEnv...) - return bazelCmd + return bazel.CmdRequest{cmdFlags, envVars} } -func printableCqueryCommand(bazelCmd *exec.Cmd) string { - outputString := strings.Join(bazelCmd.Env, " ") + " \"" + strings.Join(bazelCmd.Args, "\" \"") + "\"" +func (context *mixedBuildBazelContext) printableCqueryCommand(bazelCmd bazel.CmdRequest) string { + args := append([]string{context.paths.bazelPath}, bazelCmd.Argv...) + outputString := strings.Join(bazelCmd.Env, " ") + " \"" + strings.Join(args, "\" \"") + "\"" return outputString - } func (context *mixedBuildBazelContext) mainBzlFileContents() []byte { @@ -1106,9 +1059,10 @@ func (p *bazelPaths) outDir() string { const buildrootLabel = "@soong_injection//mixed_builds:buildroot" var ( - cqueryCmd = bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)} - aqueryCmd = bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)} - buildCmd = bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"} + cqueryCmd = bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)} + aqueryCmd = bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)} + buildCmd = bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"} + allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd} ) // Issues commands to Bazel to receive results for all cquery requests @@ -1170,12 +1124,12 @@ func (context *mixedBuildBazelContext) runCquery(config Config, ctx invokeBazelC extraFlags = append(extraFlags, "--collect_code_coverage") } - cqueryCommandWithFlag := context.createBazelCommand(config, context.paths, bazel.CqueryBuildRootRunName, cqueryCmd, extraFlags...) - cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCommandWithFlag, eventHandler) + cqueryCmdRequest := context.createBazelCommand(config, bazel.CqueryBuildRootRunName, cqueryCmd, extraFlags...) + cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCmdRequest, context.paths, eventHandler) if cqueryErr != nil { return cqueryErr } - cqueryCommandPrint := fmt.Sprintf("cquery command line:\n %s \n\n\n", printableCqueryCommand(cqueryCommandWithFlag)) + cqueryCommandPrint := fmt.Sprintf("cquery command line:\n %s \n\n\n", context.printableCqueryCommand(cqueryCmdRequest)) if err := os.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryCommandPrint+cqueryOutput), 0666); err != nil { return err } @@ -1233,8 +1187,8 @@ func (context *mixedBuildBazelContext) runAquery(config Config, ctx invokeBazelC extraFlags = append(extraFlags, "--instrumentation_filter="+strings.Join(paths, ",")) } } - aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(config, context.paths, bazel.AqueryBuildRootRunName, aqueryCmd, - extraFlags...), eventHandler) + aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.AqueryBuildRootRunName, aqueryCmd, + extraFlags...), context.paths, eventHandler) if err != nil { return err } @@ -1249,7 +1203,7 @@ func (context *mixedBuildBazelContext) generateBazelSymlinks(config Config, ctx // Issue a build command of the phony root to generate symlink forests for dependencies of the // Bazel build. This is necessary because aquery invocations do not generate this symlink forest, // but some of symlinks may be required to resolve source dependencies of the build. - _, _, err := context.issueBazelCommand(context.createBazelCommand(config, context.paths, bazel.BazelBuildPhonyRootRunName, buildCmd), eventHandler) + _, _, err := context.issueBazelCommand(context.createBazelCommand(config, bazel.BazelBuildPhonyRootRunName, buildCmd), context.paths, eventHandler) return err } diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go index c67d7fb66a..b17b7652ba 100644 --- a/android/bazel_handler_test.go +++ b/android/bazel_handler_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "android/soong/bazel" "android/soong/bazel/cquery" analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" @@ -19,6 +20,34 @@ var testConfig = TestConfig("out", nil, "", nil) type testInvokeBazelContext struct{} +type mockBazelRunner struct { + testHelper *testing.T + // Stores mock behavior. If an issueBazelCommand request is made for command + // k, and {k:v} is present in this map, then the mock will return v. + bazelCommandResults map[bazelCommand]string + // Requests actually made of the mockBazelRunner with issueBazelCommand, + // keyed by the command they represent. + bazelCommandRequests map[bazelCommand]bazel.CmdRequest +} + +func (r *mockBazelRunner) bazelCommandForRequest(cmdRequest bazel.CmdRequest) bazelCommand { + for _, arg := range cmdRequest.Argv { + for _, cmdType := range allBazelCommands { + if arg == cmdType.command { + return cmdType + } + } + } + r.testHelper.Fatalf("Unrecognized bazel request: %s", cmdRequest) + return cqueryCmd +} + +func (r *mockBazelRunner) issueBazelCommand(cmdRequest bazel.CmdRequest, paths *bazelPaths, eventHandler *metrics.EventHandler) (string, string, error) { + command := r.bazelCommandForRequest(cmdRequest) + r.bazelCommandRequests[command] = cmdRequest + return r.bazelCommandResults[command], "", nil +} + func (t *testInvokeBazelContext) GetEventHandler() *metrics.EventHandler { return &metrics.EventHandler{} } @@ -36,9 +65,7 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) { `@//foo:foo|arm64_armv8-a|android|within_apex|29>>out/foo/foo.txt`, `@//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`, } - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ - bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot, 2)"}: strings.Join(cmd_results, "\n"), - }) + bazelContext, _ := testBazelContext(t, map[bazelCommand]string{cqueryCmd: strings.Join(cmd_results, "\n")}) bazelContext.QueueBazelRequest(label_foo, cquery.GetOutputFiles, cfg_foo) bazelContext.QueueBazelRequest(label_bar, cquery.GetOutputFiles, cfg_bar) @@ -139,8 +166,7 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) { if err != nil { t.Error(err) } - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ - bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: string(data)}) + bazelContext, _ := testBazelContext(t, map[bazelCommand]string{aqueryCmd: string(data)}) err = bazelContext.InvokeBazel(testConfig, &testInvokeBazelContext{}) if err != nil { @@ -166,30 +192,26 @@ func TestCoverageFlagsAfterInvokeBazel(t *testing.T) { testConfig.productVariables.NativeCoveragePaths = []string{"foo1", "foo2"} testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1", "bar2"} - verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=+foo1,+foo2,-bar1,-bar2`) + verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1,+foo2,-bar1,-bar2") testConfig.productVariables.NativeCoveragePaths = []string{"foo1"} testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"} - verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=+foo1,-bar1`) + verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1,-bar1") testConfig.productVariables.NativeCoveragePaths = []string{"foo1"} testConfig.productVariables.NativeCoverageExcludePaths = nil - verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=+foo1`) + verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+foo1") testConfig.productVariables.NativeCoveragePaths = nil testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"} - verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=-bar1`) + verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=-bar1") testConfig.productVariables.NativeCoveragePaths = []string{"*"} testConfig.productVariables.NativeCoverageExcludePaths = nil - verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=+.*`) + verifyAqueryContainsFlags(t, testConfig, "--collect_code_coverage", "--instrumentation_filter=+.*") testConfig.productVariables.ClangCoverage = boolPtr(false) - actual := verifyExtraFlags(t, testConfig, ``) - if strings.Contains(actual, "--collect_code_coverage") || - strings.Contains(actual, "--instrumentation_filter=") { - t.Errorf("Expected code coverage disabled, but got %#v", actual) - } + verifyAqueryDoesNotContainSubstrings(t, testConfig, "collect_code_coverage", "instrumentation_filter") } func TestBazelRequestsSorted(t *testing.T) { @@ -268,7 +290,8 @@ func TestIsModuleNameAllowed(t *testing.T) { } } -func verifyExtraFlags(t *testing.T, config Config, expected string) string { +func verifyAqueryContainsFlags(t *testing.T, config Config, expected ...string) { + t.Helper() bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) err := bazelContext.InvokeBazel(config, &testInvokeBazelContext{}) @@ -276,17 +299,49 @@ func verifyExtraFlags(t *testing.T, config Config, expected string) string { t.Fatalf("Did not expect error invoking Bazel, but got %s", err) } - flags := bazelContext.bazelRunner.(*mockBazelRunner).extraFlags - if expected := 3; len(flags) != expected { - t.Errorf("Expected %d extra flags got %#v", expected, flags) + sliceContains := func(slice []string, x string) bool { + for _, s := range slice { + if s == x { + return true + } + } + return false } - actual := flags[1] - if !strings.Contains(actual, expected) { - t.Errorf("Expected %#v got %#v", expected, actual) + aqueryArgv := bazelContext.bazelRunner.(*mockBazelRunner).bazelCommandRequests[aqueryCmd].Argv + + for _, expectedFlag := range expected { + if !sliceContains(aqueryArgv, expectedFlag) { + t.Errorf("aquery does not contain expected flag %#v. Argv was: %#v", expectedFlag, aqueryArgv) + } } +} - return actual +func verifyAqueryDoesNotContainSubstrings(t *testing.T, config Config, substrings ...string) { + t.Helper() + bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) + + err := bazelContext.InvokeBazel(config, &testInvokeBazelContext{}) + if err != nil { + t.Fatalf("Did not expect error invoking Bazel, but got %s", err) + } + + sliceContainsSubstring := func(slice []string, substring string) bool { + for _, s := range slice { + if strings.Contains(s, substring) { + return true + } + } + return false + } + + aqueryArgv := bazelContext.bazelRunner.(*mockBazelRunner).bazelCommandRequests[aqueryCmd].Argv + + for _, substring := range substrings { + if sliceContainsSubstring(aqueryArgv, substring) { + t.Errorf("aquery contains unexpected substring %#v. Argv was: %#v", substring, aqueryArgv) + } + } } func testBazelContext(t *testing.T, bazelCommandResults map[bazelCommand]string) (*mixedBuildBazelContext, string) { @@ -296,11 +351,14 @@ func testBazelContext(t *testing.T, bazelCommandResults map[bazelCommand]string) outputBase: "outputbase", workspaceDir: "workspace_dir", } - aqueryCommand := bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"} - if _, exists := bazelCommandResults[aqueryCommand]; !exists { - bazelCommandResults[aqueryCommand] = "" + if _, exists := bazelCommandResults[aqueryCmd]; !exists { + bazelCommandResults[aqueryCmd] = "" + } + runner := &mockBazelRunner{ + testHelper: t, + bazelCommandResults: bazelCommandResults, + bazelCommandRequests: map[bazelCommand]bazel.CmdRequest{}, } - runner := &mockBazelRunner{bazelCommandResults: bazelCommandResults} return &mixedBuildBazelContext{ bazelRunner: runner, paths: &p, diff --git a/bazel/bazel_proxy.go b/bazel/bazel_proxy.go index d7f5e64649..2940b99955 100644 --- a/bazel/bazel_proxy.go +++ b/bazel/bazel_proxy.go @@ -128,6 +128,24 @@ func NewProxyServer(logger ServerLogger, outDir string, workspaceDir string) *Pr } } +func ExecBazel(bazelPath string, workspaceDir string, request CmdRequest) (stdout []byte, stderr []byte, cmdErr error) { + bazelCmd := exec.Command(bazelPath, request.Argv...) + bazelCmd.Dir = workspaceDir + bazelCmd.Env = request.Env + + stderrBuffer := &bytes.Buffer{} + bazelCmd.Stderr = stderrBuffer + + if output, err := bazelCmd.Output(); err != nil { + cmdErr = fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---", + err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderrBuffer) + } else { + stdout = output + } + stderr = stderrBuffer.Bytes() + return +} + func (b *ProxyServer) handleRequest(conn net.Conn) error { defer conn.Close() @@ -137,23 +155,13 @@ func (b *ProxyServer) handleRequest(conn net.Conn) error { return fmt.Errorf("Error decoding request: %s", err) } - bazelCmd := exec.Command("./build/bazel/bin/bazel", req.Argv...) - bazelCmd.Dir = b.workspaceDir - bazelCmd.Env = req.Env - - stderr := &bytes.Buffer{} - bazelCmd.Stderr = stderr - var stdout string - var bazelErrString string - - if output, err := bazelCmd.Output(); err != nil { - bazelErrString = fmt.Sprintf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---", - err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr) - } else { - stdout = string(output) + stdout, stderr, cmdErr := ExecBazel("./build/bazel/bin/bazel", b.workspaceDir, req) + errorString := "" + if cmdErr != nil { + errorString = cmdErr.Error() } - resp := CmdResponse{stdout, string(stderr.Bytes()), bazelErrString} + resp := CmdResponse{string(stdout), string(stderr), errorString} enc := gob.NewEncoder(conn) if err := enc.Encode(&resp); err != nil { return fmt.Errorf("Error encoding response: %s", err) From 00bafc669386e63ffa4b87b75b47339ef4a0063a Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Tue, 25 Apr 2023 17:33:06 +0000 Subject: [PATCH 0032/1460] Re-enable TestMixedBuildUsesStubs This was disabled because of some non-determinism in Soong, which has been been fixed in aosp/2559590 Bug: 275313114 Test: go test ./cc -run TestMixedBuildUsesStubs -count 1000 Change-Id: I261dc785c832e66f8e213132c19111b8b90e40e9 --- cc/cc_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cc/cc_test.go b/cc/cc_test.go index 787669c75e..a1d6bdce71 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -3683,9 +3683,6 @@ func TestStubsForLibraryInMultipleApexes(t *testing.T) { } func TestMixedBuildUsesStubs(t *testing.T) { - // TODO(b/275313114): Test exposes non-determinism which should be corrected and the test - // reenabled. - t.Skip() t.Parallel() bp := ` cc_library_shared { From 8a8714c781175f8f1a6c189d919ee8b0ee8c1e27 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Tue, 25 Apr 2023 18:03:54 +0000 Subject: [PATCH 0033/1460] Do not modify input in-place SortedUniqueStrings and FirstUniqueStrings dedupes repeating elements and returns the deduped list. Currently, it also modifies the input list in-place, which causes non-determinisitc failures like b/275313114 Operate on a copy of the input so that the input remains untouched. SortedUniqueStrings is O(NlogN) and FirstUniqueStrings is ~O(N), so creating a copy (O(N)) should not result in major performance regressions. Numbers for this single unit test: ``` go test . -run TestStubsForLibraryInMultipleApexes -v -count 1000 Before: 174s After: 172s ``` Test: go test ./android Test: go test . -run TestStubsForLibraryInMultipleApexes -v -count 1000 Change-Id: Id859723b2c2ebdc0023876c4b6fabe75d870bad7 --- android/apex.go | 5 ++--- android/util.go | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/android/apex.go b/android/apex.go index 5dcf73bc65..823afbb111 100644 --- a/android/apex.go +++ b/android/apex.go @@ -513,9 +513,8 @@ func (m *ApexModuleBase) checkApexAvailableProperty(mctx BaseModuleContext) { // exactly the same set of APEXes (and platform), i.e. if their apex_available // properties have the same elements. func AvailableToSameApexes(mod1, mod2 ApexModule) bool { - // Use CopyOf to prevent non-determinism (b/275313114#comment1) - mod1ApexAvail := SortedUniqueStrings(CopyOf(mod1.apexModuleBase().ApexProperties.Apex_available)) - mod2ApexAvail := SortedUniqueStrings(CopyOf(mod2.apexModuleBase().ApexProperties.Apex_available)) + mod1ApexAvail := SortedUniqueStrings(mod1.apexModuleBase().ApexProperties.Apex_available) + mod2ApexAvail := SortedUniqueStrings(mod2.apexModuleBase().ApexProperties.Apex_available) if len(mod1ApexAvail) != len(mod2ApexAvail) { return false } diff --git a/android/util.go b/android/util.go index 38e0a4dbbf..fb09128f84 100644 --- a/android/util.go +++ b/android/util.go @@ -276,6 +276,8 @@ func RemoveFromList(s string, list []string) (bool, []string) { // FirstUniqueStrings returns all unique elements of a slice of strings, keeping the first copy of // each. It modifies the slice contents in place, and returns a subslice of the original slice. func FirstUniqueStrings(list []string) []string { + // Do not moodify the input in-place, operate on a copy instead. + list = CopyOf(list) // 128 was chosen based on BenchmarkFirstUniqueStrings results. if len(list) > 128 { return firstUniqueStringsMap(list) @@ -332,6 +334,7 @@ func LastUniqueStrings(list []string) []string { // SortedUniqueStrings returns what the name says func SortedUniqueStrings(list []string) []string { + // FirstUniqueStrings creates a copy of `list`, so the input remains untouched. unique := FirstUniqueStrings(list) sort.Strings(unique) return unique From 45bd092132c05ab830ab2054dc21ffc08d08dd04 Mon Sep 17 00:00:00 2001 From: Julien Desprez Date: Tue, 25 Apr 2023 13:41:29 -0700 Subject: [PATCH 0034/1460] Remove jar target not used Test: presubmit Bug: 279638202 Change-Id: I21d6f699d665a42d84a1dd899bcfc054c2454879 --- tradefed/suite_harness/tradefed_binary.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tradefed/suite_harness/tradefed_binary.go b/tradefed/suite_harness/tradefed_binary.go index a421d8bbb6..1ce94bcb90 100644 --- a/tradefed/suite_harness/tradefed_binary.go +++ b/tradefed/suite_harness/tradefed_binary.go @@ -78,7 +78,6 @@ func tradefedBinaryLoadHook(tfb *TradefedBinaryProperties) func(ctx android.Load // Add dependencies required by all tradefed_binary modules. props.Libs = []string{ "tradefed", - "tradefed-test-framework", "loganalysis", "compatibility-host-util", } From f47e142ffdbfb8e6d39744e5a5f20127467d4d44 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Wed, 19 Apr 2023 16:47:36 +0000 Subject: [PATCH 0035/1460] Refactor MixedBuildsEnabled and add --ensure-allowlist-integrity. Currently, there is little verification around allowlisted modules actually being mixed-built. This flag would allow us to verify that a module allowlisted is mixed-built for at least one variant. Bug: 278910100 Test: m nothing --bazel-mode-staging --ensure-allowlist-integrity Test: m nothing --bazel-mode-staging --ensure-allowlist-integrity --bazel-force-enabled-modules=com.google.android.neuralnetworks (This fails, as expected) Test: build/soong/test/mixed_mode_test.sh Change-Id: Icd5976f4f44f1a8caca1e5247d986642f7995f97 --- android/bazel.go | 50 +++++++++++++++++++++++++++++++-------- android/bazel_handler.go | 6 ++--- android/config.go | 11 +++++++++ android/module.go | 2 +- cc/cc_test.go | 2 +- cmd/soong_build/main.go | 50 +++++++++++++++++++++++++++++++++++++-- tests/mixed_mode_test.sh | 2 +- ui/build/config.go | 51 +++++++++++++++++++++++----------------- ui/build/soong.go | 3 +++ 9 files changed, 136 insertions(+), 41 deletions(-) diff --git a/android/bazel.go b/android/bazel.go index 1646883965..58d9d87dbe 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -32,6 +32,22 @@ const ( Bp2BuildTopLevel = "." ) +type MixedBuildEnabledStatus int + +const ( + // This module can be mixed_built. + MixedBuildEnabled = iota + + // There is a technical incompatibility preventing this module from being + // bazel-analyzed. Note: the module might also be incompatible. + TechnicalIncompatibility + + // This module cannot be mixed_built due to some incompatibility with it + // that is not a platform incompatibility. Example: the module-type is not + // enabled, or is not bp2build-converted. + ModuleIncompatibility +) + // FileGroupAsLibrary describes a filegroup module that is converted to some library // such as aidl_library or proto_library. type FileGroupAsLibrary interface { @@ -346,24 +362,31 @@ func GetBp2BuildAllowList() Bp2BuildConversionAllowlist { }).(Bp2BuildConversionAllowlist) } -// MixedBuildsEnabled returns true if a module is ready to be replaced by a -// converted or handcrafted Bazel target. As a side effect, calling this -// method will also log whether this module is mixed build enabled for -// metrics reporting. -func MixedBuildsEnabled(ctx BaseModuleContext) bool { +// MixedBuildsEnabled returns a MixedBuildEnabledStatus regarding whether +// a module is ready to be replaced by a converted or handcrafted Bazel target. +// As a side effect, calling this method will also log whether this module is +// mixed build enabled for metrics reporting. +func MixedBuildsEnabled(ctx BaseModuleContext) MixedBuildEnabledStatus { module := ctx.Module() apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) withinApex := !apexInfo.IsForPlatform() + + platformIncompatible := isPlatformIncompatible(ctx.Os(), ctx.Arch().ArchType) + if platformIncompatible { + ctx.Config().LogMixedBuild(ctx, false) + return TechnicalIncompatibility + } + mixedBuildEnabled := ctx.Config().IsMixedBuildsEnabled() && - ctx.Os() != Windows && // Windows toolchains are not currently supported. - ctx.Os() != LinuxBionic && // Linux Bionic toolchains are not currently supported. - ctx.Os() != LinuxMusl && // Linux musl toolchains are not currently supported (b/259266326). - ctx.Arch().ArchType != Riscv64 && // TODO(b/262192655) Riscv64 toolchains are not currently supported. module.Enabled() && convertedToBazel(ctx, module) && ctx.Config().BazelContext.IsModuleNameAllowed(module.Name(), withinApex) ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled) - return mixedBuildEnabled + + if mixedBuildEnabled { + return MixedBuildEnabled + } + return ModuleIncompatibility } // ConvertedToBazel returns whether this module has been converted (with bp2build or manually) to Bazel. @@ -388,6 +411,13 @@ type bazelOtherModuleContext interface { OtherModuleDir(m blueprint.Module) string } +func isPlatformIncompatible(osType OsType, arch ArchType) bool { + return osType == Windows || // Windows toolchains are not currently supported. + osType == LinuxBionic || // Linux Bionic toolchains are not currently supported. + osType == LinuxMusl || // Linux musl toolchains are not currently supported (b/259266326). + arch == Riscv64 // TODO(b/262192655) Riscv64 toolchains are not currently supported. +} + func (b *BazelModuleBase) shouldConvertWithBp2build(ctx bazelOtherModuleContext, module blueprint.Module) bool { if !b.bazelProps().Bazel_module.CanConvertToBazel { return false diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 9c273d9a33..debc7a2fda 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -86,12 +86,10 @@ func RegisterMixedBuildsMutator(ctx RegistrationContext) { func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) { if m := ctx.Module(); m.Enabled() { if mixedBuildMod, ok := m.(MixedBuildBuildable); ok { - queueMixedBuild := mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) + mixedBuildEnabled := MixedBuildsEnabled(ctx) + queueMixedBuild := mixedBuildMod.IsMixedBuildSupported(ctx) && mixedBuildEnabled == MixedBuildEnabled if queueMixedBuild { mixedBuildMod.QueueBazelCall(ctx) - } else if _, ok := ctx.Config().bazelForceEnabledModules[m.Name()]; ok { - // TODO(b/273910287) - remove this once --ensure_allowlist_integrity is added - ctx.ModuleErrorf("Attempted to force enable an unready module: %s. Did you forget to Bp2BuildDefaultTrue its directory?\n", m.Name()) } } } diff --git a/android/config.go b/android/config.go index 032172d6ee..7141e54c3d 100644 --- a/android/config.go +++ b/android/config.go @@ -102,6 +102,8 @@ type CmdArgs struct { UseBazelProxy bool BuildFromTextStub bool + + EnsureAllowlistIntegrity bool } // Build modes that soong_build can run as. @@ -278,6 +280,11 @@ type config struct { // If buildFromTextStub is true then the Java API stubs are // built from the signature text files, not the source Java files. buildFromTextStub bool + + // If ensureAllowlistIntegrity is true, then the presence of any allowlisted + // modules that aren't mixed-built for at least one variant will cause a build + // failure + ensureAllowlistIntegrity bool } type deviceConfig struct { @@ -1904,6 +1911,10 @@ func (c *config) UseHostMusl() bool { return Bool(c.productVariables.HostMusl) } +func (c *config) GetMixedBuildsEnabledModules() map[string]struct{} { + return c.mixedBuildEnabledModules +} + func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) { moduleName := ctx.Module().Name() c.mixedBuildsLock.Lock() diff --git a/android/module.go b/android/module.go index ba474530d2..c8670c31a3 100644 --- a/android/module.go +++ b/android/module.go @@ -2438,7 +2438,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) { if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok { - if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) { + if mixedBuildMod.IsMixedBuildSupported(ctx) && (MixedBuildsEnabled(ctx) == MixedBuildEnabled) { return mixedBuildMod, true } } diff --git a/cc/cc_test.go b/cc/cc_test.go index 787669c75e..fde6bffc21 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -59,7 +59,7 @@ func registerTestMutators(ctx android.RegistrationContext) { func mixedBuildsPrepareMutator(ctx android.BottomUpMutatorContext) { if m := ctx.Module(); m.Enabled() { if mixedBuildMod, ok := m.(android.MixedBuildBuildable); ok { - if mixedBuildMod.IsMixedBuildSupported(ctx) && android.MixedBuildsEnabled(ctx) { + if mixedBuildMod.IsMixedBuildSupported(ctx) && android.MixedBuildsEnabled(ctx) == android.MixedBuildEnabled { mixedBuildMod.QueueBazelCall(ctx) } } diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 79a5ce4987..53e0e55c48 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -85,7 +85,7 @@ func init() { flag.BoolVar(&cmdlineArgs.BazelModeDev, "bazel-mode-dev", false, "use bazel for analysis of a large number of modules (less stable)") flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses") flag.BoolVar(&cmdlineArgs.BuildFromTextStub, "build-from-text-stub", false, "build Java stubs from API text files instead of source files") - + flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built") // Flags that probably shouldn't be flags of soong_build, but we haven't found // the time to remove them yet flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap") @@ -288,6 +288,46 @@ func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandl maybeQuit(err, "error writing soong_build metrics %s", metricsFile) } +// Errors out if any modules expected to be mixed_built were not, unless +// there is a platform incompatibility. +func checkForAllowlistIntegrityError(configuration android.Config, isStagingMode bool) error { + modules := findModulesNotMixedBuiltForAnyVariant(configuration, isStagingMode) + if len(modules) == 0 { + return nil + } + + return fmt.Errorf("Error: expected the following modules to be mixed_built: %s", modules) +} + +// Returns the list of modules that should have been mixed_built (per the +// allowlists and cmdline flags) but were not. +func findModulesNotMixedBuiltForAnyVariant(configuration android.Config, isStagingMode bool) []string { + retval := []string{} + forceEnabledModules := configuration.BazelModulesForceEnabledByFlag() + + mixedBuildsEnabled := configuration.GetMixedBuildsEnabledModules() + for _, module := range allowlists.ProdMixedBuildsEnabledList { + if _, ok := mixedBuildsEnabled[module]; !ok && module != "" { + retval = append(retval, module) + } + } + + if isStagingMode { + for _, module := range allowlists.StagingMixedBuildsEnabledList { + if _, ok := mixedBuildsEnabled[module]; !ok && module != "" { + retval = append(retval, module) + } + } + } + + for module, _ := range forceEnabledModules { + if _, ok := mixedBuildsEnabled[module]; !ok && module != "" { + retval = append(retval, module) + } + } + return retval +} + func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) { graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile)) maybeQuit(graphErr, "graph err") @@ -433,8 +473,14 @@ func main() { writeMetrics(configuration, ctx.EventHandler, metricsDir) default: ctx.Register() - if configuration.IsMixedBuildsEnabled() { + isMixedBuildsEnabled := configuration.IsMixedBuildsEnabled() + if isMixedBuildsEnabled { finalOutputFile = runMixedModeBuild(ctx, extraNinjaDeps) + if cmdlineArgs.EnsureAllowlistIntegrity { + if err := checkForAllowlistIntegrityError(configuration, cmdlineArgs.BazelModeStaging); err != nil { + maybeQuit(err, "") + } + } } else { finalOutputFile = runSoongOnlyBuild(ctx, extraNinjaDeps) } diff --git a/tests/mixed_mode_test.sh b/tests/mixed_mode_test.sh index a1a792de33..ca63fdfe4a 100755 --- a/tests/mixed_mode_test.sh +++ b/tests/mixed_mode_test.sh @@ -88,7 +88,7 @@ EOF fail "Bazel actions not found for force-enabled module" fi - unused=`run_soong --bazel-force-enabled-modules=unenabled-touch-file nothing >/dev/null` + unused=`run_soong --bazel-force-enabled-modules=unenabled-touch-file --ensure-allowlist-integrity nothing >/dev/null` if [[ $? -ne 1 ]]; then fail "Expected failure due to force-enabling an unenabled module " diff --git a/ui/build/config.go b/ui/build/config.go index bf4aec9b91..81ea69e86c 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -67,28 +67,29 @@ type configImpl struct { logsPrefix string // From the arguments - parallel int - keepGoing int - verbose bool - checkbuild bool - dist bool - jsonModuleGraph bool - apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs - bp2build bool - queryview bool - reportMkMetrics bool // Collect and report mk2bp migration progress metrics. - soongDocs bool - multitreeBuild bool // This is a multitree build. - skipConfig bool - skipKati bool - skipKatiNinja bool - skipSoong bool - skipNinja bool - skipSoongTests bool - searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces - skipMetricsUpload bool - buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time - buildFromTextStub bool + parallel int + keepGoing int + verbose bool + checkbuild bool + dist bool + jsonModuleGraph bool + apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs + bp2build bool + queryview bool + reportMkMetrics bool // Collect and report mk2bp migration progress metrics. + soongDocs bool + multitreeBuild bool // This is a multitree build. + skipConfig bool + skipKati bool + skipKatiNinja bool + skipSoong bool + skipNinja bool + skipSoongTests bool + searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces + skipMetricsUpload bool + buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time + buildFromTextStub bool + ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built // From the product config katiArgs []string @@ -883,6 +884,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { } else { ctx.Fatalf("Error parsing build-time-started-unix-millis", err) } + } else if arg == "--ensure-allowlist-integrity" { + c.ensureAllowlistIntegrity = true } else if len(arg) > 0 && arg[0] == '-' { parseArgNum := func(def int) int { if len(arg) > 2 { @@ -1710,6 +1713,10 @@ func (c *configImpl) SkipMetricsUpload() bool { return c.skipMetricsUpload } +func (c *configImpl) EnsureAllowlistIntegrity() bool { + return c.ensureAllowlistIntegrity +} + // Returns a Time object if one was passed via a command-line flag. // Otherwise returns the passed default. func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time { diff --git a/ui/build/soong.go b/ui/build/soong.go index 9287731f1b..563199bee8 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -288,6 +288,9 @@ func bootstrapBlueprint(ctx Context, config Config) { if config.buildFromTextStub { mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-text-stub") } + if config.ensureAllowlistIntegrity { + mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--ensure-allowlist-integrity") + } queryviewDir := filepath.Join(config.SoongOutDir(), "queryview") // The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files) From 2f1da168abf133f1bb13febfecb15fe69fbf64de Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Mon, 17 Apr 2023 15:06:56 -0700 Subject: [PATCH 0036/1460] Expand preprocessed flag to work on android_app_imports Setting the preprocessed flag will now also verify that the apk is zip-aligned and does not have compressed JNI libs or dex files. Bug: 185811447 Test: m nothing Change-Id: I01b7c25f390345b14385f6f9e1640f48a5d9dc93 --- java/app.go | 4 +-- java/app_import.go | 71 +++++++++++++++++++++++++++++++---------- java/app_import_test.go | 24 ++++++++++++++ java/builder.go | 13 ++++++++ 4 files changed, 92 insertions(+), 20 deletions(-) diff --git a/java/app.go b/java/app.go index 03e233059a..7bb8cdbd7e 100755 --- a/java/app.go +++ b/java/app.go @@ -1461,10 +1461,8 @@ func (u *usesLibrary) verifyUsesLibrariesManifest(ctx android.ModuleContext, man // verifyUsesLibrariesAPK checks the tags in the manifest of an APK against the build // system and returns the path to a copy of the APK. -func (u *usesLibrary) verifyUsesLibrariesAPK(ctx android.ModuleContext, apk android.Path) android.Path { +func (u *usesLibrary) verifyUsesLibrariesAPK(ctx android.ModuleContext, apk android.Path) { u.verifyUsesLibraries(ctx, apk, nil) // for APKs manifest_check does not write output file - outputFile := android.PathForModuleOut(ctx, "verify_uses_libraries", apk.Base()) - return outputFile } // For Bazel / bp2build diff --git a/java/app_import.go b/java/app_import.go index 85b35ebaa9..bfd67679ea 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -49,6 +49,17 @@ var ( CommandDeps: []string{"${config.Zip2ZipCmd}"}, Description: "Uncompress dex files", }) + + checkJniAndDexLibsAreUncompressedRule = pctx.AndroidStaticRule("check-jni-and-dex-libs-are-uncompressed", blueprint.RuleParams{ + // grep -v ' stor ' will search for lines that don't have ' stor '. stor means the file is stored uncompressed + Command: "if (zipinfo $in 'lib/*.so' '*.dex' 2>/dev/null | grep -v ' stor ' >/dev/null) ; then " + + "echo $in: Contains compressed JNI libraries and/or dex files >&2;" + + "exit 1; " + + "else " + + "touch $out; " + + "fi", + Description: "Check for compressed JNI libs or dex files", + }) ) func RegisterAppImportBuildComponents(ctx android.RegistrationContext) { @@ -73,8 +84,6 @@ type AndroidAppImport struct { usesLibrary usesLibrary - preprocessed bool - installPath android.InstallPath hideApexVariantFromMake bool @@ -128,6 +137,13 @@ type AndroidAppImportProperties struct { // Optional. Install to a subdirectory of the default install path for the module Relative_install_path *string + + // Whether the prebuilt apk can be installed without additional processing. Default is false. + Preprocessed *bool + + // Whether or not to skip checking the preprocessed apk for proper alignment and uncompressed + // JNI libs and dex files. Default is false + Skip_preprocessed_apk_checks *bool } func (a *AndroidAppImport) IsInstallable() bool { @@ -201,7 +217,7 @@ func (a *AndroidAppImport) uncompressEmbeddedJniLibs( ctx android.ModuleContext, inputPath android.Path, outputPath android.OutputPath) { // Test apps don't need their JNI libraries stored uncompressed. As a matter of fact, messing // with them may invalidate pre-existing signature data. - if ctx.InstallInTestcases() && (Bool(a.properties.Presigned) || a.preprocessed) { + if ctx.InstallInTestcases() && (Bool(a.properties.Presigned) || Bool(a.properties.Preprocessed)) { ctx.Build(pctx, android.BuildParams{ Rule: android.Cp, Output: outputPath, @@ -219,7 +235,7 @@ func (a *AndroidAppImport) uncompressEmbeddedJniLibs( // Returns whether this module should have the dex file stored uncompressed in the APK. func (a *AndroidAppImport) shouldUncompressDex(ctx android.ModuleContext) bool { - if ctx.Config().UnbundledBuild() || a.preprocessed { + if ctx.Config().UnbundledBuild() || proptools.Bool(a.properties.Preprocessed) { return false } @@ -297,7 +313,7 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext a.dexpreopter.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx) if a.usesLibrary.enforceUsesLibraries() { - srcApk = a.usesLibrary.verifyUsesLibrariesAPK(ctx, srcApk) + a.usesLibrary.verifyUsesLibrariesAPK(ctx, srcApk) } a.dexpreopter.dexpreopt(ctx, jnisUncompressed) @@ -317,8 +333,15 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext // Sign or align the package if package has not been preprocessed - if a.preprocessed { - a.outputFile = srcApk + if proptools.Bool(a.properties.Preprocessed) { + output := srcApk + // TODO(b/185811447) Uncomment this after all existing failing apks set skip_preprocessed_apk_checks: true + //if !proptools.Bool(a.properties.Skip_preprocessed_apk_checks) { + // writableOutput := android.PathForModuleOut(ctx, "validated-prebuilt", apkFilename) + // a.validatePreprocessedApk(ctx, srcApk, writableOutput) + // output = writableOutput + //} + a.outputFile = output a.certificate = PresignedCertificate } else if !Bool(a.properties.Presigned) { // If the certificate property is empty at this point, default_dev_cert must be set to true. @@ -352,6 +375,30 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext // TODO: androidmk converter jni libs } +func (a *AndroidAppImport) validatePreprocessedApk(ctx android.ModuleContext, srcApk android.Path, dstApk android.WritablePath) { + alignmentStamp := android.PathForModuleOut(ctx, "validated-prebuilt", "alignment.stamp") + ctx.Build(pctx, android.BuildParams{ + Rule: checkZipAlignment, + Input: srcApk, + Output: alignmentStamp, + }) + compressionStamp := android.PathForModuleOut(ctx, "validated-prebuilt", "compression.stamp") + ctx.Build(pctx, android.BuildParams{ + Rule: checkJniAndDexLibsAreUncompressedRule, + Input: srcApk, + Output: compressionStamp, + }) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: srcApk, + Output: dstApk, + Validations: []android.Path{ + alignmentStamp, + compressionStamp, + }, + }) +} + func (a *AndroidAppImport) Prebuilt() *android.Prebuilt { return &a.prebuilt } @@ -487,11 +534,6 @@ func AndroidAppImportFactory() android.Module { return module } -type androidTestImportProperties struct { - // Whether the prebuilt apk can be installed without additional processing. Default is false. - Preprocessed *bool -} - type AndroidTestImport struct { AndroidAppImport @@ -508,14 +550,10 @@ type AndroidTestImport struct { Per_testcase_directory *bool } - testImportProperties androidTestImportProperties - data android.Paths } func (a *AndroidTestImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { - a.preprocessed = Bool(a.testImportProperties.Preprocessed) - a.generateAndroidBuildActions(ctx) a.data = android.PathsForModuleSrc(ctx, a.testProperties.Data) @@ -532,7 +570,6 @@ func AndroidTestImportFactory() android.Module { module.AddProperties(&module.properties) module.AddProperties(&module.dexpreoptProperties) module.AddProperties(&module.testProperties) - module.AddProperties(&module.testImportProperties) module.populateAllVariantStructs() android.AddLoadHook(module, func(ctx android.LoadHookContext) { module.processVariants(ctx) diff --git a/java/app_import_test.go b/java/app_import_test.go index 80930248ee..845a96299b 100644 --- a/java/app_import_test.go +++ b/java/app_import_test.go @@ -657,6 +657,30 @@ func TestAndroidTestImport_Preprocessed(t *testing.T) { } } +// TODO(b/185811447) Uncomment this after all existing failing apks set skip_preprocessed_apk_checks: true +//func TestAndroidAppImport_Preprocessed(t *testing.T) { +// ctx, _ := testJava(t, ` +// android_app_import { +// name: "foo", +// apk: "prebuilts/apk/app.apk", +// presigned: true, +// preprocessed: true, +// } +// `) +// +// apkName := "foo.apk" +// variant := ctx.ModuleForTests("foo", "android_common") +// outputBuildParams := variant.Output("validated-prebuilt/" + apkName).BuildParams +// if outputBuildParams.Rule.String() != android.Cp.String() { +// t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String()) +// } +// +// // Make sure compression and aligning were validated. +// if len(outputBuildParams.Validations) != 2 { +// t.Errorf("Expected compression/alignment validation rules, found %d validations", len(outputBuildParams.Validations)) +// } +//} + func TestAndroidTestImport_UncompressDex(t *testing.T) { testCases := []struct { name string diff --git a/java/builder.go b/java/builder.go index 462626712f..0c5773823a 100644 --- a/java/builder.go +++ b/java/builder.go @@ -246,6 +246,19 @@ var ( CommandDeps: []string{"${config.ZipAlign}"}, }, ) + + checkZipAlignment = pctx.AndroidStaticRule("checkzipalign", + blueprint.RuleParams{ + Command: "if ! ${config.ZipAlign} -c -p 4 $in > /dev/null; then " + + "echo $in: Improper package alignment >&2; " + + "exit 1; " + + "else " + + "touch $out; " + + "fi", + CommandDeps: []string{"${config.ZipAlign}"}, + Description: "Check zip alignment", + }, + ) ) func init() { From b7cdbba71e12668808e4507ebc407efd916cb745 Mon Sep 17 00:00:00 2001 From: Jooyung Han Date: Wed, 26 Apr 2023 14:42:50 +0900 Subject: [PATCH 0037/1460] Run apex_sepolicy_test for supported fs_type For now ext4 is the only supported fs_type. Bug: 279689445 Test: m apex.apexd_test_erosf Test: m apex.apexd_test_f2fs Change-Id: Idd1b2e61e25110da6d3805b082195e6ec031f573 --- apex/builder.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apex/builder.go b/apex/builder.go index 3c7671b00b..04658d90dc 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -231,7 +231,7 @@ var ( apexSepolicyTestsRule = pctx.StaticRule("apexSepolicyTestsRule", blueprint.RuleParams{ Command: `${deapexer} --debugfs_path ${debugfs_static} list -Z ${in} > ${out}.fc` + - `&& ${apex_sepolicy_tests} -f ${out}.fc && touch ${out}`, + ` && ${apex_sepolicy_tests} -f ${out}.fc && touch ${out}`, CommandDeps: []string{"${apex_sepolicy_tests}", "${deapexer}", "${debugfs_static}"}, Description: "run apex_sepolicy_tests", }) @@ -876,7 +876,8 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { args["outCommaList"] = signedOutputFile.String() } var validations android.Paths - if suffix == imageApexSuffix { + // TODO(b/279688635) deapexer supports [ext4] + if suffix == imageApexSuffix && ext4 == a.payloadFsType { validations = append(validations, runApexSepolicyTests(ctx, unsignedOutputFile.OutputPath)) } ctx.Build(pctx, android.BuildParams{ From c57e3a8397b470ca9b3e7006dcf40b827a2ea314 Mon Sep 17 00:00:00 2001 From: Yi Kong Date: Wed, 26 Apr 2023 15:11:43 +0900 Subject: [PATCH 0038/1460] Update clang version to clang-r487747b Test: presubmit Change-Id: I98de36a9f301dee14c21d42e20901eb5aa157164 --- cc/config/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cc/config/global.go b/cc/config/global.go index a2d5cf439d..0c6e66d3fd 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -311,7 +311,7 @@ var ( // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r487747" + ClangDefaultVersion = "clang-r487747b" ClangDefaultShortVersion = "17" // Directories with warnings from Android.bp files. From a2b57abb4ab379ae3f3747e0c97efabeacdf4a28 Mon Sep 17 00:00:00 2001 From: Jooyung Han Date: Wed, 26 Apr 2023 16:39:20 +0900 Subject: [PATCH 0039/1460] Remove makefile_goal This was added to use 'make' target in soong. It was a hacky way and caused many problems: - can't track dependency - doesn't work with sanitizer - VSDK snapshot doesn't support - .. Now references are all gone. Let's remove it. Bug: n/a Test: m Change-Id: I0336a490780f37a4e77d343c61315686e23db643 --- android/Android.bp | 1 - android/makefile_goal.go | 98 -------------------------------------- android/neverallow.go | 15 ------ android/neverallow_test.go | 59 ----------------------- 4 files changed, 173 deletions(-) delete mode 100644 android/makefile_goal.go diff --git a/android/Android.bp b/android/Android.bp index 641c438f1f..2bc96f1e84 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -60,7 +60,6 @@ bootstrap_go_package { "license_metadata.go", "license_sdk_member.go", "licenses.go", - "makefile_goal.go", "makevars.go", "metrics.go", "module.go", diff --git a/android/makefile_goal.go b/android/makefile_goal.go deleted file mode 100644 index 07354a6486..0000000000 --- a/android/makefile_goal.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "fmt" - "io" - "path/filepath" - - "github.com/google/blueprint/proptools" -) - -func init() { - RegisterModuleType("makefile_goal", MakefileGoalFactory) -} - -type makefileGoalProperties struct { - // Sources. - - // Makefile goal output file path, relative to PRODUCT_OUT. - Product_out_path *string -} - -type makefileGoal struct { - ModuleBase - - properties makefileGoalProperties - - // Destination. Output file path of this module. - outputFilePath OutputPath -} - -var _ AndroidMkEntriesProvider = (*makefileGoal)(nil) -var _ OutputFileProducer = (*makefileGoal)(nil) - -// Input file of this makefile_goal module. Nil if none specified. May use variable names in makefiles. -func (p *makefileGoal) inputPath() *string { - if p.properties.Product_out_path != nil { - return proptools.StringPtr(filepath.Join("$(PRODUCT_OUT)", proptools.String(p.properties.Product_out_path))) - } - return nil -} - -// OutputFileProducer -func (p *makefileGoal) OutputFiles(tag string) (Paths, error) { - if tag != "" { - return nil, fmt.Errorf("unsupported tag %q", tag) - } - return Paths{p.outputFilePath}, nil -} - -// AndroidMkEntriesProvider -func (p *makefileGoal) DepsMutator(ctx BottomUpMutatorContext) { - if p.inputPath() == nil { - ctx.PropertyErrorf("product_out_path", "Path relative to PRODUCT_OUT required") - } -} - -func (p *makefileGoal) GenerateAndroidBuildActions(ctx ModuleContext) { - filename := filepath.Base(proptools.String(p.inputPath())) - p.outputFilePath = PathForModuleOut(ctx, filename).OutputPath - - ctx.InstallFile(PathForModuleInstall(ctx, "etc"), ctx.ModuleName(), p.outputFilePath) -} - -func (p *makefileGoal) AndroidMkEntries() []AndroidMkEntries { - return []AndroidMkEntries{AndroidMkEntries{ - Class: "ETC", - OutputFile: OptionalPathForPath(p.outputFilePath), - ExtraFooters: []AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string) { - // Can't use Cp because inputPath() is not a valid Path. - fmt.Fprintf(w, "$(eval $(call copy-one-file,%s,%s))\n", proptools.String(p.inputPath()), p.outputFilePath) - }, - }, - }} -} - -// Import a Makefile goal to Soong by copying the file built by -// the goal to a path visible to Soong. This rule only works on boot images. -func MakefileGoalFactory() Module { - module := &makefileGoal{} - module.AddProperties(&module.properties) - InitAndroidModule(module) - return module -} diff --git a/android/neverallow.go b/android/neverallow.go index 2139c3c3eb..5b5e613286 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -55,7 +55,6 @@ func init() { AddNeverAllowRules(createJavaDeviceForHostRules()...) AddNeverAllowRules(createCcSdkVariantRules()...) AddNeverAllowRules(createUncompressDexRules()...) - AddNeverAllowRules(createMakefileGoalRules()...) AddNeverAllowRules(createInitFirstStageRules()...) AddNeverAllowRules(createProhibitFrameworkAccessRules()...) AddNeverAllowRules(createBp2BuildRule()) @@ -236,20 +235,6 @@ func createUncompressDexRules() []Rule { } } -func createMakefileGoalRules() []Rule { - allowlist := []string{ - // libwifi_hal uses makefile_goal for its dependencies - "frameworks/opt/net/wifi/libwifi_hal", - } - return []Rule{ - NeverAllow(). - ModuleType("makefile_goal"). - WithoutMatcher("product_out_path", Regexp("^boot[0-9a-zA-Z.-]*[.]img$")). - NotIn(allowlist...). - Because("Only boot images may be imported as a makefile goal if not in allowed projects"), - } -} - func createInitFirstStageRules() []Rule { return []Rule{ NeverAllow(). diff --git a/android/neverallow_test.go b/android/neverallow_test.go index 5f5f9a1931..ddd982d155 100644 --- a/android/neverallow_test.go +++ b/android/neverallow_test.go @@ -313,45 +313,6 @@ var neverallowTests = []struct { "module \"outside_art_libraries\": violates neverallow", }, }, - { - name: "disallowed makefile_goal", - fs: map[string][]byte{ - "Android.bp": []byte(` - makefile_goal { - name: "foo", - product_out_path: "boot/trap.img" - } - `), - }, - expectedErrors: []string{ - "Only boot images.* may be imported as a makefile goal", - }, - }, - { - name: "disallowed makefile_goal outside external", - fs: map[string][]byte{ - "project/Android.bp": []byte(` - makefile_goal { - name: "foo", - product_out_path: "obj/EXE/foo", - } - `), - }, - expectedErrors: []string{ - "not in allowed projects", - }, - }, - { - name: "allow makefile_goal within external", - fs: map[string][]byte{ - "frameworks/opt/net/wifi/libwifi_hal/Android.bp": []byte(` - makefile_goal { - name: "foo", - product_out_path: "obj/EXE/foo", - } - `), - }, - }, // Tests for the rule prohibiting the use of framework { name: "prohibit framework", @@ -391,7 +352,6 @@ var prepareForNeverAllowTest = GroupFixturePreparers( ctx.RegisterModuleType("java_library", newMockJavaLibraryModule) ctx.RegisterModuleType("java_library_host", newMockJavaLibraryModule) ctx.RegisterModuleType("java_device_for_host", newMockJavaLibraryModule) - ctx.RegisterModuleType("makefile_goal", newMockMakefileGoalModule) }), ) @@ -489,22 +449,3 @@ func newMockJavaLibraryModule() Module { func (p *mockJavaLibraryModule) GenerateAndroidBuildActions(ModuleContext) { } - -type mockMakefileGoalProperties struct { - Product_out_path *string -} - -type mockMakefileGoalModule struct { - ModuleBase - properties mockMakefileGoalProperties -} - -func newMockMakefileGoalModule() Module { - m := &mockMakefileGoalModule{} - m.AddProperties(&m.properties) - InitAndroidModule(m) - return m -} - -func (p *mockMakefileGoalModule) GenerateAndroidBuildActions(ModuleContext) { -} From 50151a7489f93a1ec2b4478dd68c2c34df149c53 Mon Sep 17 00:00:00 2001 From: Anton Hansson Date: Wed, 26 Apr 2023 18:09:02 +0000 Subject: [PATCH 0040/1460] Remove --skip-kati Not used anymore. Test: No Change-Id: Ide0d29b773583d6a8faff5b41a82d650cc2f9bcb --- ui/build/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index bf4aec9b91..5543411ace 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -813,9 +813,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { // by a previous build. c.skipConfig = true c.skipKati = true - } else if arg == "--skip-kati" { - // TODO: remove --skip-kati once module builds have been migrated to --song-only - c.skipKati = true } else if arg == "--soong-only" { c.skipKati = true c.skipKatiNinja = true From 44fc9a826bbda43df97bb61b36e5333e6c575cfd Mon Sep 17 00:00:00 2001 From: David Fu Date: Wed, 26 Apr 2023 20:25:30 +0000 Subject: [PATCH 0041/1460] Changed assignee to triage_assignee in fuzzconfig Change-Id: Ifdf4e8a3d855c414c906d249674411f4f4b88396 --- fuzz/fuzz_common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/fuzz_common.go b/fuzz/fuzz_common.go index 79d2412798..f76529d316 100644 --- a/fuzz/fuzz_common.go +++ b/fuzz/fuzz_common.go @@ -339,7 +339,7 @@ type FuzzConfig struct { // List of modules for monitoring coverage drops in directories (e.g. "libicu") Target_modules []string `json:"target_modules,omitempty"` // Specifies a bug assignee to replace default ISE assignment - Assignee string `json:"assignee,omitempty"` + Triage_assignee string `json:"triage_assignee,omitempty"` } type FuzzFrameworks struct { From b9e11c5b647ada54ea59367e79fb437a2974a8f3 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Wed, 26 Apr 2023 21:28:16 +0000 Subject: [PATCH 0042/1460] Fix allowlist in order to mixed_build the entire prod allowlist. Bug: 279808471 Test: m nothing --ensure-allowlist-integrity Change-Id: Ie7c18e1457d29f300e790ea8a38fbcdc4fdcc50a --- android/allowlists/allowlists.go | 1 + 1 file changed, 1 insertion(+) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 67ca8ecf07..a4df34c3f4 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -291,6 +291,7 @@ var ( "packages/modules/Gki/libkver": Bp2BuildDefaultTrue, "packages/modules/NetworkStack/common/captiveportal": Bp2BuildDefaultTrue, "packages/modules/NeuralNetworks/apex": Bp2BuildDefaultTrue, + "packages/modules/NeuralNetworks/apex/testing": Bp2BuildDefaultTrue, "packages/providers/MediaProvider/tools/dialogs": Bp2BuildDefaultFalse, // TODO(b/242834374) "packages/screensavers/Basic": Bp2BuildDefaultTrue, "packages/services/Car/tests/SampleRearViewCamera": Bp2BuildDefaultFalse, // TODO(b/242834321) From c9508aac4ccc605d90be2c92c4e4c46db45377fc Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Tue, 7 Feb 2023 11:38:27 -0800 Subject: [PATCH 0043/1460] Load starlark files from soong There are a number of instances where we are exporting information from soong to bazel via soong_injection. This could be more bazel-centric if the information was instead held in bzl files, and both bazel and soong read it from there. Add a starlark package that will run //build/bazel/constants_exported_to_soong.bzl at initialization time, and then results can be retreived with GetStarlarkValue. Since changes to the starlark files mean that soong has to rerun, add them as ninja deps. Unfortunately, the starlark code has to be run at runtime rather than pregenerating their results, because tests run from intellij wouldn't go through any pregeneration steps. This means that starlark is run multiple times during the build, once per test package and once per primary builder invocation. (currently 3, could be reduced to 2 if we made the symlink forest generation into its own standalone tool) The starlark code we have so far in this cl is very fast, roughly half a millisecond, so it's not a big deal for now, but something to keep an eye on as we add more starlark constants. Bug: 279095899 Test: go test Change-Id: I1e7ca1df1d8d67333cbfc46e8396e229820e4476 --- android/Android.bp | 1 + android/ninja_deps.go | 12 +- bp2build/bp2build.go | 7 + cmd/soong_build/queryview.go | 9 + go.mod | 1 + go.work | 2 + starlark_import/Android.bp | 36 +++ starlark_import/README.md | 14 ++ starlark_import/starlark_import.go | 306 ++++++++++++++++++++++++ starlark_import/starlark_import_test.go | 122 ++++++++++ starlark_import/unmarshal.go | 288 ++++++++++++++++++++++ starlark_import/unmarshal_test.go | 133 ++++++++++ tests/bp2build_bazel_test.sh | 14 ++ tests/persistent_bazel_test.sh | 4 +- 14 files changed, 946 insertions(+), 3 deletions(-) create mode 100644 starlark_import/Android.bp create mode 100644 starlark_import/README.md create mode 100644 starlark_import/starlark_import.go create mode 100644 starlark_import/starlark_import_test.go create mode 100644 starlark_import/unmarshal.go create mode 100644 starlark_import/unmarshal_test.go diff --git a/android/Android.bp b/android/Android.bp index 641c438f1f..29af758501 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -17,6 +17,7 @@ bootstrap_go_package { "soong-remoteexec", "soong-response", "soong-shared", + "soong-starlark", "soong-starlark-format", "soong-ui-metrics_proto", "soong-android-allowlists", diff --git a/android/ninja_deps.go b/android/ninja_deps.go index 2f442d5f0e..1d50a47ec6 100644 --- a/android/ninja_deps.go +++ b/android/ninja_deps.go @@ -14,7 +14,10 @@ package android -import "sort" +import ( + "android/soong/starlark_import" + "sort" +) func (c *config) addNinjaFileDeps(deps ...string) { for _, dep := range deps { @@ -40,4 +43,11 @@ type ninjaDepsSingleton struct{} func (ninjaDepsSingleton) GenerateBuildActions(ctx SingletonContext) { ctx.AddNinjaFileDeps(ctx.Config().ninjaFileDeps()...) + + deps, err := starlark_import.GetNinjaDeps() + if err != nil { + ctx.Errorf("Error running starlark code: %s", err) + } else { + ctx.AddNinjaFileDeps(deps...) + } } diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go index d1dfb9d36e..b22cb28617 100644 --- a/bp2build/bp2build.go +++ b/bp2build/bp2build.go @@ -15,6 +15,7 @@ package bp2build import ( + "android/soong/starlark_import" "fmt" "os" "path/filepath" @@ -93,6 +94,12 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics { os.Exit(1) } writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles) + starlarkDeps, err := starlark_import.GetNinjaDeps() + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + ctx.AddNinjaFileDeps(starlarkDeps...) return &res.metrics } diff --git a/cmd/soong_build/queryview.go b/cmd/soong_build/queryview.go index ce32184985..67cb6cfc8a 100644 --- a/cmd/soong_build/queryview.go +++ b/cmd/soong_build/queryview.go @@ -15,6 +15,7 @@ package main import ( + "android/soong/starlark_import" "io/fs" "io/ioutil" "os" @@ -47,6 +48,14 @@ func createBazelWorkspace(ctx *bp2build.CodegenContext, outDir string, generateF } } + // Add starlark deps here, so that they apply to both queryview and apibp2build which + // both run this function. + starlarkDeps, err2 := starlark_import.GetNinjaDeps() + if err2 != nil { + return err2 + } + ctx.AddNinjaFileDeps(starlarkDeps...) + return nil } diff --git a/go.mod b/go.mod index a5d9dd51bb..4a511c528a 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/google/blueprint v0.0.0 google.golang.org/protobuf v0.0.0 prebuilts/bazel/common/proto/analysis_v2 v0.0.0 + go.starlark.net v0.0.0 ) diff --git a/go.work b/go.work index 737a9df965..67f6549087 100644 --- a/go.work +++ b/go.work @@ -4,6 +4,7 @@ use ( . ../../external/go-cmp ../../external/golang-protobuf + ../../external/starlark-go ../../prebuilts/bazel/common/proto/analysis_v2 ../../prebuilts/bazel/common/proto/build ../blueprint @@ -16,4 +17,5 @@ replace ( google.golang.org/protobuf v0.0.0 => ../../external/golang-protobuf prebuilts/bazel/common/proto/analysis_v2 v0.0.0 => ../../prebuilts/bazel/common/proto/analysis_v2 prebuilts/bazel/common/proto/build v0.0.0 => ../../prebuilts/bazel/common/proto/build + go.starlark.net v0.0.0 => ../../external/starlark-go ) diff --git a/starlark_import/Android.bp b/starlark_import/Android.bp new file mode 100644 index 0000000000..b43217b763 --- /dev/null +++ b/starlark_import/Android.bp @@ -0,0 +1,36 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-starlark", + pkgPath: "android/soong/starlark_import", + srcs: [ + "starlark_import.go", + "unmarshal.go", + ], + testSrcs: [ + "starlark_import_test.go", + "unmarshal_test.go", + ], + deps: [ + "go-starlark-starlark", + "go-starlark-starlarkstruct", + "go-starlark-starlarkjson", + "go-starlark-starlarktest", + ], +} diff --git a/starlark_import/README.md b/starlark_import/README.md new file mode 100644 index 0000000000..e444759d05 --- /dev/null +++ b/starlark_import/README.md @@ -0,0 +1,14 @@ +# starlark_import package + +This allows soong to read constant information from starlark files. At package initialization +time, soong will read `build/bazel/constants_exported_to_soong.bzl`, and then make the +variables from that file available via `starlark_import.GetStarlarkValue()`. So to import +a new variable, it must be added to `constants_exported_to_soong.bzl` and then it can +be accessed by name. + +Only constant information can be read, since this is not a full bazel execution but a +standalone starlark interpreter. This means you can't use bazel contructs like `rule`, +`provider`, `select`, `glob`, etc. + +All starlark files that were loaded must be added as ninja deps that cause soong to rerun. +The loaded files can be retrieved via `starlark_import.GetNinjaDeps()`. diff --git a/starlark_import/starlark_import.go b/starlark_import/starlark_import.go new file mode 100644 index 0000000000..ebe42472cd --- /dev/null +++ b/starlark_import/starlark_import.go @@ -0,0 +1,306 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark_import + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkjson" + "go.starlark.net/starlarkstruct" +) + +func init() { + go func() { + startTime := time.Now() + v, d, err := runStarlarkFile("//build/bazel/constants_exported_to_soong.bzl") + endTime := time.Now() + //fmt.Fprintf(os.Stderr, "starlark run time: %s\n", endTime.Sub(startTime).String()) + globalResult.Set(starlarkResult{ + values: v, + ninjaDeps: d, + err: err, + startTime: startTime, + endTime: endTime, + }) + }() +} + +type starlarkResult struct { + values starlark.StringDict + ninjaDeps []string + err error + startTime time.Time + endTime time.Time +} + +// setOnce wraps a value and exposes Set() and Get() accessors for it. +// The Get() calls will block until a Set() has been called. +// A second call to Set() will panic. +// setOnce must be created using newSetOnce() +type setOnce[T any] struct { + value T + lock sync.Mutex + wg sync.WaitGroup + isSet bool +} + +func (o *setOnce[T]) Set(value T) { + o.lock.Lock() + defer o.lock.Unlock() + if o.isSet { + panic("Value already set") + } + + o.value = value + o.isSet = true + o.wg.Done() +} + +func (o *setOnce[T]) Get() T { + if !o.isSet { + o.wg.Wait() + } + return o.value +} + +func newSetOnce[T any]() *setOnce[T] { + result := &setOnce[T]{} + result.wg.Add(1) + return result +} + +var globalResult = newSetOnce[starlarkResult]() + +func GetStarlarkValue[T any](key string) (T, error) { + result := globalResult.Get() + if result.err != nil { + var zero T + return zero, result.err + } + if !result.values.Has(key) { + var zero T + return zero, fmt.Errorf("a starlark variable by that name wasn't found, did you update //build/bazel/constants_exported_to_soong.bzl?") + } + return Unmarshal[T](result.values[key]) +} + +func GetNinjaDeps() ([]string, error) { + result := globalResult.Get() + if result.err != nil { + return nil, result.err + } + return result.ninjaDeps, nil +} + +func getTopDir() (string, error) { + // It's hard to communicate the top dir to this package in any other way than reading the + // arguments directly, because we need to know this at package initialization time. Many + // soong constants that we'd like to read from starlark are initialized during package + // initialization. + for i, arg := range os.Args { + if arg == "--top" { + if i < len(os.Args)-1 && os.Args[i+1] != "" { + return os.Args[i+1], nil + } + } + } + + // When running tests, --top is not passed. Instead, search for the top dir manually + cwd, err := os.Getwd() + if err != nil { + return "", err + } + for cwd != "/" { + if _, err := os.Stat(filepath.Join(cwd, "build/soong/soong_ui.bash")); err == nil { + return cwd, nil + } + cwd = filepath.Dir(cwd) + } + return "", fmt.Errorf("could not find top dir") +} + +const callerDirKey = "callerDir" + +type modentry struct { + globals starlark.StringDict + err error +} + +func unsupportedMethod(t *starlark.Thread, fn *starlark.Builtin, _ starlark.Tuple, _ []starlark.Tuple) (starlark.Value, error) { + return nil, fmt.Errorf("%sthis file is read by soong, and must therefore be pure starlark and include only constant information. %q is not allowed", t.CallStack().String(), fn.Name()) +} + +var builtins = starlark.StringDict{ + "aspect": starlark.NewBuiltin("aspect", unsupportedMethod), + "glob": starlark.NewBuiltin("glob", unsupportedMethod), + "json": starlarkjson.Module, + "provider": starlark.NewBuiltin("provider", unsupportedMethod), + "rule": starlark.NewBuiltin("rule", unsupportedMethod), + "struct": starlark.NewBuiltin("struct", starlarkstruct.Make), + "select": starlark.NewBuiltin("select", unsupportedMethod), + "transition": starlark.NewBuiltin("transition", unsupportedMethod), +} + +// Takes a module name (the first argument to the load() function) and returns the path +// it's trying to load, stripping out leading //, and handling leading :s. +func cleanModuleName(moduleName string, callerDir string) (string, error) { + if strings.Count(moduleName, ":") > 1 { + return "", fmt.Errorf("at most 1 colon must be present in starlark path: %s", moduleName) + } + + // We don't have full support for external repositories, but at least support skylib's dicts. + if moduleName == "@bazel_skylib//lib:dicts.bzl" { + return "external/bazel-skylib/lib/dicts.bzl", nil + } + + localLoad := false + if strings.HasPrefix(moduleName, "@//") { + moduleName = moduleName[3:] + } else if strings.HasPrefix(moduleName, "//") { + moduleName = moduleName[2:] + } else if strings.HasPrefix(moduleName, ":") { + moduleName = moduleName[1:] + localLoad = true + } else { + return "", fmt.Errorf("load path must start with // or :") + } + + if ix := strings.LastIndex(moduleName, ":"); ix >= 0 { + moduleName = moduleName[:ix] + string(os.PathSeparator) + moduleName[ix+1:] + } + + if filepath.Clean(moduleName) != moduleName { + return "", fmt.Errorf("load path must be clean, found: %s, expected: %s", moduleName, filepath.Clean(moduleName)) + } + if strings.HasPrefix(moduleName, "../") { + return "", fmt.Errorf("load path must not start with ../: %s", moduleName) + } + if strings.HasPrefix(moduleName, "/") { + return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName) + } + + if localLoad { + return filepath.Join(callerDir, moduleName), nil + } + + return moduleName, nil +} + +// loader implements load statement. The format of the loaded module URI is +// +// [//path]:base +// +// The file path is $ROOT/path/base if path is present, /base otherwise. +func loader(thread *starlark.Thread, module string, topDir string, moduleCache map[string]*modentry, moduleCacheLock *sync.Mutex, filesystem map[string]string) (starlark.StringDict, error) { + modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string)) + if err != nil { + return nil, err + } + moduleCacheLock.Lock() + e, ok := moduleCache[modulePath] + if e == nil { + if ok { + moduleCacheLock.Unlock() + return nil, fmt.Errorf("cycle in load graph") + } + + // Add a placeholder to indicate "load in progress". + moduleCache[modulePath] = nil + moduleCacheLock.Unlock() + + childThread := &starlark.Thread{Name: "exec " + module, Load: thread.Load} + + // Cheating for the sake of testing: + // propagate starlarktest's Reporter key, otherwise testing + // the load function may cause panic in starlarktest code. + const testReporterKey = "Reporter" + if v := thread.Local(testReporterKey); v != nil { + childThread.SetLocal(testReporterKey, v) + } + + childThread.SetLocal(callerDirKey, filepath.Dir(modulePath)) + + if filesystem != nil { + globals, err := starlark.ExecFile(childThread, filepath.Join(topDir, modulePath), filesystem[modulePath], builtins) + e = &modentry{globals, err} + } else { + globals, err := starlark.ExecFile(childThread, filepath.Join(topDir, modulePath), nil, builtins) + e = &modentry{globals, err} + } + + // Update the cache. + moduleCacheLock.Lock() + moduleCache[modulePath] = e + } + moduleCacheLock.Unlock() + return e.globals, e.err +} + +// Run runs the given starlark file and returns its global variables and a list of all starlark +// files that were loaded. The top dir for starlark's // is found via getTopDir(). +func runStarlarkFile(filename string) (starlark.StringDict, []string, error) { + topDir, err := getTopDir() + if err != nil { + return nil, nil, err + } + return runStarlarkFileWithFilesystem(filename, topDir, nil) +} + +func runStarlarkFileWithFilesystem(filename string, topDir string, filesystem map[string]string) (starlark.StringDict, []string, error) { + if !strings.HasPrefix(filename, "//") && !strings.HasPrefix(filename, ":") { + filename = "//" + filename + } + filename, err := cleanModuleName(filename, "") + if err != nil { + return nil, nil, err + } + moduleCache := make(map[string]*modentry) + moduleCache[filename] = nil + moduleCacheLock := &sync.Mutex{} + mainThread := &starlark.Thread{ + Name: "main", + Print: func(_ *starlark.Thread, msg string) { + // Ignore prints + }, + Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) { + return loader(thread, module, topDir, moduleCache, moduleCacheLock, filesystem) + }, + } + mainThread.SetLocal(callerDirKey, filepath.Dir(filename)) + + var result starlark.StringDict + if filesystem != nil { + result, err = starlark.ExecFile(mainThread, filepath.Join(topDir, filename), filesystem[filename], builtins) + } else { + result, err = starlark.ExecFile(mainThread, filepath.Join(topDir, filename), nil, builtins) + } + return result, sortedStringKeys(moduleCache), err +} + +func sortedStringKeys(m map[string]*modentry) []string { + s := make([]string, 0, len(m)) + for k := range m { + s = append(s, k) + } + sort.Strings(s) + return s +} diff --git a/starlark_import/starlark_import_test.go b/starlark_import/starlark_import_test.go new file mode 100644 index 0000000000..8a58e3b3d3 --- /dev/null +++ b/starlark_import/starlark_import_test.go @@ -0,0 +1,122 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark_import + +import ( + "strings" + "testing" + + "go.starlark.net/starlark" +) + +func TestBasic(t *testing.T) { + globals, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{ + "a.bzl": ` +my_string = "hello, world!" +`}) + if err != nil { + t.Error(err) + return + } + + if globals["my_string"].(starlark.String) != "hello, world!" { + t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String()) + } +} + +func TestLoad(t *testing.T) { + globals, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{ + "a.bzl": ` +load("//b.bzl", _b_string = "my_string") +my_string = "hello, " + _b_string +`, + "b.bzl": ` +my_string = "world!" +`}) + if err != nil { + t.Error(err) + return + } + + if globals["my_string"].(starlark.String) != "hello, world!" { + t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String()) + } +} + +func TestLoadRelative(t *testing.T) { + globals, ninjaDeps, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{ + "a.bzl": ` +load(":b.bzl", _b_string = "my_string") +load("//foo/c.bzl", _c_string = "my_string") +my_string = "hello, " + _b_string +c_string = _c_string +`, + "b.bzl": ` +my_string = "world!" +`, + "foo/c.bzl": ` +load(":d.bzl", _d_string = "my_string") +my_string = "hello, " + _d_string +`, + "foo/d.bzl": ` +my_string = "world!" +`}) + if err != nil { + t.Error(err) + return + } + + if globals["my_string"].(starlark.String) != "hello, world!" { + t.Errorf("Expected %q, got %q", "hello, world!", globals["my_string"].String()) + } + + expectedNinjaDeps := []string{ + "a.bzl", + "b.bzl", + "foo/c.bzl", + "foo/d.bzl", + } + if !slicesEqual(ninjaDeps, expectedNinjaDeps) { + t.Errorf("Expected %v ninja deps, got %v", expectedNinjaDeps, ninjaDeps) + } +} + +func TestLoadCycle(t *testing.T) { + _, _, err := runStarlarkFileWithFilesystem("a.bzl", "", map[string]string{ + "a.bzl": ` +load(":b.bzl", _b_string = "my_string") +my_string = "hello, " + _b_string +`, + "b.bzl": ` +load(":a.bzl", _a_string = "my_string") +my_string = "hello, " + _a_string +`}) + if err == nil || !strings.Contains(err.Error(), "cycle in load graph") { + t.Errorf("Expected cycle in load graph, got: %v", err) + return + } +} + +func slicesEqual[T comparable](a []T, b []T) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/starlark_import/unmarshal.go b/starlark_import/unmarshal.go new file mode 100644 index 0000000000..1b5443782b --- /dev/null +++ b/starlark_import/unmarshal.go @@ -0,0 +1,288 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark_import + +import ( + "fmt" + "math" + "reflect" + "unsafe" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +func Unmarshal[T any](value starlark.Value) (T, error) { + var zero T + x, err := UnmarshalReflect(value, reflect.TypeOf(zero)) + return x.Interface().(T), err +} + +func UnmarshalReflect(value starlark.Value, ty reflect.Type) (reflect.Value, error) { + zero := reflect.Zero(ty) + var result reflect.Value + if ty.Kind() == reflect.Interface { + var err error + ty, err = typeOfStarlarkValue(value) + if err != nil { + return zero, err + } + } + if ty.Kind() == reflect.Map { + result = reflect.MakeMap(ty) + } else { + result = reflect.Indirect(reflect.New(ty)) + } + + switch v := value.(type) { + case starlark.String: + if result.Type().Kind() != reflect.String { + return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String()) + } + result.SetString(v.GoString()) + case starlark.Int: + signedValue, signedOk := v.Int64() + unsignedValue, unsignedOk := v.Uint64() + switch result.Type().Kind() { + case reflect.Int64: + if !signedOk { + return zero, fmt.Errorf("starlark int didn't fit in go int64") + } + result.SetInt(signedValue) + case reflect.Int32: + if !signedOk || signedValue > math.MaxInt32 || signedValue < math.MinInt32 { + return zero, fmt.Errorf("starlark int didn't fit in go int32") + } + result.SetInt(signedValue) + case reflect.Int16: + if !signedOk || signedValue > math.MaxInt16 || signedValue < math.MinInt16 { + return zero, fmt.Errorf("starlark int didn't fit in go int16") + } + result.SetInt(signedValue) + case reflect.Int8: + if !signedOk || signedValue > math.MaxInt8 || signedValue < math.MinInt8 { + return zero, fmt.Errorf("starlark int didn't fit in go int8") + } + result.SetInt(signedValue) + case reflect.Int: + if !signedOk || signedValue > math.MaxInt || signedValue < math.MinInt { + return zero, fmt.Errorf("starlark int didn't fit in go int") + } + result.SetInt(signedValue) + case reflect.Uint64: + if !unsignedOk { + return zero, fmt.Errorf("starlark int didn't fit in go uint64") + } + result.SetUint(unsignedValue) + case reflect.Uint32: + if !unsignedOk || unsignedValue > math.MaxUint32 { + return zero, fmt.Errorf("starlark int didn't fit in go uint32") + } + result.SetUint(unsignedValue) + case reflect.Uint16: + if !unsignedOk || unsignedValue > math.MaxUint16 { + return zero, fmt.Errorf("starlark int didn't fit in go uint16") + } + result.SetUint(unsignedValue) + case reflect.Uint8: + if !unsignedOk || unsignedValue > math.MaxUint8 { + return zero, fmt.Errorf("starlark int didn't fit in go uint8") + } + result.SetUint(unsignedValue) + case reflect.Uint: + if !unsignedOk || unsignedValue > math.MaxUint { + return zero, fmt.Errorf("starlark int didn't fit in go uint") + } + result.SetUint(unsignedValue) + default: + return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String()) + } + case starlark.Float: + f := float64(v) + switch result.Type().Kind() { + case reflect.Float64: + result.SetFloat(f) + case reflect.Float32: + if f > math.MaxFloat32 || f < -math.MaxFloat32 { + return zero, fmt.Errorf("starlark float didn't fit in go float32") + } + result.SetFloat(f) + default: + return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String()) + } + case starlark.Bool: + if result.Type().Kind() != reflect.Bool { + return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String()) + } + result.SetBool(bool(v)) + case starlark.Tuple: + if result.Type().Kind() != reflect.Slice { + return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String()) + } + elemType := result.Type().Elem() + // TODO: Add this grow call when we're on go 1.20 + //result.Grow(v.Len()) + for i := 0; i < v.Len(); i++ { + elem, err := UnmarshalReflect(v.Index(i), elemType) + if err != nil { + return zero, err + } + result = reflect.Append(result, elem) + } + case *starlark.List: + if result.Type().Kind() != reflect.Slice { + return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String()) + } + elemType := result.Type().Elem() + // TODO: Add this grow call when we're on go 1.20 + //result.Grow(v.Len()) + for i := 0; i < v.Len(); i++ { + elem, err := UnmarshalReflect(v.Index(i), elemType) + if err != nil { + return zero, err + } + result = reflect.Append(result, elem) + } + case *starlark.Dict: + if result.Type().Kind() != reflect.Map { + return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String()) + } + keyType := result.Type().Key() + valueType := result.Type().Elem() + for _, pair := range v.Items() { + key := pair.Index(0) + value := pair.Index(1) + + unmarshalledKey, err := UnmarshalReflect(key, keyType) + if err != nil { + return zero, err + } + unmarshalledValue, err := UnmarshalReflect(value, valueType) + if err != nil { + return zero, err + } + + result.SetMapIndex(unmarshalledKey, unmarshalledValue) + } + case *starlarkstruct.Struct: + if result.Type().Kind() != reflect.Struct { + return zero, fmt.Errorf("starlark type was %s, but %s requested", v.Type(), result.Type().Kind().String()) + } + if result.NumField() != len(v.AttrNames()) { + return zero, fmt.Errorf("starlark struct and go struct have different number of fields (%d and %d)", len(v.AttrNames()), result.NumField()) + } + for _, attrName := range v.AttrNames() { + attr, err := v.Attr(attrName) + if err != nil { + return zero, err + } + + // TODO(b/279787235): this should probably support tags to rename the field + resultField := result.FieldByName(attrName) + if resultField == (reflect.Value{}) { + return zero, fmt.Errorf("starlark struct had field %s, but requested struct type did not", attrName) + } + // This hack allows us to change unexported fields + resultField = reflect.NewAt(resultField.Type(), unsafe.Pointer(resultField.UnsafeAddr())).Elem() + x, err := UnmarshalReflect(attr, resultField.Type()) + if err != nil { + return zero, err + } + resultField.Set(x) + } + default: + return zero, fmt.Errorf("unimplemented starlark type: %s", value.Type()) + } + + return result, nil +} + +func typeOfStarlarkValue(value starlark.Value) (reflect.Type, error) { + var err error + switch v := value.(type) { + case starlark.String: + return reflect.TypeOf(""), nil + case *starlark.List: + innerType := reflect.TypeOf("") + if v.Len() > 0 { + innerType, err = typeOfStarlarkValue(v.Index(0)) + if err != nil { + return nil, err + } + } + for i := 1; i < v.Len(); i++ { + innerTypeI, err := typeOfStarlarkValue(v.Index(i)) + if err != nil { + return nil, err + } + if innerType != innerTypeI { + return nil, fmt.Errorf("List must contain elements of entirely the same type, found %v and %v", innerType, innerTypeI) + } + } + return reflect.SliceOf(innerType), nil + case *starlark.Dict: + keyType := reflect.TypeOf("") + valueType := reflect.TypeOf("") + keys := v.Keys() + if v.Len() > 0 { + firstKey := keys[0] + keyType, err = typeOfStarlarkValue(firstKey) + if err != nil { + return nil, err + } + firstValue, found, err := v.Get(firstKey) + if !found { + err = fmt.Errorf("value not found") + } + if err != nil { + return nil, err + } + valueType, err = typeOfStarlarkValue(firstValue) + if err != nil { + return nil, err + } + } + for _, key := range keys { + keyTypeI, err := typeOfStarlarkValue(key) + if err != nil { + return nil, err + } + if keyType != keyTypeI { + return nil, fmt.Errorf("dict must contain elements of entirely the same type, found %v and %v", keyType, keyTypeI) + } + value, found, err := v.Get(key) + if !found { + err = fmt.Errorf("value not found") + } + if err != nil { + return nil, err + } + valueTypeI, err := typeOfStarlarkValue(value) + if valueType.Kind() != reflect.Interface && valueTypeI != valueType { + // If we see conflicting value types, change the result value type to an empty interface + valueType = reflect.TypeOf([]interface{}{}).Elem() + } + } + return reflect.MapOf(keyType, valueType), nil + case starlark.Int: + return reflect.TypeOf(0), nil + case starlark.Float: + return reflect.TypeOf(0.0), nil + case starlark.Bool: + return reflect.TypeOf(true), nil + default: + return nil, fmt.Errorf("unimplemented starlark type: %s", value.Type()) + } +} diff --git a/starlark_import/unmarshal_test.go b/starlark_import/unmarshal_test.go new file mode 100644 index 0000000000..ee7a9e340d --- /dev/null +++ b/starlark_import/unmarshal_test.go @@ -0,0 +1,133 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark_import + +import ( + "reflect" + "testing" + + "go.starlark.net/starlark" +) + +func createStarlarkValue(t *testing.T, code string) starlark.Value { + t.Helper() + result, err := starlark.ExecFile(&starlark.Thread{}, "main.bzl", "x = "+code, builtins) + if err != nil { + panic(err) + } + return result["x"] +} + +func TestUnmarshallConcreteType(t *testing.T) { + x, err := Unmarshal[string](createStarlarkValue(t, `"foo"`)) + if err != nil { + t.Error(err) + return + } + if x != "foo" { + t.Errorf(`Expected "foo", got %q`, x) + } +} + +func TestUnmarshallConcreteTypeWithInterfaces(t *testing.T) { + x, err := Unmarshal[map[string]map[string]interface{}](createStarlarkValue(t, + `{"foo": {"foo2": "foo3"}, "bar": {"bar2": ["bar3"]}}`)) + if err != nil { + t.Error(err) + return + } + expected := map[string]map[string]interface{}{ + "foo": {"foo2": "foo3"}, + "bar": {"bar2": []string{"bar3"}}, + } + if !reflect.DeepEqual(x, expected) { + t.Errorf(`Expected %v, got %v`, expected, x) + } +} + +func TestUnmarshall(t *testing.T) { + testCases := []struct { + input string + expected interface{} + }{ + { + input: `"foo"`, + expected: "foo", + }, + { + input: `5`, + expected: 5, + }, + { + input: `["foo", "bar"]`, + expected: []string{"foo", "bar"}, + }, + { + input: `("foo", "bar")`, + expected: []string{"foo", "bar"}, + }, + { + input: `("foo",5)`, + expected: []interface{}{"foo", 5}, + }, + { + input: `{"foo": 5, "bar": 10}`, + expected: map[string]int{"foo": 5, "bar": 10}, + }, + { + input: `{"foo": ["qux"], "bar": []}`, + expected: map[string][]string{"foo": {"qux"}, "bar": nil}, + }, + { + input: `struct(Foo="foo", Bar=5)`, + expected: struct { + Foo string + Bar int + }{Foo: "foo", Bar: 5}, + }, + { + // Unexported fields version of the above + input: `struct(foo="foo", bar=5)`, + expected: struct { + foo string + bar int + }{foo: "foo", bar: 5}, + }, + { + input: `{"foo": "foo2", "bar": ["bar2"], "baz": 5, "qux": {"qux2": "qux3"}, "quux": {"quux2": "quux3", "quux4": 5}}`, + expected: map[string]interface{}{ + "foo": "foo2", + "bar": []string{"bar2"}, + "baz": 5, + "qux": map[string]string{"qux2": "qux3"}, + "quux": map[string]interface{}{ + "quux2": "quux3", + "quux4": 5, + }, + }, + }, + } + + for _, tc := range testCases { + x, err := UnmarshalReflect(createStarlarkValue(t, tc.input), reflect.TypeOf(tc.expected)) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(x.Interface(), tc.expected) { + t.Errorf(`Expected %#v, got %#v`, tc.expected, x.Interface()) + } + } +} diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh index 68d7f8d192..71e6af009c 100755 --- a/tests/bp2build_bazel_test.sh +++ b/tests/bp2build_bazel_test.sh @@ -53,6 +53,20 @@ EOF if [[ "$buildfile_mtime1" != "$buildfile_mtime2" ]]; then fail "BUILD.bazel was updated even though contents are same" fi + + # Force bp2build to rerun by updating the timestamp of the constants_exported_to_soong.bzl file. + touch build/bazel/constants_exported_to_soong.bzl + + run_soong bp2build + local -r buildfile_mtime3=$(stat -c "%y" out/soong/bp2build/pkg/BUILD.bazel) + local -r marker_mtime3=$(stat -c "%y" out/soong/bp2build_workspace_marker) + + if [[ "$marker_mtime2" == "$marker_mtime3" ]]; then + fail "Expected bp2build marker file to change" + fi + if [[ "$buildfile_mtime2" != "$buildfile_mtime3" ]]; then + fail "BUILD.bazel was updated even though contents are same" + fi } # Tests that blueprint files that are deleted are not present when the diff --git a/tests/persistent_bazel_test.sh b/tests/persistent_bazel_test.sh index 4e2982a394..9b7b58f827 100755 --- a/tests/persistent_bazel_test.sh +++ b/tests/persistent_bazel_test.sh @@ -73,8 +73,8 @@ function test_bazel_failure { USE_PERSISTENT_BAZEL=1 run_soong nothing 1>out/failurelog.txt 2>&1 && fail "Expected build failure" || true - if ! grep -sq "'build/bazel/rules' is not a package" out/failurelog.txt ; then - fail "Expected error to contain 'build/bazel/rules' is not a package, instead got:\n$(cat out/failurelog.txt)" + if ! grep -sq "cannot load //build/bazel/rules/common/api_constants.bzl" out/failurelog.txt ; then + fail "Expected error to contain 'cannot load //build/bazel/rules/common/api_constants.bzl', instead got:\n$(cat out/failurelog.txt)" fi kill $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null || true From 4495f84f684cb6dad137ce4f3c9926ac79768f74 Mon Sep 17 00:00:00 2001 From: Jooyung Han Date: Tue, 25 Apr 2023 16:39:59 +0900 Subject: [PATCH 0044/1460] Add allowed-deps tests These tests capture - track updatable apexes - do not track apex-only modules - track transitive deps - track external deps (with mark) Bug: 274041915 Test: m nothing Change-Id: I629015f5aa4a1a7627d0ba6d92fd42bb99c96287 --- apex/apex_singleton.go | 6 ++- apex/apex_test.go | 87 ++++++++++++++++++++++++++++++++++++++++++ apex/testing.go | 1 + 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/apex/apex_singleton.go b/apex/apex_singleton.go index 1581949375..ebc35cf5c7 100644 --- a/apex/apex_singleton.go +++ b/apex/apex_singleton.go @@ -23,7 +23,11 @@ import ( ) func init() { - android.RegisterSingletonType("apex_depsinfo_singleton", apexDepsInfoSingletonFactory) + registerApexDepsInfoComponents(android.InitRegistrationContext) +} + +func registerApexDepsInfoComponents(ctx android.RegistrationContext) { + ctx.RegisterSingletonType("apex_depsinfo_singleton", apexDepsInfoSingletonFactory) } type apexDepsInfoSingleton struct { diff --git a/apex/apex_test.go b/apex/apex_test.go index d08d2466e6..31b7ef7ff0 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -1980,6 +1980,93 @@ func TestApexMinSdkVersion_InVendorApex(t *testing.T) { android.AssertStringDoesContain(t, "cflags", cflags, "-target aarch64-linux-android29") } +func TestTrackAllowedDeps(t *testing.T) { + ctx := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + updatable: true, + native_shared_libs: [ + "mylib", + "yourlib", + ], + min_sdk_version: "29", + } + + apex { + name: "myapex2", + key: "myapex.key", + updatable: false, + native_shared_libs: ["yourlib"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + shared_libs: ["libbar"], + min_sdk_version: "29", + apex_available: ["myapex"], + } + + cc_library { + name: "libbar", + stubs: { versions: ["29", "30"] }, + } + + cc_library { + name: "yourlib", + srcs: ["mylib.cpp"], + min_sdk_version: "29", + apex_available: ["myapex", "myapex2", "//apex_available:platform"], + } + `, withFiles(android.MockFS{ + "packages/modules/common/build/allowed_deps.txt": nil, + })) + + depsinfo := ctx.SingletonForTests("apex_depsinfo_singleton") + inputs := depsinfo.Rule("generateApexDepsInfoFilesRule").BuildParams.Inputs.Strings() + android.AssertStringListContains(t, "updatable myapex should generate depsinfo file", inputs, + "out/soong/.intermediates/myapex/android_common_myapex_image/depsinfo/flatlist.txt") + android.AssertStringListDoesNotContain(t, "non-updatable myapex2 should not generate depsinfo file", inputs, + "out/soong/.intermediates/myapex2/android_common_myapex2_image/depsinfo/flatlist.txt") + + myapex := ctx.ModuleForTests("myapex", "android_common_myapex_image") + flatlist := strings.Split(myapex.Output("depsinfo/flatlist.txt").BuildParams.Args["content"], "\\n") + android.AssertStringListContains(t, "deps with stubs should be tracked in depsinfo as external dep", + flatlist, "libbar(minSdkVersion:(no version)) (external)") + android.AssertStringListDoesNotContain(t, "do not track if not available for platform", + flatlist, "mylib:(minSdkVersion:29)") + android.AssertStringListContains(t, "track platform-available lib", + flatlist, "yourlib(minSdkVersion:29)") +} + +func TestTrackAllowedDeps_SkipWithoutAllowedDepsTxt(t *testing.T) { + ctx := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + updatable: true, + min_sdk_version: "29", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + depsinfo := ctx.SingletonForTests("apex_depsinfo_singleton") + if nil != depsinfo.MaybeRule("generateApexDepsInfoFilesRule").Output { + t.Error("apex_depsinfo_singleton shouldn't run when allowed_deps.txt doesn't exist") + } +} + func TestPlatformUsesLatestStubsFromApexes(t *testing.T) { ctx := testApex(t, ` apex { diff --git a/apex/testing.go b/apex/testing.go index 69bd73e5d1..3b200f05bb 100644 --- a/apex/testing.go +++ b/apex/testing.go @@ -19,6 +19,7 @@ import "android/soong/android" var PrepareForTestWithApexBuildComponents = android.GroupFixturePreparers( android.FixtureRegisterWithContext(registerApexBuildComponents), android.FixtureRegisterWithContext(registerApexKeyBuildComponents), + android.FixtureRegisterWithContext(registerApexDepsInfoComponents), // Additional files needed in tests that disallow non-existent source files. // This includes files that are needed by all, or at least most, instances of an apex module type. android.MockFS{ From f57a966b6694da970e99f519919f9b845b713163 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 12 Apr 2023 19:05:49 +0000 Subject: [PATCH 0045/1460] Ignore test apexes from bp2build generated tags The core problem I am trying to solve is making sure that stub libraries in Bazel have a single apex available. In Soong, this will be enforced using some graph walk which is not easy to port to Bazel. However, we might need to revisit this when we build the enforcement mechanism in Bazel. We likely need a `test_for` on the top level apex_test Bazel rule so that the test apex is _allowed_ to link against impl of the library it is trying to test. (This CL retricts this to cc_library, I can expand this to other modules if this is the right approach.) Bug: 277651159 Change-Id: Iaeec22c5626df79a33785c766ed29102b1da403e --- android/apex.go | 14 ++++++++++++++ android/mutator.go | 29 +++++++++++++++++++++++++++++ apex/apex.go | 4 ++++ cc/library.go | 6 +++--- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/android/apex.go b/android/apex.go index 5dcf73bc65..958b6aad43 100644 --- a/android/apex.go +++ b/android/apex.go @@ -462,6 +462,14 @@ const ( AvailableToGkiApex = "com.android.gki.*" ) +var ( + AvailableToRecognziedWildcards = []string{ + AvailableToPlatform, + AvailableToAnyApex, + AvailableToGkiApex, + } +) + // CheckAvailableForApex provides the default algorithm for checking the apex availability. When the // availability is empty, it defaults to ["//apex_available:platform"] which means "available to the // platform but not available to any APEX". When the list is not empty, `what` is matched against @@ -926,3 +934,9 @@ func CheckMinSdkVersion(ctx ModuleContext, minSdkVersion ApiLevel, walk WalkPayl return true }) } + +// Implemented by apexBundle. +type ApexTestInterface interface { + // Return true if the apex bundle is an apex_test + IsTestApex() bool +} diff --git a/android/mutator.go b/android/mutator.go index 4ec960472d..c0040b7b9a 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -758,6 +758,35 @@ func ApexAvailableTags(mod Module) bazel.StringListAttribute { return attr } +func ApexAvailableTagsWithoutTestApexes(ctx BaseModuleContext, mod Module) bazel.StringListAttribute { + attr := bazel.StringListAttribute{} + if am, ok := mod.(ApexModule); ok { + apexAvailableWithoutTestApexes := removeTestApexes(ctx, am.apexModuleBase().ApexAvailable()) + // If a user does not specify apex_available in Android.bp, then soong provides a default. + // To avoid verbosity of BUILD files, remove this default from user-facing BUILD files. + if len(am.apexModuleBase().ApexProperties.Apex_available) == 0 { + apexAvailableWithoutTestApexes = []string{} + } + attr.Value = ConvertApexAvailableToTags(apexAvailableWithoutTestApexes) + } + return attr +} + +func removeTestApexes(ctx BaseModuleContext, apex_available []string) []string { + testApexes := []string{} + for _, aa := range apex_available { + // ignore the wildcards + if InList(aa, AvailableToRecognziedWildcards) { + continue + } + mod, _ := ctx.ModuleFromName(aa) + if apex, ok := mod.(ApexTestInterface); ok && apex.IsTestApex() { + testApexes = append(testApexes, aa) + } + } + return RemoveListFromList(CopyOf(apex_available), testApexes) +} + func ConvertApexAvailableToTags(apexAvailable []string) []string { if len(apexAvailable) == 0 { // We need nil specifically to make bp2build not add the tags property at all, diff --git a/apex/apex.go b/apex/apex.go index baf4737d82..69547c3b70 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -3811,3 +3811,7 @@ func makeSharedLibsAttributes(config string, libsLabelList bazel.LabelList, func invalidCompileMultilib(ctx android.TopDownMutatorContext, value string) { ctx.PropertyErrorf("compile_multilib", "Invalid value: %s", value) } + +func (a *apexBundle) IsTestApex() bool { + return a.testApex +} diff --git a/cc/library.go b/cc/library.go index ee09389cad..13b333a121 100644 --- a/cc/library.go +++ b/cc/library.go @@ -433,11 +433,11 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { var tagsForStaticVariant bazel.StringListAttribute if compilerAttrs.stubsSymbolFile == nil && len(compilerAttrs.stubsVersions.Value) == 0 { - tagsForStaticVariant = android.ApexAvailableTags(m) + tagsForStaticVariant = android.ApexAvailableTagsWithoutTestApexes(ctx, m) } tagsForStaticVariant.Append(bazel.StringListAttribute{Value: staticAttrs.Apex_available}) - tagsForSharedVariant := android.ApexAvailableTags(m) + tagsForSharedVariant := android.ApexAvailableTagsWithoutTestApexes(ctx, m) tagsForSharedVariant.Append(bazel.StringListAttribute{Value: sharedAttrs.Apex_available}) ctx.CreateBazelTargetModuleWithRestrictions(staticProps, @@ -3002,7 +3002,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Bzl_load_location: fmt.Sprintf("//build/bazel/rules/cc:%s.bzl", modType), } - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name(), Tags: tags}, attrs) } From 1fb4ece718bd34f9c47754c14281c7ad644bb174 Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Wed, 26 Apr 2023 21:06:24 +0900 Subject: [PATCH 0046/1460] BUILD_NUMBER and BUILD_HOSTNAME doesn't affect kati regeneration * Extract BUILD_NUMBER, BUILD_HOSTNAME to file to avoid kati change * Handle FILE_NAME_TAG_PLACEHOLDER string in dist in build/make/packaging/distdir.mk Ignore-AOSP-First: there are internal use cases of FILE_NAME_TAG Test: check if kati isn't invoked even though BUILD_NUMBER, BUILD_HOSTNAME is changed Test: m && m, and check if the second m is no-op Bug: 278060169 Change-Id: I65eefc6bb86d4076098a1bf8b317b4cb88201499 --- android_sdk/sdk_repo_host.go | 2 +- ui/build/kati.go | 60 +++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/android_sdk/sdk_repo_host.go b/android_sdk/sdk_repo_host.go index 61058df098..9623a8bc46 100644 --- a/android_sdk/sdk_repo_host.go +++ b/android_sdk/sdk_repo_host.go @@ -242,7 +242,7 @@ func (s *sdkRepoHost) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, ".PHONY:", name, "sdk_repo", "sdk-repo-"+name) fmt.Fprintln(w, "sdk_repo", "sdk-repo-"+name+":", strings.Join(s.FilesToInstall().Strings(), " ")) - fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-$(FILE_NAME_TAG).zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) + fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-FILE_NAME_TAG_PLACEHOLDER.zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) }, } } diff --git a/ui/build/kati.go b/ui/build/kati.go index dad68fac2f..4297378b37 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -22,6 +22,7 @@ import ( "os/user" "path/filepath" "strings" + "time" "android/soong/ui/metrics" "android/soong/ui/status" @@ -66,6 +67,21 @@ func genKatiSuffix(ctx Context, config Config) { } } +func writeValueIfChanged(ctx Context, config Config, dir string, filename string, value string) { + filePath := filepath.Join(dir, filename) + previousValue := "" + rawPreviousValue, err := ioutil.ReadFile(filePath) + if err == nil { + previousValue = string(rawPreviousValue) + } + + if previousValue != value { + if err = ioutil.WriteFile(filePath, []byte(value), 0666); err != nil { + ctx.Fatalf("Failed to write: %v", err) + } + } +} + // Base function to construct and run the Kati command line with additional // arguments, and a custom function closure to mutate the environment Kati runs // in. @@ -157,28 +173,57 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF } cmd.Stderr = cmd.Stdout - // Apply the caller's function closure to mutate the environment variables. - envFunc(cmd.Environment) - + var username string // Pass on various build environment metadata to Kati. - if _, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { - username := "unknown" + if usernameFromEnv, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { + username = "unknown" if u, err := user.Current(); err == nil { username = u.Username } else { ctx.Println("Failed to get current user:", err) } cmd.Environment.Set("BUILD_USERNAME", username) + } else { + username = usernameFromEnv } - if _, ok := cmd.Environment.Get("BUILD_HOSTNAME"); !ok { - hostname, err := os.Hostname() + hostname, ok := cmd.Environment.Get("BUILD_HOSTNAME") + if !ok { + hostname, err = os.Hostname() if err != nil { ctx.Println("Failed to read hostname:", err) hostname = "unknown" } cmd.Environment.Set("BUILD_HOSTNAME", hostname) } + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_hostname.txt", hostname) + + // BUILD_NUMBER should be set to the source control value that + // represents the current state of the source code. E.g., a + // perforce changelist number or a git hash. Can be an arbitrary string + // (to allow for source control that uses something other than numbers), + // but must be a single word and a valid file name. + // + // If no BUILD_NUMBER is set, create a useful "I am an engineering build + // from this date/time" value. Make it start with a non-digit so that + // anyone trying to parse it as an integer will probably get "0". + cmd.Environment.Unset("HAS_BUILD_NUMBER") + buildNumber, ok := cmd.Environment.Get("BUILD_NUMBER") + if ok { + cmd.Environment.Set("HAS_BUILD_NUMBER", "true") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", buildNumber) + } else { + buildNumber = fmt.Sprintf("eng.%.6s.%s", username, time.Now().Format("20060102.150405" /* YYYYMMDD.HHMMSS */)) + cmd.Environment.Set("HAS_BUILD_NUMBER", "false") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", username) + } + // Write the build number to a file so it can be read back in + // without changing the command line every time. Avoids rebuilds + // when using ninja. + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_number.txt", buildNumber) + + // Apply the caller's function closure to mutate the environment variables. + envFunc(cmd.Environment) cmd.StartOrFatal() // Set up the ToolStatus command line reader for Kati for a consistent UI @@ -336,6 +381,7 @@ func runKatiPackage(ctx Context, config Config) { "ANDROID_BUILD_SHELL", "DIST_DIR", "OUT_DIR", + "FILE_NAME_TAG", }...) if config.Dist() { From cc4da765113299fa11dcb1e651ec4ae33e6f8f9b Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Thu, 27 Apr 2023 19:34:08 +0000 Subject: [PATCH 0047/1460] Differentiate between empty and nil input Previously, CopyOf on an empty list was returning nil. With the updates to SortedUniqueStrings and FirstUniqueStrings, we need to differentiate between empty lists and nil. Test: m nothing Change-Id: I91063ebbe5013cbda5d8f70efde4683c66581599 --- android/util.go | 6 +++++- android/util_test.go | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/android/util.go b/android/util.go index fb09128f84..08a3521a52 100644 --- a/android/util.go +++ b/android/util.go @@ -26,7 +26,11 @@ import ( // CopyOf returns a new slice that has the same contents as s. func CopyOf(s []string) []string { - return append([]string(nil), s...) + // If the input is nil, return nil and not an empty list + if s == nil { + return s + } + return append([]string{}, s...) } // Concat returns a new slice concatenated from the two input slices. It does not change the input diff --git a/android/util_test.go b/android/util_test.go index 5584b38f71..a2ef589588 100644 --- a/android/util_test.go +++ b/android/util_test.go @@ -381,6 +381,14 @@ func TestRemoveFromList(t *testing.T) { } } +func TestCopyOfEmptyAndNil(t *testing.T) { + emptyList := []string{} + copyOfEmptyList := CopyOf(emptyList) + AssertBoolEquals(t, "Copy of an empty list should be an empty list and not nil", true, copyOfEmptyList != nil) + copyOfNilList := CopyOf(nil) + AssertBoolEquals(t, "Copy of a nil list should be a nil list and not an empty list", true, copyOfNilList == nil) +} + func ExampleCopyOf() { a := []string{"1", "2", "3"} b := CopyOf(a) From 39b6cc53369a6ec132b58d99514cd7a2ea499efd Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 12 Apr 2023 19:05:49 +0000 Subject: [PATCH 0048/1460] Ignore test apexes from bp2build generated tags Soong does not enforce apex_available on the contents of test apex. To prevent special-casing test apexes in the apex validation aspect in Bazel, drop the test apexes from the tags altogether. ( The core problem I am trying to solve is making sure that stub libraries in Bazel have a single apex available. apex validation happens to be a nice side benefit) Bug: 277651159 Test: go test ./bp2build Change-Id: Ibb3cfedb5c0f2cda0464bf3758c70b67cb5885d1 --- android/mutator.go | 7 +++++++ android/proto.go | 2 +- cc/binary.go | 2 +- cc/bp2build.go | 4 ++-- cc/library_headers.go | 2 +- cc/object.go | 2 +- cc/prebuilt.go | 8 ++++---- cc/proto.go | 2 +- cc/sysprop.go | 2 +- genrule/genrule.go | 2 +- java/java.go | 2 +- 11 files changed, 21 insertions(+), 14 deletions(-) diff --git a/android/mutator.go b/android/mutator.go index c0040b7b9a..013fa77cf4 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -800,6 +800,13 @@ func ConvertApexAvailableToTags(apexAvailable []string) []string { return result } +// ConvertApexAvailableToTagsWithoutTestApexes converts a list of apex names to a list of bazel tags +// This function drops any test apexes from the input. +func ConvertApexAvailableToTagsWithoutTestApexes(ctx BaseModuleContext, apexAvailable []string) []string { + noTestApexes := removeTestApexes(ctx, apexAvailable) + return ConvertApexAvailableToTags(noTestApexes) +} + func (t *topDownMutatorContext) createBazelTargetModule( bazelProps bazel.BazelTargetModuleProperties, commonAttrs CommonAttributes, diff --git a/android/proto.go b/android/proto.go index 09e50c8c39..cebbd59cd0 100644 --- a/android/proto.go +++ b/android/proto.go @@ -234,7 +234,7 @@ func Bp2buildProtoProperties(ctx Bp2buildMutatorContext, m *ModuleBase, srcs baz } } - tags := ApexAvailableTags(ctx.Module()) + tags := ApexAvailableTagsWithoutTestApexes(ctx.(TopDownMutatorContext), ctx.Module()) ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{Rule_class: "proto_library"}, diff --git a/cc/binary.go b/cc/binary.go index 097f822529..98b9923ee6 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -661,7 +661,7 @@ func binaryBp2build(ctx android.TopDownMutatorContext, m *Module) { // shared with cc_test binaryAttrs := binaryBp2buildAttrs(ctx, m) - tags := android.ApexAvailableTags(m) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m) ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ Rule_class: "cc_binary", Bzl_load_location: "//build/bazel/rules/cc:cc_binary.bzl", diff --git a/cc/bp2build.go b/cc/bp2build.go index ad9d7021dc..820e97fa2c 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -270,7 +270,7 @@ func bp2buildParseStaticOrSharedProps(ctx android.BazelConversionPathContext, mo attrs.Srcs_c = partitionedSrcs[cSrcPartition] attrs.Srcs_as = partitionedSrcs[asSrcPartition] - attrs.Apex_available = android.ConvertApexAvailableToTags(apexAvailable) + attrs.Apex_available = android.ConvertApexAvailableToTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), apexAvailable) attrs.Features.Append(convertHiddenVisibilityToFeatureStaticOrShared(ctx, module, isStatic)) @@ -923,7 +923,7 @@ func bp2buildCcAidlLibrary( return false }) - apexAvailableTags := android.ApexAvailableTags(ctx.Module()) + apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) sdkAttrs := bp2BuildParseSdkAttributes(m) if !aidlSrcs.IsEmpty() { diff --git a/cc/library_headers.go b/cc/library_headers.go index 1dee726793..ce9c4aacf7 100644 --- a/cc/library_headers.go +++ b/cc/library_headers.go @@ -151,7 +151,7 @@ func libraryHeadersBp2Build(ctx android.TopDownMutatorContext, module *Module) { Bzl_load_location: "//build/bazel/rules/cc:cc_library_headers.bzl", } - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModule(props, android.CommonAttributes{ Name: module.Name(), diff --git a/cc/object.go b/cc/object.go index d65cdea743..5d61872333 100644 --- a/cc/object.go +++ b/cc/object.go @@ -226,7 +226,7 @@ func objectBp2Build(ctx android.TopDownMutatorContext, m *Module) { Bzl_load_location: "//build/bazel/rules/cc:cc_object.bzl", } - tags := android.ApexAvailableTags(m) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m) ctx.CreateBazelTargetModule(props, android.CommonAttributes{ Name: m.Name(), diff --git a/cc/prebuilt.go b/cc/prebuilt.go index 0b5841ef0a..44cd0d73f6 100644 --- a/cc/prebuilt.go +++ b/cc/prebuilt.go @@ -389,7 +389,7 @@ func prebuiltLibraryStaticBp2Build(ctx android.TopDownMutatorContext, module *Mo name += "_bp2build_cc_library_static" } - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModuleWithRestrictions(props, android.CommonAttributes{Name: name, Tags: tags}, attrs, prebuiltAttrs.Enabled) _true := true @@ -420,7 +420,7 @@ func prebuiltLibrarySharedBp2Build(ctx android.TopDownMutatorContext, module *Mo } name := android.RemoveOptionalPrebuiltPrefix(module.Name()) - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModuleWithRestrictions(props, android.CommonAttributes{Name: name, Tags: tags}, attrs, prebuiltAttrs.Enabled) } @@ -650,7 +650,7 @@ func prebuiltObjectBp2Build(ctx android.TopDownMutatorContext, module *Module) { } name := android.RemoveOptionalPrebuiltPrefix(module.Name()) - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: name, Tags: tags}, attrs) } @@ -813,7 +813,7 @@ func prebuiltBinaryBp2Build(ctx android.TopDownMutatorContext, module *Module) { } name := android.RemoveOptionalPrebuiltPrefix(module.Name()) - tags := android.ApexAvailableTags(module) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, module) ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: name, Tags: tags}, attrs) } diff --git a/cc/proto.go b/cc/proto.go index 97470e5ea9..5d9aef60f9 100644 --- a/cc/proto.go +++ b/cc/proto.go @@ -207,7 +207,7 @@ func bp2buildProto(ctx android.Bp2buildMutatorContext, m *Module, protoSrcs baze protoAttrs.Min_sdk_version = m.Properties.Min_sdk_version name := m.Name() + suffix - tags := android.ApexAvailableTags(m) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), m) ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: rule_class, diff --git a/cc/sysprop.go b/cc/sysprop.go index 0df290afa0..7ddd4760e4 100644 --- a/cc/sysprop.go +++ b/cc/sysprop.go @@ -38,7 +38,7 @@ type SyspropLibraryLabels struct { } func Bp2buildSysprop(ctx android.Bp2buildMutatorContext, labels SyspropLibraryLabels, srcs bazel.LabelListAttribute, minSdkVersion *string) { - apexAvailableTags := android.ApexAvailableTags(ctx.Module()) + apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: "sysprop_library", diff --git a/genrule/genrule.go b/genrule/genrule.go index f5da50ef3f..00adb7025d 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -940,7 +940,7 @@ func (m *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { } } - tags := android.ApexAvailableTags(m) + tags := android.ApexAvailableTagsWithoutTestApexes(ctx, m) if ctx.ModuleType() == "gensrcs" { // The Output_extension prop is not in an immediately accessible field diff --git a/java/java.go b/java/java.go index fe4df75204..0da73281b8 100644 --- a/java/java.go +++ b/java/java.go @@ -2839,7 +2839,7 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) return android.IsConvertedToAidlLibrary(ctx, src.OriginalModuleName) }) - apexAvailableTags := android.ApexAvailableTags(ctx.Module()) + apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx, ctx.Module()) if !aidlSrcs.IsEmpty() { aidlLibName := m.Name() + "_aidl_library" From 20fce2d340f3d84493b774a732705ddb80f56821 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 12 Apr 2023 17:21:39 +0000 Subject: [PATCH 0049/1460] Enforce stub libraries should have a single apex_available If a library contributes to an API surface, it will have only a single copy on device. Therefore, we should disallow installation to muliple apexes/platform. There are some exceptions to this rule today, and they have been relaxed using allowlists. Bug: 277651159 Test: go test ./apex Change-Id: Ice3023ecd28412a2610d8b98628cb727b58c5c3b --- apex/apex_test.go | 120 +++++++++++++++++++++++++++++++++++++++++----- cc/cc.go | 54 +++++++++++++++++++++ 2 files changed, 162 insertions(+), 12 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index d08d2466e6..f9ff0b9df3 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -3131,10 +3131,7 @@ func TestStaticLinking(t *testing.T) { stubs: { versions: ["1", "2", "3"], }, - apex_available: [ - "//apex_available:platform", - "myapex", - ], + apex_available: ["myapex"], } cc_binary { @@ -4134,7 +4131,7 @@ func TestDependenciesInApexManifest(t *testing.T) { apex { name: "myapex_selfcontained", key: "myapex.key", - native_shared_libs: ["lib_dep", "libfoo"], + native_shared_libs: ["lib_dep_on_bar", "libbar"], compile_multilib: "both", file_contexts: ":myapex-file_contexts", updatable: false, @@ -4167,6 +4164,18 @@ func TestDependenciesInApexManifest(t *testing.T) { ], } + cc_library { + name: "lib_dep_on_bar", + srcs: ["mylib.cpp"], + shared_libs: ["libbar"], + system_shared_libs: [], + stl: "none", + apex_available: [ + "myapex_selfcontained", + ], + } + + cc_library { name: "libfoo", srcs: ["mytest.cpp"], @@ -4177,9 +4186,22 @@ func TestDependenciesInApexManifest(t *testing.T) { stl: "none", apex_available: [ "myapex_provider", + ], + } + + cc_library { + name: "libbar", + srcs: ["mytest.cpp"], + stubs: { + versions: ["1"], + }, + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex_selfcontained", ], } + `) var apexManifestRule android.TestingBuildParams @@ -4206,7 +4228,7 @@ func TestDependenciesInApexManifest(t *testing.T) { apexManifestRule = ctx.ModuleForTests("myapex_selfcontained", "android_common_myapex_selfcontained_image").Rule("apexManifestRule") provideNativeLibs = names(apexManifestRule.Args["provideNativeLibs"]) requireNativeLibs = names(apexManifestRule.Args["requireNativeLibs"]) - ensureListContains(t, provideNativeLibs, "libfoo.so") + ensureListContains(t, provideNativeLibs, "libbar.so") ensureListEmpty(t, requireNativeLibs) } @@ -8488,14 +8510,14 @@ func TestTestForForLibInOtherApex(t *testing.T) { apex { name: "com.android.art", key: "myapex.key", - native_shared_libs: ["mylib"], + native_shared_libs: ["libnativebridge"], updatable: false, } apex { name: "com.android.art.debug", key: "myapex.key", - native_shared_libs: ["mylib", "mytestlib"], + native_shared_libs: ["libnativebridge", "libnativebrdige_test"], updatable: false, } @@ -8506,8 +8528,8 @@ func TestTestForForLibInOtherApex(t *testing.T) { } cc_library { - name: "mylib", - srcs: ["mylib.cpp"], + name: "libnativebridge", + srcs: ["libnativebridge.cpp"], system_shared_libs: [], stl: "none", stubs: { @@ -8517,10 +8539,10 @@ func TestTestForForLibInOtherApex(t *testing.T) { } cc_library { - name: "mytestlib", + name: "libnativebrdige_test", srcs: ["mylib.cpp"], system_shared_libs: [], - shared_libs: ["mylib"], + shared_libs: ["libnativebridge"], stl: "none", apex_available: ["com.android.art.debug"], test_for: ["com.android.art"], @@ -10192,3 +10214,77 @@ func TestCannedFsConfig_HasCustomConfig(t *testing.T) { // Ensure that canned_fs_config has "cat my_config" at the end ensureContains(t, cmd, `( echo '/ 1000 1000 0755'; echo '/apex_manifest.json 1000 1000 0644'; echo '/apex_manifest.pb 1000 1000 0644'; cat my_config ) >`) } + +func TestStubLibrariesMultipleApexViolation(t *testing.T) { + testCases := []struct { + desc string + hasStubs bool + apexAvailable string + expectedError string + }{ + { + desc: "non-stub library can have multiple apex_available", + hasStubs: false, + apexAvailable: `["myapex", "otherapex"]`, + }, + { + desc: "stub library should not be available to anyapex", + hasStubs: true, + apexAvailable: `["//apex_available:anyapex"]`, + expectedError: "Stub libraries should have a single apex_available.*anyapex", + }, + { + desc: "stub library should not be available to multiple apexes", + hasStubs: true, + apexAvailable: `["myapex", "otherapex"]`, + expectedError: "Stub libraries should have a single apex_available.*myapex.*otherapex", + }, + { + desc: "stub library can be available to a core apex and a test apex", + hasStubs: true, + apexAvailable: `["myapex", "test_myapex"]`, + }, + } + bpTemplate := ` + cc_library { + name: "libfoo", + %v + apex_available: %v, + } + apex { + name: "myapex", + key: "apex.key", + updatable: false, + native_shared_libs: ["libfoo"], + } + apex { + name: "otherapex", + key: "apex.key", + updatable: false, + } + apex_test { + name: "test_myapex", + key: "apex.key", + updatable: false, + native_shared_libs: ["libfoo"], + } + apex_key { + name: "apex.key", + } + ` + for _, tc := range testCases { + stubs := "" + if tc.hasStubs { + stubs = `stubs: {symbol_file: "libfoo.map.txt"},` + } + bp := fmt.Sprintf(bpTemplate, stubs, tc.apexAvailable) + mockFsFixturePreparer := android.FixtureModifyMockFS(func(fs android.MockFS) { + fs["system/sepolicy/apex/test_myapex-file_contexts"] = nil + }) + if tc.expectedError == "" { + testApex(t, bp, mockFsFixturePreparer) + } else { + testApexError(t, tc.expectedError, bp, mockFsFixturePreparer) + } + } +} diff --git a/cc/cc.go b/cc/cc.go index c1a1020adf..f4737c5971 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -1987,6 +1987,56 @@ func moduleContextFromAndroidModuleContext(actx android.ModuleContext, c *Module return ctx } +// TODO (b/277651159): Remove this allowlist +var ( + skipStubLibraryMultipleApexViolation = map[string]bool{ + "libclang_rt.asan": true, + "libclang_rt.hwasan": true, + // runtime apex + "libc": true, + "libc_hwasan": true, + "libdl_android": true, + "libm": true, + "libdl": true, + // art apex + "libandroidio": true, + "libdexfile": true, + "libnativebridge": true, + "libnativehelper": true, + "libnativeloader": true, + "libsigchain": true, + } +) + +// Returns true if a stub library could be installed in multiple apexes +func (c *Module) stubLibraryMultipleApexViolation(ctx android.ModuleContext) bool { + // If this is not an apex variant, no check necessary + if !c.InAnyApex() { + return false + } + // If this is not a stub library, no check necessary + if !c.HasStubsVariants() { + return false + } + // Skip the allowlist + // Use BaseModuleName so that this matches prebuilts. + if _, exists := skipStubLibraryMultipleApexViolation[c.BaseModuleName()]; exists { + return false + } + + _, aaWithoutTestApexes, _ := android.ListSetDifference(c.ApexAvailable(), c.TestApexes()) + // Stub libraries should not have more than one apex_available + if len(aaWithoutTestApexes) > 1 { + return true + } + // Stub libraries should not use the wildcard + if aaWithoutTestApexes[0] == android.AvailableToAnyApex { + return true + } + // Default: no violation + return false +} + func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { // Handle the case of a test module split by `test_per_src` mutator. // @@ -2013,6 +2063,10 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { return } + if c.stubLibraryMultipleApexViolation(actx) { + actx.PropertyErrorf("apex_available", + "Stub libraries should have a single apex_available (test apexes excluded). Got %v", c.ApexAvailable()) + } if c.Properties.Clang != nil && *c.Properties.Clang == false { ctx.PropertyErrorf("clang", "false (GCC) is no longer supported") } else if c.Properties.Clang != nil && !ctx.DeviceConfig().BuildBrokenClangProperty() { From 05d4d901146d96634ad0ff3a9982c9c87c394f18 Mon Sep 17 00:00:00 2001 From: Juan Yescas Date: Fri, 7 Apr 2023 10:35:35 -0700 Subject: [PATCH 0050/1460] 16k: Align shared libraries and executables to 4k by default The default ELF segment alignment will be 4096. This alignment can be overriden by setting the desired alignment in PRODUCT_MAX_PAGE_SIZE_SUPPORTED flag. The alignment has to be a multiple of the configured kernel page size. Test: Built changes for Pixel 4a, 6 and checked alignment. Bug: 276963698 Bug: 276801883 Change-Id: Icb380041a5b27da1fa0d86b302e1e7cde9a236d7 --- android/config.go | 7 +++++++ android/variable.go | 2 ++ cc/config/arm64_device.go | 11 ++++++++--- cc/config/arm_device.go | 7 ++++++- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/android/config.go b/android/config.go index 7141e54c3d..a6485d259f 100644 --- a/android/config.go +++ b/android/config.go @@ -176,6 +176,13 @@ func (c Config) RunningInsideUnitTest() bool { return c.config.TestProductVariables != nil } +// MaxPageSizeSupported returns the max page size supported by the device. This +// value will define the ELF segment alignment for binaries (executables and +// shared libraries). +func (c Config) MaxPageSizeSupported() string { + return String(c.config.productVariables.DeviceMaxPageSizeSupported) +} + // A DeviceConfig object represents the configuration for a particular device // being built. For now there will only be one of these, but in the future there // may be multiple devices being built. diff --git a/android/variable.go b/android/variable.go index 249d53b07a..d7152b367a 100644 --- a/android/variable.go +++ b/android/variable.go @@ -218,6 +218,7 @@ type productVariables struct { DeviceVndkVersion *string `json:",omitempty"` DeviceCurrentApiLevelForVendorModules *string `json:",omitempty"` DeviceSystemSdkVersions []string `json:",omitempty"` + DeviceMaxPageSizeSupported *string `json:",omitempty"` RecoverySnapshotVersion *string `json:",omitempty"` @@ -505,6 +506,7 @@ func (v *productVariables) SetDefaultConfig() { DeviceSecondaryArchVariant: stringPtr("armv8-a"), DeviceSecondaryCpuVariant: stringPtr("generic"), DeviceSecondaryAbi: []string{"armeabi-v7a", "armeabi"}, + DeviceMaxPageSizeSupported: stringPtr("4096"), AAPTConfig: []string{"normal", "large", "xlarge", "hdpi", "xhdpi", "xxhdpi"}, AAPTPreferredConfig: stringPtr("xhdpi"), diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go index 28f368205e..ca2e05fc3d 100644 --- a/cc/config/arm64_device.go +++ b/cc/config/arm64_device.go @@ -53,8 +53,7 @@ var ( "-Wl,-z,separate-code", } - arm64Lldflags = append(arm64Ldflags, - "-Wl,-z,max-page-size=4096") + arm64Lldflags = arm64Ldflags arm64Cppflags = []string{} @@ -93,7 +92,13 @@ var ( func init() { exportedVars.ExportStringListStaticVariable("Arm64Ldflags", arm64Ldflags) - exportedVars.ExportStringListStaticVariable("Arm64Lldflags", arm64Lldflags) + + exportedVars.ExportStringList("Arm64Lldflags", arm64Lldflags) + pctx.VariableFunc("Arm64Lldflags", func(ctx android.PackageVarContext) string { + maxPageSizeFlag := "-Wl,-z,max-page-size=" + ctx.Config().MaxPageSizeSupported() + flags := append(arm64Lldflags, maxPageSizeFlag) + return strings.Join(flags, " ") + }) exportedVars.ExportStringListStaticVariable("Arm64Cflags", arm64Cflags) exportedVars.ExportStringListStaticVariable("Arm64Cppflags", arm64Cppflags) diff --git a/cc/config/arm_device.go b/cc/config/arm_device.go index 070455021c..dec2b45522 100644 --- a/cc/config/arm_device.go +++ b/cc/config/arm_device.go @@ -185,7 +185,12 @@ func init() { exportedVars.ExportString("ArmClangTriple", clangTriple) exportedVars.ExportStringListStaticVariable("ArmLdflags", armLdflags) - exportedVars.ExportStringListStaticVariable("ArmLldflags", armLldflags) + exportedVars.ExportStringList("ArmLldflags", armLldflags) + pctx.VariableFunc("ArmLldflags", func(ctx android.PackageVarContext) string { + maxPageSizeFlag := "-Wl,-z,max-page-size=" + ctx.Config().MaxPageSizeSupported() + flags := append(armLldflags, maxPageSizeFlag) + return strings.Join(flags, " ") + }) exportedVars.ExportStringListStaticVariable("ArmFixCortexA8LdFlags", armFixCortexA8LdFlags) exportedVars.ExportStringListStaticVariable("ArmNoFixCortexA8LdFlags", armNoFixCortexA8LdFlags) From 4753b39ccaaf573910e6b7a4200e9ca03ec8d96d Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Wed, 19 Apr 2023 23:25:41 +0900 Subject: [PATCH 0051/1460] Get rid of DeviceName() from path related to dexpreopt As-is, dexpreopt config and bootjar is in the dir including DeviceName(). It causes unnecessary dexpreopt invocation when target is changed repeatly. To avoid it, rename dir to common name. Bug: 278833696 Test: m Change-Id: I2818d9ae09e6c22ca6989cc8efdb70b470ec502c --- apex/bootclasspath_fragment_test.go | 30 +- java/dexpreopt_bootjars.go | 4 +- java/dexpreopt_bootjars_test.go | 4 +- java/dexpreopt_config.go | 11 +- java/dexpreopt_config_testing.go | 572 ++++++++++++++-------------- 5 files changed, 313 insertions(+), 308 deletions(-) diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index 2ddfd03055..d784818284 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -116,18 +116,18 @@ func TestBootclasspathFragments(t *testing.T) { // Make sure that the art-bootclasspath-fragment is using the correct configuration. checkBootclasspathFragment(t, result, "art-bootclasspath-fragment", "android_common_apex10000", "com.android.art:baz,com.android.art:quuz", ` -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.art -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.oat -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.vdex -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.art -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.oat -test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.vdex +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.art +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.oat +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.vdex +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.art +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.oat +dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.vdex `) } @@ -764,7 +764,7 @@ func TestBootclasspathFragmentInPrebuiltArtApex(t *testing.T) { checkCopiesToPredefinedLocationForArt(t, result.Config, module, "bar", "foo") // Check that the right deapexer module was chosen for a boot image. - param := module.Output("out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art") + param := module.Output("out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art") android.AssertStringDoesContain(t, "didn't find the expected deapexer in the input path", param.Input.String(), "/com.android.art.deapexer") }) @@ -782,7 +782,7 @@ func checkCopiesToPredefinedLocationForArt(t *testing.T, config android.Config, bootJarLocations := []string{} for _, output := range module.AllOutputs() { output = android.StringRelativeToTop(config, output) - if strings.HasPrefix(output, "out/soong/test_device/dex_artjars_input/") { + if strings.HasPrefix(output, "out/soong/dexpreopt_arm64/dex_artjars_input/") { bootJarLocations = append(bootJarLocations, output) } } @@ -790,7 +790,7 @@ func checkCopiesToPredefinedLocationForArt(t *testing.T, config android.Config, sort.Strings(bootJarLocations) expected := []string{} for _, m := range modules { - expected = append(expected, fmt.Sprintf("out/soong/test_device/dex_artjars_input/%s.jar", m)) + expected = append(expected, fmt.Sprintf("out/soong/dexpreopt_arm64/dex_artjars_input/%s.jar", m)) } sort.Strings(expected) diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index f4827ea3a7..f4c0935101 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -507,8 +507,8 @@ func (d *dexpreoptBootJars) GenerateSingletonBuildActions(ctx android.SingletonC // No module has enabled dexpreopting, so we assume there will be no boot image to make. return } - - d.dexpreoptConfigForMake = android.PathForOutput(ctx, ctx.Config().DeviceName(), "dexpreopt.config") + archType := ctx.Config().Targets[android.Android][0].Arch.ArchType + d.dexpreoptConfigForMake = android.PathForOutput(ctx, toDexpreoptDirName(archType), "dexpreopt.config") writeGlobalConfigForMake(ctx, d.dexpreoptConfigForMake) global := dexpreopt.GetGlobalConfig(ctx) diff --git a/java/dexpreopt_bootjars_test.go b/java/dexpreopt_bootjars_test.go index bc7a55eb9c..908380813d 100644 --- a/java/dexpreopt_bootjars_test.go +++ b/java/dexpreopt_bootjars_test.go @@ -58,11 +58,11 @@ func testDexpreoptBoot(t *testing.T, ruleFile string, expectedInputs, expectedOu rule := platformBootclasspath.Output(ruleFile) for i := range expectedInputs { - expectedInputs[i] = filepath.Join("out/soong/test_device", expectedInputs[i]) + expectedInputs[i] = filepath.Join("out/soong/dexpreopt_arm64", expectedInputs[i]) } for i := range expectedOutputs { - expectedOutputs[i] = filepath.Join("out/soong/test_device", expectedOutputs[i]) + expectedOutputs[i] = filepath.Join("out/soong/dexpreopt_arm64", expectedOutputs[i]) } inputs := rule.Implicits.Strings() diff --git a/java/dexpreopt_config.go b/java/dexpreopt_config.go index 8c62c332af..8f732cf56b 100644 --- a/java/dexpreopt_config.go +++ b/java/dexpreopt_config.go @@ -108,7 +108,8 @@ func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig func genBootImageConfigs(ctx android.PathContext) map[string]*bootImageConfig { return ctx.Config().Once(bootImageConfigKey, func() interface{} { targets := dexpreoptTargets(ctx) - deviceDir := android.PathForOutput(ctx, ctx.Config().DeviceName()) + archType := ctx.Config().Targets[android.Android][0].Arch.ArchType + deviceDir := android.PathForOutput(ctx, toDexpreoptDirName(archType)) configs := genBootImageConfigRaw(ctx) @@ -220,8 +221,8 @@ var updatableBootConfigKey = android.NewOnceKey("apexBootConfig") func GetApexBootConfig(ctx android.PathContext) apexBootConfig { return ctx.Config().Once(updatableBootConfigKey, func() interface{} { apexBootJars := dexpreopt.GetGlobalConfig(ctx).ApexBootJars - - dir := android.PathForOutput(ctx, ctx.Config().DeviceName(), "apex_bootjars") + archType := ctx.Config().Targets[android.Android][0].Arch.ArchType + dir := android.PathForOutput(ctx, toDexpreoptDirName(archType), "apex_bootjars") dexPaths := apexBootJars.BuildPaths(ctx, dir) dexPathsByModuleName := apexBootJars.BuildPathsByModule(ctx, dir) @@ -261,3 +262,7 @@ func init() { func dexpreoptConfigMakevars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_BOOT_JARS_MODULES", strings.Join(defaultBootImageConfig(ctx).modules.CopyOfApexJarPairs(), ":")) } + +func toDexpreoptDirName(arch android.ArchType) string { + return "dexpreopt_" + arch.String() +} diff --git a/java/dexpreopt_config_testing.go b/java/dexpreopt_config_testing.go index 86dd329883..6b98ca58c2 100644 --- a/java/dexpreopt_config_testing.go +++ b/java/dexpreopt_config_testing.go @@ -207,65 +207,65 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b expected := &expectedConfig{ name: "art", stem: "boot", - dir: "out/soong/test_device/dex_artjars", - symbolsDir: "out/soong/test_device/dex_artjars_unstripped", + dir: "out/soong/dexpreopt_arm64/dex_artjars", + symbolsDir: "out/soong/dexpreopt_arm64/dex_artjars_unstripped", installDirOnDevice: "system/framework", installDirOnHost: "apex/art_boot_images/javalib", profileInstallPathInApex: "etc/boot-image.prof", modules: android.CreateTestConfiguredJarList([]string{"com.android.art:core1", "com.android.art:core2"}), - dexPaths: []string{"out/soong/test_device/dex_artjars_input/core1.jar", "out/soong/test_device/dex_artjars_input/core2.jar"}, - dexPathsDeps: []string{"out/soong/test_device/dex_artjars_input/core1.jar", "out/soong/test_device/dex_artjars_input/core2.jar"}, - zip: "out/soong/test_device/dex_artjars/art.zip", + dexPaths: []string{"out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar", "out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar"}, + dexPathsDeps: []string{"out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar", "out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar"}, + zip: "out/soong/dexpreopt_arm64/dex_artjars/art.zip", variants: []*expectedVariant{ { archType: android.Arm64, dexLocations: []string{"/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar"}, dexLocationsDeps: []string{"/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar"}, - imagePathOnHost: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", imagePathOnDevice: "/system/framework/arm64/boot.art", imagesDeps: []string{ - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", to: "/apex/art_boot_images/javalib/arm64/boot.art", }, { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", to: "/apex/art_boot_images/javalib/arm64/boot.oat", }, { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", to: "/apex/art_boot_images/javalib/arm64/boot-core2.art", }, { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", to: "/apex/art_boot_images/javalib/arm64/boot-core2.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", to: "/apex/art_boot_images/javalib/arm64/boot.vdex", }, { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", to: "/apex/art_boot_images/javalib/arm64/boot-core2.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot.oat", to: "/apex/art_boot_images/javalib/arm64/boot.oat", }, { - from: "out/soong/test_device/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", to: "/apex/art_boot_images/javalib/arm64/boot-core2.oat", }, }, @@ -275,51 +275,51 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b archType: android.Arm, dexLocations: []string{"/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar"}, dexLocationsDeps: []string{"/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar"}, - imagePathOnHost: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", imagePathOnDevice: "/system/framework/arm/boot.art", imagesDeps: []string{ - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", to: "/apex/art_boot_images/javalib/arm/boot.art", }, { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", to: "/apex/art_boot_images/javalib/arm/boot.oat", }, { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", to: "/apex/art_boot_images/javalib/arm/boot-core2.art", }, { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", to: "/apex/art_boot_images/javalib/arm/boot-core2.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", to: "/apex/art_boot_images/javalib/arm/boot.vdex", }, { - from: "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", + from: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", to: "/apex/art_boot_images/javalib/arm/boot-core2.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm/boot.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm/boot.oat", to: "/apex/art_boot_images/javalib/arm/boot.oat", }, { - from: "out/soong/test_device/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm/boot-core2.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm/boot-core2.oat", to: "/apex/art_boot_images/javalib/arm/boot-core2.oat", }, }, @@ -329,49 +329,49 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b archType: android.X86_64, dexLocations: []string{"host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar"}, dexLocationsDeps: []string{"host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar"}, - imagePathOnHost: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", imagePathOnDevice: "/system/framework/x86_64/boot.art", imagesDeps: []string{ - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", to: "/apex/art_boot_images/javalib/x86_64/boot.art", }, { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", to: "/apex/art_boot_images/javalib/x86_64/boot.oat", }, { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", to: "/apex/art_boot_images/javalib/x86_64/boot-core2.art", }, { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", to: "/apex/art_boot_images/javalib/x86_64/boot-core2.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", to: "/apex/art_boot_images/javalib/x86_64/boot.vdex", }, { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", to: "/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", to: "/apex/art_boot_images/javalib/x86_64/boot.oat", }, { - from: "out/soong/test_device/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", to: "/apex/art_boot_images/javalib/x86_64/boot-core2.oat", }, }, @@ -381,49 +381,49 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b archType: android.X86, dexLocations: []string{"host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar"}, dexLocationsDeps: []string{"host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar"}, - imagePathOnHost: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", imagePathOnDevice: "/system/framework/x86/boot.art", imagesDeps: []string{ - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", to: "/apex/art_boot_images/javalib/x86/boot.art", }, { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", to: "/apex/art_boot_images/javalib/x86/boot.oat", }, { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", to: "/apex/art_boot_images/javalib/x86/boot-core2.art", }, { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", to: "/apex/art_boot_images/javalib/x86/boot-core2.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", to: "/apex/art_boot_images/javalib/x86/boot.vdex", }, { - from: "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", + from: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", to: "/apex/art_boot_images/javalib/x86/boot-core2.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", to: "/apex/art_boot_images/javalib/x86/boot.oat", }, { - from: "out/soong/test_device/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", + from: "out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", to: "/apex/art_boot_images/javalib/x86/boot-core2.oat", }, }, @@ -460,15 +460,15 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut expected := &expectedConfig{ name: "boot", stem: "boot", - dir: "out/soong/test_device/dex_bootjars", - symbolsDir: "out/soong/test_device/dex_bootjars_unstripped", + dir: "out/soong/dexpreopt_arm64/dex_bootjars", + symbolsDir: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped", installDirOnDevice: "system/framework", installDirOnHost: "system/framework", profileInstallPathInApex: "", modules: android.CreateTestConfiguredJarList([]string{"platform:framework"}), - dexPaths: []string{"out/soong/test_device/dex_bootjars_input/framework.jar"}, - dexPathsDeps: []string{"out/soong/test_device/dex_artjars_input/core1.jar", "out/soong/test_device/dex_artjars_input/core2.jar", "out/soong/test_device/dex_bootjars_input/framework.jar"}, - zip: "out/soong/test_device/dex_bootjars/boot.zip", + dexPaths: []string{"out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar"}, + dexPathsDeps: []string{"out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar", "out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar", "out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar"}, + zip: "out/soong/dexpreopt_arm64/dex_bootjars/boot.zip", variants: []*expectedVariant{ { archType: android.Arm64, @@ -478,41 +478,41 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut "/apex/com.android.art/javalib/core2.jar", "/system/framework/framework.jar", }, - imagePathOnHost: "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", imagePathOnDevice: "/system/framework/arm64/boot-framework.art", imagesDeps: []string{ - "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.art", - "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.oat", - "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", }, - baseImages: []string{"out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art"}, + baseImages: []string{"out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art"}, baseImagesDeps: []string{ - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.art", + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", to: "/system/framework/arm64/boot-framework.art", }, { - from: "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.oat", + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat", to: "/system/framework/arm64/boot-framework.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", to: "/system/framework/arm64/boot-framework.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars_unstripped/android/system/framework/arm64/boot-framework.oat", + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-framework.oat", to: "/system/framework/arm64/boot-framework.oat", }, }, @@ -526,41 +526,41 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut "/apex/com.android.art/javalib/core2.jar", "/system/framework/framework.jar", }, - imagePathOnHost: "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", imagePathOnDevice: "/system/framework/arm/boot-framework.art", imagesDeps: []string{ - "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.art", - "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.oat", - "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex", }, - baseImages: []string{"out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art"}, + baseImages: []string{"out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art"}, baseImagesDeps: []string{ - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.art", + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", to: "/system/framework/arm/boot-framework.art", }, { - from: "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.oat", + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat", to: "/system/framework/arm/boot-framework.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.vdex", + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex", to: "/system/framework/arm/boot-framework.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars_unstripped/android/system/framework/arm/boot-framework.oat", + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot-framework.oat", to: "/system/framework/arm/boot-framework.oat", }, }, @@ -574,41 +574,41 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut "host/linux-x86/apex/com.android.art/javalib/core2.jar", "host/linux-x86/system/framework/framework.jar", }, - imagePathOnHost: "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", imagePathOnDevice: "/system/framework/x86_64/boot-framework.art", imagesDeps: []string{ - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", }, - baseImages: []string{"out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art"}, + baseImages: []string{"out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art"}, baseImagesDeps: []string{ - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", to: "/system/framework/x86_64/boot-framework.art", }, { - from: "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat", + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat", to: "/system/framework/x86_64/boot-framework.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", to: "/system/framework/x86_64/boot-framework.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-framework.oat", + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-framework.oat", to: "/system/framework/x86_64/boot-framework.oat", }, }, @@ -622,41 +622,41 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut "host/linux-x86/apex/com.android.art/javalib/core2.jar", "host/linux-x86/system/framework/framework.jar", }, - imagePathOnHost: "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", imagePathOnDevice: "/system/framework/x86/boot-framework.art", imagesDeps: []string{ - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", }, - baseImages: []string{"out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art"}, + baseImages: []string{"out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art"}, baseImagesDeps: []string{ - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", to: "/system/framework/x86/boot-framework.art", }, { - from: "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat", + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat", to: "/system/framework/x86/boot-framework.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", to: "/system/framework/x86/boot-framework.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-framework.oat", + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-framework.oat", to: "/system/framework/x86/boot-framework.oat", }, }, @@ -664,8 +664,8 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut }, }, profileInstalls: []normalizedInstall{ - {from: "out/soong/test_device/dex_bootjars/boot.bprof", to: "/system/etc/boot-image.bprof"}, - {from: "out/soong/test_device/dex_bootjars/boot.prof", to: "/system/etc/boot-image.prof"}, + {from: "out/soong/dexpreopt_arm64/dex_bootjars/boot.bprof", to: "/system/etc/boot-image.bprof"}, + {from: "out/soong/dexpreopt_arm64/dex_bootjars/boot.prof", to: "/system/etc/boot-image.prof"}, }, profileLicenseMetadataFile: expectedLicenseMetadataFile, } @@ -691,8 +691,8 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { expected := &expectedConfig{ name: "mainline", stem: "boot", - dir: "out/soong/test_device/dex_mainlinejars", - symbolsDir: "out/soong/test_device/dex_mainlinejars_unstripped", + dir: "out/soong/dexpreopt_arm64/dex_mainlinejars", + symbolsDir: "out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped", installDirOnDevice: "system/framework", installDirOnHost: "system/framework", profileInstallPathInApex: "", @@ -701,17 +701,17 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "com.android.bar:framework-bar", }), dexPaths: []string{ - "out/soong/test_device/dex_mainlinejars_input/framework-foo.jar", - "out/soong/test_device/dex_mainlinejars_input/framework-bar.jar", + "out/soong/dexpreopt_arm64/dex_mainlinejars_input/framework-foo.jar", + "out/soong/dexpreopt_arm64/dex_mainlinejars_input/framework-bar.jar", }, dexPathsDeps: []string{ - "out/soong/test_device/dex_artjars_input/core1.jar", - "out/soong/test_device/dex_artjars_input/core2.jar", - "out/soong/test_device/dex_bootjars_input/framework.jar", - "out/soong/test_device/dex_mainlinejars_input/framework-foo.jar", - "out/soong/test_device/dex_mainlinejars_input/framework-bar.jar", + "out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar", + "out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar", + "out/soong/dexpreopt_arm64/dex_mainlinejars_input/framework-foo.jar", + "out/soong/dexpreopt_arm64/dex_mainlinejars_input/framework-bar.jar", }, - zip: "out/soong/test_device/dex_mainlinejars/mainline.zip", + zip: "out/soong/dexpreopt_arm64/dex_mainlinejars/mainline.zip", variants: []*expectedVariant{ { archType: android.Arm64, @@ -726,47 +726,47 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "/apex/com.android.foo/javalib/framework-foo.jar", "/apex/com.android.bar/javalib/framework-bar.jar", }, - imagePathOnHost: "out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art", imagePathOnDevice: "/system/framework/arm64/boot-framework-foo.art", imagesDeps: []string{ - "out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art", - "out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat", - "out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex", + "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art", + "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat", + "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex", }, baseImages: []string{ - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", }, baseImagesDeps: []string{ - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", - "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.art", - "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.oat", - "out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art", to: "/system/framework/arm64/boot-framework-foo.art", }, { - from: "out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat", to: "/system/framework/arm64/boot-framework-foo.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex", to: "/system/framework/arm64/boot-framework-foo.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars_unstripped/android/system/framework/arm64/boot-framework-foo.oat", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/android/system/framework/arm64/boot-framework-foo.oat", to: "/system/framework/arm64/boot-framework-foo.oat", }, }, @@ -785,47 +785,47 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "/apex/com.android.foo/javalib/framework-foo.jar", "/apex/com.android.bar/javalib/framework-bar.jar", }, - imagePathOnHost: "out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art", imagePathOnDevice: "/system/framework/arm/boot-framework-foo.art", imagesDeps: []string{ - "out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art", - "out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat", - "out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex", + "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art", + "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat", + "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex", }, baseImages: []string{ - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", - "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", }, baseImagesDeps: []string{ - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", - "out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", - "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.art", - "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.oat", - "out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art", to: "/system/framework/arm/boot-framework-foo.art", }, { - from: "out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat", to: "/system/framework/arm/boot-framework-foo.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex", to: "/system/framework/arm/boot-framework-foo.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars_unstripped/android/system/framework/arm/boot-framework-foo.oat", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/android/system/framework/arm/boot-framework-foo.oat", to: "/system/framework/arm/boot-framework-foo.oat", }, }, @@ -844,47 +844,47 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "host/linux-x86/apex/com.android.foo/javalib/framework-foo.jar", "host/linux-x86/apex/com.android.bar/javalib/framework-bar.jar", }, - imagePathOnHost: "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art", imagePathOnDevice: "/system/framework/x86_64/boot-framework-foo.art", imagesDeps: []string{ - "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art", - "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.oat", - "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex", + "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art", + "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.oat", + "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex", }, baseImages: []string{ - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", }, baseImagesDeps: []string{ - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art", to: "/system/framework/x86_64/boot-framework-foo.art", }, { - from: "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.oat", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.oat", to: "/system/framework/x86_64/boot-framework-foo.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex", to: "/system/framework/x86_64/boot-framework-foo.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86_64/boot-framework-foo.oat", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86_64/boot-framework-foo.oat", to: "/system/framework/x86_64/boot-framework-foo.oat", }, }, @@ -903,47 +903,47 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "host/linux-x86/apex/com.android.foo/javalib/framework-foo.jar", "host/linux-x86/apex/com.android.bar/javalib/framework-bar.jar", }, - imagePathOnHost: "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art", imagePathOnDevice: "/system/framework/x86/boot-framework-foo.art", imagesDeps: []string{ - "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art", - "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat", - "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex", + "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art", + "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat", + "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex", }, baseImages: []string{ - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", }, baseImagesDeps: []string{ - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", - "out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat", - "out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", }, installs: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art", to: "/system/framework/x86/boot-framework-foo.art", }, { - from: "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat", to: "/system/framework/x86/boot-framework-foo.oat", }, }, vdexInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex", to: "/system/framework/x86/boot-framework-foo.vdex", }, }, unstrippedInstalls: []normalizedInstall{ { - from: "out/soong/test_device/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86/boot-framework-foo.oat", + from: "out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86/boot-framework-foo.oat", to: "/system/framework/x86/boot-framework-foo.oat", }, }, @@ -1056,34 +1056,34 @@ func checkDexpreoptMakeVars(t *testing.T, result *android.TestResult, expectedLi fmt.Fprintf(out, "%s=%s\n", v.Name(), android.StringRelativeToTop(result.Config, v.Value())) } format := ` -DEXPREOPT_BOOTCLASSPATH_DEX_FILES=out/soong/test_device/dex_artjars_input/core1.jar out/soong/test_device/dex_artjars_input/core2.jar out/soong/test_device/dex_bootjars_input/framework.jar +DEXPREOPT_BOOTCLASSPATH_DEX_FILES=out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS=/apex/com.android.art/javalib/core1.jar /apex/com.android.art/javalib/core2.jar /system/framework/framework.jar DEXPREOPT_BOOT_JARS_MODULES=platform:framework DEXPREOPT_GEN=out/host/linux-x86/bin/dexpreopt_gen -DEXPREOPT_IMAGE_BUILT_INSTALLED_art_arm=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art:/apex/art_boot_images/javalib/arm/boot.art out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat:/apex/art_boot_images/javalib/arm/boot.oat out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art:/apex/art_boot_images/javalib/arm/boot-core2.art out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat:/apex/art_boot_images/javalib/arm/boot-core2.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_art_arm64=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art:/apex/art_boot_images/javalib/arm64/boot.art out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat:/apex/art_boot_images/javalib/arm64/boot.oat out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art:/apex/art_boot_images/javalib/arm64/boot-core2.art out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat:/apex/art_boot_images/javalib/arm64/boot-core2.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_art_host_x86=out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art:/apex/art_boot_images/javalib/x86/boot.art out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat:/apex/art_boot_images/javalib/x86/boot.oat out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art:/apex/art_boot_images/javalib/x86/boot-core2.art out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat:/apex/art_boot_images/javalib/x86/boot-core2.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_art_host_x86_64=out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art:/apex/art_boot_images/javalib/x86_64/boot.art out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat:/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art:/apex/art_boot_images/javalib/x86_64/boot-core2.art out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat:/apex/art_boot_images/javalib/x86_64/boot-core2.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_arm=out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.art:/system/framework/arm/boot-framework.art out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.oat:/system/framework/arm/boot-framework.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_arm64=out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.art:/system/framework/arm64/boot-framework.art out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.oat:/system/framework/arm64/boot-framework.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_host_x86=out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art:/system/framework/x86/boot-framework.art out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat:/system/framework/x86/boot-framework.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_host_x86_64=out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art:/system/framework/x86_64/boot-framework.art out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat:/system/framework/x86_64/boot-framework.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_arm=out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art:/system/framework/arm/boot-framework-foo.art out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat:/system/framework/arm/boot-framework-foo.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_arm64=out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art:/system/framework/arm64/boot-framework-foo.art out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat:/system/framework/arm64/boot-framework-foo.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_host_x86=out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art:/system/framework/x86/boot-framework-foo.art out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat:/system/framework/x86/boot-framework-foo.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_host_x86_64=out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art:/system/framework/x86_64/boot-framework-foo.art out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.oat:/system/framework/x86_64/boot-framework-foo.oat -DEXPREOPT_IMAGE_DEPS_art_arm=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex -DEXPREOPT_IMAGE_DEPS_art_arm64=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex -DEXPREOPT_IMAGE_DEPS_art_host_x86=out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex -DEXPREOPT_IMAGE_DEPS_art_host_x86_64=out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex -DEXPREOPT_IMAGE_DEPS_boot_arm=out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.art out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.oat out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.vdex -DEXPREOPT_IMAGE_DEPS_boot_arm64=out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.art out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.oat out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.vdex -DEXPREOPT_IMAGE_DEPS_boot_host_x86=out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex -DEXPREOPT_IMAGE_DEPS_boot_host_x86_64=out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex -DEXPREOPT_IMAGE_DEPS_mainline_arm=out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex -DEXPREOPT_IMAGE_DEPS_mainline_arm64=out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex -DEXPREOPT_IMAGE_DEPS_mainline_host_x86=out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex -DEXPREOPT_IMAGE_DEPS_mainline_host_x86_64=out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.oat out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex +DEXPREOPT_IMAGE_BUILT_INSTALLED_art_arm=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art:/apex/art_boot_images/javalib/arm/boot.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat:/apex/art_boot_images/javalib/arm/boot.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art:/apex/art_boot_images/javalib/arm/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat:/apex/art_boot_images/javalib/arm/boot-core2.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_art_arm64=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art:/apex/art_boot_images/javalib/arm64/boot.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat:/apex/art_boot_images/javalib/arm64/boot.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art:/apex/art_boot_images/javalib/arm64/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat:/apex/art_boot_images/javalib/arm64/boot-core2.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art:/apex/art_boot_images/javalib/x86/boot.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat:/apex/art_boot_images/javalib/x86/boot.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art:/apex/art_boot_images/javalib/x86/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat:/apex/art_boot_images/javalib/x86/boot-core2.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art:/apex/art_boot_images/javalib/x86_64/boot.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat:/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art:/apex/art_boot_images/javalib/x86_64/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat:/apex/art_boot_images/javalib/x86_64/boot-core2.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art:/system/framework/arm/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat:/system/framework/arm/boot-framework.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art:/system/framework/arm64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat:/system/framework/arm64/boot-framework.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art:/system/framework/x86/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat:/system/framework/x86/boot-framework.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art:/system/framework/x86_64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat:/system/framework/x86_64/boot-framework.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art:/system/framework/arm/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat:/system/framework/arm/boot-framework-foo.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art:/system/framework/arm64/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat:/system/framework/arm64/boot-framework-foo.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art:/system/framework/x86/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat:/system/framework/x86/boot-framework-foo.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_host_x86_64=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art:/system/framework/x86_64/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.oat:/system/framework/x86_64/boot-framework-foo.oat +DEXPREOPT_IMAGE_DEPS_art_arm=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex +DEXPREOPT_IMAGE_DEPS_art_arm64=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex +DEXPREOPT_IMAGE_DEPS_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex +DEXPREOPT_IMAGE_DEPS_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex +DEXPREOPT_IMAGE_DEPS_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex +DEXPREOPT_IMAGE_DEPS_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex +DEXPREOPT_IMAGE_DEPS_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex +DEXPREOPT_IMAGE_DEPS_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex +DEXPREOPT_IMAGE_DEPS_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex +DEXPREOPT_IMAGE_DEPS_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex +DEXPREOPT_IMAGE_DEPS_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex +DEXPREOPT_IMAGE_DEPS_mainline_host_x86_64=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.oat out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex DEXPREOPT_IMAGE_LICENSE_METADATA_art_arm=%[1]s DEXPREOPT_IMAGE_LICENSE_METADATA_art_arm64=%[1]s DEXPREOPT_IMAGE_LICENSE_METADATA_art_host_x86=%[1]s @@ -1099,51 +1099,51 @@ DEXPREOPT_IMAGE_LICENSE_METADATA_mainline_host_x86_64=out/soong/.intermediates/f DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEart=/system/framework/boot.art DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEboot=/system/framework/boot.art:/system/framework/boot-framework.art DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEmainline=/system/framework/boot.art:/system/framework/boot-framework.art:/system/framework/boot-framework-foo.art -DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTart=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/boot.art -DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTboot=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/boot.art:out/soong/test_device/dex_bootjars/android/system/framework/boot-framework.art -DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTmainline=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/boot.art:out/soong/test_device/dex_bootjars/android/system/framework/boot-framework.art:out/soong/test_device/dex_mainlinejars/android/system/framework/boot-framework-foo.art +DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTart=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art +DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTboot=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art:out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/boot-framework.art +DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTmainline=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art:out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/boot-framework.art:out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/boot-framework-foo.art DEXPREOPT_IMAGE_NAMES=art boot mainline -DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED=out/soong/test_device/dex_bootjars/boot.bprof:/system/etc/boot-image.bprof out/soong/test_device/dex_bootjars/boot.prof:/system/etc/boot-image.prof +DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED=out/soong/dexpreopt_arm64/dex_bootjars/boot.bprof:/system/etc/boot-image.bprof out/soong/dexpreopt_arm64/dex_bootjars/boot.prof:/system/etc/boot-image.prof DEXPREOPT_IMAGE_PROFILE_LICENSE_METADATA=out/soong/.intermediates/frameworks/base/boot/platform-bootclasspath/android_common/meta_lic -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_arm=out/soong/test_device/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm/boot.oat:/apex/art_boot_images/javalib/arm/boot.oat out/soong/test_device/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm/boot-core2.oat:/apex/art_boot_images/javalib/arm/boot-core2.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_arm64=out/soong/test_device/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot.oat:/apex/art_boot_images/javalib/arm64/boot.oat out/soong/test_device/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot-core2.oat:/apex/art_boot_images/javalib/arm64/boot-core2.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_host_x86=out/soong/test_device/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat:/apex/art_boot_images/javalib/x86/boot.oat out/soong/test_device/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat:/apex/art_boot_images/javalib/x86/boot-core2.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_host_x86_64=out/soong/test_device/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat:/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/test_device/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat:/apex/art_boot_images/javalib/x86_64/boot-core2.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_arm=out/soong/test_device/dex_bootjars_unstripped/android/system/framework/arm/boot-framework.oat:/system/framework/arm/boot-framework.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_arm64=out/soong/test_device/dex_bootjars_unstripped/android/system/framework/arm64/boot-framework.oat:/system/framework/arm64/boot-framework.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_host_x86=out/soong/test_device/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-framework.oat:/system/framework/x86/boot-framework.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_host_x86_64=out/soong/test_device/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-framework.oat:/system/framework/x86_64/boot-framework.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_arm=out/soong/test_device/dex_mainlinejars_unstripped/android/system/framework/arm/boot-framework-foo.oat:/system/framework/arm/boot-framework-foo.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_arm64=out/soong/test_device/dex_mainlinejars_unstripped/android/system/framework/arm64/boot-framework-foo.oat:/system/framework/arm64/boot-framework-foo.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_host_x86=out/soong/test_device/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86/boot-framework-foo.oat:/system/framework/x86/boot-framework-foo.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_host_x86_64=out/soong/test_device/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86_64/boot-framework-foo.oat:/system/framework/x86_64/boot-framework-foo.oat -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_arm=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex:/apex/art_boot_images/javalib/arm/boot.vdex out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex:/apex/art_boot_images/javalib/arm/boot-core2.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_arm64=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex:/apex/art_boot_images/javalib/arm64/boot.vdex out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex:/apex/art_boot_images/javalib/arm64/boot-core2.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_host_x86=out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex:/apex/art_boot_images/javalib/x86/boot.vdex out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex:/apex/art_boot_images/javalib/x86/boot-core2.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_host_x86_64=out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex:/apex/art_boot_images/javalib/x86_64/boot.vdex out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex:/apex/art_boot_images/javalib/x86_64/boot-core2.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_arm=out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.vdex:/system/framework/arm/boot-framework.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_arm64=out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.vdex:/system/framework/arm64/boot-framework.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_host_x86=out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex:/system/framework/x86/boot-framework.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_host_x86_64=out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex:/system/framework/x86_64/boot-framework.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_arm=out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex:/system/framework/arm/boot-framework-foo.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_arm64=out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex:/system/framework/arm64/boot-framework-foo.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_host_x86=out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex:/system/framework/x86/boot-framework-foo.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_host_x86_64=out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex:/system/framework/x86_64/boot-framework-foo.vdex -DEXPREOPT_IMAGE_ZIP_art=out/soong/test_device/dex_artjars/art.zip -DEXPREOPT_IMAGE_ZIP_boot=out/soong/test_device/dex_bootjars/boot.zip -DEXPREOPT_IMAGE_ZIP_mainline=out/soong/test_device/dex_mainlinejars/mainline.zip -DEXPREOPT_IMAGE_art_arm=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art -DEXPREOPT_IMAGE_art_arm64=out/soong/test_device/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art -DEXPREOPT_IMAGE_art_host_x86=out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art -DEXPREOPT_IMAGE_art_host_x86_64=out/soong/test_device/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art -DEXPREOPT_IMAGE_boot_arm=out/soong/test_device/dex_bootjars/android/system/framework/arm/boot-framework.art -DEXPREOPT_IMAGE_boot_arm64=out/soong/test_device/dex_bootjars/android/system/framework/arm64/boot-framework.art -DEXPREOPT_IMAGE_boot_host_x86=out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art -DEXPREOPT_IMAGE_boot_host_x86_64=out/soong/test_device/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art -DEXPREOPT_IMAGE_mainline_arm=out/soong/test_device/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art -DEXPREOPT_IMAGE_mainline_arm64=out/soong/test_device/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art -DEXPREOPT_IMAGE_mainline_host_x86=out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art -DEXPREOPT_IMAGE_mainline_host_x86_64=out/soong/test_device/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_arm=out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm/boot.oat:/apex/art_boot_images/javalib/arm/boot.oat out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm/boot-core2.oat:/apex/art_boot_images/javalib/arm/boot-core2.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_arm64=out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot.oat:/apex/art_boot_images/javalib/arm64/boot.oat out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot-core2.oat:/apex/art_boot_images/javalib/arm64/boot-core2.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat:/apex/art_boot_images/javalib/x86/boot.oat out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat:/apex/art_boot_images/javalib/x86/boot-core2.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat:/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat:/apex/art_boot_images/javalib/x86_64/boot-core2.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot-framework.oat:/system/framework/arm/boot-framework.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-framework.oat:/system/framework/arm64/boot-framework.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-framework.oat:/system/framework/x86/boot-framework.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-framework.oat:/system/framework/x86_64/boot-framework.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/android/system/framework/arm/boot-framework-foo.oat:/system/framework/arm/boot-framework-foo.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/android/system/framework/arm64/boot-framework-foo.oat:/system/framework/arm64/boot-framework-foo.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86/boot-framework-foo.oat:/system/framework/x86/boot-framework-foo.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_host_x86_64=out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86_64/boot-framework-foo.oat:/system/framework/x86_64/boot-framework-foo.oat +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_arm=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex:/apex/art_boot_images/javalib/arm/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex:/apex/art_boot_images/javalib/arm/boot-core2.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_arm64=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex:/apex/art_boot_images/javalib/arm64/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex:/apex/art_boot_images/javalib/arm64/boot-core2.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex:/apex/art_boot_images/javalib/x86/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex:/apex/art_boot_images/javalib/x86/boot-core2.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex:/apex/art_boot_images/javalib/x86_64/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex:/apex/art_boot_images/javalib/x86_64/boot-core2.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex:/system/framework/arm/boot-framework.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex:/system/framework/arm64/boot-framework.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex:/system/framework/x86/boot-framework.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex:/system/framework/x86_64/boot-framework.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex:/system/framework/arm/boot-framework-foo.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex:/system/framework/arm64/boot-framework-foo.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex:/system/framework/x86/boot-framework-foo.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_host_x86_64=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex:/system/framework/x86_64/boot-framework-foo.vdex +DEXPREOPT_IMAGE_ZIP_art=out/soong/dexpreopt_arm64/dex_artjars/art.zip +DEXPREOPT_IMAGE_ZIP_boot=out/soong/dexpreopt_arm64/dex_bootjars/boot.zip +DEXPREOPT_IMAGE_ZIP_mainline=out/soong/dexpreopt_arm64/dex_mainlinejars/mainline.zip +DEXPREOPT_IMAGE_art_arm=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art +DEXPREOPT_IMAGE_art_arm64=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art +DEXPREOPT_IMAGE_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art +DEXPREOPT_IMAGE_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art +DEXPREOPT_IMAGE_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art +DEXPREOPT_IMAGE_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art +DEXPREOPT_IMAGE_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art +DEXPREOPT_IMAGE_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art +DEXPREOPT_IMAGE_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art +DEXPREOPT_IMAGE_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art +DEXPREOPT_IMAGE_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art +DEXPREOPT_IMAGE_mainline_host_x86_64=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.art ` expected := strings.TrimSpace(fmt.Sprintf(format, expectedLicenseMetadataFile)) actual := strings.TrimSpace(out.String()) From b5ae9bcf9067bb96c55665fe2b9bedfea2fed42d Mon Sep 17 00:00:00 2001 From: Chris Wailes Date: Fri, 28 Apr 2023 01:14:57 -0700 Subject: [PATCH 0052/1460] rustc-1.69.0 Build 10027731 Bug: https://issuetracker.google.com/issues/274972941 Test: m rust Change-Id: I03ba07802c2d77d5e3776cc49abac8b98d6ebbfe --- rust/config/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/config/global.go b/rust/config/global.go index 2d1f0c1f94..748bb3d2f3 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.68.0" + RustDefaultVersion = "1.69.0" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ From 18ed8a912be0ab54a4a56433051c5723785911e1 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Wed, 26 Apr 2023 21:36:58 +0000 Subject: [PATCH 0053/1460] Add bazel exit code to bazel metrics proto. Bug: 279754118 Test: N/A Change-Id: I52558843f40bf54b75a4255b0a9d96e9dcaa81bc --- .../bazel_metrics_proto/bazel_metrics.pb.go | 21 ++++++++++++++----- .../bazel_metrics_proto/bazel_metrics.proto | 1 + 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ui/metrics/bazel_metrics_proto/bazel_metrics.pb.go b/ui/metrics/bazel_metrics_proto/bazel_metrics.pb.go index f8b8fd6c6a..bf5e80becb 100644 --- a/ui/metrics/bazel_metrics_proto/bazel_metrics.pb.go +++ b/ui/metrics/bazel_metrics_proto/bazel_metrics.pb.go @@ -14,8 +14,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 -// protoc v3.21.7 +// protoc-gen-go v1.30.0 +// protoc v3.21.12 // source: bazel_metrics.proto package bazel_metrics_proto @@ -41,6 +41,7 @@ type BazelMetrics struct { PhaseTimings []*PhaseTiming `protobuf:"bytes,1,rep,name=phase_timings,json=phaseTimings,proto3" json:"phase_timings,omitempty"` Total *int64 `protobuf:"varint,2,opt,name=total,proto3,oneof" json:"total,omitempty"` + ExitCode *int32 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3,oneof" json:"exit_code,omitempty"` } func (x *BazelMetrics) Reset() { @@ -89,6 +90,13 @@ func (x *BazelMetrics) GetTotal() int64 { return 0 } +func (x *BazelMetrics) GetExitCode() int32 { + if x != nil && x.ExitCode != nil { + return *x.ExitCode + } + return 0 +} + type PhaseTiming struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -161,15 +169,18 @@ var file_bazel_metrics_proto_rawDesc = []byte{ 0x0a, 0x13, 0x62, 0x61, 0x7a, 0x65, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x19, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x62, 0x61, 0x7a, 0x65, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x22, 0x80, 0x01, 0x0a, 0x0c, 0x42, 0x61, 0x7a, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x22, 0xb0, 0x01, 0x0a, 0x0c, 0x42, 0x61, 0x7a, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x4b, 0x0a, 0x0d, 0x70, 0x68, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x62, 0x61, 0x7a, 0x65, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x50, 0x68, 0x61, 0x73, 0x65, 0x54, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x52, 0x0c, 0x70, 0x68, 0x61, 0x73, 0x65, 0x54, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x19, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, - 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x22, 0xd1, 0x01, 0x0a, 0x0b, 0x50, 0x68, 0x61, 0x73, 0x65, 0x54, 0x69, 0x6d, + 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x20, 0x0a, 0x09, 0x65, 0x78, 0x69, + 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, 0x08, + 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, + 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, + 0x6f, 0x64, 0x65, 0x22, 0xd1, 0x01, 0x0a, 0x0b, 0x50, 0x68, 0x61, 0x73, 0x65, 0x54, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x12, 0x22, 0x0a, 0x0a, 0x70, 0x68, 0x61, 0x73, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x70, 0x68, 0x61, 0x73, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x64, 0x75, 0x72, 0x61, 0x74, diff --git a/ui/metrics/bazel_metrics_proto/bazel_metrics.proto b/ui/metrics/bazel_metrics_proto/bazel_metrics.proto index 57eed4c288..90730809c2 100644 --- a/ui/metrics/bazel_metrics_proto/bazel_metrics.proto +++ b/ui/metrics/bazel_metrics_proto/bazel_metrics.proto @@ -20,6 +20,7 @@ option go_package = "android/soong/ui/metrics/bazel_metrics_proto"; message BazelMetrics { repeated PhaseTiming phase_timings = 1; optional int64 total = 2; + optional int32 exit_code = 3; } message PhaseTiming { From fb6a1ee68bd7b17d1a8632082de88504dddc3540 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Thu, 27 Apr 2023 16:08:26 +0000 Subject: [PATCH 0054/1460] target_sdk_version replacement should respect REL branches Currently, target_sdk_version of MTS tests were being set to 10000 even in release branches. This should only happen in active branches. Test: Unit test Test: aapt2 on CtsContentTestCases in udc-dev and verified targetSdkVersion is 34 Bug: 279507428 Change-Id: Ib79318febc0f6e771b3679c0281a97357930ec56 --- java/android_manifest.go | 4 +++- java/app_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/java/android_manifest.go b/java/android_manifest.go index dbcf09830b..f2ebfa6a23 100644 --- a/java/android_manifest.go +++ b/java/android_manifest.go @@ -63,9 +63,11 @@ func targetSdkVersionForManifestFixer(ctx android.ModuleContext, params Manifest // 2. The module is run as part of MTS, and should be testable on stable branches // Do not return 10000 if we are enforcing default targetSdkVersion and sdk has been finalised func shouldReturnFinalOrFutureInt(ctx android.ModuleContext, targetSdkVersionLevel android.ApiLevel, enforceDefaultTargetSdkVersion bool) bool { - if enforceDefaultTargetSdkVersion && ctx.Config().PlatformSdkFinal() { + // If this is a REL branch, do not return 10000 + if ctx.Config().PlatformSdkFinal() { return false } + // If this a module targeting an unreleased SDK (MTS or unbundled builds), return 10000 return targetSdkVersionLevel.IsPreview() && (ctx.Config().UnbundledBuildApps() || includedInMts(ctx.Module())) } diff --git a/java/app_test.go b/java/app_test.go index 561be684e4..7e97b0fb19 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3034,11 +3034,13 @@ func TestExportedProguardFlagFiles(t *testing.T) { func TestTargetSdkVersionManifestFixer(t *testing.T) { platform_sdk_codename := "Tiramisu" + platform_sdk_version := 33 testCases := []struct { name string targetSdkVersionInBp string targetSdkVersionExpected string unbundledBuild bool + platformSdkFinal bool }{ { name: "Non-Unbundled build: Android.bp has targetSdkVersion", @@ -3075,6 +3077,12 @@ func TestTargetSdkVersionManifestFixer(t *testing.T) { targetSdkVersionExpected: "10000", unbundledBuild: true, }, + { + name: "Bundled build in REL branches", + targetSdkVersionExpected: "33", + unbundledBuild: false, + platformSdkFinal: true, + }, } for _, testCase := range testCases { targetSdkVersionTemplate := "" @@ -3091,8 +3099,12 @@ func TestTargetSdkVersionManifestFixer(t *testing.T) { fixture := android.GroupFixturePreparers( prepareForJavaTest, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + if testCase.platformSdkFinal { + variables.Platform_sdk_final = proptools.BoolPtr(true) + } // explicitly set platform_sdk_codename to make the test deterministic variables.Platform_sdk_codename = &platform_sdk_codename + variables.Platform_sdk_version = &platform_sdk_version variables.Platform_version_active_codenames = []string{platform_sdk_codename} // create a non-empty list if unbundledBuild==true if testCase.unbundledBuild { From f0ab546f389798b953e853ded6253a294bbee835 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Tue, 4 Apr 2023 11:38:54 -0400 Subject: [PATCH 0055/1460] Allowlist more modules Test: CI Test: mixed_droid Change-Id: I9c2a816f96399401482c93cae2cb9328abbecd16 --- android/allowlists/allowlists.go | 35 +++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index c4e49c81d6..d8611f8d11 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -183,6 +183,7 @@ var ( "external/selinux/libselinux": Bp2BuildDefaultTrueRecursively, "external/selinux/libsepol": Bp2BuildDefaultTrueRecursively, "external/speex": Bp2BuildDefaultTrueRecursively, + "external/sqlite": Bp2BuildDefaultTrueRecursively, "external/tinyalsa": Bp2BuildDefaultTrueRecursively, "external/tinyalsa_new": Bp2BuildDefaultTrueRecursively, "external/toybox": Bp2BuildDefaultTrueRecursively, @@ -191,11 +192,12 @@ var ( "external/zstd": Bp2BuildDefaultTrueRecursively, "frameworks/av": Bp2BuildDefaultTrue, + "frameworks/av/media/audioaidlconversion": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/codec2/components/aom": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/codecs": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/liberror": Bp2BuildDefaultTrueRecursively, + "frameworks/av/media/libmediahelper": Bp2BuildDefaultTrue, "frameworks/av/media/libshmem": Bp2BuildDefaultTrueRecursively, - "frameworks/av/media/audioaidlconversion": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/module/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/av/services/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/base/libs/androidfw": Bp2BuildDefaultTrue, @@ -213,13 +215,13 @@ var ( "frameworks/native/libs/gui": Bp2BuildDefaultTrue, "frameworks/native/libs/math": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/nativebase": Bp2BuildDefaultTrueRecursively, + "frameworks/native/libs/permission": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/vr": Bp2BuildDefaultTrueRecursively, "frameworks/native/opengl/tests/gl2_cameraeye": Bp2BuildDefaultTrue, "frameworks/native/opengl/tests/gl2_java": Bp2BuildDefaultTrue, "frameworks/native/opengl/tests/testLatency": Bp2BuildDefaultTrue, "frameworks/native/opengl/tests/testPauseResume": Bp2BuildDefaultTrue, "frameworks/native/opengl/tests/testViewport": Bp2BuildDefaultTrue, - "frameworks/native/libs/permission": Bp2BuildDefaultTrue, "frameworks/native/services/batteryservice": Bp2BuildDefaultTrue, "frameworks/proto_logging/stats": Bp2BuildDefaultTrueRecursively, @@ -545,6 +547,14 @@ var ( "liblp", "libstorage_literals_headers", + "PluginCoreLib", + "dagger2", + "dagger2-android-annotation-stubs", + "dagger2-bootstrap-compiler", + "dagger2-producers", + "okio-lib", + "setupdesign-strings", + //external/avb "avbtool", "libavb", @@ -718,11 +728,12 @@ var ( "api_fingerprint", // allowlisting for kotlinx_coroutines + "annotations", + "kotlinx-coroutines-android-annotation-stubs", + "kotlinx-coroutines-core", "kotlinx_coroutines", "kotlinx_coroutines-device", "kotlinx_coroutines-host", - "annotations", - "kotlinx-coroutines-android-annotation-stubs", // for building com.android.neuralnetworks "libimapper_stablec", @@ -731,6 +742,8 @@ var ( // min_sdk_version in android_app "CtsShimUpgrade", + "art_cmdlineparser_headers", + // Mainline Module Apps "CaptivePortalLogin", } @@ -764,7 +777,6 @@ var ( "buffer_hub_queue_producer-test", // cc bugs - "libactivitymanager_aidl", // TODO(b/207426160): Unsupported use of aidl sources (via Dactivity_manager_procstate_aidl) in a cc_library // TODO(b/198619163) module has same name as source "logtagd.rc", @@ -777,13 +789,12 @@ var ( "libcutils_test_static", "KernelLibcutilsTest", - "linker", // TODO(b/228316882): cc_binary uses link_crt - "versioner", // TODO(b/228313961): depends on prebuilt shared library libclang-cpp_host as a shared library, which does not supply expected providers for a shared library - "art_libartbase_headers", // TODO(b/236268577): Header libraries do not support export_shared_libs_headers - "apexer_test", // Requires aapt2 - "apexer_test_host_tools", - "host_apex_verifier", - "tjbench", // TODO(b/240563612): Stem property + "linker", // TODO(b/228316882): cc_binary uses link_crt + "versioner", // TODO(b/228313961): depends on prebuilt shared library libclang-cpp_host as a shared library, which does not supply expected providers for a shared library + "tjbench", // TODO(b/240563612): Stem property + + // requires host tools for apexer + "apexer_test", "apexer_test_host_tools", "host_apex_verifier", // java bugs "libbase_ndk", // TODO(b/186826477): fails to link libctscamera2_jni for device (required for CtsCameraTestCases) From 09ddb3a73e14823d51d8227ecaf1997fa010bc14 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Wed, 19 Oct 2022 01:29:18 -0400 Subject: [PATCH 0056/1460] Restrict plugins to an existing allowlist Adds a singleton to do validation that can be disabled via a BUILD_BROKEN_PLUGIN_VALIDATION flag. Validation process: For all go modules that are a plugin for soong_build: * if path is in build/soong, allow * if path is in vendor, outside of google paths, allow * if path is in hardware, outside of google paths, allow * if name is in allowlist of current plugins, allow We extend the plugin the list for internal modules via vendor/google/build/soong/internal_plugins.json Ignore-AOSP-First: Requires an internal only change to not break when submitted Test: CI Change-Id: I264a89b3636043330711d6c996c0360b61f51d92 --- android/Android.bp | 1 + android/config.go | 4 ++ android/plugin.go | 139 ++++++++++++++++++++++++++++++++++++++++ android/variable.go | 1 + tests/bootstrap_test.sh | 24 +++---- 5 files changed, 157 insertions(+), 12 deletions(-) create mode 100644 android/plugin.go diff --git a/android/Android.bp b/android/Android.bp index 118087db7b..94d2c04f6b 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -77,6 +77,7 @@ bootstrap_go_package { "path_properties.go", "paths.go", "phony.go", + "plugin.go", "prebuilt.go", "prebuilt_build_tool.go", "proto.go", diff --git a/android/config.go b/android/config.go index 7141e54c3d..e0cf173f0e 100644 --- a/android/config.go +++ b/android/config.go @@ -1819,6 +1819,10 @@ func (c *deviceConfig) ShippingApiLevel() ApiLevel { return uncheckedFinalApiLevel(apiLevel) } +func (c *deviceConfig) BuildBrokenPluginValidation() []string { + return c.config.productVariables.BuildBrokenPluginValidation +} + func (c *deviceConfig) BuildBrokenClangAsFlags() bool { return c.config.productVariables.BuildBrokenClangAsFlags } diff --git a/android/plugin.go b/android/plugin.go new file mode 100644 index 0000000000..5be3d3ad86 --- /dev/null +++ b/android/plugin.go @@ -0,0 +1,139 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/google/blueprint" +) + +func init() { + RegisterPluginSingletonBuildComponents(InitRegistrationContext) +} + +func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { + ctx.RegisterSingletonType("plugins", pluginSingletonFactory) +} + +// pluginSingleton is a singleton to handle allowlisting of the final Android-.mk file +// output. +func pluginSingletonFactory() Singleton { + return &pluginSingleton{} +} + +type pluginSingleton struct{} + +var allowedPluginsByName = map[string]bool{ + "aidl-soong-rules": true, + "arm_compute_library_nn_driver": true, + "cuttlefish-soong-rules": true, + "gki-soong-rules": true, + "hidl-soong-rules": true, + "kernel-config-soong-rules": true, + "soong-angle-codegen": true, + "soong-api": true, + "soong-art": true, + "soong-ca-certificates": true, + "soong-clang": true, + "soong-clang-prebuilts": true, + "soong-csuite": true, + "soong-fluoride": true, + "soong-fs_config": true, + "soong-icu": true, + "soong-java-config-error_prone": true, + "soong-libchrome": true, + "soong-llvm": true, + "soong-robolectric": true, + "soong-rust-prebuilts": true, + "soong-selinux": true, + "soong-wayland-protocol-codegen": true, + "treble_report_app": true, + "treble_report_local": true, + "treble_report_module": true, + "vintf-compatibility-matrix-soong-rules": true, + "xsdc-soong-rules": true, +} + +const ( + internalPluginsPath = "vendor/google/build/soong/internal_plugins.json" +) + +type pluginProvider interface { + IsPluginFor(string) bool +} + +func maybeAddInternalPluginsToAllowlist(ctx SingletonContext) { + if path := ExistentPathForSource(ctx, internalPluginsPath); path.Valid() { + ctx.AddNinjaFileDeps(path.String()) + absPath := absolutePath(path.String()) + var moreAllowed map[string]bool + data, err := ioutil.ReadFile(absPath) + if err != nil { + ctx.Errorf("Failed to open internal plugins path %q %q", internalPluginsPath, err) + } + if err := json.Unmarshal(data, &moreAllowed); err != nil { + fmt.Fprintf(os.Stderr, "Internal plugins file %q did not parse correctly: %q", data, err) + } + for k, v := range moreAllowed { + allowedPluginsByName[k] = v + } + } +} + +func (p *pluginSingleton) GenerateBuildActions(ctx SingletonContext) { + for _, p := range ctx.DeviceConfig().BuildBrokenPluginValidation() { + allowedPluginsByName[p] = true + } + maybeAddInternalPluginsToAllowlist(ctx) + + disallowedPlugins := map[string]bool{} + ctx.VisitAllModulesBlueprint(func(module blueprint.Module) { + if ctx.ModuleType(module) != "bootstrap_go_package" { + return + } + + p, ok := module.(pluginProvider) + if !ok || !p.IsPluginFor("soong_build") { + return + } + + name := ctx.ModuleName(module) + if _, ok := allowedPluginsByName[name]; ok { + return + } + + dir := ctx.ModuleDir(module) + + // allow use of plugins within Soong to not allowlist everything + if strings.HasPrefix(dir, "build/soong") { + return + } + + // allow third party users outside of external to create new plugins, i.e. non-google paths + // under vendor or hardware + if !strings.HasPrefix(dir, "external/") && IsThirdPartyPath(dir) { + return + } + disallowedPlugins[name] = true + }) + if len(disallowedPlugins) > 0 { + ctx.Errorf("New plugins are not supported; however %q were found. Please reach out to the build team or use BUILD_BROKEN_PLUGIN_VALIDATION (see Changes.md for more info).", SortedStringKeys(disallowedPlugins)) + } +} diff --git a/android/variable.go b/android/variable.go index 249d53b07a..b5d9666683 100644 --- a/android/variable.go +++ b/android/variable.go @@ -437,6 +437,7 @@ type productVariables struct { ShippingApiLevel *string `json:",omitempty"` + BuildBrokenPluginValidation []string `json:",omitempty"` BuildBrokenClangAsFlags bool `json:",omitempty"` BuildBrokenClangCFlags bool `json:",omitempty"` BuildBrokenClangProperty bool `json:",omitempty"` diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index fda5ca0863..c5a6336a84 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -207,8 +207,8 @@ EOF function test_soong_build_rerun_iff_environment_changes() { setup - mkdir -p cherry - cat > cherry/Android.bp <<'EOF' + mkdir -p build/soong/cherry + cat > build/soong/cherry/Android.bp <<'EOF' bootstrap_go_package { name: "cherry", pkgPath: "android/soong/cherry", @@ -224,7 +224,7 @@ bootstrap_go_package { } EOF - cat > cherry/cherry.go <<'EOF' + cat > build/soong/cherry/cherry.go <<'EOF' package cherry import ( @@ -317,8 +317,8 @@ function test_add_file_to_soong_build() { run_soong local -r mtime1=$(stat -c "%y" out/soong/build.ninja) - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p vendor/foo/picard + cat > vendor/foo/picard/Android.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -334,7 +334,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > vendor/foo/picard/picard.go <<'EOF' package picard import ( @@ -390,11 +390,11 @@ EOF function test_glob_during_bootstrapping() { setup - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p build/soong/picard + cat > build/soong/picard/Android.bp <<'EOF' build=["foo*.bp"] EOF - cat > a/fooa.bp <<'EOF' + cat > build/soong/picard/fooa.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -410,7 +410,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > build/soong/picard/picard.go <<'EOF' package picard import ( @@ -459,7 +459,7 @@ EOF grep -q "Make it so" out/soong/build.ninja || fail "Original action not present" - cat > a/foob.bp <<'EOF' + cat > build/soong/picard/foob.bp <<'EOF' bootstrap_go_package { name: "worf-soong-rules", pkgPath: "android/soong/worf", @@ -476,7 +476,7 @@ bootstrap_go_package { } EOF - cat > a/worf.go <<'EOF' + cat > build/soong/picard/worf.go <<'EOF' package worf import "android/soong/picard" From d2442920d7f08683ce2dd31b3a230b9e56349f2a Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Fri, 21 Apr 2023 13:22:06 -0400 Subject: [PATCH 0057/1460] Remove alexmarquez from owners. Test: n/a Change-Id: Ic303c60bd688023bac8225c4f8435fbbb2272252 --- OWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS b/OWNERS index 964e27a316..0234f27761 100644 --- a/OWNERS +++ b/OWNERS @@ -3,7 +3,6 @@ # AMER agespino@google.com -alexmarquez@google.com ccross@android.com colefaust@google.com cparsons@google.com From 6a448ec1a346b5ace5f12687fc65933e75081a3f Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 19 Apr 2023 17:36:12 +0000 Subject: [PATCH 0058/1460] Add a function to create config_setting(s) The use case for this is creating config_setting(s) specific to an apex variant and selecting stub/impl in that config_setting. We likely need only a handful of such config_setting(s), but determining that list requires iterating the build graph. Test: go test ./bp2build Change-Id: I9aa552e3d0bcf67513023c3a7d4bbf8fae464ee4 --- android/mutator.go | 23 +++++++++++++++++++++ bazel/properties.go | 11 +++++++++++ bp2build/build_conversion.go | 5 +++++ bp2build/build_conversion_test.go | 33 +++++++++++++++++++++++++++++++ bp2build/bzl_conversion_test.go | 3 +++ bp2build/testing.go | 23 +++++++++++++++++++++ starlark_fmt/format.go | 10 ++++++++++ 7 files changed, 108 insertions(+) diff --git a/android/mutator.go b/android/mutator.go index 4ec960472d..1eac1f0f89 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -273,6 +273,12 @@ type TopDownMutatorContext interface { // This function can be used to create alias definitions in a directory that is different // from the directory of the visited Soong module. CreateBazelTargetAliasInDir(dir string, name string, actual bazel.Label) + + // CreateBazelConfigSetting creates a config_setting in /BUILD.bazel + // build/bazel has several static config_setting(s) that are used in Bazel builds. + // This function can be used to createa additional config_setting(s) based on the build graph + // (e.g. a config_setting specific to an apex variant) + CreateBazelConfigSetting(csa bazel.ConfigSettingAttributes, ca CommonAttributes, dir string) } type topDownMutatorContext struct { @@ -738,6 +744,23 @@ func (t *topDownMutatorContext) CreateBazelTargetAliasInDir( mod.base().addBp2buildInfo(info) } +func (t *topDownMutatorContext) CreateBazelConfigSetting( + csa bazel.ConfigSettingAttributes, + ca CommonAttributes, + dir string) { + mod := t.Module() + info := bp2buildInfo{ + Dir: dir, + BazelProps: bazel.BazelTargetModuleProperties{ + Rule_class: "config_setting", + }, + CommonAttrs: ca, + ConstraintAttrs: constraintAttributes{}, + Attrs: &csa, + } + mod.base().addBp2buildInfo(info) +} + // ApexAvailableTags converts the apex_available property value of an ApexModule // module and returns it as a list of keyed tags. func ApexAvailableTags(mod Module) bazel.StringListAttribute { diff --git a/bazel/properties.go b/bazel/properties.go index 40d0ba37a2..1757bad490 100644 --- a/bazel/properties.go +++ b/bazel/properties.go @@ -1424,3 +1424,14 @@ func TryVariableSubstitution(s string, productVariable string) (string, bool) { sub := productVariableSubstitutionPattern.ReplaceAllString(s, "$("+productVariable+")") return sub, s != sub } + +// StringMapAttribute is a map of strings. +// The use case for this is storing the flag_values in a config_setting object. +// Bazel rules do not support map attributes, and this should NOT be used in Bazel rules. +type StringMapAttribute map[string]string + +// ConfigSettingAttributes stores the keys of a config_setting object. +type ConfigSettingAttributes struct { + // Each key in Flag_values is a label to a custom string_setting + Flag_values StringMapAttribute +} diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go index b7678a4697..a860484182 100644 --- a/bp2build/build_conversion.go +++ b/bp2build/build_conversion.go @@ -600,6 +600,11 @@ func prettyPrint(propertyValue reflect.Value, indent int, emitZeroValues bool) ( // TODO(b/164227191): implement pretty print for interfaces. // Interfaces are used for for arch, multilib and target properties. return "", nil + case reflect.Map: + if v, ok := propertyValue.Interface().(bazel.StringMapAttribute); ok { + return starlark_fmt.PrintStringStringDict(v, indent), nil + } + return "", fmt.Errorf("bp2build expects map of type map[string]string for field: %s", propertyValue) default: return "", fmt.Errorf( "unexpected kind for property struct field: %s", propertyValue.Kind()) diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go index 73ee26b607..1b64055f7a 100644 --- a/bp2build/build_conversion_test.go +++ b/bp2build/build_conversion_test.go @@ -1898,3 +1898,36 @@ func TestGenerateApiBazelTargets(t *testing.T) { Description: "Generating API contribution Bazel targets for custom module", }) } + +func TestGenerateConfigSetting(t *testing.T) { + bp := ` + custom { + name: "foo", + test_config_setting: true, + } + ` + expectedBazelTargets := []string{ + MakeBazelTargetNoRestrictions( + "config_setting", + "foo_config_setting", + AttrNameToString{ + "flag_values": `{ + "//build/bazel/rules/my_string_setting": "foo", + }`, + }, + ), + MakeBazelTarget( + "custom", + "foo", + AttrNameToString{}, + ), + } + registerCustomModule := func(ctx android.RegistrationContext) { + ctx.RegisterModuleType("custom", customModuleFactoryHostAndDevice) + } + RunBp2BuildTestCase(t, registerCustomModule, Bp2buildTestCase{ + Blueprint: bp, + ExpectedBazelTargets: expectedBazelTargets, + Description: "Generating API contribution Bazel targets for custom module", + }) +} diff --git a/bp2build/bzl_conversion_test.go b/bp2build/bzl_conversion_test.go index a8e557deb8..fa1bf8af62 100644 --- a/bp2build/bzl_conversion_test.go +++ b/bp2build/bzl_conversion_test.go @@ -108,6 +108,7 @@ custom = rule( "string_literal_prop": attr.string(), "string_prop": attr.string(), "string_ptr_prop": attr.string(), + "test_config_setting": attr.bool(), }, ) @@ -139,6 +140,7 @@ custom_defaults = rule( "string_literal_prop": attr.string(), "string_prop": attr.string(), "string_ptr_prop": attr.string(), + "test_config_setting": attr.bool(), }, ) @@ -170,6 +172,7 @@ custom_test_ = rule( "string_literal_prop": attr.string(), "string_prop": attr.string(), "string_ptr_prop": attr.string(), + "test_config_setting": attr.bool(), # test_prop start # "test_string_prop": attr.string(), # test_prop end diff --git a/bp2build/testing.go b/bp2build/testing.go index 6e919db2b0..fd99ff007a 100644 --- a/bp2build/testing.go +++ b/bp2build/testing.go @@ -317,6 +317,8 @@ type customProps struct { One_to_many_prop *bool Api *string // File describing the APIs of this module + + Test_config_setting *bool // Used to test generation of config_setting targets } type customModule struct { @@ -490,6 +492,27 @@ func (m *customModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) { } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name()}, attrs) + + if proptools.Bool(m.props.Test_config_setting) { + m.createConfigSetting(ctx) + } + +} + +func (m *customModule) createConfigSetting(ctx android.TopDownMutatorContext) { + csa := bazel.ConfigSettingAttributes{ + Flag_values: bazel.StringMapAttribute{ + "//build/bazel/rules/my_string_setting": m.Name(), + }, + } + ca := android.CommonAttributes{ + Name: m.Name() + "_config_setting", + } + ctx.CreateBazelConfigSetting( + csa, + ca, + ctx.ModuleDir(), + ) } var _ android.ApiProvider = (*customModule)(nil) diff --git a/starlark_fmt/format.go b/starlark_fmt/format.go index a97f71b87a..42095075cd 100644 --- a/starlark_fmt/format.go +++ b/starlark_fmt/format.go @@ -99,6 +99,16 @@ func PrintStringIntDict(dict map[string]int, indentLevel int) string { return PrintDict(valDict, indentLevel) } +// PrintStringStringDict returns a Starlark-compatible string formatted as dictionary with +// string keys and string values. +func PrintStringStringDict(dict map[string]string, indentLevel int) string { + valDict := make(map[string]string, len(dict)) + for k, v := range dict { + valDict[k] = fmt.Sprintf(`"%s"`, v) + } + return PrintDict(valDict, indentLevel) +} + // PrintDict returns a starlark-compatible string containing a dictionary with string keys and // values printed with no additional formatting. func PrintDict(dict map[string]string, indentLevel int) string { From 4242f1046226d5071f5e5eff18c011231eda0c44 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 19 Apr 2023 22:31:54 +0000 Subject: [PATCH 0059/1460] Create config_setting per apex_name These are created by bp2build in /build/bazel/rules/apex. Eventually these config_settings should likely be colocated with the source apex definition. Another alternative was to make Bazel's apex a macro that generates a config_setting. I did not pursue this further for now since it requires the apex_available of every allowlisted cc_library to also be allowlisted. This might not always be true (e.g. com.android.runtime) Test: go test ./bp2build Change-Id: Ibbb14b0d9c1491b3c79b7634a18d9d35b03922c1 --- bazel/configurability.go | 10 ++-- bp2build/cc_library_conversion_test.go | 29 ++++++++++++ cc/bp2build.go | 65 ++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/bazel/configurability.go b/bazel/configurability.go index 46802565c2..d01877dd5b 100644 --- a/bazel/configurability.go +++ b/bazel/configurability.go @@ -268,9 +268,8 @@ func (ct configurationType) validateConfig(config string) { case productVariables: // do nothing case osAndInApex: - if _, ok := osAndInApexMap[config]; !ok { - panic(fmt.Errorf("Unknown os+in_apex config: %s", config)) - } + // do nothing + // this axis can contain additional per-apex keys case inApex: if _, ok := inApexMap[config]; !ok { panic(fmt.Errorf("Unknown in_apex config: %s", config)) @@ -299,7 +298,10 @@ func (ca ConfigurationAxis) SelectKey(config string) string { } return fmt.Sprintf("%s:%s", productVariableBazelPackage, config) case osAndInApex: - return osAndInApexMap[config] + if ret, exists := osAndInApexMap[config]; exists { + return ret + } + return config case inApex: return inApexMap[config] default: diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index b61b0a7cfc..9f78195a63 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -4454,3 +4454,32 @@ cc_library { }, }) } + +// Test that a config_setting specific to an apex is created by cc_library. +func TestCcLibraryCreatesInApexConfigSetting(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library creates a config_setting for each apex in apex_available", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Dir: "build/bazel/rules/apex", + Blueprint: ` +cc_library { + name: "foo", + apex_available: [ + "//apex_available:platform", // This will be skipped, since it is equivalent to //build/bazel/rules/apex:android-non_apex + "myapex" + ], +}`, + ExpectedBazelTargets: []string{ + MakeBazelTargetNoRestrictions( + "config_setting", + "android-in_myapex", + AttrNameToString{ + "flag_values": `{ + "//build/bazel/rules/apex:apex_name": "myapex", + }`, + }, + ), + }, + }) +} diff --git a/cc/bp2build.go b/cc/bp2build.go index ad9d7021dc..5ba6ea039c 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -17,6 +17,7 @@ import ( "fmt" "path/filepath" "strings" + "sync" "android/soong/android" "android/soong/bazel" @@ -1196,6 +1197,63 @@ func availableToSameApexes(a, b []string) bool { return !differ } +var ( + apexConfigSettingKey = android.NewOnceKey("apexConfigSetting") + apexConfigSettingLock sync.Mutex +) + +func getApexConfigSettingMap(config android.Config) *map[string]bool { + return config.Once(apexConfigSettingKey, func() interface{} { + return &map[string]bool{} + }).(*map[string]bool) +} + +// Create a config setting for this apex in build/bazel/rules/apex +// The use case for this is stub/impl selection in cc libraries +// Long term, these config_setting(s) should be colocated with the respective apex definitions. +// Note that this is an anti-pattern: The config_setting should be created from the apex definition +// and not from a cc_library. +// This anti-pattern is needed today since not all apexes have been allowlisted. +func createInApexConfigSetting(ctx android.TopDownMutatorContext, apexName string) { + if apexName == android.AvailableToPlatform || apexName == android.AvailableToAnyApex { + // These correspond to android-non_apex and android-in_apex + return + } + apexConfigSettingLock.Lock() + defer apexConfigSettingLock.Unlock() + + // Return if a config_setting has already been created + acsm := getApexConfigSettingMap(ctx.Config()) + if _, exists := (*acsm)[apexName]; exists { + return + } + (*acsm)[apexName] = true + + csa := bazel.ConfigSettingAttributes{ + Flag_values: bazel.StringMapAttribute{ + "//build/bazel/rules/apex:apex_name": apexName, + }, + } + ca := android.CommonAttributes{ + Name: "android-in_" + apexName, + } + ctx.CreateBazelConfigSetting( + csa, + ca, + "build/bazel/rules/apex", + ) +} + +func inApexConfigSetting(apexAvailable string) string { + if apexAvailable == android.AvailableToPlatform { + return bazel.AndroidAndNonApex + } + if apexAvailable == android.AvailableToAnyApex { + return bazel.AndroidAndInApex + } + return "//build/bazel/rules/apex:android-in_" + apexAvailable +} + func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.ConfigurationAxis, config string, apexAvailable []string, dynamicLibs bazel.LabelList, dynamicDeps *bazel.LabelListAttribute, ind int, buildNonApexWithStubs bool) { @@ -1241,6 +1299,13 @@ func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.C dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex, bazel.FirstUniqueBazelLabelList(nonApexSelectValue)) } } + + // Create a config_setting for each apex_available. + // This will be used to select impl of a dep if dep is available to the same apex. + for _, aa := range apexAvailable { + createInApexConfigSetting(ctx.(android.TopDownMutatorContext), aa) + } + } func (la *linkerAttributes) convertStripProps(ctx android.BazelConversionPathContext, module *Module) { From 9b2ca5c8c92b28e053fd8b54f4f39c60943c1d39 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Fri, 28 Apr 2023 17:39:24 -0400 Subject: [PATCH 0060/1460] Prevent sh_test from migrating as sh_binary Test: CI Change-Id: I4cb789e3f1351fcb1635d58b72c265eac4bf62d8 --- sh/sh_binary.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sh/sh_binary.go b/sh/sh_binary.go index c921ca68a6..d2eede65da 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -474,16 +474,18 @@ func (s *ShTest) AndroidMkEntries() []android.AndroidMkEntries { }} } -func InitShBinaryModule(s *ShBinary) { +func initShBinaryModule(s *ShBinary, useBazel bool) { s.AddProperties(&s.properties) - android.InitBazelModule(s) + if useBazel { + android.InitBazelModule(s) + } } // sh_binary is for a shell script or batch file to be installed as an // executable binary to /bin. func ShBinaryFactory() android.Module { module := &ShBinary{} - InitShBinaryModule(module) + initShBinaryModule(module, true) android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibFirst) return module } @@ -492,7 +494,7 @@ func ShBinaryFactory() android.Module { // to $(HOST_OUT)/bin. func ShBinaryHostFactory() android.Module { module := &ShBinary{} - InitShBinaryModule(module) + initShBinaryModule(module, true) android.InitAndroidArchModule(module, android.HostSupported, android.MultilibFirst) return module } @@ -500,7 +502,7 @@ func ShBinaryHostFactory() android.Module { // sh_test defines a shell script based test module. func ShTestFactory() android.Module { module := &ShTest{} - InitShBinaryModule(&module.ShBinary) + initShBinaryModule(&module.ShBinary, false) module.AddProperties(&module.testProperties) android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibFirst) @@ -510,7 +512,7 @@ func ShTestFactory() android.Module { // sh_test_host defines a shell script based test module that runs on a host. func ShTestHostFactory() android.Module { module := &ShTest{} - InitShBinaryModule(&module.ShBinary) + initShBinaryModule(&module.ShBinary, false) module.AddProperties(&module.testProperties) // Default sh_test_host to unit_tests = true if module.testProperties.Test_options.Unit_test == nil { From bda0ca753e0ecb152441b6e722570d1052107301 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Fri, 28 Apr 2023 12:32:27 -0700 Subject: [PATCH 0061/1460] Use api_levels_released_versions from starlark Instead of exporting it to soong_injection. Ignore-AOSP-First: Needed to resolve merge conflict Bug: 279095899 Test: m nothing Change-Id: I7b93af233b7450848a475512b5f5682ece773c09 --- android/api_levels.go | 100 +++++++++++++++++------------------- android/sdk.go | 3 ++ bp2build/conversion.go | 7 ++- bp2build/conversion_test.go | 4 -- java/java.go | 5 +- sdk/update.go | 4 ++ 6 files changed, 62 insertions(+), 61 deletions(-) diff --git a/android/api_levels.go b/android/api_levels.go index 137fd9dada..fa919fda74 100644 --- a/android/api_levels.go +++ b/android/api_levels.go @@ -15,12 +15,10 @@ package android import ( + "android/soong/starlark_import" "encoding/json" "fmt" "strconv" - - "android/soong/bazel" - "android/soong/starlark_fmt" ) func init() { @@ -288,13 +286,17 @@ var LastWithoutModuleLibCoreSystemModules = uncheckedFinalApiLevel(31) // ReplaceFinalizedCodenames returns the API level number associated with that API level // if the `raw` input is the codename of an API level has been finalized. // If the input is *not* a finalized codename, the input is returned unmodified. -func ReplaceFinalizedCodenames(config Config, raw string) string { - num, ok := getFinalCodenamesMap(config)[raw] +func ReplaceFinalizedCodenames(config Config, raw string) (string, error) { + finalCodenamesMap, err := getFinalCodenamesMap(config) + if err != nil { + return raw, err + } + num, ok := finalCodenamesMap[raw] if !ok { - return raw + return raw, nil } - return strconv.Itoa(num) + return strconv.Itoa(num), nil } // ApiLevelFrom converts the given string `raw` to an ApiLevel. @@ -344,7 +346,11 @@ func ApiLevelFromUserWithConfig(config Config, raw string) (ApiLevel, error) { } } - canonical, ok := getApiLevelsMapReleasedVersions()[raw] + apiLevelsReleasedVersions, err := getApiLevelsMapReleasedVersions() + if err != nil { + return NoneApiLevel, err + } + canonical, ok := apiLevelsReleasedVersions[raw] if !ok { asInt, err := strconv.Atoi(raw) if err != nil { @@ -410,38 +416,21 @@ func GetApiLevelsJson(ctx PathContext) WritablePath { return PathForOutput(ctx, "api_levels.json") } -func getApiLevelsMapReleasedVersions() map[string]int { - return map[string]int{ - "G": 9, - "I": 14, - "J": 16, - "J-MR1": 17, - "J-MR2": 18, - "K": 19, - "L": 21, - "L-MR1": 22, - "M": 23, - "N": 24, - "N-MR1": 25, - "O": 26, - "O-MR1": 27, - "P": 28, - "Q": 29, - "R": 30, - "S": 31, - "S-V2": 32, - "Tiramisu": 33, - "UpsideDownCake": 34, - } +func getApiLevelsMapReleasedVersions() (map[string]int, error) { + return starlark_import.GetStarlarkValue[map[string]int]("api_levels_released_versions") } var finalCodenamesMapKey = NewOnceKey("FinalCodenamesMap") -func getFinalCodenamesMap(config Config) map[string]int { +func getFinalCodenamesMap(config Config) (map[string]int, error) { + type resultStruct struct { + result map[string]int + err error + } // This logic is replicated in starlark, if changing logic here update starlark code too // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=30;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061 - return config.Once(finalCodenamesMapKey, func() interface{} { - apiLevelsMap := getApiLevelsMapReleasedVersions() + result := config.Once(finalCodenamesMapKey, func() interface{} { + apiLevelsMap, err := getApiLevelsMapReleasedVersions() // TODO: Differentiate "current" and "future". // The code base calls it FutureApiLevel, but the spelling is "current", @@ -454,41 +443,44 @@ func getFinalCodenamesMap(config Config) map[string]int { // added in S, both of these are usable when building for "current" when // neither R nor S are final, but the S APIs stop being available in a // final R build. - if Bool(config.productVariables.Platform_sdk_final) { + if err == nil && Bool(config.productVariables.Platform_sdk_final) { apiLevelsMap["current"] = config.PlatformSdkVersion().FinalOrFutureInt() } - return apiLevelsMap - }).(map[string]int) + return resultStruct{apiLevelsMap, err} + }).(resultStruct) + return result.result, result.err } var apiLevelsMapKey = NewOnceKey("ApiLevelsMap") // ApiLevelsMap has entries for preview API levels -func GetApiLevelsMap(config Config) map[string]int { +func GetApiLevelsMap(config Config) (map[string]int, error) { + type resultStruct struct { + result map[string]int + err error + } // This logic is replicated in starlark, if changing logic here update starlark code too // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=23;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061 - return config.Once(apiLevelsMapKey, func() interface{} { - apiLevelsMap := getApiLevelsMapReleasedVersions() - for i, codename := range config.PlatformVersionAllPreviewCodenames() { - apiLevelsMap[codename] = previewAPILevelBase + i + result := config.Once(apiLevelsMapKey, func() interface{} { + apiLevelsMap, err := getApiLevelsMapReleasedVersions() + if err == nil { + for i, codename := range config.PlatformVersionAllPreviewCodenames() { + apiLevelsMap[codename] = previewAPILevelBase + i + } } - return apiLevelsMap - }).(map[string]int) + return resultStruct{apiLevelsMap, err} + }).(resultStruct) + return result.result, result.err } func (a *apiLevelsSingleton) GenerateBuildActions(ctx SingletonContext) { - apiLevelsMap := GetApiLevelsMap(ctx.Config()) + apiLevelsMap, err := GetApiLevelsMap(ctx.Config()) + if err != nil { + ctx.Errorf("%s\n", err) + return + } apiLevelsJson := GetApiLevelsJson(ctx) createApiLevelsJson(ctx, apiLevelsJson, apiLevelsMap) } - -func StarlarkApiLevelConfigs(config Config) string { - return fmt.Sprintf(bazel.GeneratedBazelFileWarning+` -_api_levels_released_versions = %s - -api_levels_released_versions = _api_levels_released_versions -`, starlark_fmt.PrintStringIntDict(getApiLevelsMapReleasedVersions(), 0), - ) -} diff --git a/android/sdk.go b/android/sdk.go index 63e0bbeec2..6b598ab9ef 100644 --- a/android/sdk.go +++ b/android/sdk.go @@ -830,6 +830,9 @@ type SdkMemberContext interface { // IsTargetBuildBeforeTiramisu return true if the target build release for which this snapshot is // being generated is before Tiramisu, i.e. S. IsTargetBuildBeforeTiramisu() bool + + // ModuleErrorf reports an error at the line number of the module type in the module definition. + ModuleErrorf(fmt string, args ...interface{}) } // ExportedComponentsInfo contains information about the components that this module exports to an diff --git a/bp2build/conversion.go b/bp2build/conversion.go index 608fcd8791..f5983323c6 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -59,14 +59,17 @@ func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) ([]BazelFil files = append(files, newFile("product_config", "arch_configuration.bzl", android.StarlarkArchConfigurations())) - apiLevelsContent, err := json.Marshal(android.GetApiLevelsMap(cfg)) + apiLevelsMap, err := android.GetApiLevelsMap(cfg) + if err != nil { + return nil, err + } + apiLevelsContent, err := json.Marshal(apiLevelsMap) if err != nil { return nil, err } files = append(files, newFile("api_levels", GeneratedBuildFileName, `exports_files(["api_levels.json"])`)) // TODO(b/269691302) value of apiLevelsContent is product variable dependent and should be avoided for soong injection files = append(files, newFile("api_levels", "api_levels.json", string(apiLevelsContent))) - files = append(files, newFile("api_levels", "api_levels.bzl", android.StarlarkApiLevelConfigs(cfg))) files = append(files, newFile("api_levels", "platform_versions.bzl", platformVersionContents(cfg))) files = append(files, newFile("allowlists", GeneratedBuildFileName, "")) diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index 2f5dc3cb74..379f83bd82 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -157,10 +157,6 @@ func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { dir: "api_levels", basename: "api_levels.json", }, - { - dir: "api_levels", - basename: "api_levels.bzl", - }, { dir: "api_levels", basename: "platform_versions.bzl", diff --git a/java/java.go b/java/java.go index 26fb3a7841..e859cb4e1c 100644 --- a/java/java.go +++ b/java/java.go @@ -815,7 +815,10 @@ func (p *librarySdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberCo // If the min_sdk_version was set then add the canonical representation of the API level to the // snapshot. if j.deviceProperties.Min_sdk_version != nil { - canonical := android.ReplaceFinalizedCodenames(ctx.SdkModuleContext().Config(), j.minSdkVersion.String()) + canonical, err := android.ReplaceFinalizedCodenames(ctx.SdkModuleContext().Config(), j.minSdkVersion.String()) + if err != nil { + ctx.ModuleErrorf("%s", err) + } p.MinSdkVersion = proptools.StringPtr(canonical) } diff --git a/sdk/update.go b/sdk/update.go index d98ab683d9..d3c59b0e48 100644 --- a/sdk/update.go +++ b/sdk/update.go @@ -1963,6 +1963,10 @@ type memberContext struct { requiredTraits android.SdkMemberTraitSet } +func (m *memberContext) ModuleErrorf(fmt string, args ...interface{}) { + m.sdkMemberContext.ModuleErrorf(fmt, args...) +} + func (m *memberContext) SdkModuleContext() android.ModuleContext { return m.sdkMemberContext } From 93893ba061e774c30a647d42e011838822153511 Mon Sep 17 00:00:00 2001 From: Yu Liu Date: Mon, 1 May 2023 13:49:52 -0700 Subject: [PATCH 0062/1460] Fix a bug where CppFlags should have been Cppflags Bug: 278789968 Test: unit test and manual build Change-Id: I9abb056e6b820414707247d1b19ba6aabffc26ea --- bp2build/cc_library_conversion_test.go | 40 ++++++++++++++++++++++++++ cc/bp2build.go | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 9f78195a63..e5bf8e0ee1 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -4483,3 +4483,43 @@ cc_library { }, }) } + +func TestCcLibraryCppFlagsInProductVariables(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library cppflags in product variables", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: soongCcLibraryPreamble + `cc_library { + name: "a", + srcs: ["a.cpp"], + cppflags: [ + "-Wextra", + "-DDEBUG_ONLY_CODE=0", + ], + product_variables: { + eng: { + cppflags: [ + "-UDEBUG_ONLY_CODE", + "-DDEBUG_ONLY_CODE=1", + ], + }, + }, + include_build_directory: false, +} +`, + ExpectedBazelTargets: makeCcLibraryTargets("a", AttrNameToString{ + "cppflags": `[ + "-Wextra", + "-DDEBUG_ONLY_CODE=0", + ] + select({ + "//build/bazel/product_variables:eng": [ + "-UDEBUG_ONLY_CODE", + "-DDEBUG_ONLY_CODE=1", + ], + "//conditions:default": [], + })`, + "srcs": `["a.cpp"]`, + }), + }, + ) +} diff --git a/cc/bp2build.go b/cc/bp2build.go index 1c9474134d..1f2653e295 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -528,7 +528,7 @@ func (ca *compilerAttributes) convertProductVariables(ctx android.BazelConversio productVarPropNameToAttribute := map[string]*bazel.StringListAttribute{ "Cflags": &ca.copts, "Asflags": &ca.asFlags, - "CppFlags": &ca.cppFlags, + "Cppflags": &ca.cppFlags, } for propName, attr := range productVarPropNameToAttribute { if productConfigProps, exists := productVariableProps[propName]; exists { From 3486740cc5ae4bf9f24083237929e1c6f48c37c1 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Fri, 28 Apr 2023 12:32:27 -0700 Subject: [PATCH 0063/1460] Use api_levels_released_versions from starlark Instead of exporting it to soong_injection. Bug: 279095899 Test: m nothing Change-Id: I7b93af233b7450848a475512b5f5682ece773c09 Merged-In: I7b93af233b7450848a475512b5f5682ece773c09 --- android/api_levels.go | 99 +++++++++++++++++-------------------- android/sdk.go | 3 ++ bp2build/conversion.go | 7 ++- bp2build/conversion_test.go | 4 -- java/java.go | 5 +- sdk/update.go | 4 ++ 6 files changed, 62 insertions(+), 60 deletions(-) diff --git a/android/api_levels.go b/android/api_levels.go index 7214ccbb68..fa919fda74 100644 --- a/android/api_levels.go +++ b/android/api_levels.go @@ -15,12 +15,10 @@ package android import ( + "android/soong/starlark_import" "encoding/json" "fmt" "strconv" - - "android/soong/bazel" - "android/soong/starlark_fmt" ) func init() { @@ -288,13 +286,17 @@ var LastWithoutModuleLibCoreSystemModules = uncheckedFinalApiLevel(31) // ReplaceFinalizedCodenames returns the API level number associated with that API level // if the `raw` input is the codename of an API level has been finalized. // If the input is *not* a finalized codename, the input is returned unmodified. -func ReplaceFinalizedCodenames(config Config, raw string) string { - num, ok := getFinalCodenamesMap(config)[raw] +func ReplaceFinalizedCodenames(config Config, raw string) (string, error) { + finalCodenamesMap, err := getFinalCodenamesMap(config) + if err != nil { + return raw, err + } + num, ok := finalCodenamesMap[raw] if !ok { - return raw + return raw, nil } - return strconv.Itoa(num) + return strconv.Itoa(num), nil } // ApiLevelFrom converts the given string `raw` to an ApiLevel. @@ -344,7 +346,11 @@ func ApiLevelFromUserWithConfig(config Config, raw string) (ApiLevel, error) { } } - canonical, ok := getApiLevelsMapReleasedVersions()[raw] + apiLevelsReleasedVersions, err := getApiLevelsMapReleasedVersions() + if err != nil { + return NoneApiLevel, err + } + canonical, ok := apiLevelsReleasedVersions[raw] if !ok { asInt, err := strconv.Atoi(raw) if err != nil { @@ -410,37 +416,21 @@ func GetApiLevelsJson(ctx PathContext) WritablePath { return PathForOutput(ctx, "api_levels.json") } -func getApiLevelsMapReleasedVersions() map[string]int { - return map[string]int{ - "G": 9, - "I": 14, - "J": 16, - "J-MR1": 17, - "J-MR2": 18, - "K": 19, - "L": 21, - "L-MR1": 22, - "M": 23, - "N": 24, - "N-MR1": 25, - "O": 26, - "O-MR1": 27, - "P": 28, - "Q": 29, - "R": 30, - "S": 31, - "S-V2": 32, - "Tiramisu": 33, - } +func getApiLevelsMapReleasedVersions() (map[string]int, error) { + return starlark_import.GetStarlarkValue[map[string]int]("api_levels_released_versions") } var finalCodenamesMapKey = NewOnceKey("FinalCodenamesMap") -func getFinalCodenamesMap(config Config) map[string]int { +func getFinalCodenamesMap(config Config) (map[string]int, error) { + type resultStruct struct { + result map[string]int + err error + } // This logic is replicated in starlark, if changing logic here update starlark code too // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=30;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061 - return config.Once(finalCodenamesMapKey, func() interface{} { - apiLevelsMap := getApiLevelsMapReleasedVersions() + result := config.Once(finalCodenamesMapKey, func() interface{} { + apiLevelsMap, err := getApiLevelsMapReleasedVersions() // TODO: Differentiate "current" and "future". // The code base calls it FutureApiLevel, but the spelling is "current", @@ -453,41 +443,44 @@ func getFinalCodenamesMap(config Config) map[string]int { // added in S, both of these are usable when building for "current" when // neither R nor S are final, but the S APIs stop being available in a // final R build. - if Bool(config.productVariables.Platform_sdk_final) { + if err == nil && Bool(config.productVariables.Platform_sdk_final) { apiLevelsMap["current"] = config.PlatformSdkVersion().FinalOrFutureInt() } - return apiLevelsMap - }).(map[string]int) + return resultStruct{apiLevelsMap, err} + }).(resultStruct) + return result.result, result.err } var apiLevelsMapKey = NewOnceKey("ApiLevelsMap") // ApiLevelsMap has entries for preview API levels -func GetApiLevelsMap(config Config) map[string]int { +func GetApiLevelsMap(config Config) (map[string]int, error) { + type resultStruct struct { + result map[string]int + err error + } // This logic is replicated in starlark, if changing logic here update starlark code too // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=23;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061 - return config.Once(apiLevelsMapKey, func() interface{} { - apiLevelsMap := getApiLevelsMapReleasedVersions() - for i, codename := range config.PlatformVersionAllPreviewCodenames() { - apiLevelsMap[codename] = previewAPILevelBase + i + result := config.Once(apiLevelsMapKey, func() interface{} { + apiLevelsMap, err := getApiLevelsMapReleasedVersions() + if err == nil { + for i, codename := range config.PlatformVersionAllPreviewCodenames() { + apiLevelsMap[codename] = previewAPILevelBase + i + } } - return apiLevelsMap - }).(map[string]int) + return resultStruct{apiLevelsMap, err} + }).(resultStruct) + return result.result, result.err } func (a *apiLevelsSingleton) GenerateBuildActions(ctx SingletonContext) { - apiLevelsMap := GetApiLevelsMap(ctx.Config()) + apiLevelsMap, err := GetApiLevelsMap(ctx.Config()) + if err != nil { + ctx.Errorf("%s\n", err) + return + } apiLevelsJson := GetApiLevelsJson(ctx) createApiLevelsJson(ctx, apiLevelsJson, apiLevelsMap) } - -func StarlarkApiLevelConfigs(config Config) string { - return fmt.Sprintf(bazel.GeneratedBazelFileWarning+` -_api_levels_released_versions = %s - -api_levels_released_versions = _api_levels_released_versions -`, starlark_fmt.PrintStringIntDict(getApiLevelsMapReleasedVersions(), 0), - ) -} diff --git a/android/sdk.go b/android/sdk.go index 63e0bbeec2..6b598ab9ef 100644 --- a/android/sdk.go +++ b/android/sdk.go @@ -830,6 +830,9 @@ type SdkMemberContext interface { // IsTargetBuildBeforeTiramisu return true if the target build release for which this snapshot is // being generated is before Tiramisu, i.e. S. IsTargetBuildBeforeTiramisu() bool + + // ModuleErrorf reports an error at the line number of the module type in the module definition. + ModuleErrorf(fmt string, args ...interface{}) } // ExportedComponentsInfo contains information about the components that this module exports to an diff --git a/bp2build/conversion.go b/bp2build/conversion.go index 608fcd8791..f5983323c6 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -59,14 +59,17 @@ func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) ([]BazelFil files = append(files, newFile("product_config", "arch_configuration.bzl", android.StarlarkArchConfigurations())) - apiLevelsContent, err := json.Marshal(android.GetApiLevelsMap(cfg)) + apiLevelsMap, err := android.GetApiLevelsMap(cfg) + if err != nil { + return nil, err + } + apiLevelsContent, err := json.Marshal(apiLevelsMap) if err != nil { return nil, err } files = append(files, newFile("api_levels", GeneratedBuildFileName, `exports_files(["api_levels.json"])`)) // TODO(b/269691302) value of apiLevelsContent is product variable dependent and should be avoided for soong injection files = append(files, newFile("api_levels", "api_levels.json", string(apiLevelsContent))) - files = append(files, newFile("api_levels", "api_levels.bzl", android.StarlarkApiLevelConfigs(cfg))) files = append(files, newFile("api_levels", "platform_versions.bzl", platformVersionContents(cfg))) files = append(files, newFile("allowlists", GeneratedBuildFileName, "")) diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index 2f5dc3cb74..379f83bd82 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -157,10 +157,6 @@ func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { dir: "api_levels", basename: "api_levels.json", }, - { - dir: "api_levels", - basename: "api_levels.bzl", - }, { dir: "api_levels", basename: "platform_versions.bzl", diff --git a/java/java.go b/java/java.go index 0da73281b8..d3e74fd1ce 100644 --- a/java/java.go +++ b/java/java.go @@ -815,7 +815,10 @@ func (p *librarySdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberCo // If the min_sdk_version was set then add the canonical representation of the API level to the // snapshot. if j.deviceProperties.Min_sdk_version != nil { - canonical := android.ReplaceFinalizedCodenames(ctx.SdkModuleContext().Config(), j.minSdkVersion.String()) + canonical, err := android.ReplaceFinalizedCodenames(ctx.SdkModuleContext().Config(), j.minSdkVersion.String()) + if err != nil { + ctx.ModuleErrorf("%s", err) + } p.MinSdkVersion = proptools.StringPtr(canonical) } diff --git a/sdk/update.go b/sdk/update.go index d98ab683d9..d3c59b0e48 100644 --- a/sdk/update.go +++ b/sdk/update.go @@ -1963,6 +1963,10 @@ type memberContext struct { requiredTraits android.SdkMemberTraitSet } +func (m *memberContext) ModuleErrorf(fmt string, args ...interface{}) { + m.sdkMemberContext.ModuleErrorf(fmt, args...) +} + func (m *memberContext) SdkModuleContext() android.ModuleContext { return m.sdkMemberContext } From 17d0ee2160e79cc5b30d7b779bc595712125d23b Mon Sep 17 00:00:00 2001 From: Justin Yun Date: Tue, 2 May 2023 14:56:38 +0900 Subject: [PATCH 0064/1460] VSDK: use relative install path when generating snapshots Without respecting the relative install path, the snapshot install path may collide when they use the same stem name. To avoid this, respect the relative install path when generating the snapshots. Bug: 279652606 Test: RECOVERY_SNAPSHOT_VERSION=current m recovery-snapshot Change-Id: Ic70a5855aeb2a6eda397b4dd06113dddb6ef70f4 --- cc/vendor_snapshot.go | 4 +-- cc/vendor_snapshot_test.go | 66 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go index e6e566025a..51f23c57ec 100644 --- a/cc/vendor_snapshot.go +++ b/cc/vendor_snapshot.go @@ -324,13 +324,13 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS } } } - snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, libType, stem) + snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, libType, m.RelativeInstallPath(), stem) ret = append(ret, copyFile(ctx, libPath, snapshotLibOut, fake)) } else { stem = ctx.ModuleName(m) } - propOut = filepath.Join(snapshotArchDir, targetArch, libType, stem+".json") + propOut = filepath.Join(snapshotArchDir, targetArch, libType, m.RelativeInstallPath(), stem+".json") } else if m.Binary() { // binary flags prop.Symlinks = m.Symlinks() diff --git a/cc/vendor_snapshot_test.go b/cc/vendor_snapshot_test.go index 619500e1d1..5b69a10beb 100644 --- a/cc/vendor_snapshot_test.go +++ b/cc/vendor_snapshot_test.go @@ -1657,3 +1657,69 @@ func TestRecoverySnapshotDirected(t *testing.T) { } } } + +func TestSnapshotInRelativeInstallPath(t *testing.T) { + bp := ` + cc_library { + name: "libvendor_available", + vendor_available: true, + nocrt: true, + } + + cc_library { + name: "libvendor_available_var", + vendor_available: true, + stem: "libvendor_available", + relative_install_path: "var", + nocrt: true, + } +` + + config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("29") + ctx := testCcWithConfig(t, config) + + // Check Vendor snapshot output. + + snapshotDir := "vendor-snapshot" + snapshotVariantPath := filepath.Join("out/soong", snapshotDir, "arm64") + snapshotSingleton := ctx.SingletonForTests("vendor-snapshot") + + var jsonFiles []string + + for _, arch := range [][]string{ + []string{"arm64", "armv8-a"}, + []string{"arm", "armv7-a-neon"}, + } { + archType := arch[0] + archVariant := arch[1] + archDir := fmt.Sprintf("arch-%s-%s", archType, archVariant) + + // For shared libraries, only non-VNDK vendor_available modules are captured + sharedVariant := fmt.Sprintf("android_vendor.29_%s_%s_shared", archType, archVariant) + sharedDir := filepath.Join(snapshotVariantPath, archDir, "shared") + sharedDirVar := filepath.Join(sharedDir, "var") + CheckSnapshot(t, ctx, snapshotSingleton, "libvendor_available", "libvendor_available.so", sharedDir, sharedVariant) + CheckSnapshot(t, ctx, snapshotSingleton, "libvendor_available_var", "libvendor_available.so", sharedDirVar, sharedVariant) + jsonFiles = append(jsonFiles, + filepath.Join(sharedDir, "libvendor_available.so.json"), + filepath.Join(sharedDirVar, "libvendor_available.so.json")) + } + + for _, jsonFile := range jsonFiles { + // verify all json files exist + if snapshotSingleton.MaybeOutput(jsonFile).Rule == nil { + t.Errorf("%q expected but not found", jsonFile) + } + } + + // fake snapshot should have all outputs in the normal snapshot. + fakeSnapshotSingleton := ctx.SingletonForTests("vendor-fake-snapshot") + for _, output := range snapshotSingleton.AllOutputs() { + fakeOutput := strings.Replace(output, "/vendor-snapshot/", "/fake/vendor-snapshot/", 1) + if fakeSnapshotSingleton.MaybeOutput(fakeOutput).Rule == nil { + t.Errorf("%q expected but not found", fakeOutput) + } + } +} From befbf05350146078d0ddd60b7768c83039456451 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Fri, 28 Apr 2023 11:23:58 -0400 Subject: [PATCH 0065/1460] add media.swcodec to Bazel staging allowlist Bug: 279756270 Change-Id: Id8c9734185ff4fa263a2b4c28b1b7ecafa7bb0c3 --- android/allowlists/allowlists.go | 45 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index a4df34c3f4..aca4b20844 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -193,6 +193,8 @@ var ( "frameworks/av": Bp2BuildDefaultTrue, "frameworks/av/media/codec2/components/aom": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/codecs": Bp2BuildDefaultTrueRecursively, + "frameworks/av/media/module/codecs": Bp2BuildDefaultTrueRecursively, + "frameworks/av/media/module/foundation": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/liberror": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/libshmem": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/audioaidlconversion": Bp2BuildDefaultTrueRecursively, @@ -448,7 +450,6 @@ var ( "code_coverage.policy", "code_coverage.policy.other", "codec2_soft_exports", - "codecs_g711dec", "com.android.media.swcodec", "com.android.media.swcodec-androidManifest", "com.android.media.swcodec-ld.config.txt", @@ -493,20 +494,6 @@ var ( "libandroidio", "libandroidio_srcs", "libserviceutils", - "libstagefright_amrnbenc", - "libstagefright_amrnbdec", - "libstagefright_amrwbdec", - "libstagefright_amrwbenc", - "libstagefright_amrnb_common", - "libstagefright_enc_common", - "libstagefright_flacdec", - "libstagefright_foundation", - "libstagefright_foundation_headers", - "libstagefright_headers", - "libstagefright_m4vh263dec", - "libstagefright_m4vh263enc", - "libstagefright_mp3dec", - "libstagefright_mp3dec_headers", "libsurfaceflinger_headers", "libsync", "libtextclassifier_hash_headers", @@ -734,6 +721,8 @@ var ( // Mainline Module Apps "CaptivePortalLogin", + + "libstagefright_headers", } Bp2buildModuleTypeAlwaysConvertList = []string{ @@ -947,13 +936,11 @@ var ( // cc_test with unconverted deps, or are device-only (and not verified to pass yet) "AMRWBEncTest", - "AmrnbDecoderTest", // depends on unconverted modules: libaudioutils, libsndfile - "AmrnbEncoderTest", // depends on unconverted modules: libaudioutils, libsndfile - "AmrwbDecoderTest", // depends on unconverted modules: libsndfile, libaudioutils - "AmrwbEncoderTest", // depends on unconverted modules: libaudioutils, libsndfile - "Mp3DecoderTest", // depends on unconverted modules: libsndfile, libaudioutils - "Mpeg4H263DecoderTest", // depends on unconverted modules: libstagefright_foundation - "Mpeg4H263EncoderTest", + "AmrnbDecoderTest", // depends on unconverted modules: libaudioutils, libsndfile + "AmrnbEncoderTest", // depends on unconverted modules: libaudioutils, libsndfile + "AmrwbDecoderTest", // depends on unconverted modules: libsndfile, libaudioutils + "AmrwbEncoderTest", // depends on unconverted modules: libaudioutils, libsndfile + "Mp3DecoderTest", // depends on unconverted modules: libsndfile, libaudioutils "avcdec", "avcenc", "bionic-benchmarks-tests", @@ -1433,6 +1420,10 @@ var ( "styleprotoslite", "CtsPkgInstallerConstants", "guava-android-testlib", + + "MetaDataBaseUnitTest", // depends on libstagefright + "AVCUtilsUnitTest", // depends on libstagefright + "ColorUtilsTest", // depends on libmediandk } MixedBuildsDisabledList = []string{ @@ -1534,7 +1525,15 @@ var ( // which will soon be added to the prod allowlist. // It is implicit that all modules in ProdMixedBuildsEnabledList will // also be built - do not add them to this list. - StagingMixedBuildsEnabledList = []string{} + StagingMixedBuildsEnabledList = []string{ + // M13: media.swcodec launch + "com.android.media.swcodec", + "test_com.android.media.swcodec", + "libstagefright_foundation", + "libcodec2_hidl@1.0", + "libcodec2_hidl@1.1", + "libcodec2_hidl@1.2", + } // These should be the libs that are included by the apexes in the ProdMixedBuildsEnabledList ProdDclaMixedBuildsEnabledList = []string{ From 20f0f780df380ee713289385f538866fa45e7551 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Mon, 1 May 2023 13:46:33 -0400 Subject: [PATCH 0066/1460] Correct allowlisting for override modules Prevoiusly, we were partially correcting for override modules in bp2build/mixed builds in some but not all places. Now we always check for override modules and ensure that Bazel_module properties are propagated properly for override modules. Bug: 279609939 Test: go test soong tests Change-Id: I5445aa71f4c8013315415a2ca9ab9c6b3be6bce0 --- android/bazel.go | 4 +- android/bazel_paths.go | 4 +- android/override_module.go | 51 +++---- apex/apex.go | 3 - apex/bp2build_test.go | 278 ++++++++++++++++++++++++------------- 5 files changed, 212 insertions(+), 128 deletions(-) diff --git a/android/bazel.go b/android/bazel.go index 58d9d87dbe..3fe063c51f 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -431,7 +431,7 @@ func (b *BazelModuleBase) shouldConvertWithBp2build(ctx bazelOtherModuleContext, } propValue := b.bazelProperties.Bazel_module.Bp2build_available - packagePath := ctx.OtherModuleDir(module) + packagePath := moduleDirWithPossibleOverride(ctx, module) // Modules in unit tests which are enabled in the allowlist by type or name // trigger this conditional because unit tests run under the "." package path @@ -440,7 +440,7 @@ func (b *BazelModuleBase) shouldConvertWithBp2build(ctx bazelOtherModuleContext, return true } - moduleName := module.Name() + moduleName := moduleNameWithPossibleOverride(ctx, module) allowlist := ctx.Config().Bp2buildPackageConfig moduleNameAllowed := allowlist.moduleAlwaysConvert[moduleName] moduleTypeAllowed := allowlist.moduleTypeAlwaysConvert[ctx.OtherModuleType(module)] diff --git a/android/bazel_paths.go b/android/bazel_paths.go index bad7baf06a..ddbdbd49bb 100644 --- a/android/bazel_paths.go +++ b/android/bazel_paths.go @@ -453,8 +453,8 @@ func samePackage(label1, label2 string) bool { } func bp2buildModuleLabel(ctx BazelConversionContext, module blueprint.Module) string { - moduleName := ctx.OtherModuleName(module) - moduleDir := ctx.OtherModuleDir(module) + moduleName := moduleNameWithPossibleOverride(ctx, module) + moduleDir := moduleDirWithPossibleOverride(ctx, module) if moduleDir == Bp2BuildTopLevel { moduleDir = "" } diff --git a/android/override_module.go b/android/override_module.go index 86f582b766..9e95c0f10e 100644 --- a/android/override_module.go +++ b/android/override_module.go @@ -28,7 +28,6 @@ package android // module based on it. import ( - "fmt" "sort" "sync" @@ -121,7 +120,7 @@ type OverridableModule interface { addOverride(o OverrideModule) getOverrides() []OverrideModule - override(ctx BaseModuleContext, o OverrideModule) + override(ctx BaseModuleContext, m Module, o OverrideModule) GetOverriddenBy() string GetOverriddenByModuleDir() string @@ -192,7 +191,8 @@ func (b *OverridableModuleBase) setOverridesProperty(overridesProperty *[]string } // Overrides a base module with the given OverrideModule. -func (b *OverridableModuleBase) override(ctx BaseModuleContext, o OverrideModule) { +func (b *OverridableModuleBase) override(ctx BaseModuleContext, m Module, o OverrideModule) { + for _, p := range b.overridableProperties { for _, op := range o.getOverridingProperties() { if proptools.TypeEqual(p, op) { @@ -214,6 +214,17 @@ func (b *OverridableModuleBase) override(ctx BaseModuleContext, o OverrideModule } b.overridableModuleProperties.OverriddenBy = o.Name() b.overridableModuleProperties.OverriddenByModuleDir = o.ModuleDir() + + if oBazelable, ok := o.base().module.(Bazelable); ok { + if bBazelable, ok := m.(Bazelable); ok { + oProps := oBazelable.bazelProps() + bProps := bBazelable.bazelProps() + bProps.Bazel_module.Bp2build_available = oProps.Bazel_module.Bp2build_available + bProps.Bazel_module.Label = oProps.Bazel_module.Label + } else { + ctx.ModuleErrorf("Override type cannot be Bazelable if original module type is not Bazelable %v %v.", o.Name(), m.Name()) + } + } } // GetOverriddenBy returns the name of the override module that has overridden this module. @@ -302,7 +313,7 @@ func performOverrideMutator(ctx BottomUpMutatorContext) { // is specified. ctx.AliasVariation(variants[0]) for i, o := range overrides { - mods[i+1].(OverridableModule).override(ctx, o) + mods[i+1].(OverridableModule).override(ctx, mods[i+1], o) if o.getOverriddenByPrebuilt() { // The overriding module itself, too, is overridden by a prebuilt. // Copy the flag and hide it in make @@ -340,34 +351,26 @@ func replaceDepsOnOverridingModuleMutator(ctx BottomUpMutatorContext) { // variant of this OverridableModule, or ctx.ModuleName() if this module is not an OverridableModule // or if this variant is not overridden. func ModuleNameWithPossibleOverride(ctx BazelConversionContext) string { - if overridable, ok := ctx.Module().(OverridableModule); ok { + return moduleNameWithPossibleOverride(ctx, ctx.Module()) +} + +func moduleNameWithPossibleOverride(ctx bazelOtherModuleContext, module blueprint.Module) string { + if overridable, ok := module.(OverridableModule); ok { if o := overridable.GetOverriddenBy(); o != "" { return o } } - return ctx.OtherModuleName(ctx.Module()) + return ctx.OtherModuleName(module) } -// ModuleDirWithPossibleOverride returns the dir of the OverrideModule that overrides the current -// variant of this OverridableModule, or ctx.ModuleName() if this module is not an OverridableModule -// or if this variant is not overridden. -func moduleDirWithPossibleOverride(ctx BazelConversionContext) string { - if overridable, ok := ctx.Module().(OverridableModule); ok { +// moduleDirWithPossibleOverride returns the dir of the OverrideModule that overrides the current +// variant of the given OverridableModule, or ctx.OtherModuleName() if the module is not an +// OverridableModule or if the variant is not overridden. +func moduleDirWithPossibleOverride(ctx bazelOtherModuleContext, module blueprint.Module) string { + if overridable, ok := module.(OverridableModule); ok { if o := overridable.GetOverriddenByModuleDir(); o != "" { return o } } - return ctx.OtherModuleDir(ctx.Module()) -} - -// MaybeBp2buildLabelOfOverridingModule returns the bazel label of the -// overriding module of an OverridableModule (e.g. override_apex label of a base -// apex), or the module's label itself if not overridden. -func MaybeBp2buildLabelOfOverridingModule(ctx BazelConversionContext) string { - moduleName := ModuleNameWithPossibleOverride(ctx) - moduleDir := moduleDirWithPossibleOverride(ctx) - if moduleDir == Bp2BuildTopLevel { - moduleDir = "" - } - return fmt.Sprintf("//%s:%s", moduleDir, moduleName) + return ctx.OtherModuleDir(module) } diff --git a/apex/apex.go b/apex/apex.go index baf4737d82..3cd541ce58 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1964,9 +1964,6 @@ func (a *apexBundle) QueueBazelCall(ctx android.BaseModuleContext) { // overridden by different override_apex modules (e.g. Google or Go variants), // which is handled by the overrides mutators. func (a *apexBundle) GetBazelLabel(ctx android.BazelConversionPathContext, module blueprint.Module) string { - if _, ok := ctx.Module().(android.OverridableModule); ok { - return android.MaybeBp2buildLabelOfOverridingModule(ctx) - } return a.BazelModuleBase.GetBazelLabel(ctx, a) } diff --git a/apex/bp2build_test.go b/apex/bp2build_test.go index 2a0f6e9e25..b1b6a75e8a 100644 --- a/apex/bp2build_test.go +++ b/apex/bp2build_test.go @@ -15,7 +15,10 @@ package apex import ( "android/soong/android" + "android/soong/android/allowlists" "android/soong/bazel/cquery" + "fmt" + "path/filepath" "strings" "testing" ) @@ -326,7 +329,7 @@ apex { } func TestOverrideApexImageInMixedBuilds(t *testing.T) { - bp := ` + originalBp := ` apex_key{ name: "foo_key", } @@ -340,122 +343,203 @@ apex { min_sdk_version: "31", package_name: "pkg_name", file_contexts: ":myapex-file_contexts", - bazel_module: { label: "//:foo" }, -} + %s +}` + overrideBp := ` override_apex { name: "override_foo", key: "override_foo_key", package_name: "override_pkg_name", base: "foo", - bazel_module: { label: "//:override_foo" }, + %s } ` - outputBaseDir := "out/bazel" - result := android.GroupFixturePreparers( - prepareForApexTest, - android.FixtureModifyConfig(func(config android.Config) { - config.BazelContext = android.MockBazelContext{ - OutputBaseDir: outputBaseDir, - LabelToApexInfo: map[string]cquery.ApexInfo{ - "//:foo": cquery.ApexInfo{ - // ApexInfo Starlark provider - SignedOutput: "signed_out.apex", - UnsignedOutput: "unsigned_out.apex", - BundleKeyInfo: []string{"public_key", "private_key"}, - ContainerKeyInfo: []string{"container_cert", "container_private"}, - SymbolsUsedByApex: "foo_using.txt", - JavaSymbolsUsedByApex: "foo_using.xml", - BundleFile: "apex_bundle.zip", - InstalledFiles: "installed-files.txt", - RequiresLibs: []string{"//path/c:c", "//path/d:d"}, - - // unused - PackageName: "pkg_name", - ProvidesLibs: []string{"a", "b"}, - - // ApexMkInfo Starlark provider - MakeModulesToInstall: []string{"c"}, // d deliberately omitted - }, - "//:override_foo": cquery.ApexInfo{ - // ApexInfo Starlark provider - SignedOutput: "override_signed_out.apex", - UnsignedOutput: "override_unsigned_out.apex", - BundleKeyInfo: []string{"override_public_key", "override_private_key"}, - ContainerKeyInfo: []string{"override_container_cert", "override_container_private"}, - SymbolsUsedByApex: "override_foo_using.txt", - JavaSymbolsUsedByApex: "override_foo_using.xml", - BundleFile: "override_apex_bundle.zip", - InstalledFiles: "override_installed-files.txt", - RequiresLibs: []string{"//path/c:c", "//path/d:d"}, - - // unused - PackageName: "override_pkg_name", - ProvidesLibs: []string{"a", "b"}, + originalApexBpDir := "original" + originalApexName := "foo" + overrideApexBpDir := "override" + overrideApexName := "override_foo" + + defaultApexLabel := fmt.Sprintf("//%s:%s", originalApexBpDir, originalApexName) + defaultOverrideApexLabel := fmt.Sprintf("//%s:%s", overrideApexBpDir, overrideApexName) + + testCases := []struct { + desc string + bazelModuleProp string + apexLabel string + overrideBazelModuleProp string + overrideApexLabel string + bp2buildConfiguration android.Bp2BuildConversionAllowlist + }{ + { + desc: "both explicit labels", + bazelModuleProp: `bazel_module: { label: "//:foo" },`, + apexLabel: "//:foo", + overrideBazelModuleProp: `bazel_module: { label: "//:override_foo" },`, + overrideApexLabel: "//:override_foo", + bp2buildConfiguration: android.NewBp2BuildAllowlist(), + }, + { + desc: "both explicitly allowed", + bazelModuleProp: `bazel_module: { bp2build_available: true },`, + apexLabel: defaultApexLabel, + overrideBazelModuleProp: `bazel_module: { bp2build_available: true },`, + overrideApexLabel: defaultOverrideApexLabel, + bp2buildConfiguration: android.NewBp2BuildAllowlist(), + }, + { + desc: "original allowed by dir, override allowed by name", + apexLabel: defaultApexLabel, + overrideApexLabel: defaultOverrideApexLabel, + bp2buildConfiguration: android.NewBp2BuildAllowlist().SetDefaultConfig( + map[string]allowlists.BazelConversionConfigEntry{ + originalApexBpDir: allowlists.Bp2BuildDefaultTrue, + }).SetModuleAlwaysConvertList([]string{ + overrideApexName, + }), + }, + { + desc: "both allowed by name", + apexLabel: defaultApexLabel, + overrideApexLabel: defaultOverrideApexLabel, + bp2buildConfiguration: android.NewBp2BuildAllowlist().SetModuleAlwaysConvertList([]string{ + originalApexName, + overrideApexName, + }), + }, + { + desc: "override allowed by name", + apexLabel: defaultApexLabel, + overrideApexLabel: defaultOverrideApexLabel, + bp2buildConfiguration: android.NewBp2BuildAllowlist().SetModuleAlwaysConvertList([]string{ + overrideApexName, + }), + }, + { + desc: "override allowed by dir", + apexLabel: defaultApexLabel, + overrideApexLabel: defaultOverrideApexLabel, + bp2buildConfiguration: android.NewBp2BuildAllowlist().SetDefaultConfig( + map[string]allowlists.BazelConversionConfigEntry{ + overrideApexBpDir: allowlists.Bp2BuildDefaultTrue, + }).SetModuleAlwaysConvertList([]string{}), + }, + } - // ApexMkInfo Starlark provider - MakeModulesToInstall: []string{"c"}, // d deliberately omitted - }, - }, + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + outputBaseDir := "out/bazel" + result := android.GroupFixturePreparers( + prepareForApexTest, + android.FixtureAddTextFile(filepath.Join(originalApexBpDir, "Android.bp"), fmt.Sprintf(originalBp, tc.bazelModuleProp)), + android.FixtureAddTextFile(filepath.Join(overrideApexBpDir, "Android.bp"), fmt.Sprintf(overrideBp, tc.overrideBazelModuleProp)), + android.FixtureModifyContext(func(ctx *android.TestContext) { + ctx.RegisterBp2BuildConfig(tc.bp2buildConfiguration) + }), + android.FixtureModifyConfig(func(config android.Config) { + config.BazelContext = android.MockBazelContext{ + OutputBaseDir: outputBaseDir, + LabelToApexInfo: map[string]cquery.ApexInfo{ + tc.apexLabel: cquery.ApexInfo{ + // ApexInfo Starlark provider + SignedOutput: "signed_out.apex", + UnsignedOutput: "unsigned_out.apex", + BundleKeyInfo: []string{"public_key", "private_key"}, + ContainerKeyInfo: []string{"container_cert", "container_private"}, + SymbolsUsedByApex: "foo_using.txt", + JavaSymbolsUsedByApex: "foo_using.xml", + BundleFile: "apex_bundle.zip", + InstalledFiles: "installed-files.txt", + RequiresLibs: []string{"//path/c:c", "//path/d:d"}, + + // unused + PackageName: "pkg_name", + ProvidesLibs: []string{"a", "b"}, + + // ApexMkInfo Starlark provider + MakeModulesToInstall: []string{"c"}, // d deliberately omitted + }, + tc.overrideApexLabel: cquery.ApexInfo{ + // ApexInfo Starlark provider + SignedOutput: "override_signed_out.apex", + UnsignedOutput: "override_unsigned_out.apex", + BundleKeyInfo: []string{"override_public_key", "override_private_key"}, + ContainerKeyInfo: []string{"override_container_cert", "override_container_private"}, + SymbolsUsedByApex: "override_foo_using.txt", + JavaSymbolsUsedByApex: "override_foo_using.xml", + BundleFile: "override_apex_bundle.zip", + InstalledFiles: "override_installed-files.txt", + RequiresLibs: []string{"//path/c:c", "//path/d:d"}, + + // unused + PackageName: "override_pkg_name", + ProvidesLibs: []string{"a", "b"}, + + // ApexMkInfo Starlark provider + MakeModulesToInstall: []string{"c"}, // d deliberately omitted + }, + }, + } + }), + ).RunTest(t) + + m := result.ModuleForTests("foo", "android_common_override_foo_foo_image").Module() + ab, ok := m.(*apexBundle) + if !ok { + t.Fatalf("Expected module to be an apexBundle, was not") } - }), - ).RunTestWithBp(t, bp) - m := result.ModuleForTests("foo", "android_common_override_foo_foo_image").Module() - ab, ok := m.(*apexBundle) - if !ok { - t.Fatalf("Expected module to be an apexBundle, was not") - } - - if w, g := "out/bazel/execroot/__main__/override_public_key", ab.publicKeyFile.String(); w != g { - t.Errorf("Expected public key %q, got %q", w, g) - } + if w, g := "out/bazel/execroot/__main__/override_public_key", ab.publicKeyFile.String(); w != g { + t.Errorf("Expected public key %q, got %q", w, g) + } - if w, g := "out/bazel/execroot/__main__/override_private_key", ab.privateKeyFile.String(); w != g { - t.Errorf("Expected private key %q, got %q", w, g) - } + if w, g := "out/bazel/execroot/__main__/override_private_key", ab.privateKeyFile.String(); w != g { + t.Errorf("Expected private key %q, got %q", w, g) + } - if w, g := "out/bazel/execroot/__main__/override_container_cert", ab.containerCertificateFile.String(); w != g { - t.Errorf("Expected public container key %q, got %q", w, g) - } + if w, g := "out/bazel/execroot/__main__/override_container_cert", ab.containerCertificateFile; g != nil && w != g.String() { + t.Errorf("Expected public container key %q, got %q", w, g) + } - if w, g := "out/bazel/execroot/__main__/override_container_private", ab.containerPrivateKeyFile.String(); w != g { - t.Errorf("Expected private container key %q, got %q", w, g) - } + if w, g := "out/bazel/execroot/__main__/override_container_private", ab.containerPrivateKeyFile; g != nil && w != g.String() { + t.Errorf("Expected private container key %q, got %q", w, g) + } - if w, g := "out/bazel/execroot/__main__/override_signed_out.apex", ab.outputFile.String(); w != g { - t.Errorf("Expected output file %q, got %q", w, g) - } + if w, g := "out/bazel/execroot/__main__/override_signed_out.apex", ab.outputFile.String(); w != g { + t.Errorf("Expected output file %q, got %q", w, g) + } - if w, g := "out/bazel/execroot/__main__/override_foo_using.txt", ab.nativeApisUsedByModuleFile.String(); w != g { - t.Errorf("Expected output file %q, got %q", w, g) - } + if w, g := "out/bazel/execroot/__main__/override_foo_using.txt", ab.nativeApisUsedByModuleFile.String(); w != g { + t.Errorf("Expected output file %q, got %q", w, g) + } - if w, g := "out/bazel/execroot/__main__/override_foo_using.xml", ab.javaApisUsedByModuleFile.String(); w != g { - t.Errorf("Expected output file %q, got %q", w, g) - } + if w, g := "out/bazel/execroot/__main__/override_foo_using.xml", ab.javaApisUsedByModuleFile.String(); w != g { + t.Errorf("Expected output file %q, got %q", w, g) + } - if w, g := "out/bazel/execroot/__main__/override_installed-files.txt", ab.installedFilesFile.String(); w != g { - t.Errorf("Expected installed-files.txt %q, got %q", w, g) - } + if w, g := "out/bazel/execroot/__main__/override_installed-files.txt", ab.installedFilesFile.String(); w != g { + t.Errorf("Expected installed-files.txt %q, got %q", w, g) + } - mkData := android.AndroidMkDataForTest(t, result.TestContext, m) - var builder strings.Builder - mkData.Custom(&builder, "override_foo", "BAZEL_TARGET_", "", mkData) + mkData := android.AndroidMkDataForTest(t, result.TestContext, m) + var builder strings.Builder + mkData.Custom(&builder, "override_foo", "BAZEL_TARGET_", "", mkData) - data := builder.String() - if w := "ALL_MODULES.$(my_register_name).BUNDLE := out/bazel/execroot/__main__/override_apex_bundle.zip"; !strings.Contains(data, w) { - t.Errorf("Expected %q in androidmk data, but did not find %q", w, data) - } - if w := "$(call dist-for-goals,checkbuild,out/bazel/execroot/__main__/override_installed-files.txt:override_foo-installed-files.txt)"; !strings.Contains(data, w) { - t.Errorf("Expected %q in androidmk data, but did not find %q", w, data) - } + data := builder.String() + if w := "ALL_MODULES.$(my_register_name).BUNDLE := out/bazel/execroot/__main__/override_apex_bundle.zip"; !strings.Contains(data, w) { + t.Errorf("Expected %q in androidmk data, but did not find %q", w, data) + } + if w := "$(call dist-for-goals,checkbuild,out/bazel/execroot/__main__/override_installed-files.txt:override_foo-installed-files.txt)"; !strings.Contains(data, w) { + t.Errorf("Expected %q in androidmk data, but did not find %q", w, data) + } - // make modules to be installed to system - if len(ab.makeModulesToInstall) != 1 && ab.makeModulesToInstall[0] != "c" { - t.Errorf("Expected makeModulestoInstall slice to only contain 'c', got %q", ab.makeModulesToInstall) - } - if w := "LOCAL_REQUIRED_MODULES := c"; !strings.Contains(data, w) { - t.Errorf("Expected %q in androidmk data, but did not find it in %q", w, data) + // make modules to be installed to system + if len(ab.makeModulesToInstall) != 1 || ab.makeModulesToInstall[0] != "c" { + t.Errorf("Expected makeModulestoInstall slice to only contain 'c', got %q", ab.makeModulesToInstall) + } + if w := "LOCAL_REQUIRED_MODULES := c"; !strings.Contains(data, w) { + t.Errorf("Expected %q in androidmk data, but did not find it in %q", w, data) + } + }) } } From bd3a16b5e75e15d91d6a8cb16ccc53d9d8e202a6 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 25 Apr 2023 11:30:51 -0700 Subject: [PATCH 0067/1460] Install sdk variants in unbundled builds and package uninstallable variants This effectively undoes both If6c3ee82d588e2742c85cef7244c090c93f38b8e and I682e4f1f477f3024f7719dfaa67006ef335e0640. SDK variants are now installed again, which will fix unbundled builds of cc_test modules. The platform variants used by com.android.virt are now packagable even though they are not installable. Fix the original problem in b/194403710 by adding a flag to platform variants of modules in apexes that are not platform available, and using that to prevent install and packaging dependencies. That allows the HideFromMake flag to go back to being used for preventing install dependencies but not packaging dependencies. Test: TestPackagingWithSkipInstallDeps Test: TestFileSystemShouldInstallCoreVariantIfTargetBuildAppsIsSet Test: TestFileSystemShouldSkipApexLibraries Bug: 194403710 Bug: 268582372 Fixes: 274443025 Change-Id: If5418df3ddbb940bd631caebdf38daa81e71f40e --- android/deptag.go | 4 +- android/license_metadata.go | 2 +- android/module.go | 30 ++++++++++-- android/packaging_test.go | 4 +- apex/apex_test.go | 52 +++++++++++++++++++++ cc/androidmk.go | 13 ++---- cc/sdk.go | 6 +-- cc/sdk_test.go | 92 ------------------------------------- 8 files changed, 92 insertions(+), 111 deletions(-) diff --git a/android/deptag.go b/android/deptag.go index be5c35c8db..a15443b4a6 100644 --- a/android/deptag.go +++ b/android/deptag.go @@ -34,10 +34,10 @@ func (i InstallAlwaysNeededDependencyTag) InstallDepNeeded() bool { var _ InstallNeededDependencyTag = InstallAlwaysNeededDependencyTag{} -// IsInstallDepNeeded returns true if the dependency tag implements the InstallNeededDependencyTag +// IsInstallDepNeededTag returns true if the dependency tag implements the InstallNeededDependencyTag // interface and the InstallDepNeeded returns true, meaning that the installed files of the parent // should depend on the installed files of the child. -func IsInstallDepNeeded(tag blueprint.DependencyTag) bool { +func IsInstallDepNeededTag(tag blueprint.DependencyTag) bool { if i, ok := tag.(InstallNeededDependencyTag); ok { return i.InstallDepNeeded() } diff --git a/android/license_metadata.go b/android/license_metadata.go index 18b63d3105..73000a9fa3 100644 --- a/android/license_metadata.go +++ b/android/license_metadata.go @@ -74,7 +74,7 @@ func buildLicenseMetadata(ctx ModuleContext, licenseMetadataFile WritablePath) { if ctx.OtherModuleHasProvider(dep, LicenseMetadataProvider) { info := ctx.OtherModuleProvider(dep, LicenseMetadataProvider).(*LicenseMetadataInfo) allDepMetadataFiles = append(allDepMetadataFiles, info.LicenseMetadataPath) - if isContainer || IsInstallDepNeeded(ctx.OtherModuleDependencyTag(dep)) { + if isContainer || isInstallDepNeeded(dep, ctx.OtherModuleDependencyTag(dep)) { allDepMetadataDepSets = append(allDepMetadataDepSets, info.LicenseMetadataDepSet) } diff --git a/android/module.go b/android/module.go index c8670c31a3..db602a0aa3 100644 --- a/android/module.go +++ b/android/module.go @@ -925,6 +925,12 @@ type commonProperties struct { // and don't create a rule to install the file. SkipInstall bool `blueprint:"mutated"` + // UninstallableApexPlatformVariant is set by MakeUninstallable called by the apex + // mutator. MakeUninstallable also sets HideFromMake. UninstallableApexPlatformVariant + // is used to avoid adding install or packaging dependencies into libraries provided + // by apexes. + UninstallableApexPlatformVariant bool `blueprint:"mutated"` + // Whether the module has been replaced by a prebuilt ReplacedByPrebuilt bool `blueprint:"mutated"` @@ -2009,6 +2015,7 @@ func (m *ModuleBase) IsSkipInstall() bool { // have other side effects, in particular when it adds a NOTICE file target, // which other install targets might depend on. func (m *ModuleBase) MakeUninstallable() { + m.commonProperties.UninstallableApexPlatformVariant = true m.HideFromMake() } @@ -2038,13 +2045,19 @@ func (m *ModuleBase) EffectiveLicenseFiles() Paths { } // computeInstallDeps finds the installed paths of all dependencies that have a dependency -// tag that is annotated as needing installation via the IsInstallDepNeeded method. +// tag that is annotated as needing installation via the isInstallDepNeeded method. func (m *ModuleBase) computeInstallDeps(ctx ModuleContext) ([]*installPathsDepSet, []*packagingSpecsDepSet) { var installDeps []*installPathsDepSet var packagingSpecs []*packagingSpecsDepSet ctx.VisitDirectDeps(func(dep Module) { - if IsInstallDepNeeded(ctx.OtherModuleDependencyTag(dep)) && !dep.IsHideFromMake() && !dep.IsSkipInstall() { - installDeps = append(installDeps, dep.base().installFilesDepSet) + if isInstallDepNeeded(dep, ctx.OtherModuleDependencyTag(dep)) { + // Installation is still handled by Make, so anything hidden from Make is not + // installable. + if !dep.IsHideFromMake() && !dep.IsSkipInstall() { + installDeps = append(installDeps, dep.base().installFilesDepSet) + } + // Add packaging deps even when the dependency is not installed so that uninstallable + // modules can still be packaged. Often the package will be installed instead. packagingSpecs = append(packagingSpecs, dep.base().packagingSpecsDepSet) } }) @@ -2052,6 +2065,17 @@ func (m *ModuleBase) computeInstallDeps(ctx ModuleContext) ([]*installPathsDepSe return installDeps, packagingSpecs } +// isInstallDepNeeded returns true if installing the output files of the current module +// should also install the output files of the given dependency and dependency tag. +func isInstallDepNeeded(dep Module, tag blueprint.DependencyTag) bool { + // Don't add a dependency from the platform to a library provided by an apex. + if dep.base().commonProperties.UninstallableApexPlatformVariant { + return false + } + // Only install modules if the dependency tag is an InstallDepNeeded tag. + return IsInstallDepNeededTag(tag) +} + func (m *ModuleBase) FilesToInstall() InstallPaths { return m.installFiles } diff --git a/android/packaging_test.go b/android/packaging_test.go index 91ac1f386a..383343723d 100644 --- a/android/packaging_test.go +++ b/android/packaging_test.go @@ -373,7 +373,7 @@ func TestPackagingBaseSingleTarget(t *testing.T) { func TestPackagingWithSkipInstallDeps(t *testing.T) { // package -[dep]-> foo -[dep]-> bar -[dep]-> baz - // OK SKIPPED + // Packaging should continue transitively through modules that are not installed. multiTarget := false runPackagingTest(t, multiTarget, ` @@ -396,5 +396,5 @@ func TestPackagingWithSkipInstallDeps(t *testing.T) { name: "package", deps: ["foo"], } - `, []string{"lib64/foo"}) + `, []string{"lib64/foo", "lib64/bar", "lib64/baz"}) } diff --git a/apex/apex_test.go b/apex/apex_test.go index 3ba4d8d553..d0d4d0d937 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -33,6 +33,7 @@ import ( "android/soong/cc" "android/soong/dexpreopt" prebuilt_etc "android/soong/etc" + "android/soong/filesystem" "android/soong/java" "android/soong/rust" "android/soong/sh" @@ -10375,3 +10376,54 @@ func TestStubLibrariesMultipleApexViolation(t *testing.T) { } } } + +func TestFileSystemShouldSkipApexLibraries(t *testing.T) { + context := android.GroupFixturePreparers( + android.PrepareForIntegrationTestWithAndroid, + cc.PrepareForIntegrationTestWithCc, + PrepareForTestWithApexBuildComponents, + prepareForTestWithMyapex, + filesystem.PrepareForTestWithFilesystemBuildComponents, + ) + result := context.RunTestWithBp(t, ` + android_system_image { + name: "myfilesystem", + deps: [ + "libfoo", + ], + linker_config_src: "linker.config.json", + } + + cc_library { + name: "libfoo", + shared_libs: [ + "libbar", + ], + stl: "none", + } + + cc_library { + name: "libbar", + stl: "none", + apex_available: ["myapex"], + } + + apex { + name: "myapex", + native_shared_libs: ["libbar"], + key: "myapex.key", + updatable: false, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + + inputs := result.ModuleForTests("myfilesystem", "android_common").Output("deps.zip").Implicits + android.AssertStringListDoesNotContain(t, "filesystem should not have libbar", + inputs.Strings(), + "out/soong/.intermediates/libbar/android_arm64_armv8-a_shared/libbar.so") +} diff --git a/cc/androidmk.go b/cc/androidmk.go index 980dd0762c..ce35b5c449 100644 --- a/cc/androidmk.go +++ b/cc/androidmk.go @@ -124,17 +124,14 @@ func (c *Module) AndroidMkEntries() []android.AndroidMkEntries { } } } - if c.Properties.IsSdkVariant { + if c.Properties.IsSdkVariant && c.Properties.SdkAndPlatformVariantVisibleToMake { // Make the SDK variant uninstallable so that there are not two rules to install // to the same location. entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) - - if c.Properties.SdkAndPlatformVariantVisibleToMake { - // Add the unsuffixed name to SOONG_SDK_VARIANT_MODULES so that Make can rewrite - // dependencies to the .sdk suffix when building a module that uses the SDK. - entries.SetString("SOONG_SDK_VARIANT_MODULES", - "$(SOONG_SDK_VARIANT_MODULES) $(patsubst %.sdk,%,$(LOCAL_MODULE))") - } + // Add the unsuffixed name to SOONG_SDK_VARIANT_MODULES so that Make can rewrite + // dependencies to the .sdk suffix when building a module that uses the SDK. + entries.SetString("SOONG_SDK_VARIANT_MODULES", + "$(SOONG_SDK_VARIANT_MODULES) $(patsubst %.sdk,%,$(LOCAL_MODULE))") } }, }, diff --git a/cc/sdk.go b/cc/sdk.go index 4f361eb161..6341926ffb 100644 --- a/cc/sdk.go +++ b/cc/sdk.go @@ -47,16 +47,16 @@ func sdkMutator(ctx android.BottomUpMutatorContext) { // Mark the SDK variant. modules[1].(*Module).Properties.IsSdkVariant = true - // SDK variant is not supposed to be installed - modules[1].(*Module).Properties.PreventInstall = true if ctx.Config().UnbundledBuildApps() { // For an unbundled apps build, hide the platform variant from Make. modules[0].(*Module).Properties.HideFromMake = true + modules[0].(*Module).Properties.PreventInstall = true } else { // For a platform build, mark the SDK variant so that it gets a ".sdk" suffix when // exposed to Make. modules[1].(*Module).Properties.SdkAndPlatformVariantVisibleToMake = true + modules[1].(*Module).Properties.PreventInstall = true } ctx.AliasVariation("") } else if isCcModule && ccModule.isImportedApiLibrary() { @@ -74,8 +74,8 @@ func sdkMutator(ctx android.BottomUpMutatorContext) { if apiLibrary.hasApexStubs() { // For an unbundled apps build, hide the platform variant from Make. modules[1].(*Module).Properties.HideFromMake = true - modules[1].(*Module).Properties.PreventInstall = true } + modules[1].(*Module).Properties.PreventInstall = true } else { // For a platform build, mark the SDK variant so that it gets a ".sdk" suffix when // exposed to Make. diff --git a/cc/sdk_test.go b/cc/sdk_test.go index 790440cb9b..61925e30cc 100644 --- a/cc/sdk_test.go +++ b/cc/sdk_test.go @@ -101,95 +101,3 @@ func TestSdkMutator(t *testing.T) { assertDep(t, libsdkNDK, libcxxNDK) assertDep(t, libsdkPlatform, libcxxPlatform) } - -func TestMakeModuleNameForSdkVariant(t *testing.T) { - bp := ` - cc_library { - name: "libfoo", - srcs: ["main_test.cpp"], - sdk_version: "current", - stl: "none", - } - ` - platformVariant := "android_arm64_armv8-a_shared" - sdkVariant := "android_arm64_armv8-a_sdk_shared" - testCases := []struct { - name string - unbundledApps []string - variant string - skipInstall bool // soong skips install - hideFromMake bool // no make entry - makeUninstallable bool // make skips install - makeModuleName string - }{ - { - name: "platform variant in normal builds", - unbundledApps: nil, - variant: platformVariant, - // installable in soong - skipInstall: false, - // visiable in Make as "libfoo" - hideFromMake: false, - makeModuleName: "libfoo", - // installable in Make - makeUninstallable: false, - }, - { - name: "sdk variant in normal builds", - unbundledApps: nil, - variant: sdkVariant, - // soong doesn't install - skipInstall: true, - // visible in Make as "libfoo.sdk" - hideFromMake: false, - makeModuleName: "libfoo.sdk", - // but not installed - makeUninstallable: true, - }, - { - name: "platform variant in unbunded builds", - unbundledApps: []string{"bar"}, - variant: platformVariant, - // installable in soong - skipInstall: false, - // hidden from make - hideFromMake: true, - }, - { - name: "sdk variant in unbunded builds", - unbundledApps: []string{"bar"}, - variant: sdkVariant, - // soong doesn't install - skipInstall: true, - // visible in Make as "libfoo" - hideFromMake: false, - makeModuleName: "libfoo", - // but not installed - makeUninstallable: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - fixture := android.GroupFixturePreparers(prepareForCcTest, - android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { - variables.Unbundled_build_apps = tc.unbundledApps - }), - ) - ctx := fixture.RunTestWithBp(t, bp).TestContext - module := ctx.ModuleForTests("libfoo", tc.variant).Module().(*Module) - android.AssertBoolEquals(t, "IsSkipInstall", tc.skipInstall, module.IsSkipInstall()) - android.AssertBoolEquals(t, "HideFromMake", tc.hideFromMake, module.HiddenFromMake()) - if !tc.hideFromMake { - entries := android.AndroidMkEntriesForTest(t, ctx, module)[0] - android.AssertStringEquals(t, "LOCAL_MODULE", - tc.makeModuleName, entries.EntryMap["LOCAL_MODULE"][0]) - actualUninstallable := false - if actual, ok := entries.EntryMap["LOCAL_UNINSTALLABLE_MODULE"]; ok { - actualUninstallable = "true" == actual[0] - } - android.AssertBoolEquals(t, "LOCAL_UNINSTALLABLE_MODULE", - tc.makeUninstallable, actualUninstallable) - } - }) - } -} From fe978fd2c1d651dd11abaedf325a0bafa46c39e1 Mon Sep 17 00:00:00 2001 From: Yu Liu Date: Mon, 24 Apr 2023 16:37:18 -0700 Subject: [PATCH 0068/1460] libbuildversion should be linked as a whole archive dep as soong does. Bug: 278789968 Test: Unit tests and CI Change-Id: Ic2c44826bdc03b91cc19d93f96096ec6fdf44833 --- bp2build/cc_library_conversion_test.go | 8 ++++---- bp2build/cc_library_shared_conversion_test.go | 4 ++-- bp2build/cc_library_static_conversion_test.go | 2 +- cc/bp2build.go | 6 +----- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index d2c463dbbd..27a4c22155 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -153,10 +153,10 @@ cc_library { "//build/bazel/platforms/os:linux_glibc": ["linux.cpp"], "//conditions:default": [], })`, - "sdk_version": `"current"`, - "min_sdk_version": `"29"`, - "use_version_lib": `True`, - "implementation_whole_archive_deps": `["//build/soong/cc/libbuildversion:libbuildversion"]`, + "sdk_version": `"current"`, + "min_sdk_version": `"29"`, + "use_version_lib": `True`, + "whole_archive_deps": `["//build/soong/cc/libbuildversion:libbuildversion"]`, }), }) } diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go index cbea943e1f..59fbd439f8 100644 --- a/bp2build/cc_library_shared_conversion_test.go +++ b/bp2build/cc_library_shared_conversion_test.go @@ -516,8 +516,8 @@ func TestCcLibrarySharedUseVersionLib(t *testing.T) { }`, ExpectedBazelTargets: []string{ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ - "use_version_lib": "True", - "implementation_whole_archive_deps": `["//build/soong/cc/libbuildversion:libbuildversion"]`, + "use_version_lib": "True", + "whole_archive_deps": `["//build/soong/cc/libbuildversion:libbuildversion"]`, }), }, }) diff --git a/bp2build/cc_library_static_conversion_test.go b/bp2build/cc_library_static_conversion_test.go index cd4cf51a17..0583adaf17 100644 --- a/bp2build/cc_library_static_conversion_test.go +++ b/bp2build/cc_library_static_conversion_test.go @@ -1613,7 +1613,7 @@ func TestCcLibraryStaticUseVersionLib(t *testing.T) { }`, ExpectedBazelTargets: []string{ MakeBazelTarget("cc_library_static", "foo", AttrNameToString{ - "implementation_whole_archive_deps": `["//build/soong/cc/libbuildversion:libbuildversion"]`, + "whole_archive_deps": `["//build/soong/cc/libbuildversion:libbuildversion"]`, }), }, }) diff --git a/cc/bp2build.go b/cc/bp2build.go index c8f516cdc1..9f5da6be4d 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1036,11 +1036,7 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion _, staticLibs = android.RemoveFromList(versionLib, staticLibs) // only add the dep if it is not in progress if !versionLibAlreadyInDeps { - if isBinary { - wholeStaticLibs = append(wholeStaticLibs, versionLib) - } else { - la.implementationWholeArchiveDeps.SetSelectValue(axis, config, bazelLabelForWholeDepsExcludes(ctx, []string{versionLib}, props.Exclude_static_libs)) - } + wholeStaticLibs = append(wholeStaticLibs, versionLib) } } } From 580f7b0094bd98d3769b6cdeaaab6a9748621253 Mon Sep 17 00:00:00 2001 From: Andrei Onea Date: Wed, 17 Aug 2022 16:50:48 +0000 Subject: [PATCH 0069/1460] Add script for modifying privapp permission allowlists This script will be used to change the package that is being allowed to use privileged permissions. Test: python scripts/modify_permissions_allowlist.py Bug: 242509786 Change-Id: I1d7cb3c906ad39aef42ad4394953251db40d1bf1 --- scripts/Android.bp | 17 +++++ scripts/modify_permissions_allowlist.py | 70 ++++++++++++++++++ scripts/modify_permissions_allowlist_test.py | 76 ++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100755 scripts/modify_permissions_allowlist.py create mode 100755 scripts/modify_permissions_allowlist_test.py diff --git a/scripts/Android.bp b/scripts/Android.bp index 9367ff06aa..97f6ab424e 100644 --- a/scripts/Android.bp +++ b/scripts/Android.bp @@ -237,3 +237,20 @@ sh_binary_host { name: "jars-to-module-info-java", src: "jars-to-module-info-java.sh", } + +python_binary_host { + name: "modify_permissions_allowlist", + main: "modify_permissions_allowlist.py", + srcs: [ + "modify_permissions_allowlist.py", + ], +} + +python_test_host { + name: "modify_permissions_allowlist_test", + main: "modify_permissions_allowlist_test.py", + srcs: [ + "modify_permissions_allowlist_test.py", + "modify_permissions_allowlist.py", + ], +} diff --git a/scripts/modify_permissions_allowlist.py b/scripts/modify_permissions_allowlist.py new file mode 100755 index 0000000000..38ec7ec86e --- /dev/null +++ b/scripts/modify_permissions_allowlist.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for modifying privileged permission allowlists.""" + +from __future__ import print_function + +import argparse +import sys +from xml.dom import minidom + + +class InvalidRootNodeException(Exception): + pass + + +class InvalidNumberOfPrivappPermissionChildren(Exception): + pass + + +def modify_allowlist(allowlist_dom, package_name): + if allowlist_dom.documentElement.tagName != 'permissions': + raise InvalidRootNodeException + nodes = allowlist_dom.getElementsByTagName('privapp-permissions') + if nodes.length != 1: + raise InvalidNumberOfPrivappPermissionChildren + privapp_permissions = nodes[0] + privapp_permissions.setAttribute('package', package_name) + + +def parse_args(): + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument('input', help='input allowlist template file') + parser.add_argument( + 'package_name', help='package name to use in the allowlist' + ) + parser.add_argument('output', help='output allowlist file') + + return parser.parse_args() + + +def main(): + try: + args = parse_args() + doc = minidom.parse(args.input) + modify_allowlist(doc, args.package_name) + with open(args.output, 'w') as output_file: + doc.writexml(output_file, encoding='utf-8') + except Exception as err: + print('error: ' + str(err), file=sys.stderr) + sys.exit(-1) + + +if __name__ == '__main__': + main() diff --git a/scripts/modify_permissions_allowlist_test.py b/scripts/modify_permissions_allowlist_test.py new file mode 100755 index 0000000000..ee8b12cc83 --- /dev/null +++ b/scripts/modify_permissions_allowlist_test.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unit tests for modify_permissions_allowlist.py.""" + +from __future__ import print_function + +import unittest + +from xml.dom import minidom + +from modify_permissions_allowlist import InvalidRootNodeException, InvalidNumberOfPrivappPermissionChildren, modify_allowlist + + +class ModifyPermissionsAllowlistTest(unittest.TestCase): + + def test_invalid_root(self): + xml_data = '' + xml_dom = minidom.parseString(xml_data) + self.assertRaises(InvalidRootNodeException, modify_allowlist, xml_dom, 'x') + + def test_no_packages(self): + xml_data = '' + xml_dom = minidom.parseString(xml_data) + self.assertRaises( + InvalidNumberOfPrivappPermissionChildren, modify_allowlist, xml_dom, 'x' + ) + + def test_multiple_packages(self): + xml_data = ( + '' + ' ' + ' ' + '' + ) + xml_dom = minidom.parseString(xml_data) + self.assertRaises( + InvalidNumberOfPrivappPermissionChildren, modify_allowlist, xml_dom, 'x' + ) + + def test_modify_package_name(self): + xml_data = ( + '' + ' ' + ' ' + ' ' + '' + ) + xml_dom = minidom.parseString(xml_data) + modify_allowlist(xml_dom, 'bar.baz') + expected_data = ( + '' + '' + ' ' + ' ' + ' ' + '' + ) + self.assertEqual(expected_data, xml_dom.toxml()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 6f6b962bd09588be63957c8f6910e1a63a490a51 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Tue, 2 May 2023 16:28:55 +0000 Subject: [PATCH 0070/1460] Modify --ensure-allowlist-integrity call to avoid spurious errors. We now ensure that the module is disabled and _not_ enabled before failing the build. Bug: 280439299 Test: m nothing --ensure-allowlist-integrity --bazel-mode-staging (on master and aosp) Change-Id: I59c969a27b064f07913a511cb29bc06d23daa371 --- android/config.go | 4 ++++ cmd/soong_build/main.go | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/android/config.go b/android/config.go index 0b54ed90cc..526537406f 100644 --- a/android/config.go +++ b/android/config.go @@ -1929,6 +1929,10 @@ func (c *config) GetMixedBuildsEnabledModules() map[string]struct{} { return c.mixedBuildEnabledModules } +func (c *config) GetMixedBuildsDisabledModules() map[string]struct{} { + return c.mixedBuildDisabledModules +} + func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) { moduleName := ctx.Module().Name() c.mixedBuildsLock.Lock() diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 53e0e55c48..f347b8f564 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -289,9 +289,9 @@ func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandl } // Errors out if any modules expected to be mixed_built were not, unless -// there is a platform incompatibility. +// the modules did not exist. func checkForAllowlistIntegrityError(configuration android.Config, isStagingMode bool) error { - modules := findModulesNotMixedBuiltForAnyVariant(configuration, isStagingMode) + modules := findMisconfiguredModules(configuration, isStagingMode) if len(modules) == 0 { return nil } @@ -299,29 +299,54 @@ func checkForAllowlistIntegrityError(configuration android.Config, isStagingMode return fmt.Errorf("Error: expected the following modules to be mixed_built: %s", modules) } +// Returns true if the given module has all of the following true: +// 1. Is allowlisted to be built with Bazel. +// 2. Has a variant which is *not* built with Bazel. +// 3. Has no variant which is built with Bazel. +// +// This indicates the allowlisting of this variant had no effect. +// TODO(b/280457637): Return true for nonexistent modules. +func isAllowlistMisconfiguredForModule(module string, mixedBuildsEnabled map[string]struct{}, mixedBuildsDisabled map[string]struct{}) bool { + //TODO(dacek): Why does this occur in the allowlists? + if module == "" { + return false + } + _, enabled := mixedBuildsEnabled[module] + + if enabled { + return false + } + + _, disabled := mixedBuildsDisabled[module] + return disabled + +} + // Returns the list of modules that should have been mixed_built (per the // allowlists and cmdline flags) but were not. -func findModulesNotMixedBuiltForAnyVariant(configuration android.Config, isStagingMode bool) []string { +// Note: nonexistent modules are excluded from the list. See b/280457637 +func findMisconfiguredModules(configuration android.Config, isStagingMode bool) []string { retval := []string{} forceEnabledModules := configuration.BazelModulesForceEnabledByFlag() mixedBuildsEnabled := configuration.GetMixedBuildsEnabledModules() + mixedBuildsDisabled := configuration.GetMixedBuildsDisabledModules() for _, module := range allowlists.ProdMixedBuildsEnabledList { - if _, ok := mixedBuildsEnabled[module]; !ok && module != "" { + if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) { retval = append(retval, module) } } if isStagingMode { for _, module := range allowlists.StagingMixedBuildsEnabledList { - if _, ok := mixedBuildsEnabled[module]; !ok && module != "" { + if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) { retval = append(retval, module) } } } for module, _ := range forceEnabledModules { - if _, ok := mixedBuildsEnabled[module]; !ok && module != "" { + if isAllowlistMisconfiguredForModule(module, mixedBuildsEnabled, mixedBuildsDisabled) { retval = append(retval, module) } } From ed2253e8dfb585571f60d32ed47528c8fcb4bfd5 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Tue, 2 May 2023 21:35:39 +0000 Subject: [PATCH 0071/1460] Process bazel metrics prior to potentially exiting upload.go This is a necessary step for running b on CI with metrics collection. Bug: 280491043 Test: b build libcore:all Change-Id: I37d621c38e3f891dd63e5977befa6ce6604487ea --- ui/build/upload.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ui/build/upload.go b/ui/build/upload.go index 9959e6f9c7..1e6d94aad4 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -138,6 +138,9 @@ func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming { return phaseTiming } +// This method takes a file created by bazel's --analyze-profile mode and +// writes bazel metrics data to the provided filepath. +// TODO(b/279987768) - move this outside of upload.go func processBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context) { if bazelProfileFile == "" { return @@ -189,12 +192,13 @@ func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted t defer ctx.EndTrace() uploader := config.MetricsUploaderApp() + processBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx) + if uploader == "" { // If the uploader path was not specified, no metrics shall be uploaded. return } - processBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx) // Several of the files might be directories. metricsFiles := pruneMetricsFiles(paths) if len(metricsFiles) == 0 { From 921af32310e740ea0c4f0f83b7349771b3378916 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 26 Apr 2023 02:56:37 +0000 Subject: [PATCH 0072/1460] Print default val if all vals in axis match default val To avoid verbosity, we currently dedupe keys in axis if its value matches the value of //conditions:default. For axes where all values might match the default value, we would effectively drop the common value. To fix this, we are now dropping the select statement and not the common value. Test: go test ./bp2build Change-Id: Ic377b93ee2aba971753f6a5e7a62e15d1fcfa2bc --- bp2build/build_conversion_test.go | 15 +++++++++++++++ bp2build/configurability.go | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go index 1b64055f7a..e198a11b83 100644 --- a/bp2build/build_conversion_test.go +++ b/bp2build/build_conversion_test.go @@ -21,6 +21,7 @@ import ( "android/soong/android" "android/soong/android/allowlists" + "android/soong/bazel" "android/soong/python" ) @@ -1931,3 +1932,17 @@ func TestGenerateConfigSetting(t *testing.T) { Description: "Generating API contribution Bazel targets for custom module", }) } + +// If values of all keys in an axis are equal to //conditions:default, drop the axis and print the common value +func TestPrettyPrintSelectMapEqualValues(t *testing.T) { + lla := bazel.LabelListAttribute{ + Value: bazel.LabelList{}, + } + libFooImplLabel := bazel.Label{ + Label: ":libfoo.impl", + } + lla.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex, bazel.MakeLabelList([]bazel.Label{libFooImplLabel})) + lla.SetSelectValue(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey, bazel.MakeLabelList([]bazel.Label{libFooImplLabel})) + actual, _ := prettyPrintAttribute(lla, 0) + android.AssertStringEquals(t, "Print the common value if all keys in an axis have the same value", `[":libfoo.impl"]`, actual) +} diff --git a/bp2build/configurability.go b/bp2build/configurability.go index 8e171031cc..3d9f0a274e 100644 --- a/bp2build/configurability.go +++ b/bp2build/configurability.go @@ -279,6 +279,10 @@ func prettyPrintSelectMap(selectMap map[string]reflect.Value, defaultValue *stri } if len(selects) == 0 { + // If there is a default value, and there are no selects for this axis, print that without any selects. + if val, exists := selectMap[bazel.ConditionsDefaultSelectKey]; exists { + return prettyPrint(val, indent, emitZeroValues) + } // No conditions (or all values are empty lists), so no need for a map. return "", nil } From 1de78f3174e5a4bd3bdc27337e2b25b673f67222 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Tue, 2 May 2023 21:00:48 +0000 Subject: [PATCH 0073/1460] Fix instantiation of config.bazelForceEnabledModules Currently, uninstantiated, the field will be set to [""] instead of an empty slice. Test: m nothing --ensure-allowlist-integrity Test: m nothing --ensure-allowlist-integrity --bazel-force-enabled-modules=com.google.android.neuralnetworks (this fails, as expected) Change-Id: Ib4cd26f0cb0d40714b8c3f263ee2d22093ee15ef --- android/config.go | 9 ++++++++- cmd/soong_build/main.go | 4 ---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/android/config.go b/android/config.go index 526537406f..9e94e05177 100644 --- a/android/config.go +++ b/android/config.go @@ -606,7 +606,7 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) setBazelMode(cmdArgs.BazelMode, "--bazel-mode", BazelProdMode) setBazelMode(cmdArgs.BazelModeStaging, "--bazel-mode-staging", BazelStagingMode) - for _, module := range strings.Split(cmdArgs.BazelForceEnabledModules, ",") { + for _, module := range getForceEnabledModulesFromFlag(cmdArgs.BazelForceEnabledModules) { config.bazelForceEnabledModules[module] = struct{}{} } config.BazelContext, err = NewBazelContext(config) @@ -615,6 +615,13 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) return Config{config}, err } +func getForceEnabledModulesFromFlag(forceEnabledFlag string) []string { + if forceEnabledFlag == "" { + return []string{} + } + return strings.Split(forceEnabledFlag, ",") +} + // mockFileSystem replaces all reads with accesses to the provided map of // filenames to contents stored as a byte slice. func (c *config) mockFileSystem(bp string, fs map[string][]byte) { diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index f347b8f564..c22fd80b91 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -307,10 +307,6 @@ func checkForAllowlistIntegrityError(configuration android.Config, isStagingMode // This indicates the allowlisting of this variant had no effect. // TODO(b/280457637): Return true for nonexistent modules. func isAllowlistMisconfiguredForModule(module string, mixedBuildsEnabled map[string]struct{}, mixedBuildsDisabled map[string]struct{}) bool { - //TODO(dacek): Why does this occur in the allowlists? - if module == "" { - return false - } _, enabled := mixedBuildsEnabled[module] if enabled { From 24297765558e6e27230c0511a3733ce307b558cc Mon Sep 17 00:00:00 2001 From: Almaz Mingaleev Date: Tue, 2 May 2023 15:42:56 +0100 Subject: [PATCH 0074/1460] Add jdk.internal.access to the allowed package list. This is part of OpenJDK17 update. Bug: 270674727 Test: m Change-Id: I0708ee286cd7c3c6eef593784affaeffbdf0a76c --- scripts/check_boot_jars/package_allowed_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt index 869fd3f834..dad2b47d92 100644 --- a/scripts/check_boot_jars/package_allowed_list.txt +++ b/scripts/check_boot_jars/package_allowed_list.txt @@ -72,6 +72,7 @@ javax\.xml\.transform\.stream javax\.xml\.validation javax\.xml\.xpath jdk\.internal +jdk\.internal\.access jdk\.internal\.math jdk\.internal\.misc jdk\.internal\.ref From b95f8345c8b926035e5ed40c903f056cd106c539 Mon Sep 17 00:00:00 2001 From: Jiakai Zhang Date: Tue, 2 May 2023 14:35:44 +0100 Subject: [PATCH 0075/1460] Generate boot image profiles even if dexpreopt is disabled. Bug: 280440941 Test: - 1. Patch ag/22302622 to disable dexpreopt. 2. See boot image profiles still generated. Change-Id: I1bf05ade53fa83f3dba46f28a8f9246ba1fdf664 --- apex/apex.go | 2 +- apex/bootclasspath_fragment_test.go | 20 ++++++++++++++++++++ dexpreopt/testing.go | 7 +++++++ java/bootclasspath_fragment.go | 15 +++++++-------- java/dexpreopt_bootjars.go | 7 ++++--- java/platform_bootclasspath.go | 15 ++++++--------- sdk/java_sdk_test.go | 2 ++ 7 files changed, 47 insertions(+), 21 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index baf4737d82..5e4d3fc0e9 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -2675,7 +2675,7 @@ func apexBootclasspathFragmentFiles(ctx android.ModuleContext, module blueprint. } pathInApex := bootclasspathFragmentInfo.ProfileInstallPathInApex() - if pathInApex != "" && !java.SkipDexpreoptBootJars(ctx) { + if pathInApex != "" { pathOnHost := bootclasspathFragmentInfo.ProfilePathOnHost() tempPath := android.PathForModuleOut(ctx, "boot_image_profile", pathInApex) diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index 2ddfd03055..4a661d4ea4 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -497,6 +497,26 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { }) }) + t.Run("generate boot image profile even if dexpreopt is disabled", func(t *testing.T) { + result := android.GroupFixturePreparers( + commonPreparer, + + // Configure some libraries in the art bootclasspath_fragment that match the source + // bootclasspath_fragment's contents property. + java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), + addSource("foo", "bar"), + java.FixtureSetBootImageInstallDirOnDevice("art", "system/framework"), + dexpreopt.FixtureDisableDexpreoptBootImages(true), + ).RunTest(t) + + ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + "etc/boot-image.prof", + "etc/classpaths/bootclasspath.pb", + "javalib/bar.jar", + "javalib/foo.jar", + }) + }) + t.Run("boot image disable generate profile", func(t *testing.T) { result := android.GroupFixturePreparers( commonPreparer, diff --git a/dexpreopt/testing.go b/dexpreopt/testing.go index 47ae494e33..b3dd3cca24 100644 --- a/dexpreopt/testing.go +++ b/dexpreopt/testing.go @@ -174,3 +174,10 @@ func FixtureDisableGenerateProfile(disable bool) android.FixturePreparer { dexpreoptConfig.DisableGenerateProfile = disable }) } + +// FixtureDisableDexpreoptBootImages sets the DisablePreoptBootImages property in the global config. +func FixtureDisableDexpreoptBootImages(disable bool) android.FixturePreparer { + return FixtureModifyGlobalConfig(func(_ android.PathContext, dexpreoptConfig *GlobalConfig) { + dexpreoptConfig.DisablePreoptBootImages = disable + }) +} diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index f692563478..d1a020e4d3 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -506,10 +506,6 @@ func (b *BootclasspathFragmentModule) DepsMutator(ctx android.BottomUpMutatorCon } } - if SkipDexpreoptBootJars(ctx) { - return - } - // Add a dependency onto the dex2oat tool which is needed for creating the boot image. The // path is retrieved from the dependency by GetGlobalSoongConfig(ctx). dexpreopt.RegisterToolDeps(ctx) @@ -901,10 +897,6 @@ func (b *BootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleC // produceBootImageFiles builds the boot image files from the source if it is required. func (b *BootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig) bootImageOutputs { - if SkipDexpreoptBootJars(ctx) { - return bootImageOutputs{} - } - // Only generate the boot image if the configuration does not skip it. return b.generateBootImageBuildActions(ctx, imageConfig) } @@ -932,6 +924,13 @@ func (b *BootclasspathFragmentModule) generateBootImageBuildActions(ctx android. // Build boot image files for the host variants. buildBootImageVariantsForBuildOs(ctx, imageConfig, profile) + // If dexpreopt of boot image jars should be skipped, generate only a profile. + if SkipDexpreoptBootJars(ctx) { + return bootImageOutputs{ + profile: profile, + } + } + // Build boot image files for the android variants. bootImageFiles := buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index f4827ea3a7..fed384e9d0 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -500,9 +500,6 @@ func (d *dexpreoptBootJars) GenerateAndroidBuildActions(ctx android.ModuleContex // Generate build rules for boot images. func (d *dexpreoptBootJars) GenerateSingletonBuildActions(ctx android.SingletonContext) { - if SkipDexpreoptBootJars(ctx) { - return - } if dexpreopt.GetCachedGlobalSoongConfig(ctx) == nil { // No module has enabled dexpreopting, so we assume there will be no boot image to make. return @@ -1014,6 +1011,10 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_IMAGE_PROFILE_LICENSE_METADATA", image.profileLicenseMetadataFile.String()) } + if SkipDexpreoptBootJars(ctx) { + return + } + global := dexpreopt.GetGlobalConfig(ctx) dexPaths, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_FILES", strings.Join(dexPaths.Strings(), " ")) diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index 0ea3609798..254c40535f 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -103,10 +103,6 @@ func (b *platformBootclasspathModule) OutputFiles(tag string) (android.Paths, er func (b *platformBootclasspathModule) DepsMutator(ctx android.BottomUpMutatorContext) { b.hiddenAPIDepsMutator(ctx) - if SkipDexpreoptBootJars(ctx) { - return - } - // Add a dependency onto the dex2oat tool which is needed for creating the boot image. The // path is retrieved from the dependency by GetGlobalSoongConfig(ctx). dexpreopt.RegisterToolDeps(ctx) @@ -187,11 +183,6 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo bootDexJarByModule := b.generateHiddenAPIBuildActions(ctx, b.configuredModules, b.fragments) buildRuleForBootJarsPackageCheck(ctx, bootDexJarByModule) - // Nothing to do if skipping the dexpreopt of boot image jars. - if SkipDexpreoptBootJars(ctx) { - return - } - b.generateBootImageBuildActions(ctx, platformModules, apexModules) } @@ -429,6 +420,12 @@ func (b *platformBootclasspathModule) generateBootImage(ctx android.ModuleContex // Build a profile for the image config and then use that to build the boot image. profile := bootImageProfileRule(ctx, imageConfig) + // If dexpreopt of boot image jars should be skipped, generate only a profile. + global := dexpreopt.GetGlobalConfig(ctx) + if global.DisablePreoptBootImages { + return + } + // Build boot image files for the android variants. androidBootImageFiles := buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 3a2ecc00cc..6159ea9c24 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -19,12 +19,14 @@ import ( "testing" "android/soong/android" + "android/soong/dexpreopt" "android/soong/java" ) var prepareForSdkTestWithJava = android.GroupFixturePreparers( java.PrepareForTestWithJavaBuildComponents, PrepareForTestWithSdkBuildComponents, + dexpreopt.PrepareForTestWithFakeDex2oatd, // Ensure that all source paths are provided. This helps ensure that the snapshot generation is // consistent and all files referenced from the snapshot's Android.bp file have actually been From 580636bdd23171f31bfedd773c065e0861dd5c4a Mon Sep 17 00:00:00 2001 From: Andrei Onea Date: Wed, 17 Aug 2022 16:53:46 +0000 Subject: [PATCH 0076/1460] add privapp_allowlist property to android_app This change allows override_android_app to use the same privapp_allowlist as the non-override module so that they will always remain in sync. Test: go test ./java -v -run TestPrivappAllowlist Test: go test ./apex -v -run TestApexWithApps Test: m com.android.permission com.google.android.permission and verify manually that apex_payload.img contains correct privapp_allowlist Test: m com.android.permission before and after change && `diffoscope apex_payload_reference.img apex_payload_with_change.img` && verify that there are no semantic changes Bug: 242509786 Change-Id: Ifdcb28af40763aed7a4aac9a7f681153554bc256 --- apex/apex.go | 19 +++++++++++---- apex/apex_test.go | 3 +++ java/app.go | 58 +++++++++++++++++++++++++++++++++++++++++++--- java/app_import.go | 4 ++++ java/app_test.go | 48 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 8 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index 69547c3b70..3ca0bed324 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1828,6 +1828,7 @@ type androidApp interface { Certificate() java.Certificate BaseModuleName() string LintDepSets() java.LintDepSets + PrivAppAllowlist() android.OptionalPath } var _ androidApp = (*java.AndroidApp)(nil) @@ -1848,7 +1849,7 @@ func sanitizedBuildIdForPath(ctx android.BaseModuleContext) string { return buildId } -func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexFile { +func apexFilesForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) []apexFile { appDir := "app" if aapp.Privileged() { appDir = "priv-app" @@ -1870,7 +1871,15 @@ func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexF }); ok { af.overriddenPackageName = app.OverriddenManifestPackageName() } - return af + apexFiles := []apexFile{af} + + if allowlist := aapp.PrivAppAllowlist(); allowlist.Valid() { + dirInApex := filepath.Join("etc", "permissions") + privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"privapp", dirInApex, etc, aapp) + apexFiles = append(apexFiles, privAppAllowlist) + } + + return apexFiles } func apexFileForRuntimeResourceOverlay(ctx android.BaseModuleContext, rro java.RuntimeResourceOverlayModule) apexFile { @@ -2318,12 +2327,12 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, case androidAppTag: switch ap := child.(type) { case *java.AndroidApp: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) return true // track transitive dependencies case *java.AndroidAppImport: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) case *java.AndroidTestHelperApp: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) case *java.AndroidAppSet: appDir := "app" if ap.Privileged() { diff --git a/apex/apex_test.go b/apex/apex_test.go index 3ba4d8d553..4a4ee4408a 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6045,6 +6045,8 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, + privapp_allowlist: "perms.xml", + package_name: "com.android.AppFooPriv", stl: "none", apex_available: [ "myapex" ], } @@ -6074,6 +6076,7 @@ func TestApexWithApps(t *testing.T) { ensureContains(t, copyCmds, "image.apex/app/AppFoo@TEST.BUILD_ID/AppFoo.apk") ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPriv@TEST.BUILD_ID/AppFooPriv.apk") + ensureContains(t, copyCmds, "image.apex/etc/permissions/privapp_allowlist_com.android.AppFooPriv.xml") appZipRule := ctx.ModuleForTests("AppFoo", "android_common_apex10000").Description("zip jni libs") // JNI libraries are uncompressed diff --git a/java/app.go b/java/app.go index 7bb8cdbd7e..da9c6f3437 100755 --- a/java/app.go +++ b/java/app.go @@ -33,8 +33,17 @@ import ( func init() { RegisterAppBuildComponents(android.InitRegistrationContext) + pctx.HostBinToolVariable("ModifyAllowlistCmd", "modify_permissions_allowlist") } +var ( + modifyAllowlist = pctx.AndroidStaticRule("modifyAllowlist", + blueprint.RuleParams{ + Command: "${ModifyAllowlistCmd} $in $packageName $out", + CommandDeps: []string{"${ModifyAllowlistCmd}"}, + }, "packageName") +) + func RegisterAppBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("android_app", AndroidAppFactory) ctx.RegisterModuleType("android_test", AndroidTestFactory) @@ -115,6 +124,9 @@ type appProperties struct { // Prefer using other specific properties if build behaviour must be changed; avoid using this // flag for anything but neverallow rules (unless the behaviour change is invisible to owners). Updatable *bool + + // Specifies the file that contains the allowlist for this app. + Privapp_allowlist *string `android:"path"` } // android_app properties that can be overridden by override_android_app @@ -179,6 +191,8 @@ type AndroidApp struct { android.ApexBundleDepsInfo javaApiUsedByOutputFile android.ModuleOutPath + + privAppAllowlist android.OptionalPath } func (a *AndroidApp) IsInstallable() bool { @@ -205,6 +219,10 @@ func (a *AndroidApp) JniCoverageOutputs() android.Paths { return a.jniCoverageOutputs } +func (a *AndroidApp) PrivAppAllowlist() android.OptionalPath { + return a.privAppAllowlist +} + var _ AndroidLibraryDependency = (*AndroidApp)(nil) type Certificate struct { @@ -269,6 +287,10 @@ func (a *AndroidApp) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato ctx.AddDependency(ctx.Module(), certificateTag, cert) } + if a.appProperties.Privapp_allowlist != nil && !Bool(a.appProperties.Privileged) { + ctx.PropertyErrorf("privapp_allowlist", "privileged must be set in order to use privapp_allowlist") + } + for _, cert := range a.appProperties.Additional_certificates { cert = android.SrcIsModule(cert) if cert != "" { @@ -598,6 +620,27 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { + if a.appProperties.Privapp_allowlist == nil { + return nil + } + if a.overridableAppProperties.Package_name == nil { + ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") + } + packageName := *a.overridableAppProperties.Package_name + fileName := "privapp_allowlist_" + packageName + ".xml" + outPath := android.PathForModuleOut(ctx, fileName).OutputPath + ctx.Build(pctx, android.BuildParams{ + Rule: modifyAllowlist, + Input: android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist), + Output: outPath, + Args: map[string]string{ + "packageName": packageName, + }, + }) + return &outPath +} + func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { var apkDeps android.Paths @@ -733,18 +776,27 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { BuildBundleModule(ctx, bundleFile, a.exportPackage, jniJarFile, dexJarFile) a.bundleFile = bundleFile + allowlist := a.createPrivappAllowlist(ctx) + if allowlist != nil { + a.privAppAllowlist = android.OptionalPathForPath(allowlist) + } + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) // Install the app package. - if (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && - !a.appProperties.PreventInstall { - + shouldInstallAppPackage := (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && !a.appProperties.PreventInstall + if shouldInstallAppPackage { var extraInstalledPaths android.Paths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) } ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile, extraInstalledPaths...) + + if a.privAppAllowlist.Valid() { + installPath := android.PathForModuleInstall(ctx, "etc", "permissions") + ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) + } } a.buildAppDependencyInfo(ctx) diff --git a/java/app_import.go b/java/app_import.go index bfd67679ea..52ae0247ae 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -423,6 +423,10 @@ func (a *AndroidAppImport) ProvenanceMetaDataFile() android.OutputPath { return a.provenanceMetaDataFile } +func (a *AndroidAppImport) PrivAppAllowlist() android.OptionalPath { + return android.OptionalPath{} +} + var dpiVariantGroupType reflect.Type var archVariantGroupType reflect.Type var supportedDpis = []string{"ldpi", "mdpi", "hdpi", "xhdpi", "xxhdpi", "xxxhdpi"} diff --git a/java/app_test.go b/java/app_test.go index 7e97b0fb19..daff94ca9e 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3539,3 +3539,51 @@ func TestTargetSdkVersionMtsTests(t *testing.T) { android.AssertStringDoesContain(t, testCase.desc, manifestFixerArgs, "--targetSdkVersion "+testCase.targetSdkVersionExpected) } } + +func TestPrivappAllowlist(t *testing.T) { + testJavaError(t, "privileged must be set in order to use privapp_allowlist", ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "perms.xml", + } + `) + + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp( + t, + ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "perms.xml", + privileged: true, + package_name: "com.android.foo", + sdk_version: "current", + } + override_android_app { + name: "bar", + base: "foo", + package_name: "com.google.android.foo", + } + `, + ) + app := result.ModuleForTests("foo", "android_common") + overrideApp := result.ModuleForTests("foo", "android_common_bar") + + // verify that privapp allowlist is created + app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") + expectedAllowlist := "perms.xml" + actualAllowlist := app.Rule("modifyAllowlist").Input.String() + if expectedAllowlist != actualAllowlist { + t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) + } + overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlist != overrideActualAllowlist { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + } + + // verify that permissions are copied to device + app.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml") + overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml") +} From 74e3e003148d202749cc79456e61f65ba1f4f25a Mon Sep 17 00:00:00 2001 From: Wei Li Date: Wed, 26 Apr 2023 12:15:55 -0700 Subject: [PATCH 0077/1460] Allowlist SBOM generation tool so it can be used in Bazel. Bug: 275472038 Test: b build //build/make/tools/sbom:generate-sbom && bazel-out/aosp_cf_x86_64_phone-userdebug_linux_x86_64-opt/bin/build/make/tools/sbom/generate-sbom Change-Id: I4ad68b0613410f40c79ee24d3eef779c2ad57815 --- android/allowlists/allowlists.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 433c06357e..6f7302222e 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -59,7 +59,9 @@ var ( "build/bazel": Bp2BuildDefaultTrueRecursively, "build/make/target/product/security": Bp2BuildDefaultTrue, + "build/make/tools/protos": Bp2BuildDefaultTrue, "build/make/tools/releasetools": Bp2BuildDefaultTrue, + "build/make/tools/sbom": Bp2BuildDefaultTrue, "build/make/tools/signapk": Bp2BuildDefaultTrue, "build/make/tools/zipalign": Bp2BuildDefaultTrueRecursively, "build/soong": Bp2BuildDefaultTrue, @@ -1435,6 +1437,9 @@ var ( "MetaDataBaseUnitTest", // depends on libstagefright "AVCUtilsUnitTest", // depends on libstagefright "ColorUtilsTest", // depends on libmediandk + + // python_test_host with test data + "sbom_writers_test", } MixedBuildsDisabledList = []string{ From 0428f71b304371db98caff9700ad7ea6aa45f311 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 3 May 2023 18:21:05 -0700 Subject: [PATCH 0078/1460] Remove "dumping rbe metrics" message The issue where RBE was taking a long time has been fixed, so this message isn't that important anymore, and it was overriding the last ephemeral status message that soong output. Test: Presubmits Change-Id: I019f351a3307176ec7b4786d4387fde9061ba08b --- ui/build/rbe.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 6479925dd1..3b9d301f5b 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -183,8 +183,6 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { return } - ctx.Status.Status("Dumping rbe metrics...") - outputDir := config.rbeProxyLogsDir() if outputDir == "" { ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.") From ebaa5733173f687ed14c0b7aedfa72bcc990138d Mon Sep 17 00:00:00 2001 From: Jooyung Han Date: Tue, 2 May 2023 11:43:14 +0900 Subject: [PATCH 0079/1460] Prebuilt replacing source should not change partition This reveals unintended mistake (setting a wrong target partition) at build-time instead of runtime, which is much harder to debug. Bug: 280368661 Test: m nothing (soong test) Change-Id: Ic5e5e97ba918e24f7a59aceb405c2b105e28cccc --- android/override_module.go | 18 ++++++++------- android/prebuilt.go | 16 +++++++++++++ android/prebuilt_test.go | 46 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/android/override_module.go b/android/override_module.go index 9e95c0f10e..a4b7431f6a 100644 --- a/android/override_module.go +++ b/android/override_module.go @@ -46,8 +46,8 @@ type OverrideModule interface { // Internal funcs to handle interoperability between override modules and prebuilts. // i.e. cases where an overriding module, too, is overridden by a prebuilt module. - setOverriddenByPrebuilt(overridden bool) - getOverriddenByPrebuilt() bool + setOverriddenByPrebuilt(prebuilt Module) + getOverriddenByPrebuilt() Module // Directory containing the Blueprint definition of the overriding module setModuleDir(string) @@ -60,7 +60,7 @@ type OverrideModuleBase struct { overridingProperties []interface{} - overriddenByPrebuilt bool + overriddenByPrebuilt Module moduleDir string } @@ -96,11 +96,11 @@ func (o *OverrideModuleBase) GetOverriddenModuleName() string { return proptools.String(o.moduleProperties.Base) } -func (o *OverrideModuleBase) setOverriddenByPrebuilt(overridden bool) { - o.overriddenByPrebuilt = overridden +func (o *OverrideModuleBase) setOverriddenByPrebuilt(prebuilt Module) { + o.overriddenByPrebuilt = prebuilt } -func (o *OverrideModuleBase) getOverriddenByPrebuilt() bool { +func (o *OverrideModuleBase) getOverriddenByPrebuilt() Module { return o.overriddenByPrebuilt } @@ -281,7 +281,7 @@ func overrideModuleDepsMutator(ctx BottomUpMutatorContext) { panic("PrebuiltDepTag leads to a non-prebuilt module " + dep.Name()) } if prebuilt.UsePrebuilt() { - module.setOverriddenByPrebuilt(true) + module.setOverriddenByPrebuilt(dep) return } }) @@ -314,8 +314,10 @@ func performOverrideMutator(ctx BottomUpMutatorContext) { ctx.AliasVariation(variants[0]) for i, o := range overrides { mods[i+1].(OverridableModule).override(ctx, mods[i+1], o) - if o.getOverriddenByPrebuilt() { + if prebuilt := o.getOverriddenByPrebuilt(); prebuilt != nil { // The overriding module itself, too, is overridden by a prebuilt. + // Perform the same check for replacement + checkInvariantsForSourceAndPrebuilt(ctx, mods[i+1], prebuilt) // Copy the flag and hide it in make mods[i+1].ReplacedByPrebuilt() } diff --git a/android/prebuilt.go b/android/prebuilt.go index 9b5c0e919a..95b772d414 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -419,6 +419,20 @@ func PrebuiltSourceDepsMutator(ctx BottomUpMutatorContext) { } } +// checkInvariantsForSourceAndPrebuilt checks if invariants are kept when replacing +// source with prebuilt. Note that the current module for the context is the source module. +func checkInvariantsForSourceAndPrebuilt(ctx BaseModuleContext, s, p Module) { + if _, ok := s.(OverrideModule); ok { + // skip the check when the source module is `override_X` because it's only a placeholder + // for the actual source module. The check will be invoked for the actual module. + return + } + if sourcePartition, prebuiltPartition := s.PartitionTag(ctx.DeviceConfig()), p.PartitionTag(ctx.DeviceConfig()); sourcePartition != prebuiltPartition { + ctx.OtherModuleErrorf(p, "partition is different: %s(%s) != %s(%s)", + sourcePartition, ctx.ModuleName(), prebuiltPartition, ctx.OtherModuleName(p)) + } +} + // PrebuiltSelectModuleMutator marks prebuilts that are used, either overriding source modules or // because the source module doesn't exist. It also disables installing overridden source modules. func PrebuiltSelectModuleMutator(ctx TopDownMutatorContext) { @@ -434,6 +448,8 @@ func PrebuiltSelectModuleMutator(ctx TopDownMutatorContext) { ctx.VisitDirectDepsWithTag(PrebuiltDepTag, func(prebuiltModule Module) { p := GetEmbeddedPrebuilt(prebuiltModule) if p.usePrebuilt(ctx, s, prebuiltModule) { + checkInvariantsForSourceAndPrebuilt(ctx, s, prebuiltModule) + p.properties.UsePrebuilt = true s.ReplacedByPrebuilt() } diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index fa40d1fb35..fc47cfd97f 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -497,6 +497,52 @@ func TestPrebuilts(t *testing.T) { } } +func testPrebuiltError(t *testing.T, expectedError, bp string) { + t.Helper() + fs := MockFS{ + "prebuilt_file": nil, + } + GroupFixturePreparers( + PrepareForTestWithArchMutator, + PrepareForTestWithPrebuilts, + PrepareForTestWithOverrides, + fs.AddToFixture(), + FixtureRegisterWithContext(registerTestPrebuiltModules), + ). + ExtendWithErrorHandler(FixtureExpectsAtLeastOneErrorMatchingPattern(expectedError)). + RunTestWithBp(t, bp) +} + +func TestPrebuiltShouldNotChangePartition(t *testing.T) { + testPrebuiltError(t, `partition is different`, ` + source { + name: "foo", + vendor: true, + } + prebuilt { + name: "foo", + prefer: true, + srcs: ["prebuilt_file"], + }`) +} + +func TestPrebuiltShouldNotChangePartition_WithOverride(t *testing.T) { + testPrebuiltError(t, `partition is different`, ` + source { + name: "foo", + vendor: true, + } + override_source { + name: "bar", + base: "foo", + } + prebuilt { + name: "bar", + prefer: true, + srcs: ["prebuilt_file"], + }`) +} + func registerTestPrebuiltBuildComponents(ctx RegistrationContext) { registerTestPrebuiltModules(ctx) From 73d491196bae765a211d88af5c4e48accc8c10cc Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Thu, 4 May 2023 18:16:11 +0900 Subject: [PATCH 0080/1460] Measure 'ninja_hint' time Bug: 273282046 Test: m nothing and check out/soong_build_metrics.pb Change-Id: I6440c3279b141c1f057145b668f8b96c45eaa75d --- cmd/soong_build/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 2723dfd12c..96f65a44b1 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -257,6 +257,8 @@ func apiBuildFileExcludes(ctx *android.Context) []string { } func writeNinjaHint(ctx *android.Context) error { + ctx.BeginEvent("ninja_hint") + defer ctx.EndEvent("ninja_hint") // The current predictor focuses on reducing false negatives. // If there are too many false positives (e.g., most modules are marked as positive), // real long-running jobs cannot run early. From 7b845e808f07a607a9ecf630eea00a334636f0c5 Mon Sep 17 00:00:00 2001 From: Jiakai Zhang Date: Tue, 2 May 2023 14:35:47 +0100 Subject: [PATCH 0081/1460] Generate app profiles even if dexpreopt is disabled. Bug: 280440941 Test: - 1. Patch ag/22302622 to disable dexpreopt. 2. lunch aosp_cf_x86_64_phone-userdebug && m 3. See app profiles still generated. Test: - 1. Patch ag/20592051 to enable profile for service-art. 2. banchan com.android.art x86_64 && m 3. See the profile for service-art generated. Change-Id: I4e721b475b84a2f667bbccc030a8947078f26bb0 --- android/testing.go | 21 +++++++++++++++------ dexpreopt/dexpreopt.go | 8 ++++++++ dexpreopt/testing.go | 7 +++++++ java/dex_test.go | 10 +++++----- java/dexpreopt.go | 10 ++-------- java/dexpreopt_test.go | 25 +++++++++++++++++++++++++ 6 files changed, 62 insertions(+), 19 deletions(-) diff --git a/android/testing.go b/android/testing.go index fc39a9c7af..2a9c6584e0 100644 --- a/android/testing.go +++ b/android/testing.go @@ -813,6 +813,20 @@ func normalizePathRelativeToTop(path Path) Path { return path.RelativeToTop() } +func allOutputs(p BuildParams) []string { + outputs := append(WritablePaths(nil), p.Outputs...) + outputs = append(outputs, p.ImplicitOutputs...) + if p.Output != nil { + outputs = append(outputs, p.Output) + } + return outputs.Strings() +} + +// AllOutputs returns all 'BuildParams.Output's and 'BuildParams.Outputs's in their full path string forms. +func (p TestingBuildParams) AllOutputs() []string { + return allOutputs(p.BuildParams) +} + // baseTestingComponent provides functionality common to both TestingModule and TestingSingleton. type baseTestingComponent struct { config Config @@ -954,12 +968,7 @@ func (b baseTestingComponent) buildParamsFromOutput(file string) TestingBuildPar func (b baseTestingComponent) allOutputs() []string { var outputFullPaths []string for _, p := range b.provider.BuildParamsForTests() { - outputs := append(WritablePaths(nil), p.Outputs...) - outputs = append(outputs, p.ImplicitOutputs...) - if p.Output != nil { - outputs = append(outputs, p.Output) - } - outputFullPaths = append(outputFullPaths, outputs.Strings()...) + outputFullPaths = append(outputFullPaths, allOutputs(p)...) } return outputFullPaths } diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index a590c72a51..2b38793ffd 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -100,11 +100,19 @@ func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongC return rule, nil } +// If dexpreopt is applicable to the module, returns whether dexpreopt is disabled. Otherwise, the +// behavior is undefined. +// When it returns true, dexpreopt artifacts will not be generated, but profile will still be +// generated if profile-guided compilation is requested. func dexpreoptDisabled(ctx android.PathContext, global *GlobalConfig, module *ModuleConfig) bool { if ctx.Config().UnbundledBuild() { return true } + if global.DisablePreopt { + return true + } + if contains(global.DisablePreoptModules, module.Name) { return true } diff --git a/dexpreopt/testing.go b/dexpreopt/testing.go index b3dd3cca24..6ed0736f77 100644 --- a/dexpreopt/testing.go +++ b/dexpreopt/testing.go @@ -181,3 +181,10 @@ func FixtureDisableDexpreoptBootImages(disable bool) android.FixturePreparer { dexpreoptConfig.DisablePreoptBootImages = disable }) } + +// FixtureDisableDexpreopt sets the DisablePreopt property in the global config. +func FixtureDisableDexpreopt(disable bool) android.FixturePreparer { + return FixtureModifyGlobalConfig(func(_ android.PathContext, dexpreoptConfig *GlobalConfig) { + dexpreoptConfig.DisablePreopt = disable + }) +} diff --git a/java/dex_test.go b/java/dex_test.go index 97fc3d0dda..2ba3831f49 100644 --- a/java/dex_test.go +++ b/java/dex_test.go @@ -23,7 +23,7 @@ import ( ) func TestR8(t *testing.T) { - result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` android_app { name: "app", srcs: ["foo.java"], @@ -191,7 +191,7 @@ func TestR8TransitiveDeps(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - fixturePreparer := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd + fixturePreparer := PrepareForTestWithJavaDefaultModules if tc.unbundled { fixturePreparer = android.GroupFixturePreparers( fixturePreparer, @@ -258,7 +258,7 @@ func TestR8TransitiveDeps(t *testing.T) { } func TestR8Flags(t *testing.T) { - result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` android_app { name: "app", srcs: ["foo.java"], @@ -287,7 +287,7 @@ func TestR8Flags(t *testing.T) { } func TestD8(t *testing.T) { - result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` java_library { name: "foo", srcs: ["foo.java"], @@ -328,7 +328,7 @@ func TestD8(t *testing.T) { } func TestProguardFlagsInheritance(t *testing.T) { - result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` android_app { name: "app", static_libs: [ diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 0ffedf6c40..f0bb9a32c9 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -180,6 +180,8 @@ func moduleName(ctx android.BaseModuleContext) string { return android.RemoveOptionalPrebuiltPrefix(ctx.ModuleName()) } +// Returns whether dexpreopt is applicable to the module. +// When it returns true, neither profile nor dexpreopt artifacts will be generated. func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { if !ctx.Device() { return true @@ -205,14 +207,6 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { global := dexpreopt.GetGlobalConfig(ctx) - if global.DisablePreopt { - return true - } - - if inList(moduleName(ctx), global.DisablePreoptModules) { - return true - } - isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) if isApexVariant(ctx) { // Don't preopt APEX variant module unless the module is an APEX system server jar. diff --git a/java/dexpreopt_test.go b/java/dexpreopt_test.go index 3d2c5c3a10..f91ac5cc3b 100644 --- a/java/dexpreopt_test.go +++ b/java/dexpreopt_test.go @@ -438,3 +438,28 @@ func TestAndroidMkEntriesForApex(t *testing.T) { android.AssertIntEquals(t, "entries count", 0, len(entriesList)) } + +func TestGenerateProfileEvenIfDexpreoptIsDisabled(t *testing.T) { + preparers := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + PrepareForTestWithFakeApexMutator, + dexpreopt.FixtureDisableDexpreopt(true), + ) + + result := preparers.RunTestWithBp(t, ` + java_library { + name: "foo", + installable: true, + dex_preopt: { + profile: "art-profile", + }, + srcs: ["a.java"], + }`) + + ctx := result.TestContext + dexpreopt := ctx.ModuleForTests("foo", "android_common").MaybeRule("dexpreopt") + + expected := []string{"out/soong/.intermediates/foo/android_common/dexpreopt/profile.prof"} + + android.AssertArrayString(t, "outputs", expected, dexpreopt.AllOutputs()) +} From a7bea31bee7ac12faaebb61fb005d35c1f7ac650 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Thu, 4 May 2023 09:51:56 -0400 Subject: [PATCH 0082/1460] Allowlist test_com.android.media.swcodec Test: b build --config=android test_com.android.media.swcodec Change-Id: I6fa18f8739c476ec6d5c8a09681e7acc74bf463c --- android/allowlists/allowlists.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 433c06357e..9cd50fa903 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -449,9 +449,8 @@ var ( "libidmap2_policies", "libSurfaceFlingerProp", // cc mainline modules - "code_coverage.policy", - "code_coverage.policy.other", - "codec2_soft_exports", + + // com.android.media.swcodec "com.android.media.swcodec", "com.android.media.swcodec-androidManifest", "com.android.media.swcodec-ld.config.txt", @@ -459,6 +458,12 @@ var ( "com.android.media.swcodec-mediaswcodec.rc", "com.android.media.swcodec.certificate", "com.android.media.swcodec.key", + "test_com.android.media.swcodec", + + // deps + "code_coverage.policy", + "code_coverage.policy.other", + "codec2_soft_exports", "flatbuffer_headers", "framework-connectivity-protos", "gemmlowp_headers", From ebb25bd22e87df67006f07d116d3bfc43c91866d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Kosi=C5=84ski?= Date: Tue, 20 Sep 2022 02:16:33 +0000 Subject: [PATCH 0083/1460] Delete Python 2 embedded launcher test. There are no remaining Python 2 binaries using an embedded launcher in Android. Bug: 245854393 Test: m par_test py2-cmd py3-cmd && build/soong/python/tests/runtest.sh Change-Id: I241bbaa417060b51b4d2883011ccb43e22ace4c8 --- python/python.go | 1 - python/tests/Android.bp | 23 ----------------------- python/tests/py-cmd_test.py | 2 +- python/tests/runtest.sh | 9 +-------- python/tests/testpkg/par_test.py | 6 +----- 5 files changed, 3 insertions(+), 38 deletions(-) diff --git a/python/python.go b/python/python.go index 1a129737a8..8fde638b4a 100644 --- a/python/python.go +++ b/python/python.go @@ -265,7 +265,6 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) { } if proptools.BoolDefault(props.Version.Py2.Enabled, false) { if !mctx.DeviceConfig().BuildBrokenUsesSoongPython2Modules() && - mctx.ModuleName() != "par_test" && mctx.ModuleName() != "py2-cmd" && mctx.ModuleName() != "py2-stdlib" { mctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3. This error can be temporarily overridden by setting BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES := true in the product configuration") diff --git a/python/tests/Android.bp b/python/tests/Android.bp index a656859609..e5569ba87b 100644 --- a/python/tests/Android.bp +++ b/python/tests/Android.bp @@ -27,29 +27,6 @@ python_test_host { test_options: { unit_test: false, }, - version: { - py2: { - enabled: true, - embedded_launcher: true, - }, - py3: { - enabled: false, - embedded_launcher: true, - }, - }, -} - -python_test_host { - name: "par_test3", - main: "par_test.py", - srcs: [ - "par_test.py", - "testpkg/par_test.py", - ], - // Is not implemented as a python unittest - test_options: { - unit_test: false, - }, version: { py3: { embedded_launcher: true, diff --git a/python/tests/py-cmd_test.py b/python/tests/py-cmd_test.py index acda2d7420..c7ba0ab4b1 100644 --- a/python/tests/py-cmd_test.py +++ b/python/tests/py-cmd_test.py @@ -57,7 +57,7 @@ def assert_equal(what, a, b): if sys.version_info[0] == 2: assert_equal("len(sys.path)", len(sys.path), 4) - assert_equal("sys.path[0]", sys.path[0], os.path.dirname(__file__)) + assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__))) assert_equal("sys.path[1]", sys.path[1], "/extra") assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, "internal")) assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, "internal", "stdlib")) diff --git a/python/tests/runtest.sh b/python/tests/runtest.sh index 35941dc881..f4abae5874 100755 --- a/python/tests/runtest.sh +++ b/python/tests/runtest.sh @@ -24,10 +24,9 @@ if [ -z $ANDROID_HOST_OUT ]; then fi if [[ ( ! -f $ANDROID_HOST_OUT/nativetest64/par_test/par_test ) || - ( ! -f $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 ) || ( ! -f $ANDROID_HOST_OUT/bin/py2-cmd ) || ( ! -f $ANDROID_HOST_OUT/bin/py3-cmd )]]; then - echo "Run 'm par_test par_test3 py2-cmd py3-cmd' first" + echo "Run 'm par_test py2-cmd py3-cmd' first" exit 1 fi @@ -41,12 +40,6 @@ PYTHONPATH=/usr $ANDROID_HOST_OUT/nativetest64/par_test/par_test ARGTEST=true $ANDROID_HOST_OUT/nativetest64/par_test/par_test --arg1 arg2 -PYTHONHOME= PYTHONPATH= $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 -PYTHONHOME=/usr $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 -PYTHONPATH=/usr $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 - -ARGTEST=true $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 --arg1 arg2 - cd $(dirname ${BASH_SOURCE[0]}) PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py diff --git a/python/tests/testpkg/par_test.py b/python/tests/testpkg/par_test.py index b51340907b..e12c527ad2 100644 --- a/python/tests/testpkg/par_test.py +++ b/python/tests/testpkg/par_test.py @@ -33,11 +33,7 @@ def assert_equal(what, a, b): fileName = fileName[:-1] assert_equal("__file__", fileName, os.path.join(archive, "testpkg/par_test.py")) -# Python3 is returning None here for me, and I haven't found any problems caused by this. -if sys.version_info[0] == 2: - assert_equal("__package__", __package__, "testpkg") -else: - assert_equal("__package__", __package__, None) +assert_equal("__package__", __package__, "testpkg") assert_equal("__loader__.archive", __loader__.archive, archive) assert_equal("__loader__.prefix", __loader__.prefix, "testpkg/") From 83a8bc47d2454052ed47577cf71ecfedeeb6f75f Mon Sep 17 00:00:00 2001 From: Alix Date: Thu, 4 May 2023 18:03:23 +0000 Subject: [PATCH 0084/1460] Delete java/rules.bzl file Bug: 277800767 Test: treehugger Change-Id: Idc80cd51317d167fca8d5543bb53c585adeaddad --- java/java.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/java.go b/java/java.go index d3e74fd1ce..c9dac6c880 100644 --- a/java/java.go +++ b/java/java.go @@ -2975,7 +2975,7 @@ func ktJvmLibraryBazelTargetModuleProperties() bazel.BazelTargetModuleProperties func javaLibraryBazelTargetModuleProperties() bazel.BazelTargetModuleProperties { return bazel.BazelTargetModuleProperties{ Rule_class: "java_library", - Bzl_load_location: "//build/bazel/rules/java:rules.bzl", + Bzl_load_location: "//build/bazel/rules/java:library.bzl", } } @@ -3084,7 +3084,7 @@ func javaBinaryHostBp2Build(ctx android.TopDownMutatorContext, m *Binary) { props := bazel.BazelTargetModuleProperties{ Rule_class: "java_binary", - Bzl_load_location: "//build/bazel/rules/java:rules.bzl", + Bzl_load_location: "@rules_java//java:defs.bzl", } binAttrs := &javaBinaryHostAttributes{ Runtime_deps: runtimeDeps, @@ -3140,7 +3140,7 @@ func (i *Import) ConvertWithBp2build(ctx android.TopDownMutatorContext) { } props := bazel.BazelTargetModuleProperties{ Rule_class: "java_import", - Bzl_load_location: "//build/bazel/rules/java:rules.bzl", + Bzl_load_location: "//build/bazel/rules/java:import.bzl", } name := android.RemoveOptionalPrebuiltPrefix(i.Name()) From d5fe1330774ee9e6342b08e371fdefce5c0d6214 Mon Sep 17 00:00:00 2001 From: Romain Jobredeaux Date: Thu, 4 May 2023 14:54:45 -0400 Subject: [PATCH 0085/1460] Provide a resource_prefix_strip for java_resources in bp2build. Although paths to resource files in a Bazel java_library should be relative to the package, the directory structure in the resulting jar will have resources under the full path from the top-level of the workspace, e.g. if a library in "a/BUILD" has java_resouces as "res/res.txt" then by default the res.txt file would appear under "a/res/res.txt". Fix this by adding a resource_strip_prefix in that case. Test: Unit tests Change-Id: If4325126f5c19a2a8fb83ee09bc3a95a18673fe3 --- bp2build/java_binary_host_conversion_test.go | 14 +++++++++----- bp2build/java_library_conversion_test.go | 16 ++++++++++------ java/java.go | 5 +++++ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go index c821f590ef..e51f608bd5 100644 --- a/bp2build/java_binary_host_conversion_test.go +++ b/bp2build/java_binary_host_conversion_test.go @@ -259,17 +259,20 @@ func TestJavaBinaryHostKotlinWithResources(t *testing.T) { runJavaBinaryHostTestCase(t, Bp2buildTestCase{ Description: "java_binary_host with srcs, libs, resources.", Filesystem: map[string]string{ - "test.mf": "Main-Class: com.android.test.MainClass", - "res/a.res": "", - "res/b.res": "", - }, - Blueprint: `java_binary_host { + "adir/test.mf": "Main-Class: com.android.test.MainClass", + "adir/res/a.res": "", + "adir/res/b.res": "", + "adir/Android.bp": `java_binary_host { name: "java-binary-host", manifest: "test.mf", srcs: ["a.java", "b.kt"], java_resources: ["res/a.res", "res/b.res"], + bazel_module: { bp2build_available: true }, } `, + }, + Dir: "adir", + Blueprint: "", ExpectedBazelTargets: []string{ MakeBazelTarget("kt_jvm_library", "java-binary-host_lib", AttrNameToString{ "srcs": `[ @@ -280,6 +283,7 @@ func TestJavaBinaryHostKotlinWithResources(t *testing.T) { "res/a.res", "res/b.res", ]`, + "resource_strip_prefix": `"adir"`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go index 24b763bcf5..60766f42ad 100644 --- a/bp2build/java_library_conversion_test.go +++ b/bp2build/java_library_conversion_test.go @@ -283,21 +283,25 @@ func TestJavaLibraryLogTags(t *testing.T) { func TestJavaLibraryResources(t *testing.T) { runJavaLibraryTestCase(t, Bp2buildTestCase{ + Dir: "adir", Filesystem: map[string]string{ - "res/a.res": "", - "res/b.res": "", - "res/dir1/b.res": "", - }, - Blueprint: `java_library { + "adir/res/a.res": "", + "adir/res/b.res": "", + "adir/res/dir1/b.res": "", + "adir/Android.bp": `java_library { name: "java-lib-1", - java_resources: ["res/a.res", "res/b.res"], + java_resources: ["res/a.res", "res/b.res"], + bazel_module: { bp2build_available: true }, }`, + }, + Blueprint: "", ExpectedBazelTargets: []string{ MakeBazelTarget("java_library", "java-lib-1", AttrNameToString{ "resources": `[ "res/a.res", "res/b.res", ]`, + "resource_strip_prefix": `"adir"`, }), MakeNeverlinkDuplicateTarget("java_library", "java-lib-1"), }, diff --git a/java/java.go b/java/java.go index 0da73281b8..cb8003dcc3 100644 --- a/java/java.go +++ b/java/java.go @@ -2707,8 +2707,13 @@ func (m *Library) convertJavaResourcesAttributes(ctx android.TopDownMutatorConte var resources bazel.LabelList var resourceStripPrefix *string + if m.properties.Java_resources != nil && len(m.properties.Java_resource_dirs) > 0 { + ctx.ModuleErrorf("bp2build doesn't support both java_resources and java_resource_dirs being set on the same module.") + } + if m.properties.Java_resources != nil { resources.Append(android.BazelLabelForModuleSrc(ctx, m.properties.Java_resources)) + resourceStripPrefix = proptools.StringPtr(ctx.ModuleDir()) } //TODO(b/179889880) handle case where glob includes files outside package From ccb20f47d5318a9146a365daa2b2dc3549ad3763 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 4 May 2023 12:38:24 -0700 Subject: [PATCH 0086/1460] Enable prebuilt apk checks Now that existing violations set the skip flag. Bug: 185811447 Test: Presubmits Change-Id: Ia513b71be1cd878f36fdb28c94ef3cdc3f2a6bc6 --- java/app_import.go | 11 +++++----- java/app_import_test.go | 45 ++++++++++++++++++++--------------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/java/app_import.go b/java/app_import.go index 52ae0247ae..9c01960390 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -335,12 +335,11 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext if proptools.Bool(a.properties.Preprocessed) { output := srcApk - // TODO(b/185811447) Uncomment this after all existing failing apks set skip_preprocessed_apk_checks: true - //if !proptools.Bool(a.properties.Skip_preprocessed_apk_checks) { - // writableOutput := android.PathForModuleOut(ctx, "validated-prebuilt", apkFilename) - // a.validatePreprocessedApk(ctx, srcApk, writableOutput) - // output = writableOutput - //} + if !proptools.Bool(a.properties.Skip_preprocessed_apk_checks) { + writableOutput := android.PathForModuleOut(ctx, "validated-prebuilt", apkFilename) + a.validatePreprocessedApk(ctx, srcApk, writableOutput) + output = writableOutput + } a.outputFile = output a.certificate = PresignedCertificate } else if !Bool(a.properties.Presigned) { diff --git a/java/app_import_test.go b/java/app_import_test.go index 845a96299b..bb8fab93ba 100644 --- a/java/app_import_test.go +++ b/java/app_import_test.go @@ -657,29 +657,28 @@ func TestAndroidTestImport_Preprocessed(t *testing.T) { } } -// TODO(b/185811447) Uncomment this after all existing failing apks set skip_preprocessed_apk_checks: true -//func TestAndroidAppImport_Preprocessed(t *testing.T) { -// ctx, _ := testJava(t, ` -// android_app_import { -// name: "foo", -// apk: "prebuilts/apk/app.apk", -// presigned: true, -// preprocessed: true, -// } -// `) -// -// apkName := "foo.apk" -// variant := ctx.ModuleForTests("foo", "android_common") -// outputBuildParams := variant.Output("validated-prebuilt/" + apkName).BuildParams -// if outputBuildParams.Rule.String() != android.Cp.String() { -// t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String()) -// } -// -// // Make sure compression and aligning were validated. -// if len(outputBuildParams.Validations) != 2 { -// t.Errorf("Expected compression/alignment validation rules, found %d validations", len(outputBuildParams.Validations)) -// } -//} +func TestAndroidAppImport_Preprocessed(t *testing.T) { + ctx, _ := testJava(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + presigned: true, + preprocessed: true, + } + `) + + apkName := "foo.apk" + variant := ctx.ModuleForTests("foo", "android_common") + outputBuildParams := variant.Output("validated-prebuilt/" + apkName).BuildParams + if outputBuildParams.Rule.String() != android.Cp.String() { + t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String()) + } + + // Make sure compression and aligning were validated. + if len(outputBuildParams.Validations) != 2 { + t.Errorf("Expected compression/alignment validation rules, found %d validations", len(outputBuildParams.Validations)) + } +} func TestAndroidTestImport_UncompressDex(t *testing.T) { testCases := []struct { From 290fc2c1b1c3b4d47e8254eb1edada6e2fff8b95 Mon Sep 17 00:00:00 2001 From: Yu Liu Date: Thu, 4 May 2023 12:48:50 -0700 Subject: [PATCH 0087/1460] Remove the codec2 hidl libs from mixed build This is to investigate some CI failures. Bug: 280845386 Test: manual build Change-Id: Ic2e8a5a146caead0955f5c73cb6ddb51fc0162cd --- android/allowlists/allowlists.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 67f3132a1e..df47a5c914 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -1557,9 +1557,6 @@ var ( "com.android.media.swcodec", "test_com.android.media.swcodec", "libstagefright_foundation", - "libcodec2_hidl@1.0", - "libcodec2_hidl@1.1", - "libcodec2_hidl@1.2", } // These should be the libs that are included by the apexes in the ProdMixedBuildsEnabledList From 036afabcfb148d468e710145f822ce6f944a9847 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Thu, 4 May 2023 16:43:36 -0400 Subject: [PATCH 0088/1460] export WarningAllowedProjects to bazel Bug: 187088227 Test: b test Change-Id: I668f2c3e834fd03ad2b6d01f07f79ec84dea54eb --- cc/config/global.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cc/config/global.go b/cc/config/global.go index 20298dd5b1..dbc43f78af 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -444,6 +444,8 @@ func init() { pctx.StaticVariableWithEnvOverride("ClangShortVersion", "LLVM_RELEASE_VERSION", ClangDefaultShortVersion) pctx.StaticVariable("ClangAsanLibDir", "${ClangBase}/linux-x86/${ClangVersion}/lib/clang/${ClangShortVersion}/lib/linux") + exportedVars.ExportStringListStaticVariable("WarningAllowedProjects", WarningAllowedProjects) + // These are tied to the version of LLVM directly in external/llvm, so they might trail the host prebuilts // being used for the rest of the build process. pctx.SourcePathVariable("RSClangBase", "prebuilts/clang/host") From daa314ac97d5ab8076113ee7f269b4e4cc3cf620 Mon Sep 17 00:00:00 2001 From: Gurpreet Singh Date: Fri, 21 Apr 2023 16:30:03 +0000 Subject: [PATCH 0089/1460] Add genrule to build *.latest.version build target. Build a new target *.latest.version which will contain a text file containing the last finalized version. Bug: 242316893 Test: atest prebuilt_apis_test Change-Id: I41fa91c9ec273f342b7807c66c4d65ba13260124 --- java/prebuilt_apis.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go index 206d995276..0740467eb8 100644 --- a/java/prebuilt_apis.go +++ b/java/prebuilt_apis.go @@ -135,6 +135,19 @@ func createApiModule(mctx android.LoadHookContext, name string, path string) { mctx.CreateModule(genrule.GenRuleFactory, &genruleProps) } +func createLatestApiModuleExtensionVersionFile(mctx android.LoadHookContext, name string, version string) { + genruleProps := struct { + Name *string + Srcs []string + Out []string + Cmd *string + }{} + genruleProps.Name = proptools.StringPtr(name) + genruleProps.Out = []string{name} + genruleProps.Cmd = proptools.StringPtr("echo " + version + " > $(out)") + mctx.CreateModule(genrule.GenRuleFactory, &genruleProps) +} + func createEmptyFile(mctx android.LoadHookContext, name string) { props := struct { Name *string @@ -233,9 +246,10 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { type latestApiInfo struct { module, scope, path string version int + isExtensionApiFile bool } - getLatest := func(files []string) map[string]latestApiInfo { + getLatest := func(files []string, isExtensionApiFile bool) map[string]latestApiInfo { m := make(map[string]latestApiInfo) for _, f := range files { module, version, scope := parseFinalizedPrebuiltPath(mctx, f) @@ -245,16 +259,16 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { key := module + "." + scope info, exists := m[key] if !exists || version > info.version { - m[key] = latestApiInfo{module, scope, f, version} + m[key] = latestApiInfo{module, scope, f, version, isExtensionApiFile} } } return m } - latest := getLatest(apiLevelFiles) + latest := getLatest(apiLevelFiles, false) if p.properties.Extensions_dir != nil { extensionApiFiles := globExtensionDirs(mctx, p, "api/*.txt") - for k, v := range getLatest(extensionApiFiles) { + for k, v := range getLatest(extensionApiFiles, true) { if _, exists := latest[k]; !exists { mctx.ModuleErrorf("Module %v finalized for extension %d but never during an API level; likely error", v.module, v.version) } @@ -267,6 +281,12 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { for _, k := range android.SortedKeys(latest) { info := latest[k] name := PrebuiltApiModuleName(info.module, info.scope, "latest") + latestExtensionVersionModuleName := PrebuiltApiModuleName(info.module, info.scope, "latest.extension_version") + if info.isExtensionApiFile { + createLatestApiModuleExtensionVersionFile(mctx, latestExtensionVersionModuleName, strconv.Itoa(info.version)) + } else { + createLatestApiModuleExtensionVersionFile(mctx, latestExtensionVersionModuleName, "-1") + } createApiModule(mctx, name, info.path) } From d33c2fd5a2a1c8bfcf97a1e9b103232776a73325 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Thu, 4 May 2023 20:40:04 +0000 Subject: [PATCH 0090/1460] Add --bazel-exit-code parameter to soong_ui. This is for use in tracking bazel exit codes from b invocations. Bug: 279754118 Test: b build libcore:all (with prints to verify) Change-Id: If36b5b73ff93cd15647e5c1c1f676e95137210ff --- cmd/soong_ui/main.go | 12 ++---------- ui/build/config.go | 18 ++++++++++++++++-- ui/build/upload.go | 5 +++-- ui/metrics/metrics.go | 5 ++++- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 301246a88c..bd774c6456 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -94,7 +94,7 @@ var commands = []command{ }, { flag: "--upload-metrics-only", description: "upload metrics without building anything", - config: uploadOnlyConfig, + config: build.UploadOnlyConfig, stdio: stdio, // Upload-only mode mostly skips to the metrics-uploading phase of soong_ui. // However, this invocation marks the true "end of the build", and thus we @@ -451,14 +451,6 @@ func dumpVarConfig(ctx build.Context, args ...string) build.Config { return build.NewConfig(ctx) } -// uploadOnlyConfig explicitly requires no arguments. -func uploadOnlyConfig(ctx build.Context, args ...string) build.Config { - if len(args) > 0 { - fmt.Printf("--upload-only does not require arguments.") - } - return build.UploadOnlyConfig(ctx) -} - func buildActionConfig(ctx build.Context, args ...string) build.Config { flags := flag.NewFlagSet("build-mode", flag.ContinueOnError) flags.SetOutput(ctx.Writer) @@ -710,7 +702,7 @@ func updateTotalRealTime(ctx build.Context, config build.Config, args []string) } met := ctx.ContextImpl.Metrics - err = met.UpdateTotalRealTime(data) + err = met.UpdateTotalRealTimeAndNonZeroExit(data, config.BazelExitCode()) if err != nil { ctx.Fatal(err) } diff --git a/ui/build/config.go b/ui/build/config.go index 2dda52a01f..8ec96800f3 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -89,7 +89,8 @@ type configImpl struct { skipMetricsUpload bool buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time buildFromTextStub bool - ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built + ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built + bazelExitCode int32 // For b-runs - necessary for updating NonZeroExit // From the product config katiArgs []string @@ -298,11 +299,12 @@ func defaultBazelProdMode(cfg *configImpl) bool { return true } -func UploadOnlyConfig(ctx Context, _ ...string) Config { +func UploadOnlyConfig(ctx Context, args ...string) Config { ret := &configImpl{ environ: OsEnvironment(), sandboxConfig: &SandboxConfig{}, } + ret.parseArgs(ctx, args) srcDir := absPath(ctx, ".") bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") if err := loadEnvConfig(ctx, ret, bc); err != nil { @@ -883,6 +885,14 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { } } else if arg == "--ensure-allowlist-integrity" { c.ensureAllowlistIntegrity = true + } else if strings.HasPrefix(arg, "--bazel-exit-code=") { + bazelExitCodeStr := strings.TrimPrefix(arg, "--bazel-exit-code=") + val, err := strconv.Atoi(bazelExitCodeStr) + if err == nil { + c.bazelExitCode = int32(val) + } else { + ctx.Fatalf("Error parsing bazel-exit-code", err) + } } else if len(arg) > 0 && arg[0] == '-' { parseArgNum := func(def int) int { if len(arg) > 2 { @@ -1723,6 +1733,10 @@ func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time return time.UnixMilli(c.buildStartedTime) } +func (c *configImpl) BazelExitCode() int32 { + return c.bazelExitCode +} + func GetMetricsUploader(topDir string, env *Environment) string { if p, ok := env.Get("METRICS_UPLOADER"); ok { metricsUploader := filepath.Join(topDir, p) diff --git a/ui/build/upload.go b/ui/build/upload.go index 1e6d94aad4..ee4a5b3450 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -141,7 +141,7 @@ func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming { // This method takes a file created by bazel's --analyze-profile mode and // writes bazel metrics data to the provided filepath. // TODO(b/279987768) - move this outside of upload.go -func processBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context) { +func processBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context, config Config) { if bazelProfileFile == "" { return } @@ -179,6 +179,7 @@ func processBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx C return } bazelProto := readBazelProto(bazelProfileFile) + bazelProto.ExitCode = proto.Int32(config.bazelExitCode) shared.Save(&bazelProto, bazelMetricsFile) } @@ -192,7 +193,7 @@ func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted t defer ctx.EndTrace() uploader := config.MetricsUploaderApp() - processBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx) + processBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx, config) if uploader == "" { // If the uploader path was not specified, no metrics shall be uploaded. diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go index 82d11ed305..a282e2030b 100644 --- a/ui/metrics/metrics.go +++ b/ui/metrics/metrics.go @@ -228,7 +228,7 @@ func (m *Metrics) SetBuildDateTime(buildTimestamp time.Time) { m.metrics.BuildDateTimestamp = proto.Int64(buildTimestamp.UnixNano() / int64(time.Second)) } -func (m *Metrics) UpdateTotalRealTime(data []byte) error { +func (m *Metrics) UpdateTotalRealTimeAndNonZeroExit(data []byte, bazelExitCode int32) error { if err := proto.Unmarshal(data, &m.metrics); err != nil { return fmt.Errorf("Failed to unmarshal proto", err) } @@ -236,6 +236,9 @@ func (m *Metrics) UpdateTotalRealTime(data []byte) error { endTime := uint64(time.Now().UnixNano()) *m.metrics.Total.RealTime = *proto.Uint64(endTime - startTime) + + bazelError := bazelExitCode != 0 + m.metrics.NonZeroExit = proto.Bool(bazelError) return nil } From 9cad90f966cb3245accfb1cd788cc0ee1c6048f9 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Thu, 4 May 2023 17:15:44 +0000 Subject: [PATCH 0091/1460] Broaden the granularity of config_setting from apex_name to api_domain The use case for this is for creating a select statement for stub/impl selection. Since Bazel propagates apex_name from the top-level apex, the source apex and test apex builds a specific library in two different configurations. We need select statements for both these two configurations, but this metadata might not always exist in Android.bp since test apexes are not necessary to be listed in Android.bp files. To overcome this, the select statements will be created per api domain instead. This CL uses a naming convention to infer the api domain of apexes. Test: go test ./bp2build Change-Id: Iad2b45f736bc98a24d2ce1cf2f69aad67973092d --- bazel/properties.go | 2 ++ bp2build/cc_library_conversion_test.go | 5 +-- cc/bp2build.go | 48 +++++++++++++++++++------- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/bazel/properties.go b/bazel/properties.go index 1757bad490..c3b68a47e8 100644 --- a/bazel/properties.go +++ b/bazel/properties.go @@ -1434,4 +1434,6 @@ type StringMapAttribute map[string]string type ConfigSettingAttributes struct { // Each key in Flag_values is a label to a custom string_setting Flag_values StringMapAttribute + // Each element in Constraint_values is a label to a constraint_value + Constraint_values LabelListAttribute } diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 776129f582..6d5fd7ec9e 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -4473,11 +4473,12 @@ cc_library { ExpectedBazelTargets: []string{ MakeBazelTargetNoRestrictions( "config_setting", - "android-in_myapex", + "myapex", AttrNameToString{ "flag_values": `{ - "//build/bazel/rules/apex:apex_name": "myapex", + "//build/bazel/rules/apex:api_domain": "myapex", }`, + "constraint_values": `["//build/bazel/platforms/os:android"]`, }, ), }, diff --git a/cc/bp2build.go b/cc/bp2build.go index 749fce59a3..6e91bffa24 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1194,16 +1194,30 @@ func availableToSameApexes(a, b []string) bool { } var ( - apexConfigSettingKey = android.NewOnceKey("apexConfigSetting") - apexConfigSettingLock sync.Mutex + apiDomainConfigSettingKey = android.NewOnceKey("apiDomainConfigSettingKey") + apiDomainConfigSettingLock sync.Mutex ) -func getApexConfigSettingMap(config android.Config) *map[string]bool { - return config.Once(apexConfigSettingKey, func() interface{} { +func getApiDomainConfigSettingMap(config android.Config) *map[string]bool { + return config.Once(apiDomainConfigSettingKey, func() interface{} { return &map[string]bool{} }).(*map[string]bool) } +var ( + testApexNameToApiDomain = map[string]string{ + "test_broken_com.android.art": "com.android.art", + } +) + +func getApiDomain(apexName string) string { + if apiDomain, exists := testApexNameToApiDomain[apexName]; exists { + return apiDomain + } + // Remove `test_` prefix + return strings.TrimPrefix(apexName, "test_") +} + // Create a config setting for this apex in build/bazel/rules/apex // The use case for this is stub/impl selection in cc libraries // Long term, these config_setting(s) should be colocated with the respective apex definitions. @@ -1215,23 +1229,32 @@ func createInApexConfigSetting(ctx android.TopDownMutatorContext, apexName strin // These correspond to android-non_apex and android-in_apex return } - apexConfigSettingLock.Lock() - defer apexConfigSettingLock.Unlock() + apiDomainConfigSettingLock.Lock() + defer apiDomainConfigSettingLock.Unlock() // Return if a config_setting has already been created - acsm := getApexConfigSettingMap(ctx.Config()) - if _, exists := (*acsm)[apexName]; exists { + apiDomain := getApiDomain(apexName) + acsm := getApiDomainConfigSettingMap(ctx.Config()) + if _, exists := (*acsm)[apiDomain]; exists { return } - (*acsm)[apexName] = true + (*acsm)[apiDomain] = true csa := bazel.ConfigSettingAttributes{ Flag_values: bazel.StringMapAttribute{ - "//build/bazel/rules/apex:apex_name": apexName, + "//build/bazel/rules/apex:api_domain": apiDomain, }, + // Constraint this to android + Constraint_values: bazel.MakeLabelListAttribute( + bazel.MakeLabelList( + []bazel.Label{ + bazel.Label{Label: "//build/bazel/platforms/os:android"}, + }, + ), + ), } ca := android.CommonAttributes{ - Name: "android-in_" + apexName, + Name: apiDomain, } ctx.CreateBazelConfigSetting( csa, @@ -1247,7 +1270,8 @@ func inApexConfigSetting(apexAvailable string) string { if apexAvailable == android.AvailableToAnyApex { return bazel.AndroidAndInApex } - return "//build/bazel/rules/apex:android-in_" + apexAvailable + apiDomain := getApiDomain(apexAvailable) + return "//build/bazel/rules/apex:" + apiDomain } func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.ConfigurationAxis, From 8f065e6c5dc6dbd084d31e23764c8f208202c2cb Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Thu, 4 May 2023 18:02:22 -0700 Subject: [PATCH 0092/1460] Delete rbc-run script It was inlined into the usage in the other cl in this topic. Bug: 280685526 Test: ./build/bazel/ci/rbc_dashboard.py --quick aosp_arm64-userdebug Change-Id: I68af9186e17a996baf4676d97cdc9895abcb7e06 --- scripts/rbc-run | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100755 scripts/rbc-run diff --git a/scripts/rbc-run b/scripts/rbc-run deleted file mode 100755 index 8d93f0e992..0000000000 --- a/scripts/rbc-run +++ /dev/null @@ -1,18 +0,0 @@ -#! /bin/bash -# Convert and run one configuration -# Args: a product/board makefile optionally followed by additional arguments -# that will be passed to rbcrun. -[[ $# -gt 1 && -f "$1" && -f "$2" ]] || { echo "Usage: ${0##*/} product.mk input_variables.mk [Additional rbcrun arguments]" >&2; exit 1; } -set -eu - -declare -r output_root="${OUT_DIR:-out}" -declare -r runner="${output_root}/rbcrun" -declare -r converter="${output_root}/mk2rbc" -declare -r launcher="${output_root}/rbc/launcher.rbc" -declare -r makefile_list="${output_root}/.module_paths/configuration.list" -declare -r makefile="$1" -declare -r input_variables="$2" -shift 2 -"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" --makefile_list="${makefile_list}" "${makefile}" -"${runner}" RBC_OUT="make,global" RBC_DEBUG="${RBC_DEBUG:-}" $@ "${launcher}" - From c642d688d6a091766536f0e02696d43e393f93e0 Mon Sep 17 00:00:00 2001 From: Wei Li Date: Thu, 4 May 2023 09:06:06 -0700 Subject: [PATCH 0093/1460] Export some make variables to Bazel through product variables. Bug: 275472038 Test: CIs Change-Id: I4fdd0f989d75045644e8f2e32d1eb77543a3b9f3 --- android/variable.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/android/variable.go b/android/variable.go index 496f523b29..aaf0606e17 100644 --- a/android/variable.go +++ b/android/variable.go @@ -469,6 +469,10 @@ type productVariables struct { SourceRootDirs []string `json:",omitempty"` AfdoProfiles []string `json:",omitempty"` + + ProductManufacturer string `json:",omitempty"` + ProductBrand string `json:",omitempty"` + BuildVersionTags []string `json:",omitempty"` } func boolPtr(v bool) *bool { From 3d08c388b9e00cf98510cbd1408b5ffc3524e2ff Mon Sep 17 00:00:00 2001 From: Qiao Yang Date: Fri, 5 May 2023 15:03:24 +0000 Subject: [PATCH 0094/1460] Revert "Generate app profiles even if dexpreopt is disabled." Revert submission 2574032 Reason for revert: DroidMonitor-triggered revert due to breakage , bug Reverted changes: /q/submissionid:2574032 Change-Id: Ia9d05f3b7439604eb4a4b4100f46879fe11f5820 BUG: <280902279> --- android/testing.go | 21 ++++++--------------- dexpreopt/dexpreopt.go | 8 -------- dexpreopt/testing.go | 7 ------- java/dex_test.go | 10 +++++----- java/dexpreopt.go | 10 ++++++++-- java/dexpreopt_test.go | 25 ------------------------- 6 files changed, 19 insertions(+), 62 deletions(-) diff --git a/android/testing.go b/android/testing.go index 2a9c6584e0..fc39a9c7af 100644 --- a/android/testing.go +++ b/android/testing.go @@ -813,20 +813,6 @@ func normalizePathRelativeToTop(path Path) Path { return path.RelativeToTop() } -func allOutputs(p BuildParams) []string { - outputs := append(WritablePaths(nil), p.Outputs...) - outputs = append(outputs, p.ImplicitOutputs...) - if p.Output != nil { - outputs = append(outputs, p.Output) - } - return outputs.Strings() -} - -// AllOutputs returns all 'BuildParams.Output's and 'BuildParams.Outputs's in their full path string forms. -func (p TestingBuildParams) AllOutputs() []string { - return allOutputs(p.BuildParams) -} - // baseTestingComponent provides functionality common to both TestingModule and TestingSingleton. type baseTestingComponent struct { config Config @@ -968,7 +954,12 @@ func (b baseTestingComponent) buildParamsFromOutput(file string) TestingBuildPar func (b baseTestingComponent) allOutputs() []string { var outputFullPaths []string for _, p := range b.provider.BuildParamsForTests() { - outputFullPaths = append(outputFullPaths, allOutputs(p)...) + outputs := append(WritablePaths(nil), p.Outputs...) + outputs = append(outputs, p.ImplicitOutputs...) + if p.Output != nil { + outputs = append(outputs, p.Output) + } + outputFullPaths = append(outputFullPaths, outputs.Strings()...) } return outputFullPaths } diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index 2b38793ffd..a590c72a51 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -100,19 +100,11 @@ func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongC return rule, nil } -// If dexpreopt is applicable to the module, returns whether dexpreopt is disabled. Otherwise, the -// behavior is undefined. -// When it returns true, dexpreopt artifacts will not be generated, but profile will still be -// generated if profile-guided compilation is requested. func dexpreoptDisabled(ctx android.PathContext, global *GlobalConfig, module *ModuleConfig) bool { if ctx.Config().UnbundledBuild() { return true } - if global.DisablePreopt { - return true - } - if contains(global.DisablePreoptModules, module.Name) { return true } diff --git a/dexpreopt/testing.go b/dexpreopt/testing.go index 6ed0736f77..b3dd3cca24 100644 --- a/dexpreopt/testing.go +++ b/dexpreopt/testing.go @@ -181,10 +181,3 @@ func FixtureDisableDexpreoptBootImages(disable bool) android.FixturePreparer { dexpreoptConfig.DisablePreoptBootImages = disable }) } - -// FixtureDisableDexpreopt sets the DisablePreopt property in the global config. -func FixtureDisableDexpreopt(disable bool) android.FixturePreparer { - return FixtureModifyGlobalConfig(func(_ android.PathContext, dexpreoptConfig *GlobalConfig) { - dexpreoptConfig.DisablePreopt = disable - }) -} diff --git a/java/dex_test.go b/java/dex_test.go index 2ba3831f49..97fc3d0dda 100644 --- a/java/dex_test.go +++ b/java/dex_test.go @@ -23,7 +23,7 @@ import ( ) func TestR8(t *testing.T) { - result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` android_app { name: "app", srcs: ["foo.java"], @@ -191,7 +191,7 @@ func TestR8TransitiveDeps(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - fixturePreparer := PrepareForTestWithJavaDefaultModules + fixturePreparer := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd if tc.unbundled { fixturePreparer = android.GroupFixturePreparers( fixturePreparer, @@ -258,7 +258,7 @@ func TestR8TransitiveDeps(t *testing.T) { } func TestR8Flags(t *testing.T) { - result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` android_app { name: "app", srcs: ["foo.java"], @@ -287,7 +287,7 @@ func TestR8Flags(t *testing.T) { } func TestD8(t *testing.T) { - result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` java_library { name: "foo", srcs: ["foo.java"], @@ -328,7 +328,7 @@ func TestD8(t *testing.T) { } func TestProguardFlagsInheritance(t *testing.T) { - result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` android_app { name: "app", static_libs: [ diff --git a/java/dexpreopt.go b/java/dexpreopt.go index f0bb9a32c9..0ffedf6c40 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -180,8 +180,6 @@ func moduleName(ctx android.BaseModuleContext) string { return android.RemoveOptionalPrebuiltPrefix(ctx.ModuleName()) } -// Returns whether dexpreopt is applicable to the module. -// When it returns true, neither profile nor dexpreopt artifacts will be generated. func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { if !ctx.Device() { return true @@ -207,6 +205,14 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { global := dexpreopt.GetGlobalConfig(ctx) + if global.DisablePreopt { + return true + } + + if inList(moduleName(ctx), global.DisablePreoptModules) { + return true + } + isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) if isApexVariant(ctx) { // Don't preopt APEX variant module unless the module is an APEX system server jar. diff --git a/java/dexpreopt_test.go b/java/dexpreopt_test.go index f91ac5cc3b..3d2c5c3a10 100644 --- a/java/dexpreopt_test.go +++ b/java/dexpreopt_test.go @@ -438,28 +438,3 @@ func TestAndroidMkEntriesForApex(t *testing.T) { android.AssertIntEquals(t, "entries count", 0, len(entriesList)) } - -func TestGenerateProfileEvenIfDexpreoptIsDisabled(t *testing.T) { - preparers := android.GroupFixturePreparers( - PrepareForTestWithJavaDefaultModules, - PrepareForTestWithFakeApexMutator, - dexpreopt.FixtureDisableDexpreopt(true), - ) - - result := preparers.RunTestWithBp(t, ` - java_library { - name: "foo", - installable: true, - dex_preopt: { - profile: "art-profile", - }, - srcs: ["a.java"], - }`) - - ctx := result.TestContext - dexpreopt := ctx.ModuleForTests("foo", "android_common").MaybeRule("dexpreopt") - - expected := []string{"out/soong/.intermediates/foo/android_common/dexpreopt/profile.prof"} - - android.AssertArrayString(t, "outputs", expected, dexpreopt.AllOutputs()) -} From 8d8c660710037e87c7094a612246d64e5af74846 Mon Sep 17 00:00:00 2001 From: Qiao Yang Date: Fri, 5 May 2023 15:03:24 +0000 Subject: [PATCH 0095/1460] Revert "Generate boot image profiles even if dexpreopt is disabled." Revert submission 2574032 Reason for revert: DroidMonitor-triggered revert due to breakage , bug Reverted changes: /q/submissionid:2574032 Change-Id: I8e99f8231639198b149ea8d822ee7f9a5b391a89 BUG: <280902279> --- apex/apex.go | 2 +- apex/bootclasspath_fragment_test.go | 20 -------------------- dexpreopt/testing.go | 7 ------- java/bootclasspath_fragment.go | 15 ++++++++------- java/dexpreopt_bootjars.go | 7 +++---- java/platform_bootclasspath.go | 15 +++++++++------ sdk/java_sdk_test.go | 2 -- 7 files changed, 21 insertions(+), 47 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index 5e4d3fc0e9..baf4737d82 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -2675,7 +2675,7 @@ func apexBootclasspathFragmentFiles(ctx android.ModuleContext, module blueprint. } pathInApex := bootclasspathFragmentInfo.ProfileInstallPathInApex() - if pathInApex != "" { + if pathInApex != "" && !java.SkipDexpreoptBootJars(ctx) { pathOnHost := bootclasspathFragmentInfo.ProfilePathOnHost() tempPath := android.PathForModuleOut(ctx, "boot_image_profile", pathInApex) diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index 4a661d4ea4..2ddfd03055 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -497,26 +497,6 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { }) }) - t.Run("generate boot image profile even if dexpreopt is disabled", func(t *testing.T) { - result := android.GroupFixturePreparers( - commonPreparer, - - // Configure some libraries in the art bootclasspath_fragment that match the source - // bootclasspath_fragment's contents property. - java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), - addSource("foo", "bar"), - java.FixtureSetBootImageInstallDirOnDevice("art", "system/framework"), - dexpreopt.FixtureDisableDexpreoptBootImages(true), - ).RunTest(t) - - ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ - "etc/boot-image.prof", - "etc/classpaths/bootclasspath.pb", - "javalib/bar.jar", - "javalib/foo.jar", - }) - }) - t.Run("boot image disable generate profile", func(t *testing.T) { result := android.GroupFixturePreparers( commonPreparer, diff --git a/dexpreopt/testing.go b/dexpreopt/testing.go index b3dd3cca24..47ae494e33 100644 --- a/dexpreopt/testing.go +++ b/dexpreopt/testing.go @@ -174,10 +174,3 @@ func FixtureDisableGenerateProfile(disable bool) android.FixturePreparer { dexpreoptConfig.DisableGenerateProfile = disable }) } - -// FixtureDisableDexpreoptBootImages sets the DisablePreoptBootImages property in the global config. -func FixtureDisableDexpreoptBootImages(disable bool) android.FixturePreparer { - return FixtureModifyGlobalConfig(func(_ android.PathContext, dexpreoptConfig *GlobalConfig) { - dexpreoptConfig.DisablePreoptBootImages = disable - }) -} diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index d1a020e4d3..f692563478 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -506,6 +506,10 @@ func (b *BootclasspathFragmentModule) DepsMutator(ctx android.BottomUpMutatorCon } } + if SkipDexpreoptBootJars(ctx) { + return + } + // Add a dependency onto the dex2oat tool which is needed for creating the boot image. The // path is retrieved from the dependency by GetGlobalSoongConfig(ctx). dexpreopt.RegisterToolDeps(ctx) @@ -897,6 +901,10 @@ func (b *BootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleC // produceBootImageFiles builds the boot image files from the source if it is required. func (b *BootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig) bootImageOutputs { + if SkipDexpreoptBootJars(ctx) { + return bootImageOutputs{} + } + // Only generate the boot image if the configuration does not skip it. return b.generateBootImageBuildActions(ctx, imageConfig) } @@ -924,13 +932,6 @@ func (b *BootclasspathFragmentModule) generateBootImageBuildActions(ctx android. // Build boot image files for the host variants. buildBootImageVariantsForBuildOs(ctx, imageConfig, profile) - // If dexpreopt of boot image jars should be skipped, generate only a profile. - if SkipDexpreoptBootJars(ctx) { - return bootImageOutputs{ - profile: profile, - } - } - // Build boot image files for the android variants. bootImageFiles := buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index fed384e9d0..f4827ea3a7 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -500,6 +500,9 @@ func (d *dexpreoptBootJars) GenerateAndroidBuildActions(ctx android.ModuleContex // Generate build rules for boot images. func (d *dexpreoptBootJars) GenerateSingletonBuildActions(ctx android.SingletonContext) { + if SkipDexpreoptBootJars(ctx) { + return + } if dexpreopt.GetCachedGlobalSoongConfig(ctx) == nil { // No module has enabled dexpreopting, so we assume there will be no boot image to make. return @@ -1011,10 +1014,6 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_IMAGE_PROFILE_LICENSE_METADATA", image.profileLicenseMetadataFile.String()) } - if SkipDexpreoptBootJars(ctx) { - return - } - global := dexpreopt.GetGlobalConfig(ctx) dexPaths, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_FILES", strings.Join(dexPaths.Strings(), " ")) diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index 254c40535f..0ea3609798 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -103,6 +103,10 @@ func (b *platformBootclasspathModule) OutputFiles(tag string) (android.Paths, er func (b *platformBootclasspathModule) DepsMutator(ctx android.BottomUpMutatorContext) { b.hiddenAPIDepsMutator(ctx) + if SkipDexpreoptBootJars(ctx) { + return + } + // Add a dependency onto the dex2oat tool which is needed for creating the boot image. The // path is retrieved from the dependency by GetGlobalSoongConfig(ctx). dexpreopt.RegisterToolDeps(ctx) @@ -183,6 +187,11 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo bootDexJarByModule := b.generateHiddenAPIBuildActions(ctx, b.configuredModules, b.fragments) buildRuleForBootJarsPackageCheck(ctx, bootDexJarByModule) + // Nothing to do if skipping the dexpreopt of boot image jars. + if SkipDexpreoptBootJars(ctx) { + return + } + b.generateBootImageBuildActions(ctx, platformModules, apexModules) } @@ -420,12 +429,6 @@ func (b *platformBootclasspathModule) generateBootImage(ctx android.ModuleContex // Build a profile for the image config and then use that to build the boot image. profile := bootImageProfileRule(ctx, imageConfig) - // If dexpreopt of boot image jars should be skipped, generate only a profile. - global := dexpreopt.GetGlobalConfig(ctx) - if global.DisablePreoptBootImages { - return - } - // Build boot image files for the android variants. androidBootImageFiles := buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 6159ea9c24..3a2ecc00cc 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -19,14 +19,12 @@ import ( "testing" "android/soong/android" - "android/soong/dexpreopt" "android/soong/java" ) var prepareForSdkTestWithJava = android.GroupFixturePreparers( java.PrepareForTestWithJavaBuildComponents, PrepareForTestWithSdkBuildComponents, - dexpreopt.PrepareForTestWithFakeDex2oatd, // Ensure that all source paths are provided. This helps ensure that the snapshot generation is // consistent and all files referenced from the snapshot's Android.bp file have actually been From 34f2bc56939dd685e102dca322482424cea2e224 Mon Sep 17 00:00:00 2001 From: Chris Wailes Date: Fri, 5 May 2023 18:18:01 +0000 Subject: [PATCH 0096/1460] Revert "rustc-1.69.0 Build 10027731" Revert submission 2579357-rust-update-prebuilts-1.69.0 Reason for revert: Patch got dropped accidentally that broke a postsubmit test Reverted changes: /q/submissionid:2579357-rust-update-prebuilts-1.69.0 Change-Id: I3fdd44a9df6c9c775c4f4157ea58df5e4215c47e --- rust/config/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/config/global.go b/rust/config/global.go index 748bb3d2f3..2d1f0c1f94 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.69.0" + RustDefaultVersion = "1.68.0" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ From 8d72714736d0cf6cc591fa6440dc02c9e903c74f Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Fri, 5 May 2023 20:51:46 +0000 Subject: [PATCH 0097/1460] Drop androidx libraries from baselineApexAvailable These are not needed in this map since the Android.bp files for these modules are created by pom2bp script which explicitly set apex_available as ['//apex_available:apex', '//apex_available:platform']. Bug: 281077552 Test: m nothing Change-Id: I53bb9a104b1e7a2535fe568e27431b1a48a885e3 --- apex/apex.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index cf18ec31dd..e30f674a74 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -3423,15 +3423,6 @@ func makeApexAvailableBaseline() map[string][]string { // Module separator // m[android.AvailableToAnyApex] = []string{ - // TODO(b/156996905) Set apex_available/min_sdk_version for androidx/extras support libraries - "androidx", - "androidx-constraintlayout_constraintlayout", - "androidx-constraintlayout_constraintlayout-nodeps", - "androidx-constraintlayout_constraintlayout-solver", - "androidx-constraintlayout_constraintlayout-solver-nodeps", - "com.google.android.material_material", - "com.google.android.material_material-nodeps", - "libclang_rt", "libprofile-clang-extras", "libprofile-clang-extras_ndk", From 34c6d7d56f3b77195519320447af8a79102e57e6 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Fri, 5 May 2023 20:40:20 +0000 Subject: [PATCH 0098/1460] Fix Proguard configuration output from R8 invocation The output when using `--pg-conf-output` seems to be flaky. In the meantime, use `-printconfiguration` to generate full config summaries for each R8 invocation. Bug: 281081834 Test: m && \ find $ANDROID_BUILD_TOP/out -name proguard_configuration -empty Change-Id: I0c8c36212e338b50505184076f4c470a5d3d9b94 --- java/dex.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/dex.go b/java/dex.go index 4d6aa34567..f7c1361e24 100644 --- a/java/dex.go +++ b/java/dex.go @@ -140,7 +140,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", `$r8Template${config.R8Cmd} ${config.R8Flags} -injars $tmpJar --output $outDir ` + `--no-data-resources ` + `-printmapping ${outDict} ` + - `--pg-conf-output ${outConfig} ` + + `-printconfiguration ${outConfig} ` + `-printusage ${outUsage} ` + `--deps-file ${out}.d ` + `$r8Flags && ` + From 4de7b49cd71f18b1d9da5b669d0f3684a9da366b Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Fri, 5 May 2023 21:13:01 +0000 Subject: [PATCH 0099/1460] Drop redundant libraries from baseline apexAvailable map These libraries now set the correct apex_available in their Android.bp files. Therefore, they do not a separate entry in the baseline apexAvailable map. Test: m nothing #passes Bug: 281077552 Change-Id: Icf6b97c3bcc4bda96eefa14ba6d4bf10614bc378 --- apex/apex.go | 48 ----------------------------------------------- apex/apex_test.go | 8 ++++++++ cc/testing.go | 4 ++++ 3 files changed, 12 insertions(+), 48 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index e30f674a74..73265d2185 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -3233,7 +3233,6 @@ func makeApexAvailableBaseline() map[string][]string { // m["com.android.appsearch"] = []string{ "icing-java-proto-lite", - "libprotobuf-java-lite", } // // Module separator @@ -3249,11 +3248,8 @@ func makeApexAvailableBaseline() map[string][]string { // Module separator // m["com.android.extservices"] = []string{ - "error_prone_annotations", "ExtServices-core", - "ExtServices", "libtextclassifier-java", - "libz_current", "textclassifier-statsd", "TextClassifierNotificationLibNoManifest", "TextClassifierServiceLibNoManifest", @@ -3271,8 +3267,6 @@ func makeApexAvailableBaseline() map[string][]string { "android.hidl.memory@1.0", "android.hidl.safe_union@1.0", "libarect", - "libbuildversion", - "libmath", "libprocpartition", } // @@ -3302,15 +3296,12 @@ func makeApexAvailableBaseline() map[string][]string { // Module separator // m["com.android.runtime"] = []string{ - "bionic_libc_platform_headers", - "libarm-optimized-routines-math", "libc_aeabi", "libc_bionic", "libc_bionic_ndk", "libc_bootstrap", "libc_common", "libc_common_shared", - "libc_common_static", "libc_dns", "libc_dynamic_dispatch", "libc_fortify", @@ -3327,19 +3318,16 @@ func makeApexAvailableBaseline() map[string][]string { "libc_openbsd_large_stack", "libc_openbsd_ndk", "libc_pthread", - "libc_static_dispatch", "libc_syscalls", "libc_tzcode", "libc_unwind_static", "libdebuggerd", "libdebuggerd_common_headers", "libdebuggerd_handler_core", - "libdebuggerd_handler_fallback", "libdl_static", "libjemalloc5", "liblinker_main", "liblinker_malloc", - "liblz4", "liblzma", "libprocinfo", "libpropertyinfoparser", @@ -3357,17 +3345,7 @@ func makeApexAvailableBaseline() map[string][]string { m["com.android.tethering"] = []string{ "android.hardware.tetheroffload.config-V1.0-java", "android.hardware.tetheroffload.control-V1.0-java", - "android.hidl.base-V1.0-java", - "libcgrouprc", - "libcgrouprc_format", - "libtetherutilsjni", - "libvndksupport", "net-utils-framework-common", - "netd_aidl_interface-V3-java", - "netlink-client", - "networkstack-aidl-interfaces-java", - "tethering-aidl-interfaces-java", - "TetheringApiCurrentLib", } // // Module separator @@ -3387,48 +3365,22 @@ func makeApexAvailableBaseline() map[string][]string { "android.hardware.wifi.supplicant-V1.1-java", "android.hardware.wifi.supplicant-V1.2-java", "android.hardware.wifi.supplicant-V1.3-java", - "android.hidl.base-V1.0-java", - "android.hidl.manager-V1.0-java", - "android.hidl.manager-V1.1-java", - "android.hidl.manager-V1.2-java", "bouncycastle-unbundled", - "dnsresolver_aidl_interface-V2-java", - "error_prone_annotations", - "framework-wifi-pre-jarjar", "framework-wifi-util-lib", - "ipmemorystore-aidl-interfaces-V3-java", - "ipmemorystore-aidl-interfaces-java", "ksoap2", "libnanohttpd", - "libwifi-jni", - "net-utils-services-common", - "netd_aidl_interface-V2-java", - "netd_aidl_interface-unstable-java", - "netd_event_listener_interface-java", - "netlink-client", - "networkstack-client", - "services.net", "wifi-lite-protos", "wifi-nano-protos", "wifi-service-pre-jarjar", - "wifi-service-resources", - } - // - // Module separator - // - m["com.android.os.statsd"] = []string{ - "libstatssocket", } // // Module separator // m[android.AvailableToAnyApex] = []string{ - "libclang_rt", "libprofile-clang-extras", "libprofile-clang-extras_ndk", "libprofile-extras", "libprofile-extras_ndk", - "libunwind", } return m } diff --git a/apex/apex_test.go b/apex/apex_test.go index 3ba4d8d553..c4708a8c0c 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -1460,6 +1460,10 @@ func TestRuntimeApexShouldInstallHwasanIfLibcDependsOnIt(t *testing.T) { sanitize: { never: true, }, + apex_available: [ + "//apex_available:anyapex", + "//apex_available:platform", + ], } `) ctx := result.TestContext @@ -1508,6 +1512,10 @@ func TestRuntimeApexShouldInstallHwasanIfHwaddressSanitized(t *testing.T) { sanitize: { never: true, }, + apex_available: [ + "//apex_available:anyapex", + "//apex_available:platform", + ], } `) ctx := result.TestContext diff --git a/cc/testing.go b/cc/testing.go index ced09290fe..d346739bdf 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -82,6 +82,10 @@ func commonDefaultModules() string { sanitize: { never: true, }, + apex_available: [ + "//apex_available:anyapex", + "//apex_available:platform", + ], } cc_prebuilt_library_static { From 072f7bc5f2905833fdd6dcdd15b4b576c3af0066 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Fri, 5 May 2023 21:06:23 +0000 Subject: [PATCH 0100/1460] Remove CellBroadcast apps from baseline apex available map Since these apps explicitly set com.android.cellbroadcast in their Android.bp files, they do not need to be present in the baseline apex_available map. Test: m nothing Bug: 281077552 Change-Id: Ibea86bfbc7e460e27c5e780b611674e66e9cf727 --- apex/apex.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apex/apex.go b/apex/apex.go index 73265d2185..79a3465461 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -3243,7 +3243,7 @@ func makeApexAvailableBaseline() map[string][]string { // // Module separator // - m["com.android.cellbroadcast"] = []string{"CellBroadcastApp", "CellBroadcastServiceModule"} + m["com.android.cellbroadcast"] = []string{} // // Module separator // From 591366dd63303dff6773f6d48cc71666fc9d1baa Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Mon, 8 May 2023 11:32:52 +0900 Subject: [PATCH 0101/1460] Invoke writeNinjaHint for only actual build Bug: 273282046 Test: m nothing Change-Id: I13754646c786b0cfee8c8d75a8b61e62b54c32ba --- cmd/soong_build/main.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 5de232664f..27219b1be1 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -133,6 +133,10 @@ func runMixedModeBuild(ctx *android.Context, extraNinjaDeps []string) string { ninjaDeps = append(ninjaDeps, writeBuildGlobsNinjaFile(ctx)...) writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps) + + if ctx.Config().IsEnvTrue("SOONG_GENERATES_NINJA_HINT") { + writeNinjaHint(ctx) + } return cmdlineArgs.OutFile } @@ -455,6 +459,9 @@ func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string { // The actual output (build.ninja) was written in the RunBlueprint() call // above writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps) + if ctx.Config().IsEnvTrue("SOONG_GENERATES_NINJA_HINT") { + writeNinjaHint(ctx) + } return cmdlineArgs.OutFile } } @@ -535,9 +542,6 @@ func main() { } else { finalOutputFile = runSoongOnlyBuild(ctx, extraNinjaDeps) } - if ctx.Config().IsEnvTrue("SOONG_GENERATES_NINJA_HINT") { - writeNinjaHint(ctx) - } writeMetrics(configuration, ctx.EventHandler, metricsDir) } writeUsedEnvironmentFile(configuration, finalOutputFile) From d27fedf8e91b7a1426a2aa1b1deef487df899e3d Mon Sep 17 00:00:00 2001 From: Jeff Hamilton Date: Mon, 8 May 2023 03:12:51 +0000 Subject: [PATCH 0102/1460] Add TARGET_RELEASE to the banner printout Test: run lunch with and without a release and confirmed it's printed Change-Id: I67e86aa1328ae464a1bb10dfdcd8e024e72d77a8 --- ui/build/dumpvars.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index efe747819b..b83b5cba3b 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -150,6 +150,7 @@ var BannerVars = []string{ "PRODUCT_INCLUDE_TAGS", "PRODUCT_SOURCE_ROOT_DIRS", "TARGET_PRODUCT", + "TARGET_RELEASE", "TARGET_BUILD_VARIANT", "TARGET_BUILD_APPS", "TARGET_BUILD_UNBUNDLED", From e3f0281b8897da1fe23b2f4f3a05f1dc87bcc902 Mon Sep 17 00:00:00 2001 From: Jooyung Han Date: Mon, 8 May 2023 13:54:50 +0900 Subject: [PATCH 0103/1460] Mark LLNDK prebuilts as LLNDK so that Vendor APEXes don't embed the LLNDK prebuilts (which are just stubs of LLNDK libraries). Bug: 280697209 Test: m nothing (soong test) Change-Id: I9e6c123e73fa0ab56d8494a01652ee32a9b6a6cd --- apex/apex_test.go | 168 ++++++++++++++++++++++++++++++++++++++++++++++ cc/vndk.go | 7 +- 2 files changed, 174 insertions(+), 1 deletion(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index e4b0323e31..bf9c71b2e8 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -4187,6 +4187,174 @@ func TestVndkApexShouldNotProvideNativeLibs(t *testing.T) { }) } +func TestVendorApexWithVndkPrebuilts(t *testing.T) { + ctx := testApex(t, "", + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.DeviceVndkVersion = proptools.StringPtr("27") + }), + android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + cc.RegisterVendorSnapshotModules(ctx) + }), + withFiles(map[string][]byte{ + "vendor/foo/Android.bp": []byte(` + apex { + name: "myapex", + binaries: ["foo"], + key: "myapex.key", + min_sdk_version: "27", + vendor: true, + } + + cc_binary { + name: "foo", + vendor: true, + srcs: ["abc.cpp"], + shared_libs: [ + "libllndk", + "libvndk", + ], + nocrt: true, + system_shared_libs: [], + min_sdk_version: "27", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `), + // Simulate VNDK prebuilts with vendor_snapshot + "prebuilts/vndk/Android.bp": []byte(` + vndk_prebuilt_shared { + name: "libllndk", + version: "27", + vendor_available: true, + product_available: true, + target_arch: "arm64", + arch: { + arm64: { + srcs: ["libllndk.so"], + }, + }, + } + + vndk_prebuilt_shared { + name: "libvndk", + version: "27", + vendor_available: true, + product_available: true, + target_arch: "arm64", + arch: { + arm64: { + srcs: ["libvndk.so"], + }, + }, + vndk: { + enabled: true, + }, + min_sdk_version: "27", + } + + vndk_prebuilt_shared { + name: "libc++", + version: "27", + target_arch: "arm64", + vendor_available: true, + product_available: true, + vndk: { + enabled: true, + support_system_process: true, + }, + arch: { + arm64: { + srcs: ["libc++.so"], + }, + }, + min_sdk_version: "apex_inherit", + } + + vendor_snapshot { + name: "vendor_snapshot", + version: "27", + arch: { + arm64: { + vndk_libs: [ + "libc++", + "libllndk", + "libvndk", + ], + static_libs: [ + "libc++demangle", + "libclang_rt.builtins", + "libunwind", + ], + }, + } + } + + vendor_snapshot_static { + name: "libclang_rt.builtins", + version: "27", + target_arch: "arm64", + vendor: true, + arch: { + arm64: { + src: "libclang_rt.builtins-aarch64-android.a", + }, + }, + } + + vendor_snapshot_static { + name: "libc++demangle", + version: "27", + target_arch: "arm64", + compile_multilib: "64", + vendor: true, + arch: { + arm64: { + src: "libc++demangle.a", + }, + }, + min_sdk_version: "apex_inherit", + } + + vendor_snapshot_static { + name: "libunwind", + version: "27", + target_arch: "arm64", + compile_multilib: "64", + vendor: true, + arch: { + arm64: { + src: "libunwind.a", + }, + }, + min_sdk_version: "apex_inherit", + } + `), + })) + + // Should embed the prebuilt VNDK libraries in the apex + ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ + "bin/foo", + "prebuilts/vndk/libc++.so:lib64/libc++.so", + "prebuilts/vndk/libvndk.so:lib64/libvndk.so", + }) + + // Should link foo with prebuilt libraries (shared/static) + ldRule := ctx.ModuleForTests("foo", "android_vendor.27_arm64_armv8-a_myapex").Rule("ld") + android.AssertStringDoesContain(t, "should link to prebuilt llndk", ldRule.Args["libFlags"], "prebuilts/vndk/libllndk.so") + android.AssertStringDoesContain(t, "should link to prebuilt vndk", ldRule.Args["libFlags"], "prebuilts/vndk/libvndk.so") + android.AssertStringDoesContain(t, "should link to prebuilt libc++demangle", ldRule.Args["libFlags"], "prebuilts/vndk/libc++demangle.a") + android.AssertStringDoesContain(t, "should link to prebuilt libunwind", ldRule.Args["libFlags"], "prebuilts/vndk/libunwind.a") + + // Should declare the LLNDK library as a "required" external dependency + manifestRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexManifestRule") + requireNativeLibs := names(manifestRule.Args["requireNativeLibs"]) + ensureListContains(t, requireNativeLibs, "libllndk.so") +} + func TestDependenciesInApexManifest(t *testing.T) { ctx := testApex(t, ` apex { diff --git a/cc/vndk.go b/cc/vndk.go index 30bfdd89f2..9b70004c52 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -241,7 +241,7 @@ var ( func vndkModuleLister(predicate func(*Module) bool) moduleListerFunc { return func(ctx android.SingletonContext) (moduleNames, fileNames []string) { ctx.VisitAllModules(func(m android.Module) { - if c, ok := m.(*Module); ok && predicate(c) { + if c, ok := m.(*Module); ok && predicate(c) && !c.IsVndkPrebuiltLibrary() { filename, err := getVndkFileName(c) if err != nil { ctx.ModuleErrorf(m, "%s", err) @@ -402,6 +402,11 @@ func VndkMutator(mctx android.BottomUpMutatorContext) { m.VendorProperties.IsVNDKPrivate = Bool(prebuiltLib.Properties.Llndk.Private) } + if m.IsVndkPrebuiltLibrary() && !m.IsVndk() { + m.VendorProperties.IsLLNDK = true + // TODO(b/280697209): copy "llndk.private" flag to vndk_prebuilt_shared + } + if (isLib && lib.buildShared()) || (isPrebuiltLib && prebuiltLib.buildShared()) { if m.vndkdep != nil && m.vndkdep.isVndk() && !m.vndkdep.isVndkExt() { processVndkLibrary(mctx, m) From 3cc7846b5887c130ba5ccb5a8f77d8a1a7358ab2 Mon Sep 17 00:00:00 2001 From: Justin Yun Date: Mon, 8 May 2023 21:06:59 +0900 Subject: [PATCH 0104/1460] APEX can use any Prebuilt libraries Vendor snapshot libraries generated from prebuilt libraries set "apex_inherit" to their min_sdk_version property to allow them to be used by vendor APEXes. Bug: 277403349 Test: m vendor-snapshot Change-Id: Idd9e5f8e94b3fa2adf47a04507bf0c50c59edfb7 --- cc/vendor_snapshot.go | 6 +++++- cc/vendor_snapshot_test.go | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go index 51f23c57ec..d2531c03da 100644 --- a/cc/vendor_snapshot.go +++ b/cc/vendor_snapshot.go @@ -250,7 +250,11 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS for _, path := range m.VintfFragments() { prop.VintfFragments = append(prop.VintfFragments, filepath.Join("configs", path.Base())) } - prop.MinSdkVersion = m.MinSdkVersion() + if m.IsPrebuilt() { + prop.MinSdkVersion = "apex_inherit" + } else { + prop.MinSdkVersion = m.MinSdkVersion() + } // install config files. ignores any duplicates. for _, path := range append(m.InitRc(), m.VintfFragments()...) { diff --git a/cc/vendor_snapshot_test.go b/cc/vendor_snapshot_test.go index 5b69a10beb..c5431b3d3d 100644 --- a/cc/vendor_snapshot_test.go +++ b/cc/vendor_snapshot_test.go @@ -23,6 +23,17 @@ import ( "testing" ) +func checkJsonContents(t *testing.T, ctx android.TestingSingleton, jsonPath string, key string, value string) { + jsonOut := ctx.MaybeOutput(jsonPath) + if jsonOut.Rule == nil { + t.Errorf("%q expected but not found", jsonPath) + return + } + if !strings.Contains(jsonOut.Args["content"], fmt.Sprintf("%q:%q", key, value)) { + t.Errorf("%q must include %q:%q but it only has %v", jsonPath, key, value, jsonOut.Args["content"]) + } +} + func TestVendorSnapshotCapture(t *testing.T) { bp := ` cc_library { @@ -52,6 +63,7 @@ func TestVendorSnapshotCapture(t *testing.T) { name: "libvendor_available", vendor_available: true, nocrt: true, + min_sdk_version: "29", } cc_library_headers { @@ -155,6 +167,9 @@ func TestVendorSnapshotCapture(t *testing.T) { filepath.Join(staticDir, "libvendor_available.a.json"), filepath.Join(staticDir, "libvendor_available.cfi.a.json")) + checkJsonContents(t, snapshotSingleton, filepath.Join(staticDir, "libb.a.json"), "MinSdkVersion", "apex_inherit") + checkJsonContents(t, snapshotSingleton, filepath.Join(staticDir, "libvendor_available.a.json"), "MinSdkVersion", "29") + // For binary executables, all vendor:true and vendor_available modules are captured. if archType == "arm64" { binaryVariant := fmt.Sprintf("android_vendor.29_%s_%s", archType, archVariant) From a28acb5a16e7c16e11b393a84f686397ded0eb54 Mon Sep 17 00:00:00 2001 From: Alix Date: Mon, 8 May 2023 15:41:01 +0000 Subject: [PATCH 0105/1460] Delete kotlin/rules.bzl file Bug: 277800767 Test: treehugger Change-Id: I5cff1e4bbb18b9145335169bc4edb0d569525b35 --- java/java.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/java.go b/java/java.go index d3e74fd1ce..16d9ee7c3b 100644 --- a/java/java.go +++ b/java/java.go @@ -2968,7 +2968,7 @@ type kotlinAttributes struct { func ktJvmLibraryBazelTargetModuleProperties() bazel.BazelTargetModuleProperties { return bazel.BazelTargetModuleProperties{ Rule_class: "kt_jvm_library", - Bzl_load_location: "//build/bazel/rules/kotlin:rules.bzl", + Bzl_load_location: "//build/bazel/rules/kotlin:kt_jvm_library.bzl", } } From a43ae1366ea312f1865ad672d82ed2480f4109e8 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Mon, 8 May 2023 18:33:16 +0000 Subject: [PATCH 0106/1460] For test apexes, base_apex_name is the api domain apex_test do not "override" the source apexes that they test. However, similar to override_apexes, test apexes have the same path (/apex/) on device. Instead of creating another property, reuse the existing base_apex_name property. The use case for this will be for creating selection statements for stub/impl selection. Test: go test ./bp2build Change-Id: I4b7548e0e0fc920e407e1c5e5c00e87efc6e70c9 --- apex/apex.go | 2 ++ bp2build/apex_conversion_test.go | 9 +++++---- cc/bp2build.go | 10 +++++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index 69547c3b70..e01406bc65 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -3703,6 +3703,8 @@ func convertWithBp2build(a *apexBundle, ctx android.TopDownMutatorContext) (baze commonAttrs := android.CommonAttributes{} if a.testApex { commonAttrs.Testonly = proptools.BoolPtr(true) + // Set the api_domain of the test apex + attrs.Base_apex_name = proptools.StringPtr(cc.GetApiDomain(a.Name())) } return attrs, props, commonAttrs diff --git a/bp2build/apex_conversion_test.go b/bp2build/apex_conversion_test.go index 1cc3f22ed3..390cabe1f8 100644 --- a/bp2build/apex_conversion_test.go +++ b/bp2build/apex_conversion_test.go @@ -1475,10 +1475,11 @@ apex_test { `, ExpectedBazelTargets: []string{ MakeBazelTarget("apex", "test_com.android.apogee", AttrNameToString{ - "file_contexts": `"file_contexts_file"`, - "manifest": `"apex_manifest.json"`, - "testonly": `True`, - "tests": `[":cc_test_1"]`, + "file_contexts": `"file_contexts_file"`, + "base_apex_name": `"com.android.apogee"`, + "manifest": `"apex_manifest.json"`, + "testonly": `True`, + "tests": `[":cc_test_1"]`, }), }}) } diff --git a/cc/bp2build.go b/cc/bp2build.go index 6e91bffa24..ca8d60902b 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1210,7 +1210,11 @@ var ( } ) -func getApiDomain(apexName string) string { +// GetApiDomain returns the canonical name of the apex. This is synonymous to the apex_name definition. +// https://cs.android.com/android/_/android/platform/build/soong/+/e3f0281b8897da1fe23b2f4f3a05f1dc87bcc902:apex/prebuilt.go;l=81-83;drc=2dc7244af985a6ad701b22f1271e606cabba527f;bpv=1;bpt=0 +// For test apexes, it uses a naming convention heuristic to determine the api domain. +// TODO (b/281548611): Move this build/soong/android +func GetApiDomain(apexName string) string { if apiDomain, exists := testApexNameToApiDomain[apexName]; exists { return apiDomain } @@ -1233,7 +1237,7 @@ func createInApexConfigSetting(ctx android.TopDownMutatorContext, apexName strin defer apiDomainConfigSettingLock.Unlock() // Return if a config_setting has already been created - apiDomain := getApiDomain(apexName) + apiDomain := GetApiDomain(apexName) acsm := getApiDomainConfigSettingMap(ctx.Config()) if _, exists := (*acsm)[apiDomain]; exists { return @@ -1270,7 +1274,7 @@ func inApexConfigSetting(apexAvailable string) string { if apexAvailable == android.AvailableToAnyApex { return bazel.AndroidAndInApex } - apiDomain := getApiDomain(apexAvailable) + apiDomain := GetApiDomain(apexAvailable) return "//build/bazel/rules/apex:" + apiDomain } From 8242b43df84c883db8f7231a3d0e82849583e94d Mon Sep 17 00:00:00 2001 From: Romain Jobredeaux Date: Thu, 4 May 2023 10:16:26 -0400 Subject: [PATCH 0107/1460] Map prebuilt_apis labels to Bazel's manually generated ones. Change-Id: I5bec807da9b39e2b166af1bd83da43df91f997ef Bug: 237810289 Test: check that android-non-updatable.stubs.module_lib uses manual bazel target for its sdk_system_current_android dependency Change-Id: I6f5fb348e1dcaad10eb756382359f34de792b6e0 --- android/allowlists/allowlists.go | 1 + android/bazel.go | 4 +-- android/bazel_test.go | 4 +-- java/prebuilt_apis.go | 55 +++++++++++++++++++++++++------- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 6bd4e26e88..10c3aef83f 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -303,6 +303,7 @@ var ( "prebuilts/clang/host/linux-x86": Bp2BuildDefaultTrueRecursively, "prebuilts/gradle-plugin": Bp2BuildDefaultTrueRecursively, "prebuilts/runtime/mainline/platform/sdk": Bp2BuildDefaultTrueRecursively, + "prebuilts/sdk": Bp2BuildDefaultTrue, "prebuilts/sdk/current/androidx": Bp2BuildDefaultTrue, "prebuilts/sdk/current/androidx-legacy": Bp2BuildDefaultTrue, "prebuilts/sdk/current/extras/constraint-layout-x": Bp2BuildDefaultTrue, diff --git a/android/bazel.go b/android/bazel.go index 58d9d87dbe..a9ba79044e 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -71,7 +71,7 @@ type BazelConversionStatus struct { MissingDeps []string `blueprint:"mutated"` } -type bazelModuleProperties struct { +type BazelModuleProperties struct { // The label of the Bazel target replacing this Soong module. When run in conversion mode, this // will import the handcrafted build target into the autogenerated file. Note: this may result in // a conflict due to duplicate targets if bp2build_available is also set. @@ -96,7 +96,7 @@ type bazelModuleProperties struct { type properties struct { // In "Bazel mixed build" mode, this represents the Bazel target replacing // this Soong module. - Bazel_module bazelModuleProperties + Bazel_module BazelModuleProperties } // namespacedVariableProperties is a map from a string representing a Soong diff --git a/android/bazel_test.go b/android/bazel_test.go index 87b2c8f429..77e251575c 100644 --- a/android/bazel_test.go +++ b/android/bazel_test.go @@ -218,7 +218,7 @@ func (bcc *TestBazelConversionContext) Config() Config { var bazelableBazelModuleBase = BazelModuleBase{ bazelProperties: properties{ - Bazel_module: bazelModuleProperties{ + Bazel_module: BazelModuleProperties{ CanConvertToBazel: true, }, }, @@ -344,7 +344,7 @@ func TestBp2BuildAllowlist(t *testing.T) { }, BazelModuleBase: BazelModuleBase{ bazelProperties: properties{ - Bazel_module: bazelModuleProperties{ + Bazel_module: BazelModuleProperties{ CanConvertToBazel: true, Bp2build_available: proptools.BoolPtr(true), }, diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go index 206d995276..7964f4cc67 100644 --- a/java/prebuilt_apis.go +++ b/java/prebuilt_apis.go @@ -104,20 +104,51 @@ func prebuiltApiModuleName(mctx android.LoadHookContext, module, scope, version return fmt.Sprintf("%s_%s_%s_%s", mctx.ModuleName(), scope, version, module) } +func hasBazelPrebuilt(module string) bool { + return module == "android" || module == "core-for-system-modules" +} + +func bazelPrebuiltApiModuleName(module, scope, version string) string { + bazelModule := module + switch module { + case "android": + bazelModule = "android_jar" + case "core-for-system-modules": + bazelModule = "core_jar" + } + bazelVersion := version + if version == "current" { + bazelVersion = strconv.Itoa(android.FutureApiLevelInt) + } + bazelScope := scope + switch scope { + case "module-lib": + bazelScope = "module" + case "system-server": + bazelScope = "system_server" + } + return fmt.Sprintf("//prebuilts/sdk:%s_%s_%s", bazelScope, bazelVersion, bazelModule) +} + func createImport(mctx android.LoadHookContext, module, scope, version, path, sdkVersion string, compileDex bool) { props := struct { - Name *string - Jars []string - Sdk_version *string - Installable *bool - Compile_dex *bool - }{} - props.Name = proptools.StringPtr(prebuiltApiModuleName(mctx, module, scope, version)) - props.Jars = append(props.Jars, path) - props.Sdk_version = proptools.StringPtr(sdkVersion) - props.Installable = proptools.BoolPtr(false) - props.Compile_dex = proptools.BoolPtr(compileDex) - + Name *string + Jars []string + Sdk_version *string + Installable *bool + Compile_dex *bool + Bazel_module android.BazelModuleProperties + }{ + Name: proptools.StringPtr(prebuiltApiModuleName(mctx, module, scope, version)), + Jars: []string{path}, + Sdk_version: proptools.StringPtr(sdkVersion), + Installable: proptools.BoolPtr(false), + Compile_dex: proptools.BoolPtr(compileDex), + } + if hasBazelPrebuilt(module) { + props.Bazel_module = android.BazelModuleProperties{ + Label: proptools.StringPtr(bazelPrebuiltApiModuleName(module, scope, version))} + } mctx.CreateModule(ImportFactory, &props) } From bc698cd28aa2ee17af208bfd56f003347116ddb0 Mon Sep 17 00:00:00 2001 From: Jiakai Zhang Date: Mon, 8 May 2023 16:28:38 +0000 Subject: [PATCH 0108/1460] Revert^2 "Generate boot image profiles even if dexpreopt is disabled." Revert submission 2580631-revert-2574032-XXTWCJDTDQ Reason for revert: Fixed build breakages Reverted changes: /q/submissionid:2580631-revert-2574032-XXTWCJDTDQ Bug: 280440941 Test: lunch aosp_cf_riscv64_minidroid-userdebug && m UNSAFE_DISABLE_HIDDENAPI_FLAGS=true dist Test: Disable dex2oat on host (to simulate macOS) and build Change-Id: I5f7f746ca1d4da660fe0c40115e6c71750dfdccc --- apex/apex.go | 2 +- apex/bootclasspath_fragment_test.go | 20 ++++++++++++++++++++ dexpreopt/config.go | 9 +++++++++ dexpreopt/testing.go | 7 +++++++ java/bootclasspath_fragment.go | 13 ++++++++----- java/dexpreopt_bootjars.go | 9 +++++---- java/platform_bootclasspath.go | 13 +++++++------ sdk/java_sdk_test.go | 2 ++ 8 files changed, 59 insertions(+), 16 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index 50c9957b0b..6a64ad6cd4 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -2681,7 +2681,7 @@ func apexBootclasspathFragmentFiles(ctx android.ModuleContext, module blueprint. } pathInApex := bootclasspathFragmentInfo.ProfileInstallPathInApex() - if pathInApex != "" && !java.SkipDexpreoptBootJars(ctx) { + if pathInApex != "" { pathOnHost := bootclasspathFragmentInfo.ProfilePathOnHost() tempPath := android.PathForModuleOut(ctx, "boot_image_profile", pathInApex) diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index d784818284..3e55ccc117 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -497,6 +497,26 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { }) }) + t.Run("generate boot image profile even if dexpreopt is disabled", func(t *testing.T) { + result := android.GroupFixturePreparers( + commonPreparer, + + // Configure some libraries in the art bootclasspath_fragment that match the source + // bootclasspath_fragment's contents property. + java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), + addSource("foo", "bar"), + java.FixtureSetBootImageInstallDirOnDevice("art", "system/framework"), + dexpreopt.FixtureDisableDexpreoptBootImages(true), + ).RunTest(t) + + ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + "etc/boot-image.prof", + "etc/classpaths/bootclasspath.pb", + "javalib/bar.jar", + "javalib/foo.jar", + }) + }) + t.Run("boot image disable generate profile", func(t *testing.T) { result := android.GroupFixturePreparers( commonPreparer, diff --git a/dexpreopt/config.go b/dexpreopt/config.go index 609a29c2da..0cc3bd63bc 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -475,7 +475,16 @@ func RegisterToolDeps(ctx android.BottomUpMutatorContext) { ctx.AddFarVariationDependencies(v, Dex2oatDepTag, dex2oatBin) } +func IsDex2oatNeeded(ctx android.PathContext) bool { + global := GetGlobalConfig(ctx) + return !global.DisablePreopt || !global.DisablePreoptBootImages +} + func dex2oatPathFromDep(ctx android.ModuleContext) android.Path { + if !IsDex2oatNeeded(ctx) { + return nil + } + dex2oatBin := dex2oatModuleName(ctx.Config()) // Find the right dex2oat module, trying to follow PrebuiltDepTag from source diff --git a/dexpreopt/testing.go b/dexpreopt/testing.go index 47ae494e33..b3dd3cca24 100644 --- a/dexpreopt/testing.go +++ b/dexpreopt/testing.go @@ -174,3 +174,10 @@ func FixtureDisableGenerateProfile(disable bool) android.FixturePreparer { dexpreoptConfig.DisableGenerateProfile = disable }) } + +// FixtureDisableDexpreoptBootImages sets the DisablePreoptBootImages property in the global config. +func FixtureDisableDexpreoptBootImages(disable bool) android.FixturePreparer { + return FixtureModifyGlobalConfig(func(_ android.PathContext, dexpreoptConfig *GlobalConfig) { + dexpreoptConfig.DisablePreoptBootImages = disable + }) +} diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index f692563478..6ccf5a3314 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -506,7 +506,7 @@ func (b *BootclasspathFragmentModule) DepsMutator(ctx android.BottomUpMutatorCon } } - if SkipDexpreoptBootJars(ctx) { + if !dexpreopt.IsDex2oatNeeded(ctx) { return } @@ -901,10 +901,6 @@ func (b *BootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleC // produceBootImageFiles builds the boot image files from the source if it is required. func (b *BootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig) bootImageOutputs { - if SkipDexpreoptBootJars(ctx) { - return bootImageOutputs{} - } - // Only generate the boot image if the configuration does not skip it. return b.generateBootImageBuildActions(ctx, imageConfig) } @@ -929,6 +925,13 @@ func (b *BootclasspathFragmentModule) generateBootImageBuildActions(ctx android. // Build a profile for the image config and then use that to build the boot image. profile := bootImageProfileRule(ctx, imageConfig) + // If dexpreopt of boot image jars should be skipped, generate only a profile. + if SkipDexpreoptBootJars(ctx) { + return bootImageOutputs{ + profile: profile, + } + } + // Build boot image files for the host variants. buildBootImageVariantsForBuildOs(ctx, imageConfig, profile) diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index f4c0935101..8e79674fb9 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -500,9 +500,6 @@ func (d *dexpreoptBootJars) GenerateAndroidBuildActions(ctx android.ModuleContex // Generate build rules for boot images. func (d *dexpreoptBootJars) GenerateSingletonBuildActions(ctx android.SingletonContext) { - if SkipDexpreoptBootJars(ctx) { - return - } if dexpreopt.GetCachedGlobalSoongConfig(ctx) == nil { // No module has enabled dexpreopting, so we assume there will be no boot image to make. return @@ -1002,7 +999,7 @@ func writeGlobalConfigForMake(ctx android.SingletonContext, path android.Writabl // (make/core/dex_preopt_libart.mk) to generate install rules that copy boot image files to the // correct output directories. func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { - if d.dexpreoptConfigForMake != nil { + if d.dexpreoptConfigForMake != nil && !SkipDexpreoptBootJars(ctx) { ctx.Strict("DEX_PREOPT_CONFIG_FOR_MAKE", d.dexpreoptConfigForMake.String()) ctx.Strict("DEX_PREOPT_SOONG_CONFIG_FOR_MAKE", android.PathForOutput(ctx, "dexpreopt_soong.config").String()) } @@ -1014,6 +1011,10 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_IMAGE_PROFILE_LICENSE_METADATA", image.profileLicenseMetadataFile.String()) } + if SkipDexpreoptBootJars(ctx) { + return + } + global := dexpreopt.GetGlobalConfig(ctx) dexPaths, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_FILES", strings.Join(dexPaths.Strings(), " ")) diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index 0ea3609798..d5779f75b1 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -103,7 +103,7 @@ func (b *platformBootclasspathModule) OutputFiles(tag string) (android.Paths, er func (b *platformBootclasspathModule) DepsMutator(ctx android.BottomUpMutatorContext) { b.hiddenAPIDepsMutator(ctx) - if SkipDexpreoptBootJars(ctx) { + if !dexpreopt.IsDex2oatNeeded(ctx) { return } @@ -187,11 +187,6 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo bootDexJarByModule := b.generateHiddenAPIBuildActions(ctx, b.configuredModules, b.fragments) buildRuleForBootJarsPackageCheck(ctx, bootDexJarByModule) - // Nothing to do if skipping the dexpreopt of boot image jars. - if SkipDexpreoptBootJars(ctx) { - return - } - b.generateBootImageBuildActions(ctx, platformModules, apexModules) } @@ -429,6 +424,12 @@ func (b *platformBootclasspathModule) generateBootImage(ctx android.ModuleContex // Build a profile for the image config and then use that to build the boot image. profile := bootImageProfileRule(ctx, imageConfig) + // If dexpreopt of boot image jars should be skipped, generate only a profile. + global := dexpreopt.GetGlobalConfig(ctx) + if global.DisablePreoptBootImages { + return + } + // Build boot image files for the android variants. androidBootImageFiles := buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 3a2ecc00cc..6159ea9c24 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -19,12 +19,14 @@ import ( "testing" "android/soong/android" + "android/soong/dexpreopt" "android/soong/java" ) var prepareForSdkTestWithJava = android.GroupFixturePreparers( java.PrepareForTestWithJavaBuildComponents, PrepareForTestWithSdkBuildComponents, + dexpreopt.PrepareForTestWithFakeDex2oatd, // Ensure that all source paths are provided. This helps ensure that the snapshot generation is // consistent and all files referenced from the snapshot's Android.bp file have actually been From cf61e3c59133227894ac7d72091b48451a3e6761 Mon Sep 17 00:00:00 2001 From: Jiakai Zhang Date: Mon, 8 May 2023 16:28:38 +0000 Subject: [PATCH 0109/1460] Revert^2 "Generate app profiles even if dexpreopt is disabled." Revert submission 2580631-revert-2574032-XXTWCJDTDQ Reason for revert: Fixed build breakages Reverted changes: /q/submissionid:2580631-revert-2574032-XXTWCJDTDQ Bug: 280440941 Test: lunch aosp_cf_riscv64_minidroid-userdebug && m UNSAFE_DISABLE_HIDDENAPI_FLAGS=true dist Test: Disable dex2oat on host (to simulate macOS) and build Change-Id: I6090b4b74cedb6d129fcbeef58d075c8ccdcc4e2 --- android/testing.go | 21 +++++++++++++++------ dexpreopt/dexpreopt.go | 8 ++++++++ dexpreopt/testing.go | 7 +++++++ java/dex_test.go | 10 +++++----- java/dexpreopt.go | 12 +++--------- java/dexpreopt_test.go | 25 +++++++++++++++++++++++++ 6 files changed, 63 insertions(+), 20 deletions(-) diff --git a/android/testing.go b/android/testing.go index fc39a9c7af..2a9c6584e0 100644 --- a/android/testing.go +++ b/android/testing.go @@ -813,6 +813,20 @@ func normalizePathRelativeToTop(path Path) Path { return path.RelativeToTop() } +func allOutputs(p BuildParams) []string { + outputs := append(WritablePaths(nil), p.Outputs...) + outputs = append(outputs, p.ImplicitOutputs...) + if p.Output != nil { + outputs = append(outputs, p.Output) + } + return outputs.Strings() +} + +// AllOutputs returns all 'BuildParams.Output's and 'BuildParams.Outputs's in their full path string forms. +func (p TestingBuildParams) AllOutputs() []string { + return allOutputs(p.BuildParams) +} + // baseTestingComponent provides functionality common to both TestingModule and TestingSingleton. type baseTestingComponent struct { config Config @@ -954,12 +968,7 @@ func (b baseTestingComponent) buildParamsFromOutput(file string) TestingBuildPar func (b baseTestingComponent) allOutputs() []string { var outputFullPaths []string for _, p := range b.provider.BuildParamsForTests() { - outputs := append(WritablePaths(nil), p.Outputs...) - outputs = append(outputs, p.ImplicitOutputs...) - if p.Output != nil { - outputs = append(outputs, p.Output) - } - outputFullPaths = append(outputFullPaths, outputs.Strings()...) + outputFullPaths = append(outputFullPaths, allOutputs(p)...) } return outputFullPaths } diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index a590c72a51..2b38793ffd 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -100,11 +100,19 @@ func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongC return rule, nil } +// If dexpreopt is applicable to the module, returns whether dexpreopt is disabled. Otherwise, the +// behavior is undefined. +// When it returns true, dexpreopt artifacts will not be generated, but profile will still be +// generated if profile-guided compilation is requested. func dexpreoptDisabled(ctx android.PathContext, global *GlobalConfig, module *ModuleConfig) bool { if ctx.Config().UnbundledBuild() { return true } + if global.DisablePreopt { + return true + } + if contains(global.DisablePreoptModules, module.Name) { return true } diff --git a/dexpreopt/testing.go b/dexpreopt/testing.go index b3dd3cca24..6ed0736f77 100644 --- a/dexpreopt/testing.go +++ b/dexpreopt/testing.go @@ -181,3 +181,10 @@ func FixtureDisableDexpreoptBootImages(disable bool) android.FixturePreparer { dexpreoptConfig.DisablePreoptBootImages = disable }) } + +// FixtureDisableDexpreopt sets the DisablePreopt property in the global config. +func FixtureDisableDexpreopt(disable bool) android.FixturePreparer { + return FixtureModifyGlobalConfig(func(_ android.PathContext, dexpreoptConfig *GlobalConfig) { + dexpreoptConfig.DisablePreopt = disable + }) +} diff --git a/java/dex_test.go b/java/dex_test.go index 97fc3d0dda..2ba3831f49 100644 --- a/java/dex_test.go +++ b/java/dex_test.go @@ -23,7 +23,7 @@ import ( ) func TestR8(t *testing.T) { - result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` android_app { name: "app", srcs: ["foo.java"], @@ -191,7 +191,7 @@ func TestR8TransitiveDeps(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - fixturePreparer := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd + fixturePreparer := PrepareForTestWithJavaDefaultModules if tc.unbundled { fixturePreparer = android.GroupFixturePreparers( fixturePreparer, @@ -258,7 +258,7 @@ func TestR8TransitiveDeps(t *testing.T) { } func TestR8Flags(t *testing.T) { - result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` android_app { name: "app", srcs: ["foo.java"], @@ -287,7 +287,7 @@ func TestR8Flags(t *testing.T) { } func TestD8(t *testing.T) { - result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` java_library { name: "foo", srcs: ["foo.java"], @@ -328,7 +328,7 @@ func TestD8(t *testing.T) { } func TestProguardFlagsInheritance(t *testing.T) { - result := PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd.RunTestWithBp(t, ` + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp(t, ` android_app { name: "app", static_libs: [ diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 0ffedf6c40..a96b312819 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -180,6 +180,8 @@ func moduleName(ctx android.BaseModuleContext) string { return android.RemoveOptionalPrebuiltPrefix(ctx.ModuleName()) } +// Returns whether dexpreopt is applicable to the module. +// When it returns true, neither profile nor dexpreopt artifacts will be generated. func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { if !ctx.Device() { return true @@ -205,14 +207,6 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { global := dexpreopt.GetGlobalConfig(ctx) - if global.DisablePreopt { - return true - } - - if inList(moduleName(ctx), global.DisablePreoptModules) { - return true - } - isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) if isApexVariant(ctx) { // Don't preopt APEX variant module unless the module is an APEX system server jar. @@ -232,7 +226,7 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { } func dexpreoptToolDepsMutator(ctx android.BottomUpMutatorContext) { - if d, ok := ctx.Module().(DexpreopterInterface); !ok || d.dexpreoptDisabled(ctx) { + if d, ok := ctx.Module().(DexpreopterInterface); !ok || d.dexpreoptDisabled(ctx) || !dexpreopt.IsDex2oatNeeded(ctx) { return } dexpreopt.RegisterToolDeps(ctx) diff --git a/java/dexpreopt_test.go b/java/dexpreopt_test.go index 3d2c5c3a10..f91ac5cc3b 100644 --- a/java/dexpreopt_test.go +++ b/java/dexpreopt_test.go @@ -438,3 +438,28 @@ func TestAndroidMkEntriesForApex(t *testing.T) { android.AssertIntEquals(t, "entries count", 0, len(entriesList)) } + +func TestGenerateProfileEvenIfDexpreoptIsDisabled(t *testing.T) { + preparers := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + PrepareForTestWithFakeApexMutator, + dexpreopt.FixtureDisableDexpreopt(true), + ) + + result := preparers.RunTestWithBp(t, ` + java_library { + name: "foo", + installable: true, + dex_preopt: { + profile: "art-profile", + }, + srcs: ["a.java"], + }`) + + ctx := result.TestContext + dexpreopt := ctx.ModuleForTests("foo", "android_common").MaybeRule("dexpreopt") + + expected := []string{"out/soong/.intermediates/foo/android_common/dexpreopt/profile.prof"} + + android.AssertArrayString(t, "outputs", expected, dexpreopt.AllOutputs()) +} From 6d4d9da47f8c9d5a947c03554853dec12c02bddd Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Tue, 18 Apr 2023 06:20:40 +0000 Subject: [PATCH 0110/1460] Select stub/impl per apex variant Create a select statement for every api_domain a library could be included in. The stub/impl selection heuristics per apex domain are 1. If dep has stubs and the api domain appears in dep's apex_available, use impl 2. If dep has stubs and the api domain does not appear in dep's apex_available, use stubs (Category 3: If dep does not have stubs and the apex does not appear in dep's apex_available, then a separate apex_available validation in Bazel will emit an error). Platform variants have been special-cased for now to use equality of apex_available for stub/impl selection Test: go test ./bp2build Bug: 272378496 Change-Id: Ibd29efd763c8863c7e6d2a9af0da30bbde07175d --- bazel/configurability.go | 6 +- bp2build/build_conversion_test.go | 2 +- bp2build/cc_library_conversion_test.go | 41 +++--- bp2build/cc_library_shared_conversion_test.go | 94 +++++++++++- bp2build/cc_library_static_conversion_test.go | 4 +- cc/bp2build.go | 135 ++++++++++++------ 6 files changed, 206 insertions(+), 76 deletions(-) diff --git a/bazel/configurability.go b/bazel/configurability.go index d01877dd5b..29299e9b9e 100644 --- a/bazel/configurability.go +++ b/bazel/configurability.go @@ -69,8 +69,8 @@ const ( productVariableBazelPackage = "//build/bazel/product_variables" - AndroidAndInApex = "android-in_apex" - AndroidAndNonApex = "android-non_apex" + AndroidAndInApex = "android-in_apex" + AndroidPlatform = "system" InApex = "in_apex" NonApex = "non_apex" @@ -202,7 +202,7 @@ var ( osAndInApexMap = map[string]string{ AndroidAndInApex: "//build/bazel/rules/apex:android-in_apex", - AndroidAndNonApex: "//build/bazel/rules/apex:android-non_apex", + AndroidPlatform: "//build/bazel/rules/apex:system", osDarwin: "//build/bazel/platforms/os:darwin", osLinux: "//build/bazel/platforms/os:linux_glibc", osLinuxMusl: "//build/bazel/platforms/os:linux_musl", diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go index e198a11b83..e127fd542f 100644 --- a/bp2build/build_conversion_test.go +++ b/bp2build/build_conversion_test.go @@ -1941,7 +1941,7 @@ func TestPrettyPrintSelectMapEqualValues(t *testing.T) { libFooImplLabel := bazel.Label{ Label: ":libfoo.impl", } - lla.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex, bazel.MakeLabelList([]bazel.Label{libFooImplLabel})) + lla.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidPlatform, bazel.MakeLabelList([]bazel.Label{libFooImplLabel})) lla.SetSelectValue(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey, bazel.MakeLabelList([]bazel.Label{libFooImplLabel})) actual, _ := prettyPrintAttribute(lla, 0) android.AssertStringEquals(t, "Print the common value if all keys in an axis have the same value", `[":libfoo.impl"]`, actual) diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 6d5fd7ec9e..1b681efb0c 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -3042,7 +3042,8 @@ cc_library { }`, ExpectedBazelTargets: makeCcLibraryTargets("foolib", AttrNameToString{ "implementation_dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:barlib"], + "//build/bazel/rules/apex:foo": ["@api_surfaces//module-libapi/current:barlib"], + "//build/bazel/rules/apex:system": ["@api_surfaces//module-libapi/current:barlib"], "//conditions:default": [":barlib"], })`, "local_includes": `["."]`, @@ -3096,7 +3097,11 @@ cc_library { "//build/bazel/platforms/os:linux_glibc": [":quxlib"], "//build/bazel/platforms/os:linux_musl": [":quxlib"], "//build/bazel/platforms/os:windows": [":quxlib"], - "//build/bazel/rules/apex:android-in_apex": [ + "//build/bazel/rules/apex:foo": [ + "@api_surfaces//module-libapi/current:barlib", + "@api_surfaces//module-libapi/current:quxlib", + ], + "//build/bazel/rules/apex:system": [ "@api_surfaces//module-libapi/current:barlib", "@api_surfaces//module-libapi/current:quxlib", ], @@ -4139,44 +4144,34 @@ cc_library { name: "barlib", stubs: { symbol_file: "bar.map.txt", versions: ["28", "29", "current"] }, bazel_module: { bp2build_available: false }, + apex_available: ["//apex_available:platform",], } cc_library { name: "bazlib", stubs: { symbol_file: "bar.map.txt", versions: ["28", "29", "current"] }, bazel_module: { bp2build_available: false }, + apex_available: ["//apex_available:platform",], } cc_library { name: "foo", shared_libs: ["barlib", "bazlib"], export_shared_lib_headers: ["bazlib"], apex_available: [ - "apex_available:platform", + "//apex_available:platform", ], }`, ExpectedBazelTargets: []string{ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ - "implementation_dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:barlib"], - "//conditions:default": [":barlib"], - })`, - "dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:bazlib"], - "//conditions:default": [":bazlib"], - })`, - "local_includes": `["."]`, - "tags": `["apex_available=apex_available:platform"]`, + "implementation_dynamic_deps": `[":barlib"]`, + "dynamic_deps": `[":bazlib"]`, + "local_includes": `["."]`, + "tags": `["apex_available=//apex_available:platform"]`, }), MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ - "implementation_dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:barlib"], - "//conditions:default": [":barlib"], - })`, - "dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:bazlib"], - "//conditions:default": [":bazlib"], - })`, - "local_includes": `["."]`, - "tags": `["apex_available=apex_available:platform"]`, + "implementation_dynamic_deps": `[":barlib"]`, + "dynamic_deps": `[":bazlib"]`, + "local_includes": `["."]`, + "tags": `["apex_available=//apex_available:platform"]`, }), }, }) diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go index 47dff8a570..2ee9c99f9c 100644 --- a/bp2build/cc_library_shared_conversion_test.go +++ b/bp2build/cc_library_shared_conversion_test.go @@ -609,7 +609,8 @@ cc_library_shared { ExpectedBazelTargets: []string{ MakeBazelTarget("cc_library_shared", "b", AttrNameToString{ "implementation_dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:a"], + "//build/bazel/rules/apex:apex_b": ["@api_surfaces//module-libapi/current:a"], + "//build/bazel/rules/apex:system": ["@api_surfaces//module-libapi/current:a"], "//conditions:default": [":a"], })`, "tags": `["apex_available=apex_b"]`, @@ -618,6 +619,64 @@ cc_library_shared { }) } +// Tests that library in apexfoo links against stubs of platform_lib and otherapex_lib +func TestCcLibrarySharedStubs_UseStubsFromMultipleApiDomains(t *testing.T) { + runCcLibrarySharedTestCase(t, Bp2buildTestCase{ + Description: "cc_library_shared stubs", + ModuleTypeUnderTest: "cc_library_shared", + ModuleTypeUnderTestFactory: cc.LibrarySharedFactory, + Blueprint: soongCcLibrarySharedPreamble + ` +cc_library_shared { + name: "libplatform_stable", + stubs: { symbol_file: "libplatform_stable.map.txt", versions: ["28", "29", "current"] }, + apex_available: ["//apex_available:platform"], + bazel_module: { bp2build_available: false }, + include_build_directory: false, +} +cc_library_shared { + name: "libapexfoo_stable", + stubs: { symbol_file: "libapexfoo_stable.map.txt", versions: ["28", "29", "current"] }, + apex_available: ["apexfoo"], + bazel_module: { bp2build_available: false }, + include_build_directory: false, +} +cc_library_shared { + name: "libutils", + shared_libs: ["libplatform_stable", "libapexfoo_stable",], + apex_available: ["//apex_available:platform", "apexfoo", "apexbar"], + include_build_directory: false, +} +`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_shared", "libutils", AttrNameToString{ + "implementation_dynamic_deps": `select({ + "//build/bazel/rules/apex:apexbar": [ + "@api_surfaces//module-libapi/current:libplatform_stable", + "@api_surfaces//module-libapi/current:libapexfoo_stable", + ], + "//build/bazel/rules/apex:apexfoo": [ + "@api_surfaces//module-libapi/current:libplatform_stable", + ":libapexfoo_stable", + ], + "//build/bazel/rules/apex:system": [ + "@api_surfaces//module-libapi/current:libplatform_stable", + "@api_surfaces//module-libapi/current:libapexfoo_stable", + ], + "//conditions:default": [ + ":libplatform_stable", + ":libapexfoo_stable", + ], + })`, + "tags": `[ + "apex_available=//apex_available:platform", + "apex_available=apexfoo", + "apex_available=apexbar", + ]`, + }), + }, + }) +} + func TestCcLibrarySharedStubs_IgnorePlatformAvailable(t *testing.T) { runCcLibrarySharedTestCase(t, Bp2buildTestCase{ Description: "cc_library_shared stubs", @@ -641,7 +700,8 @@ cc_library_shared { ExpectedBazelTargets: []string{ MakeBazelTarget("cc_library_shared", "b", AttrNameToString{ "implementation_dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:a"], + "//build/bazel/rules/apex:apex_b": ["@api_surfaces//module-libapi/current:a"], + "//build/bazel/rules/apex:system": ["@api_surfaces//module-libapi/current:a"], "//conditions:default": [":a"], })`, "tags": `[ @@ -653,6 +713,34 @@ cc_library_shared { }) } +func TestCcLibraryDoesNotDropStubDepIfNoVariationAcrossAxis(t *testing.T) { + runCcLibrarySharedTestCase(t, Bp2buildTestCase{ + Description: "cc_library depeends on impl for all configurations", + ModuleTypeUnderTest: "cc_library_shared", + ModuleTypeUnderTestFactory: cc.LibrarySharedFactory, + Blueprint: soongCcLibrarySharedPreamble + ` +cc_library_shared { + name: "a", + stubs: { symbol_file: "a.map.txt", versions: ["28", "29", "current"] }, + bazel_module: { bp2build_available: false }, + apex_available: ["//apex_available:platform"], +} +cc_library_shared { + name: "b", + shared_libs: [":a"], + include_build_directory: false, + apex_available: ["//apex_available:platform"], +} +`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_shared", "b", AttrNameToString{ + "implementation_dynamic_deps": `[":a"]`, + "tags": `["apex_available=//apex_available:platform"]`, + }), + }, + }) +} + func TestCcLibrarySharedStubs_MultipleApexAvailable(t *testing.T) { runCcLibrarySharedTestCase(t, Bp2buildTestCase{ ModuleTypeUnderTest: "cc_library_shared", @@ -682,7 +770,7 @@ cc_library_shared { ExpectedBazelTargets: []string{ MakeBazelTarget("cc_library_shared", "b", AttrNameToString{ "implementation_dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:a"], + "//build/bazel/rules/apex:system": ["@api_surfaces//module-libapi/current:a"], "//conditions:default": [":a"], })`, "tags": `[ diff --git a/bp2build/cc_library_static_conversion_test.go b/bp2build/cc_library_static_conversion_test.go index 948801486d..2705aafe52 100644 --- a/bp2build/cc_library_static_conversion_test.go +++ b/bp2build/cc_library_static_conversion_test.go @@ -1504,6 +1504,7 @@ cc_library { versions: ["current"], }, bazel_module: { bp2build_available: false }, + apex_available: ["com.android.runtime"], } cc_library_static { @@ -1561,7 +1562,8 @@ cc_library_static { }), MakeBazelTarget("cc_library_static", "keep_with_stubs", AttrNameToString{ "implementation_dynamic_deps": `select({ - "//build/bazel/rules/apex:android-in_apex": ["@api_surfaces//module-libapi/current:libm"], + "//build/bazel/rules/apex:foo": ["@api_surfaces//module-libapi/current:libm"], + "//build/bazel/rules/apex:system": ["@api_surfaces//module-libapi/current:libm"], "//conditions:default": [":libm"], })`, "system_dynamic_deps": `[]`, diff --git a/cc/bp2build.go b/cc/bp2build.go index ca8d60902b..c42a7e8ba6 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1269,7 +1269,7 @@ func createInApexConfigSetting(ctx android.TopDownMutatorContext, apexName strin func inApexConfigSetting(apexAvailable string) string { if apexAvailable == android.AvailableToPlatform { - return bazel.AndroidAndNonApex + return bazel.AndroidPlatform } if apexAvailable == android.AvailableToAnyApex { return bazel.AndroidAndInApex @@ -1278,51 +1278,49 @@ func inApexConfigSetting(apexAvailable string) string { return "//build/bazel/rules/apex:" + apiDomain } -func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.ConfigurationAxis, - config string, apexAvailable []string, dynamicLibs bazel.LabelList, dynamicDeps *bazel.LabelListAttribute, ind int, buildNonApexWithStubs bool) { - - depsWithStubs := []bazel.Label{} - for _, l := range dynamicLibs.Includes { - dep, _ := ctx.ModuleFromName(l.OriginalModuleName) - if d, ok := dep.(*Module); ok && d.HasStubsVariants() { - depApexAvailable := d.ApexAvailable() - if !availableToSameApexes(apexAvailable, depApexAvailable) { - depsWithStubs = append(depsWithStubs, l) - } +// Inputs to stub vs impl selection. +type stubSelectionInfo struct { + // Label of the implementation library (e.g. //bionic/libc:libc) + impl bazel.Label + // Axis containing the implementation library + axis bazel.ConfigurationAxis + // Axis key containing the implementation library + config string + // API domain of the apex + // For test apexes (test_com.android.foo), this will be the source apex (com.android.foo) + apiDomain string + // List of dep labels + dynamicDeps *bazel.LabelListAttribute + // Boolean value for determining if the dep is in the same api domain + // If false, the label will be rewritten to to the stub label + sameApiDomain bool +} + +func useStubOrImplInApexWithName(ssi stubSelectionInfo) { + lib := ssi.impl + if !ssi.sameApiDomain { + lib = bazel.Label{ + Label: apiSurfaceModuleLibCurrentPackage + strings.TrimPrefix(lib.OriginalModuleName, ":"), } } - if len(depsWithStubs) > 0 { - implDynamicDeps := bazel.SubtractBazelLabelList(dynamicLibs, bazel.MakeLabelList(depsWithStubs)) - dynamicDeps.SetSelectValue(axis, config, implDynamicDeps) - - stubLibLabels := []bazel.Label{} - for _, l := range depsWithStubs { - stubLabelInApiSurfaces := bazel.Label{ - Label: apiSurfaceModuleLibCurrentPackage + strings.TrimPrefix(l.OriginalModuleName, ":"), - } - stubLibLabels = append(stubLibLabels, stubLabelInApiSurfaces) - } - inApexSelectValue := dynamicDeps.SelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndInApex) - nonApexSelectValue := dynamicDeps.SelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex) - defaultSelectValue := dynamicDeps.SelectValue(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey) - nonApexDeps := depsWithStubs - if buildNonApexWithStubs { - nonApexDeps = stubLibLabels - } - if axis == bazel.NoConfigAxis { - (&inApexSelectValue).Append(bazel.MakeLabelList(stubLibLabels)) - (&nonApexSelectValue).Append(bazel.MakeLabelList(nonApexDeps)) - (&defaultSelectValue).Append(bazel.MakeLabelList(depsWithStubs)) - dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndInApex, bazel.FirstUniqueBazelLabelList(inApexSelectValue)) - dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex, bazel.FirstUniqueBazelLabelList(nonApexSelectValue)) - dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey, bazel.FirstUniqueBazelLabelList(defaultSelectValue)) - } else if config == bazel.OsAndroid { - (&inApexSelectValue).Append(bazel.MakeLabelList(stubLibLabels)) - (&nonApexSelectValue).Append(bazel.MakeLabelList(nonApexDeps)) - dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndInApex, bazel.FirstUniqueBazelLabelList(inApexSelectValue)) - dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex, bazel.FirstUniqueBazelLabelList(nonApexSelectValue)) - } + // Create a select statement specific to this apex + inApexSelectValue := ssi.dynamicDeps.SelectValue(bazel.OsAndInApexAxis, inApexConfigSetting(ssi.apiDomain)) + (&inApexSelectValue).Append(bazel.MakeLabelList([]bazel.Label{lib})) + ssi.dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, inApexConfigSetting(ssi.apiDomain), bazel.FirstUniqueBazelLabelList(inApexSelectValue)) + // Delete the library from the common config for this apex + implDynamicDeps := ssi.dynamicDeps.SelectValue(ssi.axis, ssi.config) + implDynamicDeps = bazel.SubtractBazelLabelList(implDynamicDeps, bazel.MakeLabelList([]bazel.Label{ssi.impl})) + ssi.dynamicDeps.SetSelectValue(ssi.axis, ssi.config, implDynamicDeps) + if ssi.axis == bazel.NoConfigAxis { + // Set defaults. Defaults (i.e. host) should use impl and not stubs. + defaultSelectValue := ssi.dynamicDeps.SelectValue(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey) + (&defaultSelectValue).Append(bazel.MakeLabelList([]bazel.Label{ssi.impl})) + ssi.dynamicDeps.SetSelectValue(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey, bazel.FirstUniqueBazelLabelList(defaultSelectValue)) } +} + +func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.ConfigurationAxis, + config string, apexAvailable []string, dynamicLibs bazel.LabelList, dynamicDeps *bazel.LabelListAttribute, ind int, buildNonApexWithStubs bool) { // Create a config_setting for each apex_available. // This will be used to select impl of a dep if dep is available to the same apex. @@ -1330,6 +1328,49 @@ func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.C createInApexConfigSetting(ctx.(android.TopDownMutatorContext), aa) } + apiDomainForSelects := []string{} + for _, apex := range apexAvailable { + apiDomainForSelects = append(apiDomainForSelects, GetApiDomain(apex)) + } + // Always emit a select statement for the platform variant. + // This ensures that b build //foo --config=android works + // Soong always creates a platform variant even when the library might not be available to platform. + if !android.InList(android.AvailableToPlatform, apiDomainForSelects) { + apiDomainForSelects = append(apiDomainForSelects, android.AvailableToPlatform) + } + apiDomainForSelects = android.SortedUniqueStrings(apiDomainForSelects) + + // Create a select for each apex this library could be included in. + for _, l := range dynamicLibs.Includes { + dep, _ := ctx.ModuleFromName(l.OriginalModuleName) + if c, ok := dep.(*Module); !ok || !c.HasStubsVariants() { + continue + } + // TODO (b/280339069): Decrease the verbosity of the generated BUILD files + for _, apiDomain := range apiDomainForSelects { + var sameApiDomain bool + if apiDomain == android.AvailableToPlatform { + // Platform variants in Soong use equality of apex_available for stub/impl selection. + // https://cs.android.com/android/_/android/platform/build/soong/+/316b0158fe57ee7764235923e7c6f3d530da39c6:cc/cc.go;l=3393-3404;drc=176271a426496fa2688efe2b40d5c74340c63375;bpv=1;bpt=0 + // One of the factors behind this design choice is cc_test + // Tests only have a platform variant, and using equality of apex_available ensures + // that tests of an apex library gets its implementation and not stubs. + // TODO (b/280343104): Discuss if we can drop this special handling for platform variants. + sameApiDomain = availableToSameApexes(apexAvailable, dep.(*Module).ApexAvailable()) + } else { + sameApiDomain = android.InList(apiDomain, dep.(*Module).ApexAvailable()) + } + ssi := stubSelectionInfo{ + impl: l, + axis: axis, + config: config, + apiDomain: apiDomain, + dynamicDeps: dynamicDeps, + sameApiDomain: sameApiDomain, + } + useStubOrImplInApexWithName(ssi) + } + } } func (la *linkerAttributes) convertStripProps(ctx android.BazelConversionPathContext, module *Module) { @@ -1424,7 +1465,6 @@ func (la *linkerAttributes) finalize(ctx android.BazelConversionPathContext) { la.implementationDynamicDeps.Exclude(bazel.OsConfigurationAxis, "linux_bionic", toRemove) la.implementationDynamicDeps.Exclude(bazel.OsAndInApexAxis, bazel.ConditionsDefaultConfigKey, toRemove) - la.implementationDynamicDeps.Exclude(bazel.OsAndInApexAxis, bazel.AndroidAndNonApex, toRemove) stubsToRemove := make([]bazel.Label, 0, len(la.usedSystemDynamicDepAsDynamicDep)) for _, lib := range toRemove.Includes { stubLabelInApiSurfaces := bazel.Label{ @@ -1432,7 +1472,12 @@ func (la *linkerAttributes) finalize(ctx android.BazelConversionPathContext) { } stubsToRemove = append(stubsToRemove, stubLabelInApiSurfaces) } - la.implementationDynamicDeps.Exclude(bazel.OsAndInApexAxis, bazel.AndroidAndInApex, bazel.MakeLabelList(stubsToRemove)) + // system libraries (e.g. libc, libm, libdl) belong the com.android.runtime api domain + // dedupe the stubs of these libraries from the other api domains (platform, other_apexes...) + for _, aa := range ctx.Module().(*Module).ApexAvailable() { + la.implementationDynamicDeps.Exclude(bazel.OsAndInApexAxis, inApexConfigSetting(aa), bazel.MakeLabelList(stubsToRemove)) + } + la.implementationDynamicDeps.Exclude(bazel.OsAndInApexAxis, bazel.AndroidPlatform, bazel.MakeLabelList(stubsToRemove)) } la.deps.ResolveExcludes() From 6aaab9d2aa4f529d04f8bcbb4f87e3f1841e314e Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Thu, 4 May 2023 17:27:30 +0000 Subject: [PATCH 0111/1460] Special case platform variant of bootstrap libs Platform variants of libraries that set `bootstrap: true` in their Android.bp file gets impl in Soong. This CL ports this behavior to bp2build. Note that even after this CL, there will be still be some other cases where stub/impl logic does not match Soong perfectly (most notably the platform_apis property which is propagated top-down from the parent apex). Test: bp2build.sh Change-Id: I3da284ab42631d6de1c0d52e56ccbfd4e4a09f1d --- cc/bp2build.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cc/bp2build.go b/cc/bp2build.go index c42a7e8ba6..33eafe3e7b 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1357,6 +1357,9 @@ func setStubsForDynamicDeps(ctx android.BazelConversionPathContext, axis bazel.C // that tests of an apex library gets its implementation and not stubs. // TODO (b/280343104): Discuss if we can drop this special handling for platform variants. sameApiDomain = availableToSameApexes(apexAvailable, dep.(*Module).ApexAvailable()) + if linkable, ok := ctx.Module().(LinkableInterface); ok && linkable.Bootstrap() { + sameApiDomain = true + } } else { sameApiDomain = android.InList(apiDomain, dep.(*Module).ApexAvailable()) } From 150f9a5a6318412c4e1d1f2a8ed650e510bd24ab Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Wed, 26 Apr 2023 10:52:24 -0700 Subject: [PATCH 0112/1460] Bp2build support for soong config variables + os For converting the art plugins to pure soong, it would be useful to have a property that's qualified on both a soong config variable and the OS. Soong had very little-known support for this by saying your soong config variable changes the "target.android.cflags" property, and we didn't supporting bp2building that. Add the bp2build support. This cl also refactors product variable and soong variable bp2building so that they're separate from each other, which I think makes the code easier to understand. Test: go test Change-Id: Ic74dc75da8103fa2523da95c3560c9ce3c5e5672 --- android/variable.go | 401 +++++++++--------- bazel/configurability.go | 11 +- bazel/properties.go | 2 +- bazel/properties_test.go | 12 +- ...oong_config_module_type_conversion_test.go | 108 +++++ cc/bp2build.go | 4 +- 6 files changed, 329 insertions(+), 209 deletions(-) diff --git a/android/variable.go b/android/variable.go index aaf0606e17..bf66135b55 100644 --- a/android/variable.go +++ b/android/variable.go @@ -541,124 +541,102 @@ type ProductConfigContext interface { Module() Module } -// ProductConfigProperty contains the information for a single property (may be a struct) paired -// with the appropriate ProductConfigVariable. +// ProductConfigOrSoongConfigProperty represents either a soong config variable + its value +// or a product config variable. You can get both a ConfigurationAxis and a SelectKey from it +// for use in bazel attributes. ProductVariableProperties() will return a map from properties -> +// this interface -> property structs for use in bp2build converters +type ProductConfigOrSoongConfigProperty interface { + // Name of the product variable or soong config variable + Name() string + // AlwaysEmit returns true for soong config variables but false for product variables. This + // is intended to indicate if we need to always emit empty lists in the select statements. + AlwaysEmit() bool + // ConfigurationAxis returns the bazel.ConfigurationAxis that represents this variable. The + // configuration axis will change depending on the variable and whether it's arch/os variant + // as well. + ConfigurationAxis() bazel.ConfigurationAxis + // SelectKey returns a string that represents the key of a select branch, however, it is not + // actually the real label written out to the build file. + // this.ConfigurationAxis().SelectKey(this.SelectKey()) will give the actual label. + SelectKey() string +} + +// ProductConfigProperty represents a product config variable, and if it is arch-variant or not. type ProductConfigProperty struct { // The name of the product variable, e.g. "safestack", "malloc_not_svelte", // "board" - Name string + name string - // Namespace of the variable, if this is a soong_config_module_type variable - // e.g. "acme", "ANDROID", "vendor_name" - Namespace string + arch string +} - // Unique configuration to identify this product config property (i.e. a - // primary key), as just using the product variable name is not sufficient. - // - // For product variables, this is the product variable name + optional - // archvariant information. e.g. - // - // product_variables: { - // foo: { - // cflags: ["-Dfoo"], - // }, - // }, - // - // FullConfig would be "foo". - // - // target: { - // android: { - // product_variables: { - // foo: { - // cflags: ["-Dfoo-android"], - // }, - // }, - // }, - // }, - // - // FullConfig would be "foo-android". - // - // For soong config variables, this is the namespace + product variable name - // + value of the variable, if applicable. The value can also be - // conditions_default. - // - // e.g. - // - // soong_config_variables: { - // feature1: { - // conditions_default: { - // cflags: ["-DDEFAULT1"], - // }, - // cflags: ["-DFEATURE1"], - // }, - // } - // - // where feature1 is created in the "acme" namespace, so FullConfig would be - // "acme__feature1" and "acme__feature1__conditions_default". - // - // e.g. - // - // soong_config_variables: { - // board: { - // soc_a: { - // cflags: ["-DSOC_A"], - // }, - // soc_b: { - // cflags: ["-DSOC_B"], - // }, - // soc_c: {}, - // conditions_default: { - // cflags: ["-DSOC_DEFAULT"] - // }, - // }, - // } - // - // where board is created in the "acme" namespace, so FullConfig would be - // "acme__board__soc_a", "acme__board__soc_b", and - // "acme__board__conditions_default" - FullConfig string +func (p ProductConfigProperty) Name() string { + return p.name +} - // keeps track of whether this product variable is nested under an arch variant - OuterAxis bazel.ConfigurationAxis +func (p ProductConfigProperty) AlwaysEmit() bool { + return false } -func (p *ProductConfigProperty) AlwaysEmit() bool { - return p.Namespace != "" +func (p ProductConfigProperty) ConfigurationAxis() bazel.ConfigurationAxis { + return bazel.ProductVariableConfigurationAxis(p.arch != "", p.name+"__"+p.arch) } -func (p *ProductConfigProperty) ConfigurationAxis() bazel.ConfigurationAxis { - if p.Namespace == "" { - return bazel.ProductVariableConfigurationAxis(p.FullConfig, p.OuterAxis) +func (p ProductConfigProperty) SelectKey() string { + if p.arch == "" { + return strings.ToLower(p.name) } else { - // Soong config variables can be uniquely identified by the namespace - // (e.g. acme, android) and the product variable name (e.g. board, size) - return bazel.ProductVariableConfigurationAxis(p.Namespace+"__"+p.Name, bazel.NoConfigAxis) + return strings.ToLower(p.name + "-" + p.arch) } } +// SoongConfigProperty represents a soong config variable, its value if it's a string variable, +// and if it's dependent on the OS or not +type SoongConfigProperty struct { + name string + namespace string + // Can be an empty string for bool/value soong config variables + value string + // If there is a target: field inside a soong config property struct, the os that it selects + // on will be represented here. + os string +} + +func (p SoongConfigProperty) Name() string { + return p.name +} + +func (p SoongConfigProperty) AlwaysEmit() bool { + return true +} + +func (p SoongConfigProperty) ConfigurationAxis() bazel.ConfigurationAxis { + return bazel.ProductVariableConfigurationAxis(false, p.namespace+"__"+p.name+"__"+p.os) +} + // SelectKey returns the literal string that represents this variable in a BUILD // select statement. -func (p *ProductConfigProperty) SelectKey() string { - if p.Namespace == "" { - return strings.ToLower(p.FullConfig) - } - - if p.FullConfig == bazel.ConditionsDefaultConfigKey { +func (p SoongConfigProperty) SelectKey() string { + // p.value being conditions_default can happen with or without a desired os. When not using + // an os, we want to emit literally just //conditions:default in the select statement, but + // when using an os, we want to emit namespace__name__conditions_default__os, so that + // the branch is only taken if the variable is not set, and we're on the desired os. + // ConfigurationAxis#SelectKey will map the conditions_default result of this function to + // //conditions:default. + if p.value == bazel.ConditionsDefaultConfigKey && p.os == "" { return bazel.ConditionsDefaultConfigKey } - value := p.FullConfig - if value == p.Name { - value = "" + parts := []string{p.namespace, p.name} + if p.value != "" && p.value != bazel.ConditionsDefaultSelectKey { + parts = append(parts, p.value) } - - // e.g. acme__feature1, android__board__soc_a - selectKey := strings.ToLower(strings.Join([]string{p.Namespace, p.Name}, "__")) - if value != "" { - selectKey = strings.ToLower(strings.Join([]string{selectKey, value}, "__")) + if p.os != "" { + parts = append(parts, p.os) } - return selectKey + // e.g. acme__feature1, android__board__soc_a, my_namespace__my_variables__my_value__my_os + return strings.ToLower(strings.Join(parts, "__")) } // ProductConfigProperties is a map of maps to group property values according @@ -674,7 +652,7 @@ func (p *ProductConfigProperty) SelectKey() string { // // The value of the map is the interface{} representing the value of the // property, like ["-DDEFINES"] for cflags. -type ProductConfigProperties map[string]map[ProductConfigProperty]interface{} +type ProductConfigProperties map[string]map[ProductConfigOrSoongConfigProperty]interface{} // ProductVariableProperties returns a ProductConfigProperties containing only the properties which // have been set for the given module. @@ -685,26 +663,10 @@ func ProductVariableProperties(ctx ArchVariantContext, module Module) ProductCon if moduleBase.variableProperties != nil { productVariablesProperty := proptools.FieldNameForProperty("product_variables") - productVariableValues( - productVariablesProperty, - moduleBase.variableProperties, - "", - "", - &productConfigProperties, - bazel.ConfigurationAxis{}, - ) - - for axis, configToProps := range moduleBase.GetArchVariantProperties(ctx, moduleBase.variableProperties) { + for /* axis */ _, configToProps := range moduleBase.GetArchVariantProperties(ctx, moduleBase.variableProperties) { for config, props := range configToProps { - // GetArchVariantProperties is creating an instance of the requested type - // and productVariablesValues expects an interface, so no need to cast - productVariableValues( - productVariablesProperty, - props, - "", - config, - &productConfigProperties, - axis) + variableValues := reflect.ValueOf(props).Elem().FieldByName(productVariablesProperty) + productConfigProperties.AddProductConfigProperties(variableValues, config) } } } @@ -712,13 +674,8 @@ func ProductVariableProperties(ctx ArchVariantContext, module Module) ProductCon if m, ok := module.(Bazelable); ok && m.namespacedVariableProps() != nil { for namespace, namespacedVariableProps := range m.namespacedVariableProps() { for _, namespacedVariableProp := range namespacedVariableProps { - productVariableValues( - soongconfig.SoongConfigProperty, - namespacedVariableProp, - namespace, - "", - &productConfigProperties, - bazel.NoConfigAxis) + variableValues := reflect.ValueOf(namespacedVariableProp).Elem().FieldByName(soongconfig.SoongConfigProperty) + productConfigProperties.AddSoongConfigProperties(namespace, variableValues) } } } @@ -727,30 +684,49 @@ func ProductVariableProperties(ctx ArchVariantContext, module Module) ProductCon } func (p *ProductConfigProperties) AddProductConfigProperty( - propertyName, namespace, productVariableName, config string, property interface{}, outerAxis bazel.ConfigurationAxis) { - if (*p)[propertyName] == nil { - (*p)[propertyName] = make(map[ProductConfigProperty]interface{}) - } + propertyName, productVariableName, arch string, propertyValue interface{}) { productConfigProp := ProductConfigProperty{ - Namespace: namespace, // e.g. acme, android - Name: productVariableName, // e.g. size, feature1, feature2, FEATURE3, board - FullConfig: config, // e.g. size, feature1-x86, size__conditions_default - OuterAxis: outerAxis, + name: productVariableName, // e.g. size, feature1, feature2, FEATURE3, board + arch: arch, // e.g. "", x86, arm64 } - if existing, ok := (*p)[propertyName][productConfigProp]; ok && namespace != "" { + p.AddEitherProperty(propertyName, productConfigProp, propertyValue) +} + +func (p *ProductConfigProperties) AddSoongConfigProperty( + propertyName, namespace, variableName, value, os string, propertyValue interface{}) { + + soongConfigProp := SoongConfigProperty{ + namespace: namespace, + name: variableName, // e.g. size, feature1, feature2, FEATURE3, board + value: value, + os: os, // e.g. android, linux_x86 + } + + p.AddEitherProperty(propertyName, soongConfigProp, propertyValue) +} + +func (p *ProductConfigProperties) AddEitherProperty( + propertyName string, key ProductConfigOrSoongConfigProperty, propertyValue interface{}) { + if (*p)[propertyName] == nil { + (*p)[propertyName] = make(map[ProductConfigOrSoongConfigProperty]interface{}) + } + + if existing, ok := (*p)[propertyName][key]; ok { switch dst := existing.(type) { case []string: - if src, ok := property.([]string); ok { - dst = append(dst, src...) - (*p)[propertyName][productConfigProp] = dst + src, ok := propertyValue.([]string) + if !ok { + panic("Conflicting types") } + dst = append(dst, src...) + (*p)[propertyName][key] = dst default: - panic(fmt.Errorf("TODO: handle merging value %s", existing)) + panic(fmt.Errorf("TODO: handle merging value %#v", existing)) } } else { - (*p)[propertyName][productConfigProp] = property + (*p)[propertyName][key] = propertyValue } } @@ -787,10 +763,7 @@ func maybeExtractConfigVarProp(v reflect.Value) (reflect.Value, bool) { return v, true } -func (productConfigProperties *ProductConfigProperties) AddProductConfigProperties(namespace, suffix string, variableValues reflect.Value, outerAxis bazel.ConfigurationAxis) { - // variableValues can either be a product_variables or - // soong_config_variables struct. - // +func (productConfigProperties *ProductConfigProperties) AddProductConfigProperties(variableValues reflect.Value, arch string) { // Example of product_variables: // // product_variables: { @@ -803,19 +776,44 @@ func (productConfigProperties *ProductConfigProperties) AddProductConfigProperti // ], // }, // }, + + for i := 0; i < variableValues.NumField(); i++ { + // e.g. Platform_sdk_version, Unbundled_build, Malloc_not_svelte, etc. + productVariableName := variableValues.Type().Field(i).Name + + variableValue := variableValues.Field(i) + // Check if any properties were set for the module + if variableValue.IsZero() { + // e.g. feature1: {}, malloc_not_svelte: {} + continue + } + + for j := 0; j < variableValue.NumField(); j++ { + property := variableValue.Field(j) + // e.g. Asflags, Cflags, Enabled, etc. + propertyName := variableValue.Type().Field(j).Name + if property.Kind() != reflect.Interface { + productConfigProperties.AddProductConfigProperty(propertyName, productVariableName, arch, property.Interface()) + } + } + } + +} + +func (productConfigProperties *ProductConfigProperties) AddSoongConfigProperties(namespace string, soongConfigVariablesStruct reflect.Value) { // // Example of soong_config_variables: // // soong_config_variables: { // feature1: { - // conditions_default: { + // conditions_default: { // ... // }, // cflags: ... // }, // feature2: { // cflags: ... - // conditions_default: { + // conditions_default: { // ... // }, // }, @@ -823,7 +821,7 @@ func (productConfigProperties *ProductConfigProperties) AddProductConfigProperti // soc_a: { // ... // }, - // soc_a: { + // soc_b: { // ... // }, // soc_c: {}, @@ -832,36 +830,36 @@ func (productConfigProperties *ProductConfigProperties) AddProductConfigProperti // }, // }, // } - for i := 0; i < variableValues.NumField(); i++ { - // e.g. Platform_sdk_version, Unbundled_build, Malloc_not_svelte, etc. - productVariableName := variableValues.Type().Field(i).Name - - variableValue := variableValues.Field(i) + for i := 0; i < soongConfigVariablesStruct.NumField(); i++ { + // e.g. feature1, feature2, board + variableName := soongConfigVariablesStruct.Type().Field(i).Name + variableStruct := soongConfigVariablesStruct.Field(i) // Check if any properties were set for the module - if variableValue.IsZero() { - // e.g. feature1: {}, malloc_not_svelte: {} + if variableStruct.IsZero() { + // e.g. feature1: {} continue } // Unlike product variables, config variables require a few more // indirections to extract the struct from the reflect.Value. - if v, ok := maybeExtractConfigVarProp(variableValue); ok { - variableValue = v + if v, ok := maybeExtractConfigVarProp(variableStruct); ok { + variableStruct = v } - for j := 0; j < variableValue.NumField(); j++ { - property := variableValue.Field(j) - // e.g. Asflags, Cflags, Enabled, etc. - propertyName := variableValue.Type().Field(j).Name - // config can also be "conditions_default". - config := proptools.PropertyNameForField(propertyName) + for j := 0; j < variableStruct.NumField(); j++ { + propertyOrStruct := variableStruct.Field(j) + // propertyOrValueName can either be: + // - A property, like: Asflags, Cflags, Enabled, etc. + // - A soong config string variable's value, like soc_a, soc_b, soc_c in the example above + // - "conditions_default" + propertyOrValueName := variableStruct.Type().Field(j).Name // If the property wasn't set, no need to pass it along - if property.IsZero() { + if propertyOrStruct.IsZero() { continue } - if v, ok := maybeExtractConfigVarProp(property); ok { + if v, ok := maybeExtractConfigVarProp(propertyOrStruct); ok { // The field is a struct, which is used by: // 1) soong_config_string_variables // @@ -879,6 +877,9 @@ func (productConfigProperties *ProductConfigProperties) AddProductConfigProperti // cflags: ..., // static_libs: ... // } + // + // This means that propertyOrValueName is either conditions_default, or a soong + // config string variable's value. field := v // Iterate over fields of this struct prop. for k := 0; k < field.NumField(); k++ { @@ -888,47 +889,59 @@ func (productConfigProperties *ProductConfigProperties) AddProductConfigProperti if field.Field(k).IsZero() && namespace == "" { continue } - actualPropertyName := field.Type().Field(k).Name - - productConfigProperties.AddProductConfigProperty( - actualPropertyName, // e.g. cflags, static_libs - namespace, // e.g. acme, android - productVariableName, // e.g. size, feature1, FEATURE2, board - config, - field.Field(k).Interface(), // e.g. ["-DDEFAULT"], ["foo", "bar"], - outerAxis, - ) + + propertyName := field.Type().Field(k).Name + if propertyName == "Target" { + productConfigProperties.AddSoongConfigPropertiesFromTargetStruct(namespace, variableName, proptools.PropertyNameForField(propertyOrValueName), field.Field(k)) + } else if propertyName == "Arch" || propertyName == "Multilib" { + panic("Arch/Multilib are not currently supported in soong config variable structs") + } else { + productConfigProperties.AddSoongConfigProperty(propertyName, namespace, variableName, proptools.PropertyNameForField(propertyOrValueName), "", field.Field(k).Interface()) + } } - } else if property.Kind() != reflect.Interface { + } else if propertyOrStruct.Kind() != reflect.Interface { // If not an interface, then this is not a conditions_default or - // a struct prop. That is, this is a regular product variable, - // or a bool/value config variable. - config := productVariableName + suffix - productConfigProperties.AddProductConfigProperty( - propertyName, - namespace, - productVariableName, - config, - property.Interface(), - outerAxis, - ) + // a struct prop. That is, this is a bool/value config variable. + if propertyOrValueName == "Target" { + productConfigProperties.AddSoongConfigPropertiesFromTargetStruct(namespace, variableName, "", propertyOrStruct) + } else if propertyOrValueName == "Arch" || propertyOrValueName == "Multilib" { + panic("Arch/Multilib are not currently supported in soong config variable structs") + } else { + productConfigProperties.AddSoongConfigProperty(propertyOrValueName, namespace, variableName, "", "", propertyOrStruct.Interface()) + } } } } } -// productVariableValues uses reflection to convert a property struct for -// product_variables and soong_config_variables to structs that can be generated -// as select statements. -func productVariableValues( - fieldName string, variableProps interface{}, namespace, suffix string, productConfigProperties *ProductConfigProperties, outerAxis bazel.ConfigurationAxis) { - if suffix != "" { - suffix = "-" + suffix +func (productConfigProperties *ProductConfigProperties) AddSoongConfigPropertiesFromTargetStruct(namespace, soongConfigVariableName string, soongConfigVariableValue string, targetStruct reflect.Value) { + // targetStruct will be a struct with fields like "android", "host", "arm", "x86", + // "android_arm", etc. The values of each of those fields will be a regular property struct. + for i := 0; i < targetStruct.NumField(); i++ { + targetFieldName := targetStruct.Type().Field(i).Name + archOrOsSpecificStruct := targetStruct.Field(i) + for j := 0; j < archOrOsSpecificStruct.NumField(); j++ { + property := archOrOsSpecificStruct.Field(j) + // e.g. Asflags, Cflags, Enabled, etc. + propertyName := archOrOsSpecificStruct.Type().Field(j).Name + + if targetFieldName == "Android" { + productConfigProperties.AddSoongConfigProperty(propertyName, namespace, soongConfigVariableName, soongConfigVariableValue, "android", property.Interface()) + } else if targetFieldName == "Host" { + for _, os := range osTypeList { + if os.Class == Host { + productConfigProperties.AddSoongConfigProperty(propertyName, namespace, soongConfigVariableName, soongConfigVariableValue, os.Name, property.Interface()) + } + } + } else { + // One problem with supporting additional fields is that if multiple branches of + // "target" overlap, we don't want them to be in the same select statement (aka + // configuration axis). "android" and "host" are disjoint, so it's ok that we only + // have 2 axes right now. (soongConfigVariables and soongConfigVariablesPlusOs) + panic("TODO: support other target types in soong config variable structs: " + targetFieldName) + } + } } - - // variableValues represent the product_variables or soong_config_variables struct. - variableValues := reflect.ValueOf(variableProps).Elem().FieldByName(fieldName) - productConfigProperties.AddProductConfigProperties(namespace, suffix, variableValues, outerAxis) } func VariableMutator(mctx BottomUpMutatorContext) { diff --git a/bazel/configurability.go b/bazel/configurability.go index d01877dd5b..8f4f965f3f 100644 --- a/bazel/configurability.go +++ b/bazel/configurability.go @@ -292,8 +292,7 @@ func (ca ConfigurationAxis) SelectKey(config string) string { case osArch: return platformOsArchMap[config] case productVariables: - if strings.HasSuffix(config, ConditionsDefaultConfigKey) { - // e.g. "acme__feature1__conditions_default" or "android__board__conditions_default" + if config == ConditionsDefaultConfigKey { return ConditionsDefaultSelectKey } return fmt.Sprintf("%s:%s", productVariableBazelPackage, config) @@ -325,11 +324,11 @@ var ( ) // ProductVariableConfigurationAxis returns an axis for the given product variable -func ProductVariableConfigurationAxis(variable string, outerAxis ConfigurationAxis) ConfigurationAxis { +func ProductVariableConfigurationAxis(archVariant bool, variable string) ConfigurationAxis { return ConfigurationAxis{ configurationType: productVariables, subType: variable, - outerAxisType: outerAxis.configurationType, + archVariant: archVariant, } } @@ -340,8 +339,8 @@ type ConfigurationAxis struct { // some configuration types (e.g. productVariables) have multiple independent axes, subType helps // distinguish between them without needing to list all 17 product variables. subType string - // used to keep track of which product variables are arch variant - outerAxisType configurationType + + archVariant bool } func (ca *ConfigurationAxis) less(other ConfigurationAxis) bool { diff --git a/bazel/properties.go b/bazel/properties.go index 1757bad490..77db1c4ec0 100644 --- a/bazel/properties.go +++ b/bazel/properties.go @@ -334,7 +334,7 @@ func (la *LabelAttribute) Collapse() error { if containsArch { allProductVariablesAreArchVariant := true for k := range la.ConfigurableValues { - if k.configurationType == productVariables && k.outerAxisType != arch { + if k.configurationType == productVariables && !k.archVariant { allProductVariablesAreArchVariant = false } } diff --git a/bazel/properties_test.go b/bazel/properties_test.go index cf03eb5f61..c56d11f3ff 100644 --- a/bazel/properties_test.go +++ b/bazel/properties_test.go @@ -248,13 +248,13 @@ func TestResolveExcludes(t *testing.T) { OsArchConfigurationAxis: labelListSelectValues{ "linux_x86": makeLabelList([]string{"linux_x86_include"}, []string{}), }, - ProductVariableConfigurationAxis("product_with_defaults", NoConfigAxis): labelListSelectValues{ + ProductVariableConfigurationAxis(false, "product_with_defaults"): labelListSelectValues{ "a": makeLabelList([]string{}, []string{"not_in_value"}), "b": makeLabelList([]string{"b_val"}, []string{}), "c": makeLabelList([]string{"c_val"}, []string{}), ConditionsDefaultConfigKey: makeLabelList([]string{"c_val", "default", "default2", "all_exclude"}, []string{}), }, - ProductVariableConfigurationAxis("product_only_with_excludes", NoConfigAxis): labelListSelectValues{ + ProductVariableConfigurationAxis(false, "product_only_with_excludes"): labelListSelectValues{ "a": makeLabelList([]string{}, []string{"product_config_exclude"}), }, }, @@ -282,13 +282,13 @@ func TestResolveExcludes(t *testing.T) { "linux_x86": makeLabels("linux_x86_include"), ConditionsDefaultConfigKey: nilLabels, }, - ProductVariableConfigurationAxis("product_with_defaults", NoConfigAxis): { + ProductVariableConfigurationAxis(false, "product_with_defaults"): { "a": nilLabels, "b": makeLabels("b_val"), "c": makeLabels("c_val"), ConditionsDefaultConfigKey: makeLabels("c_val", "default", "default2"), }, - ProductVariableConfigurationAxis("product_only_with_excludes", NoConfigAxis): { + ProductVariableConfigurationAxis(false, "product_only_with_excludes"): { "a": nilLabels, ConditionsDefaultConfigKey: makeLabels("product_config_exclude"), }, @@ -679,7 +679,7 @@ func TestDeduplicateAxesFromBase(t *testing.T) { OsArchConfigurationAxis: stringListSelectValues{ "linux_x86": {"linux_x86_include"}, }, - ProductVariableConfigurationAxis("a", NoConfigAxis): stringListSelectValues{ + ProductVariableConfigurationAxis(false, "a"): stringListSelectValues{ "a": []string{"not_in_value"}, }, }, @@ -704,7 +704,7 @@ func TestDeduplicateAxesFromBase(t *testing.T) { "linux": []string{"linux_include"}, }, OsArchConfigurationAxis: stringListSelectValues{}, - ProductVariableConfigurationAxis("a", NoConfigAxis): stringListSelectValues{ + ProductVariableConfigurationAxis(false, "a"): stringListSelectValues{ "a": []string{"not_in_value"}, }, } diff --git a/bp2build/soong_config_module_type_conversion_test.go b/bp2build/soong_config_module_type_conversion_test.go index ba42f34df5..ad07f68335 100644 --- a/bp2build/soong_config_module_type_conversion_test.go +++ b/bp2build/soong_config_module_type_conversion_test.go @@ -1251,3 +1251,111 @@ cc_binary { srcs = ["main.cc"], )`}}) } + +func TestSoongConfigModuleType_CombinedWithArchVariantProperties(t *testing.T) { + bp := ` +soong_config_bool_variable { + name: "my_bool_variable", +} + +soong_config_string_variable { + name: "my_string_variable", + values: [ + "value1", + "value2", + ], +} + +soong_config_module_type { + name: "special_build_cc_defaults", + module_type: "cc_defaults", + config_namespace: "my_namespace", + bool_variables: ["my_bool_variable"], + variables: ["my_string_variable"], + properties: ["target.android.cflags", "cflags"], +} + +special_build_cc_defaults { + name: "sample_cc_defaults", + target: { + android: { + cflags: ["-DFOO"], + }, + }, + soong_config_variables: { + my_bool_variable: { + target: { + android: { + cflags: ["-DBAR"], + }, + }, + conditions_default: { + target: { + android: { + cflags: ["-DBAZ"], + }, + }, + }, + }, + my_string_variable: { + value1: { + cflags: ["-DVALUE1_NOT_ANDROID"], + target: { + android: { + cflags: ["-DVALUE1"], + }, + }, + }, + value2: { + target: { + android: { + cflags: ["-DVALUE2"], + }, + }, + }, + conditions_default: { + target: { + android: { + cflags: ["-DSTRING_VAR_CONDITIONS_DEFAULT"], + }, + }, + }, + }, + }, +} + +cc_binary { + name: "my_binary", + srcs: ["main.cc"], + defaults: ["sample_cc_defaults"], +}` + + runSoongConfigModuleTypeTest(t, Bp2buildTestCase{ + Description: "soong config variables - generates selects for library_linking_strategy", + ModuleTypeUnderTest: "cc_binary", + ModuleTypeUnderTestFactory: cc.BinaryFactory, + Blueprint: bp, + Filesystem: map[string]string{}, + ExpectedBazelTargets: []string{`cc_binary( + name = "my_binary", + copts = select({ + "//build/bazel/platforms/os:android": ["-DFOO"], + "//conditions:default": [], + }) + select({ + "//build/bazel/product_variables:my_namespace__my_bool_variable__android": ["-DBAR"], + "//build/bazel/product_variables:my_namespace__my_bool_variable__conditions_default__android": ["-DBAZ"], + "//conditions:default": [], + }) + select({ + "//build/bazel/product_variables:my_namespace__my_string_variable__value1": ["-DVALUE1_NOT_ANDROID"], + "//conditions:default": [], + }) + select({ + "//build/bazel/product_variables:my_namespace__my_string_variable__conditions_default__android": ["-DSTRING_VAR_CONDITIONS_DEFAULT"], + "//build/bazel/product_variables:my_namespace__my_string_variable__value1__android": ["-DVALUE1"], + "//build/bazel/product_variables:my_namespace__my_string_variable__value2__android": ["-DVALUE2"], + "//conditions:default": [], + }), + local_includes = ["."], + srcs = ["main.cc"], + target_compatible_with = ["//build/bazel/platforms/os:android"], +)`}}) +} diff --git a/cc/bp2build.go b/cc/bp2build.go index 749fce59a3..3bb00adee6 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -537,7 +537,7 @@ func (ca *compilerAttributes) convertProductVariables(ctx android.BazelConversio if !ok { ctx.ModuleErrorf("Could not convert product variable %s property", proptools.PropertyNameForField(propName)) } - newFlags, _ := bazel.TryVariableSubstitutions(flags, productConfigProp.Name) + newFlags, _ := bazel.TryVariableSubstitutions(flags, productConfigProp.Name()) attr.SetSelectValue(productConfigProp.ConfigurationAxis(), productConfigProp.SelectKey(), newFlags) } } @@ -1350,7 +1350,7 @@ func (la *linkerAttributes) convertProductVariables(ctx android.BazelConversionP // Collect all the configurations that an include or exclude property exists for. // We want to iterate all configurations rather than either the include or exclude because, for a // particular configuration, we may have either only an include or an exclude to handle. - productConfigProps := make(map[android.ProductConfigProperty]bool, len(props)+len(excludeProps)) + productConfigProps := make(map[android.ProductConfigOrSoongConfigProperty]bool, len(props)+len(excludeProps)) for p := range props { productConfigProps[p] = true } From 733b77c2de3ec91125c42e7cb70adb8fdf44a719 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Tue, 9 May 2023 18:21:36 +0000 Subject: [PATCH 0113/1460] Refactor upload-only to process bazel metrics outside of upload.go Bug: 279987768 Test: m nothing Test: b build libcore:all (with prints to verify similiar data) Change-Id: I01d960a7215be30728c9e90f52c2455fd21cf1b8 --- cmd/soong_ui/main.go | 15 +++-- ui/build/Android.bp | 1 + ui/build/bazel_metrics.go | 135 ++++++++++++++++++++++++++++++++++++++ ui/build/upload.go | 114 +------------------------------- ui/build/upload_test.go | 4 +- 5 files changed, 150 insertions(+), 119 deletions(-) create mode 100644 ui/build/bazel_metrics.go diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index bd774c6456..1c6aaad35d 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -99,7 +99,7 @@ var commands = []command{ // Upload-only mode mostly skips to the metrics-uploading phase of soong_ui. // However, this invocation marks the true "end of the build", and thus we // need to update the total runtime of the build to include this upload step. - run: updateTotalRealTime, + run: finalizeBazelMetrics, }, } @@ -203,8 +203,6 @@ func main() { bazelMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bazel_metrics.pb") soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb") - //the profile file generated by Bazel" - bazelProfileFile := filepath.Join(logsDir, c.logsPrefix+"analyzed_bazel_profile.txt") metricsFiles := []string{ buildErrorFile, // build error strings rbeMetricsFile, // high level metrics related to remote build execution. @@ -226,7 +224,7 @@ func main() { criticalPath.WriteToMetrics(met) met.Dump(soongMetricsFile) if !config.SkipMetricsUpload() { - build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, bazelProfileFile, bazelMetricsFile, metricsFiles...) + build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, metricsFiles...) } }() c.run(buildCtx, config, args) @@ -692,6 +690,15 @@ func setMaxFiles(ctx build.Context) { } } +func finalizeBazelMetrics(ctx build.Context, config build.Config, args []string) { + updateTotalRealTime(ctx, config, args) + + logsDir := config.LogsDir() + logsPrefix := config.GetLogsPrefix() + bazelMetricsFile := filepath.Join(logsDir, logsPrefix+"bazel_metrics.pb") + bazelProfileFile := filepath.Join(logsDir, logsPrefix+"analyzed_bazel_profile.txt") + build.ProcessBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx, config) +} func updateTotalRealTime(ctx build.Context, config build.Config, args []string) { soongMetricsFile := filepath.Join(config.LogsDir(), "soong_metrics") diff --git a/ui/build/Android.bp b/ui/build/Android.bp index b79754cbeb..959ae4c69b 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -46,6 +46,7 @@ bootstrap_go_package { "soong-ui-tracer", ], srcs: [ + "bazel_metrics.go", "build.go", "cleanbuild.go", "config.go", diff --git a/ui/build/bazel_metrics.go b/ui/build/bazel_metrics.go new file mode 100644 index 0000000000..c0690c1e72 --- /dev/null +++ b/ui/build/bazel_metrics.go @@ -0,0 +1,135 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +// This file contains functionality to parse bazel profile data into +// a bazel_metrics proto, defined in build/soong/ui/metrics/bazel_metrics_proto +// These metrics are later uploaded in upload.go + +import ( + "bufio" + "os" + "strconv" + "strings" + + "android/soong/shared" + "google.golang.org/protobuf/proto" + + bazel_metrics_proto "android/soong/ui/metrics/bazel_metrics_proto" +) + +func parseTimingToNanos(str string) int64 { + millisString := removeDecimalPoint(str) + timingMillis, _ := strconv.ParseInt(millisString, 10, 64) + return timingMillis * 1000000 +} + +func parsePercentageToTenThousandths(str string) int32 { + percentageString := removeDecimalPoint(str) + //remove the % at the end of the string + percentage := strings.ReplaceAll(percentageString, "%", "") + percentagePortion, _ := strconv.ParseInt(percentage, 10, 32) + return int32(percentagePortion) +} + +func removeDecimalPoint(numString string) string { + // The format is always 0.425 or 10.425 + return strings.ReplaceAll(numString, ".", "") +} + +func parseTotal(line string) int64 { + words := strings.Fields(line) + timing := words[3] + return parseTimingToNanos(timing) +} + +func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming { + words := strings.Fields(line) + getPhaseNameAndTimingAndPercentage := func([]string) (string, int64, int32) { + // Sample lines include: + // Total launch phase time 0.011 s 2.59% + // Total target pattern evaluation phase time 0.011 s 2.59% + var beginning int + var end int + for ind, word := range words { + if word == "Total" { + beginning = ind + 1 + } else if beginning > 0 && word == "phase" { + end = ind + break + } + } + phaseName := strings.Join(words[beginning:end], " ") + + // end is now "phase" - advance by 2 for timing and 4 for percentage + percentageString := words[end+4] + timingString := words[end+2] + timing := parseTimingToNanos(timingString) + percentagePortion := parsePercentageToTenThousandths(percentageString) + return phaseName, timing, percentagePortion + } + + phaseName, timing, portion := getPhaseNameAndTimingAndPercentage(words) + phaseTiming := bazel_metrics_proto.PhaseTiming{} + phaseTiming.DurationNanos = &timing + phaseTiming.PortionOfBuildTime = &portion + + phaseTiming.PhaseName = &phaseName + return phaseTiming +} + +// This method takes a file created by bazel's --analyze-profile mode and +// writes bazel metrics data to the provided filepath. +func ProcessBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context, config Config) { + if bazelProfileFile == "" { + return + } + + readBazelProto := func(filepath string) bazel_metrics_proto.BazelMetrics { + //serialize the proto, write it + bazelMetrics := bazel_metrics_proto.BazelMetrics{} + + file, err := os.ReadFile(filepath) + if err != nil { + ctx.Fatalln("Error reading metrics file\n", err) + } + + scanner := bufio.NewScanner(strings.NewReader(string(file))) + scanner.Split(bufio.ScanLines) + + var phaseTimings []*bazel_metrics_proto.PhaseTiming + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Total run time") { + total := parseTotal(line) + bazelMetrics.Total = &total + } else if strings.HasPrefix(line, "Total") { + phaseTiming := parsePhaseTiming(line) + phaseTimings = append(phaseTimings, &phaseTiming) + } + } + bazelMetrics.PhaseTimings = phaseTimings + + return bazelMetrics + } + + if _, err := os.Stat(bazelProfileFile); err != nil { + // We can assume bazel didn't run if the profile doesn't exist + return + } + bazelProto := readBazelProto(bazelProfileFile) + bazelProto.ExitCode = proto.Int32(config.bazelExitCode) + shared.Save(&bazelProto, bazelMetricsFile) +} diff --git a/ui/build/upload.go b/ui/build/upload.go index ee4a5b3450..9f14bdd7c6 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -18,21 +18,16 @@ package build // another. import ( - "bufio" "fmt" "io/ioutil" "os" "path/filepath" - "strconv" - "strings" "time" - "android/soong/shared" "android/soong/ui/metrics" "google.golang.org/protobuf/proto" - bazel_metrics_proto "android/soong/ui/metrics/bazel_metrics_proto" upload_proto "android/soong/ui/metrics/upload_proto" ) @@ -78,123 +73,16 @@ func pruneMetricsFiles(paths []string) []string { return metricsFiles } -func parseTimingToNanos(str string) int64 { - millisString := removeDecimalPoint(str) - timingMillis, _ := strconv.ParseInt(millisString, 10, 64) - return timingMillis * 1000000 -} - -func parsePercentageToTenThousandths(str string) int32 { - percentageString := removeDecimalPoint(str) - //remove the % at the end of the string - percentage := strings.ReplaceAll(percentageString, "%", "") - percentagePortion, _ := strconv.ParseInt(percentage, 10, 32) - return int32(percentagePortion) -} - -func removeDecimalPoint(numString string) string { - // The format is always 0.425 or 10.425 - return strings.ReplaceAll(numString, ".", "") -} - -func parseTotal(line string) int64 { - words := strings.Fields(line) - timing := words[3] - return parseTimingToNanos(timing) -} - -func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming { - words := strings.Fields(line) - getPhaseNameAndTimingAndPercentage := func([]string) (string, int64, int32) { - // Sample lines include: - // Total launch phase time 0.011 s 2.59% - // Total target pattern evaluation phase time 0.011 s 2.59% - var beginning int - var end int - for ind, word := range words { - if word == "Total" { - beginning = ind + 1 - } else if beginning > 0 && word == "phase" { - end = ind - break - } - } - phaseName := strings.Join(words[beginning:end], " ") - - // end is now "phase" - advance by 2 for timing and 4 for percentage - percentageString := words[end+4] - timingString := words[end+2] - timing := parseTimingToNanos(timingString) - percentagePortion := parsePercentageToTenThousandths(percentageString) - return phaseName, timing, percentagePortion - } - - phaseName, timing, portion := getPhaseNameAndTimingAndPercentage(words) - phaseTiming := bazel_metrics_proto.PhaseTiming{} - phaseTiming.DurationNanos = &timing - phaseTiming.PortionOfBuildTime = &portion - - phaseTiming.PhaseName = &phaseName - return phaseTiming -} - -// This method takes a file created by bazel's --analyze-profile mode and -// writes bazel metrics data to the provided filepath. -// TODO(b/279987768) - move this outside of upload.go -func processBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context, config Config) { - if bazelProfileFile == "" { - return - } - - readBazelProto := func(filepath string) bazel_metrics_proto.BazelMetrics { - //serialize the proto, write it - bazelMetrics := bazel_metrics_proto.BazelMetrics{} - - file, err := os.ReadFile(filepath) - if err != nil { - ctx.Fatalln("Error reading metrics file\n", err) - } - - scanner := bufio.NewScanner(strings.NewReader(string(file))) - scanner.Split(bufio.ScanLines) - - var phaseTimings []*bazel_metrics_proto.PhaseTiming - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "Total run time") { - total := parseTotal(line) - bazelMetrics.Total = &total - } else if strings.HasPrefix(line, "Total") { - phaseTiming := parsePhaseTiming(line) - phaseTimings = append(phaseTimings, &phaseTiming) - } - } - bazelMetrics.PhaseTimings = phaseTimings - - return bazelMetrics - } - - if _, err := os.Stat(bazelProfileFile); err != nil { - // We can assume bazel didn't run if the profile doesn't exist - return - } - bazelProto := readBazelProto(bazelProfileFile) - bazelProto.ExitCode = proto.Int32(config.bazelExitCode) - shared.Save(&bazelProto, bazelMetricsFile) -} - // UploadMetrics uploads a set of metrics files to a server for analysis. // The metrics files are first copied to a temporary directory // and the uploader is then executed in the background to allow the user/system // to continue working. Soong communicates to the uploader through the // upload_proto raw protobuf file. -func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, bazelProfileFile string, bazelMetricsFile string, paths ...string) { +func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) { ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") defer ctx.EndTrace() uploader := config.MetricsUploaderApp() - processBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx, config) - if uploader == "" { // If the uploader path was not specified, no metrics shall be uploaded. return diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go index 58d923702a..1fcded921a 100644 --- a/ui/build/upload_test.go +++ b/ui/build/upload_test.go @@ -166,7 +166,7 @@ func TestUploadMetrics(t *testing.T) { metricsUploader: tt.uploader, }} - UploadMetrics(ctx, config, false, time.Now(), "out/bazel_metrics.txt", "out/bazel_metrics.pb", metricsFiles...) + UploadMetrics(ctx, config, false, time.Now(), metricsFiles...) }) } } @@ -221,7 +221,7 @@ func TestUploadMetricsErrors(t *testing.T) { metricsUploader: "echo", }} - UploadMetrics(ctx, config, true, time.Now(), "", "", metricsFile) + UploadMetrics(ctx, config, true, time.Now(), metricsFile) t.Errorf("got nil, expecting %q as a failure", tt.expectedErr) }) } From 883507a7c9da72adace8af6abd5f67da9bbfe051 Mon Sep 17 00:00:00 2001 From: Chris Wailes Date: Tue, 9 May 2023 13:19:31 -0700 Subject: [PATCH 0114/1460] rustc-1.69.0 Build 10095304 Bug: https://issuetracker.google.com/issues/274972941 Test: m rust Change-Id: I5f376bd661eee014793aceeeb37a012dfad1fd80 --- rust/config/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/config/global.go b/rust/config/global.go index 2d1f0c1f94..748bb3d2f3 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.68.0" + RustDefaultVersion = "1.69.0" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ From 56d41d0725db796671d661118e9b769a77b644e4 Mon Sep 17 00:00:00 2001 From: Romain Jobredeaux Date: Tue, 9 May 2023 16:34:16 -0400 Subject: [PATCH 0115/1460] Don't translate "required" to unsupported "data" in android_app. Bug: 228514588 Test: Presubmits Change-Id: Ib964073d24f03cc3cdd389ba28b44ed96b61d841 --- bp2build/android_app_conversion_test.go | 1 + java/app.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bp2build/android_app_conversion_test.go b/bp2build/android_app_conversion_test.go index 928a1f2e18..7f7aa6a3a7 100644 --- a/bp2build/android_app_conversion_test.go +++ b/bp2build/android_app_conversion_test.go @@ -80,6 +80,7 @@ android_app { static_libs: ["static_lib_dep"], java_version: "7", certificate: "foocert", + required: ["static_lib_dep"], } `, ExpectedBazelTargets: []string{ diff --git a/java/app.go b/java/app.go index 7bb8cdbd7e..2083772e43 100755 --- a/java/app.go +++ b/java/app.go @@ -1570,7 +1570,7 @@ func (a *AndroidApp) ConvertWithBp2build(ctx android.TopDownMutatorContext) { ctx.CreateBazelTargetModule( props, - android.CommonAttributes{Name: a.Name()}, + android.CommonAttributes{Name: a.Name(), SkipData: proptools.BoolPtr(true)}, appAttrs, ) From c85d08f63be6b7a762797415ef222cc73e15dbf7 Mon Sep 17 00:00:00 2001 From: Cole Faust Date: Tue, 9 May 2023 15:00:58 -0700 Subject: [PATCH 0116/1460] Make mk2rbc output paths relative to android root Previously, mk2rbc would output absolute paths in the load statements. This caused issues with the new module resolution logic in rbcrun, and is not something we'd want to check in anyways. Bug: 280685526 Test: go test Change-Id: Iae6e3f6be9cecc8e19b6e1fd33c5575ff6864654 --- mk2rbc/mk2rbc.go | 23 +++++++++++++++++++++++ mk2rbc/mk2rbc/mk2rbc.go | 13 +++++++++++-- mk2rbc/soong_variables_test.go | 4 ++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index 77394d9646..8225df60d4 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -163,6 +163,21 @@ var ignoredDefines = map[string]bool{ var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") +func RelativeToCwd(path string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + path, err = filepath.Rel(cwd, path) + if err != nil { + return "", err + } + if strings.HasPrefix(path, "../") { + return "", fmt.Errorf("Could not make path relative to current working directory: " + path) + } + return path, nil +} + // Conversion request parameters type Request struct { MkFile string // file to convert @@ -320,6 +335,14 @@ func (gctx *generationContext) emitPreamble() { loadedSubConfigs := make(map[string]string) for _, mi := range gctx.starScript.inherited { uri := mi.path + if strings.HasPrefix(uri, "/") && !strings.HasPrefix(uri, "//") { + var err error + uri, err = RelativeToCwd(uri) + if err != nil { + panic(err) + } + uri = "//" + uri + } if m, ok := loadedSubConfigs[uri]; ok { // No need to emit load statement, but fix module name. mi.moduleLocalName = m diff --git a/mk2rbc/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc/mk2rbc.go index cc83430a89..08c363f1be 100644 --- a/mk2rbc/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc/mk2rbc.go @@ -187,7 +187,7 @@ func main() { quit(fmt.Errorf("the product launcher input variables file failed to convert")) } - err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(files[0]), outputFilePath(*inputVariables), + err := writeGenerated(*launcher, mk2rbc.Launcher(outputModulePath(files[0]), outputModulePath(*inputVariables), mk2rbc.MakePath2ModuleName(files[0]))) if err != nil { fmt.Fprintf(os.Stderr, "%s: %s", files[0], err) @@ -205,7 +205,7 @@ func main() { quit(fmt.Errorf("the board launcher input variables file failed to convert")) } err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher( - outputFilePath(files[0]), outputFilePath(*inputVariables))) + outputModulePath(files[0]), outputModulePath(*inputVariables))) if err != nil { fmt.Fprintf(os.Stderr, "%s: %s", files[0], err) ok = false @@ -402,6 +402,15 @@ func outputFilePath(mkFile string) string { return path } +func outputModulePath(mkFile string) string { + path := outputFilePath(mkFile) + path, err := mk2rbc.RelativeToCwd(path) + if err != nil { + panic(err) + } + return "//" + path +} + func writeGenerated(path string, contents string) error { if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil { return err diff --git a/mk2rbc/soong_variables_test.go b/mk2rbc/soong_variables_test.go index c883882c47..58e98f68e3 100644 --- a/mk2rbc/soong_variables_test.go +++ b/mk2rbc/soong_variables_test.go @@ -42,8 +42,8 @@ func TestSoongVariables(t *testing.T) { {"BUILD_ID", VarClassSoong, starlarkTypeString}, {"PLATFORM_SDK_VERSION", VarClassSoong, starlarkTypeInt}, {"DEVICE_PACKAGE_OVERLAYS", VarClassSoong, starlarkTypeList}, - {"ENABLE_CFI", VarClassSoong, starlarkTypeBool}, - {"ENABLE_PREOPT", VarClassSoong, starlarkTypeBool}, + {"ENABLE_CFI", VarClassSoong, starlarkTypeString}, + {"ENABLE_PREOPT", VarClassSoong, starlarkTypeString}, }} if !reflect.DeepEqual(expected, actual) { t.Errorf("\nExpected: %v\n Actual: %v", expected, actual) From 38917496e5e87fd638dc2a5a8dff937bb436480c Mon Sep 17 00:00:00 2001 From: Hamzeh Zawawy Date: Wed, 5 Apr 2023 22:08:46 +0000 Subject: [PATCH 0117/1460] Support removing suffix for device specific prebuilt during build Some device specific prebuilts have a suffix that varies depending on the lunch target. The resulting install dependency binary has the suffix. The fuzzer build expects the presubmit binary without the suffix. This change is to drop the suffix. Test: make haiku Change-Id: I8eadd84bf5797659b17bc428dca47b7c3f382aff --- cc/cc.go | 2 +- cc/fuzz.go | 49 +++++++++++++++++++++++++++++++++---------------- cc/linkable.go | 2 +- rust/fuzz.go | 10 ++++++---- rust/rust.go | 2 +- 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/cc/cc.go b/cc/cc.go index 3fbefcd511..7237686e4c 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -1086,7 +1086,7 @@ func (c *Module) FuzzPackagedModule() fuzz.FuzzPackagedModule { panic(fmt.Errorf("FuzzPackagedModule called on non-fuzz module: %q", c.BaseModuleName())) } -func (c *Module) FuzzSharedLibraries() android.Paths { +func (c *Module) FuzzSharedLibraries() android.RuleBuilderInstalls { if fuzzer, ok := c.compiler.(*fuzzBinary); ok { return fuzzer.sharedLibraries } diff --git a/cc/fuzz.go b/cc/fuzz.go index 7aa8b91c75..38792e6d5b 100644 --- a/cc/fuzz.go +++ b/cc/fuzz.go @@ -103,7 +103,7 @@ type fuzzBinary struct { *baseCompiler fuzzPackagedModule fuzz.FuzzPackagedModule installedSharedDeps []string - sharedLibraries android.Paths + sharedLibraries android.RuleBuilderInstalls } func (fuzz *fuzzBinary) fuzzBinary() bool { @@ -213,19 +213,19 @@ func IsValidSharedDependency(dependency android.Module) bool { } func SharedLibraryInstallLocation( - libraryPath android.Path, isHost bool, fuzzDir string, archString string) string { + libraryBase string, isHost bool, fuzzDir string, archString string) string { installLocation := "$(PRODUCT_OUT)/data" if isHost { installLocation = "$(HOST_OUT)" } installLocation = filepath.Join( - installLocation, fuzzDir, archString, "lib", libraryPath.Base()) + installLocation, fuzzDir, archString, "lib", libraryBase) return installLocation } // Get the device-only shared library symbols install directory. -func SharedLibrarySymbolsInstallLocation(libraryPath android.Path, fuzzDir string, archString string) string { - return filepath.Join("$(PRODUCT_OUT)/symbols/data/", fuzzDir, archString, "/lib/", libraryPath.Base()) +func SharedLibrarySymbolsInstallLocation(libraryBase string, fuzzDir string, archString string) string { + return filepath.Join("$(PRODUCT_OUT)/symbols/data/", fuzzDir, archString, "/lib/", libraryBase) } func (fuzzBin *fuzzBinary) install(ctx ModuleContext, file android.Path) { @@ -242,15 +242,16 @@ func (fuzzBin *fuzzBinary) install(ctx ModuleContext, file android.Path) { // Grab the list of required shared libraries. fuzzBin.sharedLibraries, _ = CollectAllSharedDependencies(ctx) - for _, lib := range fuzzBin.sharedLibraries { + for _, ruleBuilderInstall := range fuzzBin.sharedLibraries { + install := ruleBuilderInstall.To fuzzBin.installedSharedDeps = append(fuzzBin.installedSharedDeps, SharedLibraryInstallLocation( - lib, ctx.Host(), installBase, ctx.Arch().ArchType.String())) + install, ctx.Host(), installBase, ctx.Arch().ArchType.String())) // Also add the dependency on the shared library symbols dir. if !ctx.Host() { fuzzBin.installedSharedDeps = append(fuzzBin.installedSharedDeps, - SharedLibrarySymbolsInstallLocation(lib, installBase, ctx.Arch().ArchType.String())) + SharedLibrarySymbolsInstallLocation(install, installBase, ctx.Arch().ArchType.String())) } } } @@ -453,19 +454,21 @@ func (s *ccRustFuzzPackager) MakeVars(ctx android.MakeVarsContext) { // GetSharedLibsToZip finds and marks all the transiently-dependent shared libraries for // packaging. -func GetSharedLibsToZip(sharedLibraries android.Paths, module LinkableInterface, s *fuzz.FuzzPackager, archString string, destinationPathPrefix string, sharedLibraryInstalled *map[string]bool) []fuzz.FileToZip { +func GetSharedLibsToZip(sharedLibraries android.RuleBuilderInstalls, module LinkableInterface, s *fuzz.FuzzPackager, archString string, destinationPathPrefix string, sharedLibraryInstalled *map[string]bool) []fuzz.FileToZip { var files []fuzz.FileToZip fuzzDir := "fuzz" - for _, library := range sharedLibraries { + for _, ruleBuilderInstall := range sharedLibraries { + library := ruleBuilderInstall.From + install := ruleBuilderInstall.To files = append(files, fuzz.FileToZip{library, destinationPathPrefix}) // For each architecture-specific shared library dependency, we need to // install it to the output directory. Setup the install destination here, // which will be used by $(copy-many-files) in the Make backend. installDestination := SharedLibraryInstallLocation( - library, module.Host(), fuzzDir, archString) + install, module.Host(), fuzzDir, archString) if (*sharedLibraryInstalled)[installDestination] { continue } @@ -483,7 +486,7 @@ func GetSharedLibsToZip(sharedLibraries android.Paths, module LinkableInterface, // we want symbolization tools (like `stack`) to be able to find the symbols // in $ANDROID_PRODUCT_OUT/symbols automagically. if !module.Host() { - symbolsInstallDestination := SharedLibrarySymbolsInstallLocation(library, fuzzDir, archString) + symbolsInstallDestination := SharedLibrarySymbolsInstallLocation(install, fuzzDir, archString) symbolsInstallDestination = strings.ReplaceAll(symbolsInstallDestination, "$", "$$") s.SharedLibInstallStrings = append(s.SharedLibInstallStrings, library.String()+":"+symbolsInstallDestination) @@ -497,12 +500,12 @@ func GetSharedLibsToZip(sharedLibraries android.Paths, module LinkableInterface, // VisitDirectDeps is used first to avoid incorrectly using the core libraries (sanitizer // runtimes, libc, libdl, etc.) from a dependency. This may cause issues when dependencies // have explicit sanitizer tags, as we may get a dependency on an unsanitized libc, etc. -func CollectAllSharedDependencies(ctx android.ModuleContext) (android.Paths, []android.Module) { +func CollectAllSharedDependencies(ctx android.ModuleContext) (android.RuleBuilderInstalls, []android.Module) { seen := make(map[string]bool) recursed := make(map[string]bool) deps := []android.Module{} - var sharedLibraries android.Paths + var sharedLibraries android.RuleBuilderInstalls // Enumerate the first level of dependencies, as we discard all non-library // modules in the BFS loop below. @@ -510,22 +513,36 @@ func CollectAllSharedDependencies(ctx android.ModuleContext) (android.Paths, []a if !IsValidSharedDependency(dep) { return } + if !ctx.OtherModuleHasProvider(dep, SharedLibraryInfoProvider) { + return + } if seen[ctx.OtherModuleName(dep)] { return } seen[ctx.OtherModuleName(dep)] = true deps = append(deps, dep) - sharedLibraries = append(sharedLibraries, android.OutputFileForModule(ctx, dep, "unstripped")) + + sharedLibraryInfo := ctx.OtherModuleProvider(dep, SharedLibraryInfoProvider).(SharedLibraryInfo) + installDestination := sharedLibraryInfo.SharedLibrary.Base() + ruleBuilderInstall := android.RuleBuilderInstall{android.OutputFileForModule(ctx, dep, "unstripped"), installDestination} + sharedLibraries = append(sharedLibraries, ruleBuilderInstall) }) ctx.WalkDeps(func(child, parent android.Module) bool { if !IsValidSharedDependency(child) { return false } + if !ctx.OtherModuleHasProvider(child, SharedLibraryInfoProvider) { + return false + } if !seen[ctx.OtherModuleName(child)] { seen[ctx.OtherModuleName(child)] = true deps = append(deps, child) - sharedLibraries = append(sharedLibraries, android.OutputFileForModule(ctx, child, "unstripped")) + + sharedLibraryInfo := ctx.OtherModuleProvider(child, SharedLibraryInfoProvider).(SharedLibraryInfo) + installDestination := sharedLibraryInfo.SharedLibrary.Base() + ruleBuilderInstall := android.RuleBuilderInstall{android.OutputFileForModule(ctx, child, "unstripped"), installDestination} + sharedLibraries = append(sharedLibraries, ruleBuilderInstall) } if recursed[ctx.OtherModuleName(child)] { diff --git a/cc/linkable.go b/cc/linkable.go index 9578807f81..557f5d2b31 100644 --- a/cc/linkable.go +++ b/cc/linkable.go @@ -130,7 +130,7 @@ type LinkableInterface interface { // FuzzSharedLibraries returns the shared library dependencies for this module. // Expects that IsFuzzModule returns true. - FuzzSharedLibraries() android.Paths + FuzzSharedLibraries() android.RuleBuilderInstalls Device() bool Host() bool diff --git a/rust/fuzz.go b/rust/fuzz.go index d7e7ddfff6..c2b9405255 100644 --- a/rust/fuzz.go +++ b/rust/fuzz.go @@ -31,7 +31,7 @@ type fuzzDecorator struct { *binaryDecorator fuzzPackagedModule fuzz.FuzzPackagedModule - sharedLibraries android.Paths + sharedLibraries android.RuleBuilderInstalls installedSharedDeps []string } @@ -119,15 +119,17 @@ func (fuzz *fuzzDecorator) install(ctx ModuleContext) { // Grab the list of required shared libraries. fuzz.sharedLibraries, _ = cc.CollectAllSharedDependencies(ctx) - for _, lib := range fuzz.sharedLibraries { + for _, ruleBuilderInstall := range fuzz.sharedLibraries { + install := ruleBuilderInstall.To + fuzz.installedSharedDeps = append(fuzz.installedSharedDeps, cc.SharedLibraryInstallLocation( - lib, ctx.Host(), installBase, ctx.Arch().ArchType.String())) + install, ctx.Host(), installBase, ctx.Arch().ArchType.String())) // Also add the dependency on the shared library symbols dir. if !ctx.Host() { fuzz.installedSharedDeps = append(fuzz.installedSharedDeps, - cc.SharedLibrarySymbolsInstallLocation(lib, installBase, ctx.Arch().ArchType.String())) + cc.SharedLibrarySymbolsInstallLocation(install, installBase, ctx.Arch().ArchType.String())) } } } diff --git a/rust/rust.go b/rust/rust.go index 7b520cdb0c..dc53cc0280 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -649,7 +649,7 @@ func (mod *Module) FuzzPackagedModule() fuzz.FuzzPackagedModule { panic(fmt.Errorf("FuzzPackagedModule called on non-fuzz module: %q", mod.BaseModuleName())) } -func (mod *Module) FuzzSharedLibraries() android.Paths { +func (mod *Module) FuzzSharedLibraries() android.RuleBuilderInstalls { if fuzzer, ok := mod.compiler.(*fuzzDecorator); ok { return fuzzer.sharedLibraries } From 25ff305370712354a8a6f903825833893be275cc Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 8 May 2023 15:05:29 -0700 Subject: [PATCH 0118/1460] Add -e argument to soong_zip to allow setting an explicit filename soong_zip normally takes the name of the input source file as the name of the file in the zip, which is ideal for zipping up directories but not for constructing arbitrary zip files. Add a -e argument that explicitly sets the path in the zip file for the next -f argument. Bug: 254867347 Test: zip_test.go Change-Id: If9d62c1a0064a485aebddf6d2a661f63f3e60b0f --- zip/cmd/main.go | 10 +++++++ zip/zip.go | 15 ++++++++++- zip/zip_test.go | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/zip/cmd/main.go b/zip/cmd/main.go index def76aa629..a2ccc2011e 100644 --- a/zip/cmd/main.go +++ b/zip/cmd/main.go @@ -78,6 +78,15 @@ func (rspFiles) Set(s string) error { return nil } +type explicitFile struct{} + +func (explicitFile) String() string { return `""` } + +func (explicitFile) Set(s string) error { + fileArgsBuilder.ExplicitPathInZip(s) + return nil +} + type dir struct{} func (dir) String() string { return `""` } @@ -173,6 +182,7 @@ func main() { flags.Var(&nonDeflatedFiles, "s", "file path to be stored within the zip without compression") flags.Var(&relativeRoot{}, "C", "path to use as relative root of files in following -f, -l, or -D arguments") flags.Var(&junkPaths{}, "j", "junk paths, zip files without directory names") + flags.Var(&explicitFile{}, "e", "filename to use in the zip file for the next -f argument") flags.Parse(expandedArgs[1:]) diff --git a/zip/zip.go b/zip/zip.go index 6f1a8adaf2..5e1a10462f 100644 --- a/zip/zip.go +++ b/zip/zip.go @@ -80,6 +80,7 @@ type pathMapping struct { type FileArg struct { PathPrefixInZip, SourcePrefixToStrip string + ExplicitPathInZip string SourceFiles []string JunkPaths bool GlobDir string @@ -124,6 +125,10 @@ func (b *FileArgsBuilder) File(name string) *FileArgsBuilder { arg := b.state arg.SourceFiles = []string{name} b.fileArgs = append(b.fileArgs, arg) + + if b.state.ExplicitPathInZip != "" { + b.state.ExplicitPathInZip = "" + } return b } @@ -189,6 +194,12 @@ func (b *FileArgsBuilder) RspFile(name string) *FileArgsBuilder { return b } +// ExplicitPathInZip sets the path in the zip file for the next File call. +func (b *FileArgsBuilder) ExplicitPathInZip(s string) *FileArgsBuilder { + b.state.ExplicitPathInZip = s + return b +} + func (b *FileArgsBuilder) Error() error { if b == nil { return nil @@ -425,7 +436,9 @@ func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping, var dest string - if fa.JunkPaths { + if fa.ExplicitPathInZip != "" { + dest = fa.ExplicitPathInZip + } else if fa.JunkPaths { dest = filepath.Base(src) } else { var err error diff --git a/zip/zip_test.go b/zip/zip_test.go index e7fdea8399..c64c3f499f 100644 --- a/zip/zip_test.go +++ b/zip/zip_test.go @@ -454,6 +454,60 @@ func TestZip(t *testing.T) { fhWithSHA256("c", fileC, zip.Deflate, sha256FileC), }, }, + { + name: "explicit path", + args: fileArgsBuilder(). + ExplicitPathInZip("foo"). + File("a/a/a"). + File("a/a/b"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("foo", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + }, + }, + { + name: "explicit path with prefix", + args: fileArgsBuilder(). + PathPrefixInZip("prefix"). + ExplicitPathInZip("foo"). + File("a/a/a"). + File("a/a/b"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("prefix/foo", fileA, zip.Deflate), + fh("prefix/a/a/b", fileB, zip.Deflate), + }, + }, + { + name: "explicit path with glob", + args: fileArgsBuilder(). + ExplicitPathInZip("foo"). + File("a/a/a*"). + File("a/a/b"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("foo", fileA, zip.Deflate), + fh("a/a/b", fileB, zip.Deflate), + }, + }, + { + name: "explicit path with junk paths", + args: fileArgsBuilder(). + JunkPaths(true). + ExplicitPathInZip("foo/bar"). + File("a/a/a*"). + File("a/a/b"), + compressionLevel: 9, + + files: []zip.FileHeader{ + fh("foo/bar", fileA, zip.Deflate), + fh("b", fileB, zip.Deflate), + }, + }, // errors { @@ -490,6 +544,22 @@ func TestZip(t *testing.T) { File("d/a/a"), err: ConflictingFileError{}, }, + { + name: "error explicit path conflicting", + args: fileArgsBuilder(). + ExplicitPathInZip("foo"). + File("a/a/a"). + ExplicitPathInZip("foo"). + File("a/a/b"), + err: ConflictingFileError{}, + }, + { + name: "error explicit path conflicting glob", + args: fileArgsBuilder(). + ExplicitPathInZip("foo"). + File("a/a/*"), + err: ConflictingFileError{}, + }, } for _, test := range testCases { From 80462dc04065495ead13fdaa9b92f537117ef25e Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 8 May 2023 15:09:31 -0700 Subject: [PATCH 0119/1460] Propagate shared library destination path through FilesToZip Shared libraries in fuzz zip files may have a different filename than their source file, for example when the source file is a prebuilt in the source tree. Propagate the computed library name through FilesToZip to soong_zip to use the correct name in the zip file. Bug: 254867347 Test: builds Change-Id: Ib0def8decdbec7842b8e1a9056fac373d743bac0 --- cc/fuzz.go | 8 ++++++-- fuzz/fuzz_common.go | 14 +++++++++----- java/fuzz.go | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/cc/fuzz.go b/cc/fuzz.go index 38792e6d5b..dfefc11f22 100644 --- a/cc/fuzz.go +++ b/cc/fuzz.go @@ -423,7 +423,7 @@ func (s *ccRustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) files = append(files, GetSharedLibsToZip(ccModule.FuzzSharedLibraries(), ccModule, &s.FuzzPackager, archString, sharedLibsInstallDirPrefix, &sharedLibraryInstalled)...) // The executable. - files = append(files, fuzz.FileToZip{android.OutputFileForModule(ctx, ccModule, "unstripped"), ""}) + files = append(files, fuzz.FileToZip{SourceFilePath: android.OutputFileForModule(ctx, ccModule, "unstripped")}) archDirs[archOs], ok = s.BuildZipFile(ctx, module, fpm, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs) if !ok { @@ -462,7 +462,11 @@ func GetSharedLibsToZip(sharedLibraries android.RuleBuilderInstalls, module Link for _, ruleBuilderInstall := range sharedLibraries { library := ruleBuilderInstall.From install := ruleBuilderInstall.To - files = append(files, fuzz.FileToZip{library, destinationPathPrefix}) + files = append(files, fuzz.FileToZip{ + SourceFilePath: library, + DestinationPathPrefix: destinationPathPrefix, + DestinationPath: install, + }) // For each architecture-specific shared library dependency, we need to // install it to the output directory. Setup the install destination here, diff --git a/fuzz/fuzz_common.go b/fuzz/fuzz_common.go index f76529d316..2a1b40452f 100644 --- a/fuzz/fuzz_common.go +++ b/fuzz/fuzz_common.go @@ -61,6 +61,7 @@ type FuzzPackager struct { type FileToZip struct { SourceFilePath android.Path DestinationPathPrefix string + DestinationPath string } type ArchOs struct { @@ -443,7 +444,7 @@ func (s *FuzzPackager) PackageArtifacts(ctx android.SingletonContext, module and FlagWithOutput("-o ", corpusZip) rspFile := corpusZip.ReplaceExtension(ctx, "rsp") command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.Corpus) - files = append(files, FileToZip{corpusZip, ""}) + files = append(files, FileToZip{SourceFilePath: corpusZip}) } // Package the data into a zipfile. @@ -456,17 +457,17 @@ func (s *FuzzPackager) PackageArtifacts(ctx android.SingletonContext, module and command.FlagWithArg("-C ", intermediateDir) command.FlagWithInput("-f ", f) } - files = append(files, FileToZip{dataZip, ""}) + files = append(files, FileToZip{SourceFilePath: dataZip}) } // The dictionary. if fuzzModule.Dictionary != nil { - files = append(files, FileToZip{fuzzModule.Dictionary, ""}) + files = append(files, FileToZip{SourceFilePath: fuzzModule.Dictionary}) } // Additional fuzz config. if fuzzModule.Config != nil && IsValidConfig(fuzzModule, module.Name()) { - files = append(files, FileToZip{fuzzModule.Config, ""}) + files = append(files, FileToZip{SourceFilePath: fuzzModule.Config}) } return files @@ -485,6 +486,9 @@ func (s *FuzzPackager) BuildZipFile(ctx android.SingletonContext, module android } else { command.Flag("-P ''") } + if file.DestinationPath != "" { + command.FlagWithArg("-e ", file.DestinationPath) + } command.FlagWithInput("-f ", file.SourceFilePath) } @@ -502,7 +506,7 @@ func (s *FuzzPackager) BuildZipFile(ctx android.SingletonContext, module android } s.FuzzTargets[module.Name()] = true - archDirs[archOs] = append(archDirs[archOs], FileToZip{fuzzZip, ""}) + archDirs[archOs] = append(archDirs[archOs], FileToZip{SourceFilePath: fuzzZip}) return archDirs[archOs], true } diff --git a/java/fuzz.go b/java/fuzz.go index 1d6b913460..9a0c908a03 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -250,11 +250,11 @@ func generateBuildActions(s *fuzz.FuzzPackager, hostOrTargetString string, ctx a files = s.PackageArtifacts(ctx, module, javaFuzzModule.fuzzPackagedModule, archDir, builder) // Add .jar - files = append(files, fuzz.FileToZip{javaFuzzModule.implementationJarFile, ""}) + files = append(files, fuzz.FileToZip{SourceFilePath: javaFuzzModule.implementationJarFile}) // Add jni .so files for _, fPath := range javaFuzzModule.jniFilePaths { - files = append(files, fuzz.FileToZip{fPath, ""}) + files = append(files, fuzz.FileToZip{SourceFilePath: fPath}) } archDirs[archOs], ok = s.BuildZipFile(ctx, module, javaFuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs) From 2c9e8d612819840020b2d110d23fb2771b6c92f0 Mon Sep 17 00:00:00 2001 From: Wei Li Date: Fri, 5 May 2023 01:07:15 -0700 Subject: [PATCH 0120/1460] Change bp2build converter of module "package". 1) Use attribute name "default_package_metadata" instead of "default_applicable_licenses" in packages to better describe its use. 2) Add a filegroup "default_metadata_file" in packages to search for METADATA file in each package. 3) Include "default_metadata_file" in each package's "default_package_metadata" attribute. Bug: 275472038 Test: CIs Change-Id: I645c013c39e3190fd96c4a549d39a331aced16bd --- android/filegroup.go | 3 +- android/package.go | 27 +++++++++++-- bp2build/package_conversion_test.go | 60 +++++++++++++++++++++++++++-- ui/build/finder.go | 9 +++++ ui/build/soong.go | 2 + 5 files changed, 94 insertions(+), 7 deletions(-) diff --git a/android/filegroup.go b/android/filegroup.go index 0ca5dc50ed..f30ee51438 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -75,7 +75,8 @@ func isFilegroupWithPattern(pattern *regexp.Regexp) bazel.LabelMapper { // https://docs.bazel.build/versions/master/be/general.html#filegroup type bazelFilegroupAttributes struct { - Srcs bazel.LabelListAttribute + Srcs bazel.LabelListAttribute + Applicable_licenses bazel.LabelListAttribute } type bazelAidlLibraryAttributes struct { diff --git a/android/package.go b/android/package.go index 2bf6521260..7fbc700417 100644 --- a/android/package.go +++ b/android/package.go @@ -15,6 +15,8 @@ package android import ( + "path/filepath" + "android/soong/bazel" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -39,8 +41,8 @@ type packageProperties struct { } type bazelPackageAttributes struct { - Default_visibility []string - Default_applicable_licenses bazel.LabelListAttribute + Default_visibility []string + Default_package_metadata bazel.LabelListAttribute } type packageModule struct { @@ -53,13 +55,32 @@ type packageModule struct { var _ Bazelable = &packageModule{} func (p *packageModule) ConvertWithBp2build(ctx TopDownMutatorContext) { + defaultPackageMetadata := bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, p.properties.Default_applicable_licenses)) + // If METADATA file exists in the package, add it to package(default_package_metadata=) using a + // filegroup(name="default_metadata_file") which can be accessed later on each module in Bazel + // using attribute "applicable_licenses". + // Attribute applicable_licenses of filegroup "default_metadata_file" has to be set to [], + // otherwise Bazel reports cyclic reference error. + if existed, _, _ := ctx.Config().fs.Exists(filepath.Join(ctx.ModuleDir(), "METADATA")); existed { + ctx.CreateBazelTargetModule( + bazel.BazelTargetModuleProperties{ + Rule_class: "filegroup", + }, + CommonAttributes{Name: "default_metadata_file"}, + &bazelFilegroupAttributes{ + Srcs: bazel.MakeLabelListAttribute(BazelLabelForModuleSrc(ctx, []string{"METADATA"})), + Applicable_licenses: bazel.LabelListAttribute{Value: bazel.LabelList{Includes: []bazel.Label{}}, EmitEmptyList: true}, + }) + defaultPackageMetadata.Value.Add(&bazel.Label{Label: ":default_metadata_file"}) + } + ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: "package", }, CommonAttributes{}, &bazelPackageAttributes{ - Default_applicable_licenses: bazel.MakeLabelListAttribute(BazelLabelForModuleDeps(ctx, p.properties.Default_applicable_licenses)), + Default_package_metadata: defaultPackageMetadata, // FIXME(asmundak): once b/221436821 is resolved Default_visibility: []string{"//visibility:public"}, }) diff --git a/bp2build/package_conversion_test.go b/bp2build/package_conversion_test.go index 3704b2d7b1..ce848e42d9 100644 --- a/bp2build/package_conversion_test.go +++ b/bp2build/package_conversion_test.go @@ -15,9 +15,10 @@ package bp2build import ( + "testing" + "android/soong/android" "android/soong/genrule" - "testing" ) func registerDependentModules(ctx android.RegistrationContext) { @@ -29,6 +30,7 @@ func TestPackage(t *testing.T) { tests := []struct { description string modules string + fs map[string]string expected []ExpectedRuleTarget }{ { @@ -50,8 +52,50 @@ package { "package", "", AttrNameToString{ - "default_applicable_licenses": `[":my_license"]`, - "default_visibility": `["//visibility:public"]`, + "default_package_metadata": `[":my_license"]`, + "default_visibility": `["//visibility:public"]`, + }, + android.HostAndDeviceDefault, + }, + { + "android_license", + "my_license", + AttrNameToString{ + "license_kinds": `["SPDX-license-identifier-Apache-2.0"]`, + "license_text": `"NOTICE"`, + "visibility": `[":__subpackages__"]`, + }, + android.HostAndDeviceDefault, + }, + }, + }, + { + description: "package has METADATA file", + fs: map[string]string{ + "METADATA": ``, + }, + modules: ` +license { + name: "my_license", + visibility: [":__subpackages__"], + license_kinds: ["SPDX-license-identifier-Apache-2.0"], + license_text: ["NOTICE"], +} + +package { + default_applicable_licenses: ["my_license"], +} +`, + expected: []ExpectedRuleTarget{ + { + "package", + "", + AttrNameToString{ + "default_package_metadata": `[ + ":my_license", + ":default_metadata_file", + ]`, + "default_visibility": `["//visibility:public"]`, }, android.HostAndDeviceDefault, }, @@ -65,6 +109,15 @@ package { }, android.HostAndDeviceDefault, }, + { + "filegroup", + "default_metadata_file", + AttrNameToString{ + "applicable_licenses": `[]`, + "srcs": `["METADATA"]`, + }, + android.HostAndDeviceDefault, + }, }, }, } @@ -80,6 +133,7 @@ package { ModuleTypeUnderTestFactory: android.PackageFactory, Blueprint: test.modules, ExpectedBazelTargets: expected, + Filesystem: test.fs, }) } } diff --git a/ui/build/finder.go b/ui/build/finder.go index 3f628cf7bc..62079fee92 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -87,6 +87,8 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { "TEST_MAPPING", // Bazel top-level file to mark a directory as a Bazel workspace. "WORKSPACE", + // METADATA file of packages + "METADATA", }, // Bazel Starlark configuration files and all .mk files for product/board configuration. IncludeSuffixes: []string{".bzl", ".mk"}, @@ -189,6 +191,13 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { ctx.Fatalf("Could not find OWNERS: %v", err) } + // Recursively look for all METADATA files. + metadataFiles := f.FindNamedAt(".", "METADATA") + err = dumpListToFile(ctx, config, metadataFiles, filepath.Join(dumpDir, "METADATA.list")) + if err != nil { + ctx.Fatalf("Could not find METADATA: %v", err) + } + // Recursively look for all TEST_MAPPING files. testMappings := f.FindNamedAt(".", "TEST_MAPPING") err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list")) diff --git a/ui/build/soong.go b/ui/build/soong.go index 563199bee8..18bf3b9121 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -400,6 +400,8 @@ func bootstrapBlueprint(ctx Context, config Config) { pbi.Inputs = append(pbi.Inputs, config.Bp2BuildFilesMarkerFile(), filepath.Join(config.FileListDir(), "bazel.list")) + case bp2buildFilesTag: + pbi.Inputs = append(pbi.Inputs, filepath.Join(config.FileListDir(), "METADATA.list")) } invocations = append(invocations, pbi) } From 1c223017351e0a8d3fe490d78f4a8252ee914bf9 Mon Sep 17 00:00:00 2001 From: Yi Kong Date: Wed, 10 May 2023 17:24:44 +0900 Subject: [PATCH 0121/1460] Update clang version to clang-r487747c Test: presubmit Bug: 279955553 Change-Id: Ibe3b5e81f7850560b340c89a1f83d5ebdf4fc25f --- cc/config/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cc/config/global.go b/cc/config/global.go index 20298dd5b1..530b79aefe 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -311,7 +311,7 @@ var ( // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r487747b" + ClangDefaultVersion = "clang-r487747c" ClangDefaultShortVersion = "17" // Directories with warnings from Android.bp files. From 98e986542b47cc45fdd0be313f7e1129d3646acc Mon Sep 17 00:00:00 2001 From: Joe Onorato Date: Wed, 10 May 2023 05:56:57 -0700 Subject: [PATCH 0122/1460] TARGET_RELEASE is unavailable here, so don't print it. Test: m Change-Id: Iacb00db54882ed2578ca1aa1d8e71d5732fbe247 --- ui/build/dumpvars.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index b83b5cba3b..efe747819b 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -150,7 +150,6 @@ var BannerVars = []string{ "PRODUCT_INCLUDE_TAGS", "PRODUCT_SOURCE_ROOT_DIRS", "TARGET_PRODUCT", - "TARGET_RELEASE", "TARGET_BUILD_VARIANT", "TARGET_BUILD_APPS", "TARGET_BUILD_UNBUNDLED", From a381cd19737b5276c04ee33fce2b0602da099cf9 Mon Sep 17 00:00:00 2001 From: Alix Date: Wed, 10 May 2023 14:49:38 +0000 Subject: [PATCH 0123/1460] Delete android/rules.bzl file Bug: 277800767 Test: treehugger Change-Id: I578bf63e33d15219812c2c17117186c74323d8d3 --- java/aar.go | 4 ++-- java/app.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/java/aar.go b/java/aar.go index f1b137de19..29e86e6785 100644 --- a/java/aar.go +++ b/java/aar.go @@ -1056,7 +1056,7 @@ func (a *AARImport) ConvertWithBp2build(ctx android.TopDownMutatorContext) { ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: "aar_import", - Bzl_load_location: "//build/bazel/rules/android:rules.bzl", + Bzl_load_location: "//build/bazel/rules/android:aar_import.bzl", }, android.CommonAttributes{Name: name}, &bazelAndroidLibraryImport{ @@ -1086,7 +1086,7 @@ func (a *AARImport) ConvertWithBp2build(ctx android.TopDownMutatorContext) { func AndroidLibraryBazelTargetModuleProperties() bazel.BazelTargetModuleProperties { return bazel.BazelTargetModuleProperties{ Rule_class: "android_library", - Bzl_load_location: "//build/bazel/rules/android:rules.bzl", + Bzl_load_location: "//build/bazel/rules/android:android_library.bzl", } } diff --git a/java/app.go b/java/app.go index da9c6f3437..9ba2bd6333 100755 --- a/java/app.go +++ b/java/app.go @@ -1539,7 +1539,7 @@ func androidAppCertificateBp2Build(ctx android.TopDownMutatorContext, module *An props := bazel.BazelTargetModuleProperties{ Rule_class: "android_app_certificate", - Bzl_load_location: "//build/bazel/rules/android:rules.bzl", + Bzl_load_location: "//build/bazel/rules/android:android_app_certificate.bzl", } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name()}, attrs) @@ -1592,7 +1592,7 @@ func (a *AndroidApp) ConvertWithBp2build(ctx android.TopDownMutatorContext) { props := bazel.BazelTargetModuleProperties{ Rule_class: "android_binary", - Bzl_load_location: "//build/bazel/rules/android:rules.bzl", + Bzl_load_location: "//build/bazel/rules/android:android_binary.bzl", } if !bp2BuildInfo.hasKotlin { From 3cd6a25953bec12cb6e5e0f3e88991b8505d6611 Mon Sep 17 00:00:00 2001 From: MarkDacek Date: Wed, 10 May 2023 18:39:58 +0000 Subject: [PATCH 0124/1460] Rename soong_ui's upload-metrics-only mode. It also processes bazel metrics, and is thus misnamed. Bug: 279988780 Test: b build libcore:all (with prints to verify that it's called) Change-Id: I99828e759872ba2dfb583c20a01b8101395b3558 --- cmd/soong_ui/main.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 1c6aaad35d..eea41e6f36 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -92,13 +92,12 @@ var commands = []command{ stdio: stdio, run: runMake, }, { - flag: "--upload-metrics-only", - description: "upload metrics without building anything", + flag: "--finalize-bazel-metrics", + description: "finalize b metrics and upload", config: build.UploadOnlyConfig, stdio: stdio, - // Upload-only mode mostly skips to the metrics-uploading phase of soong_ui. - // However, this invocation marks the true "end of the build", and thus we - // need to update the total runtime of the build to include this upload step. + // Finalize-bazel-metrics mode updates metrics files and calls the metrics + // uploader. This marks the end of a b invocation. run: finalizeBazelMetrics, }, } From 0e7fd8a14b5e1ebc2f2516639e9cea5dfdefce63 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 0125/1460] Implement aidl_library module type We currently specifies aidl files directly to the srcs prop on a filegroup or other module types such as cc_library or java_library. We use aidl.include_dirs prop to specify paths to aidl headers. This include_dirs pattern isn't migratable to Bazel because Bazel requires explicit dependencies. This CL introduces aidl_library to better map with Bazel's aidl_library rule and to enable aidl headers to be specified in a separate aidl_library or the hdrs prop. A follow-up CL will turn on inputs sandbox to enforce all aidl headers be explicitly specified in Android.bp Test: go test Bug: 278704136 Change-Id: I2c99af080525bf8a6c5724ed5ee2001842969098 --- aidl_library/Android.bp | 32 ++++++++ aidl_library/aidl_library.go | 120 ++++++++++++++++++++++++++++ aidl_library/aidl_library_test.go | 126 ++++++++++++++++++++++++++++++ 3 files changed, 278 insertions(+) create mode 100644 aidl_library/Android.bp create mode 100644 aidl_library/aidl_library.go create mode 100644 aidl_library/aidl_library_test.go diff --git a/aidl_library/Android.bp b/aidl_library/Android.bp new file mode 100644 index 0000000000..ec21504271 --- /dev/null +++ b/aidl_library/Android.bp @@ -0,0 +1,32 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-aidl-library", + pkgPath: "android/soong/aidl_library", + deps: [ + "soong-android", + ], + srcs: [ + "aidl_library.go", + ], + testSrcs: [ + "aidl_library_test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go new file mode 100644 index 0000000000..2a42170f23 --- /dev/null +++ b/aidl_library/aidl_library.go @@ -0,0 +1,120 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aidl_library + +import ( + "android/soong/android" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +func init() { + registerAidlLibraryBuildComponents(android.InitRegistrationContext) +} + +func registerAidlLibraryBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("aidl_library", AidlLibraryFactory) +} + +type aidlLibraryProperties struct { + // srcs lists files that are included in this module for aidl compilation + Srcs []string `android:"path"` + + // hdrs lists the headers that are imported by srcs but are not compiled by aidl to language binding code + // hdrs is provided to support Bazel migration. It is a no-op until + // we enable input sandbox in aidl compilation action + Hdrs []string `android:"path"` + + // The prefix to strip from the paths of the .aidl files + // The remaining path is the package path of the aidl interface + Strip_import_prefix *string + + // List of aidl files or aidl_library depended on by the module + Deps []string `android:"arch_variant"` +} + +type AidlLibrary struct { + android.ModuleBase + properties aidlLibraryProperties +} + +type AidlLibraryInfo struct { + // The direct aidl files of the module + Srcs android.Paths + // The include dirs to the direct aidl files and those provided from aidl_library deps + IncludeDirs android.DepSet +} + +// AidlLibraryProvider provides the srcs and the transitive include dirs +var AidlLibraryProvider = blueprint.NewProvider(AidlLibraryInfo{}) + +func (lib *AidlLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + includeDirsDepSetBuilder := android.NewDepSetBuilder(android.PREORDER) + + if len(lib.properties.Srcs) == 0 && len(lib.properties.Hdrs) == 0 { + ctx.ModuleErrorf("at least srcs or hdrs prop must be non-empty") + } + + srcs := android.PathsForModuleSrc(ctx, lib.properties.Srcs) + if lib.properties.Strip_import_prefix != nil { + srcs = android.PathsWithModuleSrcSubDir( + ctx, + srcs, + android.String(lib.properties.Strip_import_prefix)) + } + + includeDir := android.PathForModuleSrc( + ctx, + proptools.StringDefault(lib.properties.Strip_import_prefix, ""), + ) + + includeDirsDepSetBuilder.Direct(includeDir) + + for _, dep := range ctx.GetDirectDepsWithTag(aidlLibraryTag) { + if ctx.OtherModuleHasProvider(dep, AidlLibraryProvider) { + info := ctx.OtherModuleProvider(dep, AidlLibraryProvider).(AidlLibraryInfo) + includeDirsDepSetBuilder.Transitive(&info.IncludeDirs) + } + } + + // TODO(b/279960133) Propagate direct and transitive headers/srcs when aidl action sandboxes inputs + ctx.SetProvider(AidlLibraryProvider, AidlLibraryInfo{ + Srcs: srcs, + IncludeDirs: *includeDirsDepSetBuilder.Build(), + }) +} + +// aidl_library contains a list of .aidl files and the strip_import_prefix to +// to strip from the paths of the .aidl files. The sub-path left-over after stripping +// corresponds to the aidl package path the aidl interfaces are scoped in +func AidlLibraryFactory() android.Module { + module := &AidlLibrary{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + return module +} + +type aidlDependencyTag struct { + blueprint.BaseDependencyTag +} + +var aidlLibraryTag = aidlDependencyTag{} + +func (lib *AidlLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { + for _, dep := range lib.properties.Deps { + ctx.AddDependency(lib, aidlLibraryTag, dep) + } +} diff --git a/aidl_library/aidl_library_test.go b/aidl_library/aidl_library_test.go new file mode 100644 index 0000000000..42fa5367e3 --- /dev/null +++ b/aidl_library/aidl_library_test.go @@ -0,0 +1,126 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aidl_library + +import ( + "android/soong/android" + "testing" +) + +var PrepareForTestWithAidlLibrary = android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + registerAidlLibraryBuildComponents(ctx) +}) + +func TestAidlLibrary(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + PrepareForTestWithAidlLibrary, + android.MockFS{ + "package_bar/Android.bp": []byte(` + aidl_library { + name: "bar", + srcs: ["x/y/Bar.aidl"], + strip_import_prefix: "x", + } + `), + }.AddToFixture(), + android.MockFS{ + "package_foo/Android.bp": []byte(` + aidl_library { + name: "foo", + srcs: ["a/b/Foo.aidl"], + hdrs: ["Header.aidl"], + strip_import_prefix: "a", + deps: ["bar"], + } + `), + }.AddToFixture(), + ).RunTest(t).TestContext + + foo := ctx.ModuleForTests("foo", "").Module().(*AidlLibrary) + actualInfo := ctx.ModuleProvider(foo, AidlLibraryProvider).(AidlLibraryInfo) + + android.AssertArrayString( + t, + "aidl include dirs", + []string{"package_foo/a", "package_bar/x"}, + actualInfo.IncludeDirs.ToList().Strings(), + ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl srcs paths", + []string{"package_foo/a/b/Foo.aidl"}, + actualInfo.Srcs, + ) +} + +func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + PrepareForTestWithAidlLibrary, + android.MockFS{ + "package_bar/Android.bp": []byte(` + aidl_library { + name: "bar", + srcs: ["x/y/Bar.aidl"], + } + `), + }.AddToFixture(), + android.MockFS{ + "package_foo/Android.bp": []byte(` + aidl_library { + name: "foo", + srcs: ["a/b/Foo.aidl"], + hdrs: ["Header.aidl"], + deps: ["bar"], + } + `), + }.AddToFixture(), + ).RunTest(t).TestContext + + foo := ctx.ModuleForTests("foo", "").Module().(*AidlLibrary) + actualInfo := ctx.ModuleProvider(foo, AidlLibraryProvider).(AidlLibraryInfo) + + android.AssertArrayString( + t, + "aidl include dirs", + []string{"package_foo", "package_bar"}, + actualInfo.IncludeDirs.ToList().Strings(), + ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl srcs paths", + []string{"package_foo/a/b/Foo.aidl"}, + actualInfo.Srcs, + ) +} + +func TestAidlLibraryWithNoSrcsHdrsDeps(t *testing.T) { + t.Parallel() + android.GroupFixturePreparers( + PrepareForTestWithAidlLibrary, + android.MockFS{ + "package_bar/Android.bp": []byte(` + aidl_library { + name: "bar", + } + `), + }.AddToFixture(), + ). + ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern("at least srcs or hdrs prop must be non-empty")). + RunTest(t) +} From 4026558dc7711d2fcc6be5cf568d867dbc1c66fb Mon Sep 17 00:00:00 2001 From: Pirama Arumuga Nainar Date: Thu, 11 May 2023 00:01:25 +0000 Subject: [PATCH 0126/1460] [cc] Remove deprecated clang flag for auto-var-init Bug: http://b/259559364 Bug: http://b/267819531 The extra flag is no longer needed by the scan-deps used by RBE. Test: `USE_RBE m` Change-Id: Ia35b78f7b28f9008479256d9fead253910627254 --- cc/config/global.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cc/config/global.go b/cc/config/global.go index 20298dd5b1..835fa46c3b 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -360,14 +360,14 @@ func init() { // Automatically initialize any uninitialized stack variables. // Prefer zero-init if multiple options are set. if ctx.Config().IsEnvTrue("AUTO_ZERO_INITIALIZE") { - flags = append(flags, "-ftrivial-auto-var-init=zero -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang -Wno-unused-command-line-argument") + flags = append(flags, "-ftrivial-auto-var-init=zero") } else if ctx.Config().IsEnvTrue("AUTO_PATTERN_INITIALIZE") { flags = append(flags, "-ftrivial-auto-var-init=pattern") } else if ctx.Config().IsEnvTrue("AUTO_UNINITIALIZE") { flags = append(flags, "-ftrivial-auto-var-init=uninitialized") } else { // Default to zero initialization. - flags = append(flags, "-ftrivial-auto-var-init=zero -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang -Wno-unused-command-line-argument") + flags = append(flags, "-ftrivial-auto-var-init=zero") } // Workaround for ccache with clang. From df4c2134ea59886cde76916b8da79863499482d1 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Tue, 9 May 2023 23:58:52 +0000 Subject: [PATCH 0127/1460] bp2build converter for cc libraries containing yacc For cc_libraries containing .y/.yy srcs, we will generate an additional cc_yacc_static_library target. This target will produce an .a file that will be statically linked to the parent target. Test: go test ./bp2build Bug: 281546029 Change-Id: I094ba56f0a95869e5bc1e1d38b83c777192b7ddd --- bp2build/cc_library_conversion_test.go | 56 +++++++++++++++++++++ cc/bp2build.go | 67 ++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 776129f582..4ff525404f 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -4523,3 +4523,59 @@ func TestCcLibraryCppFlagsInProductVariables(t *testing.T) { }, ) } + +func TestCcLibraryYaccConversion(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library is built from .y/.yy files", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: soongCcLibraryPreamble + `cc_library { + name: "a", + srcs: [ + "a.cpp", + "a.yy", + ], + shared_libs: ["sharedlib"], + static_libs: ["staticlib"], + yacc: { + flags: ["someYaccFlag"], + gen_location_hh: true, + gen_position_hh: true, + }, +} +cc_library_static { + name: "staticlib", + bazel_module: { bp2build_available: false }, +} +cc_library { + name: "sharedlib", + bazel_module: { bp2build_available: false }, +} +`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_yacc_static_library", "a_yacc", AttrNameToString{ + "src": `"a.yy"`, + "implementation_deps": `[":staticlib"]`, + "implementation_dynamic_deps": `[":sharedlib"]`, + "flags": `["someYaccFlag"]`, + "gen_location_hh": "True", + "gen_position_hh": "True", + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "a", AttrNameToString{ + "srcs": `["a.cpp"]`, + "implementation_deps": `[":staticlib"]`, + "implementation_dynamic_deps": `[":sharedlib"]`, + "implementation_whole_archive_deps": `[":a_yacc"]`, + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_static", "a_bp2build_cc_library_static", AttrNameToString{ + "srcs": `["a.cpp"]`, + "implementation_deps": `[":staticlib"]`, + "implementation_dynamic_deps": `[":sharedlib"]`, + "implementation_whole_archive_deps": `[":a_yacc"]`, + "local_includes": `["."]`, + }), + }, + }) +} diff --git a/cc/bp2build.go b/cc/bp2build.go index 749fce59a3..55bca1a178 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -38,6 +38,7 @@ const ( protoSrcPartition = "proto" aidlSrcPartition = "aidl" syspropSrcPartition = "sysprop" + yaccSrcPartition = "yacc" stubsSuffix = "_stub_libs_current" ) @@ -154,6 +155,7 @@ func groupSrcsByExtension(ctx android.BazelConversionPathContext, srcs bazel.Lab // know the language of these sources until the genrule is executed. cppSrcPartition: bazel.LabelPartition{Extensions: []string{".cpp", ".cc", ".cxx", ".mm"}, LabelMapper: addSuffixForFilegroup("_cpp_srcs"), Keep_remainder: true}, syspropSrcPartition: bazel.LabelPartition{Extensions: []string{".sysprop"}}, + yaccSrcPartition: bazel.LabelPartition{Extensions: []string{".y", "yy"}}, } return bazel.PartitionLabelListAttribute(ctx, &srcs, labels) @@ -404,6 +406,12 @@ type compilerAttributes struct { // Sysprop sources syspropSrcs bazel.LabelListAttribute + // Yacc sources + yaccSrc *bazel.LabelAttribute + yaccFlags bazel.StringListAttribute + yaccGenLocationHeader bazel.BoolAttribute + yaccGenPositionHeader bazel.BoolAttribute + hdrs bazel.LabelListAttribute rtti bazel.BoolAttribute @@ -566,6 +574,12 @@ func (ca *compilerAttributes) finalize(ctx android.BazelConversionPathContext, i ca.asmSrcs = partitionedSrcs[asmSrcPartition] ca.lSrcs = partitionedSrcs[lSrcPartition] ca.llSrcs = partitionedSrcs[llSrcPartition] + if yacc := partitionedSrcs[yaccSrcPartition]; !yacc.IsEmpty() { + if len(yacc.Value.Includes) > 1 { + ctx.PropertyErrorf("srcs", "Found multiple yacc (.y/.yy) files in library") + } + ca.yaccSrc = bazel.MakeLabelAttribute(yacc.Value.Includes[0].Label) + } ca.syspropSrcs = partitionedSrcs[syspropSrcPartition] ca.absoluteIncludes.DeduplicateAxesFromBase() @@ -742,6 +756,11 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) if baseCompilerProps.Lex != nil { compilerAttrs.lexopts.SetSelectValue(axis, cfg, baseCompilerProps.Lex.Flags) } + if baseCompilerProps.Yacc != nil { + compilerAttrs.yaccFlags.SetSelectValue(axis, cfg, baseCompilerProps.Yacc.Flags) + compilerAttrs.yaccGenLocationHeader.SetSelectValue(axis, cfg, baseCompilerProps.Yacc.Gen_location_hh) + compilerAttrs.yaccGenPositionHeader.SetSelectValue(axis, cfg, baseCompilerProps.Yacc.Gen_position_hh) + } (&compilerAttrs).bp2buildForAxisAndConfig(ctx, axis, cfg, baseCompilerProps) } @@ -826,6 +845,12 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) } } + // Create a cc_yacc_static_library if srcs contains .y/.yy files + // This internal target will produce an .a file that will be statically linked to the parent library + if yaccDep := bp2buildCcYaccLibrary(ctx, compilerAttrs, linkerAttrs); yaccDep != nil { + (&linkerAttrs).implementationWholeArchiveDeps.Add(yaccDep) + } + convertedLSrcs := bp2BuildLex(ctx, module.Name(), compilerAttrs) (&compilerAttrs).srcs.Add(&convertedLSrcs.srcName) (&compilerAttrs).cSrcs.Add(&convertedLSrcs.cSrcName) @@ -864,6 +889,48 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) } } +type ccYaccLibraryAttributes struct { + Src bazel.LabelAttribute + Flags bazel.StringListAttribute + Gen_location_hh bazel.BoolAttribute + Gen_position_hh bazel.BoolAttribute + Local_includes bazel.StringListAttribute + Implementation_deps bazel.LabelListAttribute + Implementation_dynamic_deps bazel.LabelListAttribute +} + +func bp2buildCcYaccLibrary(ctx android.Bp2buildMutatorContext, ca compilerAttributes, la linkerAttributes) *bazel.LabelAttribute { + if ca.yaccSrc == nil { + return nil + } + yaccLibraryLabel := ctx.Module().Name() + "_yacc" + ctx.CreateBazelTargetModule( + bazel.BazelTargetModuleProperties{ + Rule_class: "cc_yacc_static_library", + Bzl_load_location: "//build/bazel/rules/cc:cc_yacc_library.bzl", + }, + android.CommonAttributes{ + Name: yaccLibraryLabel, + }, + &ccYaccLibraryAttributes{ + Src: *ca.yaccSrc, + Flags: ca.yaccFlags, + Gen_location_hh: ca.yaccGenLocationHeader, + Gen_position_hh: ca.yaccGenPositionHeader, + Local_includes: ca.localIncludes, + Implementation_deps: la.implementationDeps, + Implementation_dynamic_deps: la.implementationDynamicDeps, + }, + ) + + yaccLibrary := &bazel.LabelAttribute{ + Value: &bazel.Label{ + Label: ":" + yaccLibraryLabel, + }, + } + return yaccLibrary +} + // As a workaround for b/261657184, we are manually adding the default value // of system_dynamic_deps for the linux_musl os. // TODO: Solve this properly From ebbea2a8c91a295fe554bc28dcae0a33bae4c38a Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 10 May 2023 02:05:43 +0000 Subject: [PATCH 0128/1460] Allowlist aidl Test: b build //system/tools/aidl Change-Id: I3ddca72343822acf0a3d581061b32a37da6e1790 --- android/allowlists/allowlists.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index df47a5c914..cccf6ad4b8 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -749,6 +749,10 @@ var ( "CaptivePortalLogin", "libstagefright_headers", + + // aidl + "aidl", + "libaidl-common", } Bp2buildModuleTypeAlwaysConvertList = []string{ From f36b2fae67f5379b8a42024de8dc2ac5cac2ba29 Mon Sep 17 00:00:00 2001 From: Android Culprit Assistant Date: Thu, 11 May 2023 06:36:11 +0000 Subject: [PATCH 0129/1460] Revert "rustc-1.69.0 Build 10095304" This revert was created by Android Culprit Assistant. The culprit was identified in the following culprit search session (http://go/aca-get/1fa75415-4722-46e3-b61b-7de26844a12d). Change-Id: I7aabd3ff76c255ca13d594cc0f2f331ed8edc01b --- rust/config/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/config/global.go b/rust/config/global.go index 748bb3d2f3..2d1f0c1f94 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.69.0" + RustDefaultVersion = "1.68.0" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ From 3d16990b29fe2a4a8deed9769908f8496755adde Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 0130/1460] Implement bp2build converter for aidl_library Test: go test Bug: 278704136 Change-Id: Ia9c3772257af58e1de9041ba465130740b555fe4 --- aidl_library/aidl_library.go | 50 ++++++++++ bp2build/Android.bp | 2 + bp2build/aidl_library_conversion_test.go | 119 +++++++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 bp2build/aidl_library_conversion_test.go diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go index 2a42170f23..9b5f0a8141 100644 --- a/aidl_library/aidl_library.go +++ b/aidl_library/aidl_library.go @@ -16,6 +16,7 @@ package aidl_library import ( "android/soong/android" + "android/soong/bazel" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -48,9 +49,57 @@ type aidlLibraryProperties struct { type AidlLibrary struct { android.ModuleBase + android.BazelModuleBase properties aidlLibraryProperties } +type bazelAidlLibraryAttributes struct { + Srcs bazel.LabelListAttribute + Hdrs bazel.LabelListAttribute + Strip_import_prefix *string + Deps bazel.LabelListAttribute +} + +func (lib *AidlLibrary) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + srcs := bazel.MakeLabelListAttribute( + android.BazelLabelForModuleSrc( + ctx, + lib.properties.Srcs, + ), + ) + + hdrs := bazel.MakeLabelListAttribute( + android.BazelLabelForModuleSrc( + ctx, + lib.properties.Hdrs, + ), + ) + + tags := []string{"apex_available=//apex_available:anyapex"} + deps := bazel.MakeLabelListAttribute(android.BazelLabelForModuleDeps(ctx, lib.properties.Deps)) + + attrs := &bazelAidlLibraryAttributes{ + Srcs: srcs, + Hdrs: hdrs, + Strip_import_prefix: lib.properties.Strip_import_prefix, + Deps: deps, + } + + props := bazel.BazelTargetModuleProperties{ + Rule_class: "aidl_library", + Bzl_load_location: "//build/bazel/rules/aidl:aidl_library.bzl", + } + + ctx.CreateBazelTargetModule( + props, + android.CommonAttributes{ + Name: lib.Name(), + Tags: bazel.MakeStringListAttribute(tags), + }, + attrs, + ) +} + type AidlLibraryInfo struct { // The direct aidl files of the module Srcs android.Paths @@ -104,6 +153,7 @@ func AidlLibraryFactory() android.Module { module := &AidlLibrary{} module.AddProperties(&module.properties) android.InitAndroidModule(module) + android.InitBazelModule(module) return module } diff --git a/bp2build/Android.bp b/bp2build/Android.bp index b6635c4308..9ec3a40b9f 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -19,6 +19,7 @@ bootstrap_go_package { "testing.go", ], deps: [ + "soong-aidl-library", "soong-android", "soong-android-allowlists", "soong-android-soongconfig", @@ -37,6 +38,7 @@ bootstrap_go_package { ], testSrcs: [ "aar_conversion_test.go", + "aidl_library_conversion_test.go", "android_app_certificate_conversion_test.go", "android_app_conversion_test.go", "apex_conversion_test.go", diff --git a/bp2build/aidl_library_conversion_test.go b/bp2build/aidl_library_conversion_test.go new file mode 100644 index 0000000000..0522da45e6 --- /dev/null +++ b/bp2build/aidl_library_conversion_test.go @@ -0,0 +1,119 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "testing" + + "android/soong/aidl_library" + "android/soong/android" +) + +func runAidlLibraryTestCase(t *testing.T, tc Bp2buildTestCase) { + t.Helper() + (&tc).ModuleTypeUnderTest = "aidl_library" + (&tc).ModuleTypeUnderTestFactory = aidl_library.AidlLibraryFactory + RunBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, tc) +} + +func TestAidlLibrary(t *testing.T) { + testcases := []struct { + name string + bp string + expectedBazelAttrs AttrNameToString + }{ + { + name: "aidl_library with strip_import_prefix", + bp: ` + aidl_library { + name: "foo", + srcs: ["aidl/foo.aidl"], + hdrs: ["aidl/header.aidl"], + strip_import_prefix: "aidl", + }`, + expectedBazelAttrs: AttrNameToString{ + "srcs": `["aidl/foo.aidl"]`, + "hdrs": `["aidl/header.aidl"]`, + "strip_import_prefix": `"aidl"`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }, + }, + { + name: "aidl_library without strip_import_prefix", + bp: ` + aidl_library { + name: "foo", + srcs: ["aidl/foo.aidl"], + hdrs: ["aidl/header.aidl"], + }`, + expectedBazelAttrs: AttrNameToString{ + "srcs": `["aidl/foo.aidl"]`, + "hdrs": `["aidl/header.aidl"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }, + }, + } + + for _, test := range testcases { + t.Run(test.name, func(t *testing.T) { + expectedBazelTargets := []string{ + MakeBazelTargetNoRestrictions("aidl_library", "foo", test.expectedBazelAttrs), + } + runAidlLibraryTestCase(t, Bp2buildTestCase{ + Description: test.name, + Blueprint: test.bp, + ExpectedBazelTargets: expectedBazelTargets, + }) + }) + } +} + +func TestAidlLibraryWithDeps(t *testing.T) { + bp := ` + aidl_library { + name: "bar", + srcs: ["Bar.aidl"], + hdrs: ["aidl/BarHeader.aidl"], + } + aidl_library { + name: "foo", + srcs: ["aidl/Foo.aidl"], + hdrs: ["aidl/FooHeader.aidl"], + strip_import_prefix: "aidl", + deps: ["bar"], + }` + + t.Run("aidl_library with deps", func(t *testing.T) { + expectedBazelTargets := []string{ + MakeBazelTargetNoRestrictions("aidl_library", "bar", AttrNameToString{ + "srcs": `["Bar.aidl"]`, + "hdrs": `["aidl/BarHeader.aidl"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + MakeBazelTargetNoRestrictions("aidl_library", "foo", AttrNameToString{ + "srcs": `["aidl/Foo.aidl"]`, + "hdrs": `["aidl/FooHeader.aidl"]`, + "strip_import_prefix": `"aidl"`, + "deps": `[":bar"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + } + runAidlLibraryTestCase(t, Bp2buildTestCase{ + Description: "aidl_library with deps", + Blueprint: bp, + ExpectedBazelTargets: expectedBazelTargets, + }) + }) +} From bd58e746928bdfd618ba81e66a19dc42f7b15380 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Thu, 11 May 2023 15:58:13 +0000 Subject: [PATCH 0131/1460] Revert "Make min_sdk overridable in override_apex" This reverts commit 133c55b48abe73a51efc6f7cd30b6b6e318ba6f9. Reason for revert: change was untested and not working Change-Id: I06d6e3c439ba541ae85dc148f91b184b9731b92d --- apex/apex.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index a9c8afce75..ae5dd3bf62 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -124,6 +124,10 @@ type apexBundleProperties struct { // List of filesystem images that are embedded inside this APEX bundle. Filesystems []string + // The minimum SDK version that this APEX must support at minimum. This is usually set to + // the SDK version that the APEX was first introduced. + Min_sdk_version *string + // Whether this APEX is considered updatable or not. When set to true, this will enforce // additional rules for making sure that the APEX is truly updatable. To be updatable, // min_sdk_version should be set as well. This will also disable the size optimizations like @@ -389,10 +393,6 @@ type overridableProperties struct { // Trim against a specific Dynamic Common Lib APEX Trim_against *string - - // The minimum SDK version that this APEX must support at minimum. This is usually set to - // the SDK version that the APEX was first introduced. - Min_sdk_version *string } type apexBundle struct { @@ -2880,7 +2880,7 @@ func (a *apexBundle) minSdkVersionValue(ctx android.EarlyModuleContext) string { // Only override the minSdkVersion value on Apexes which already specify // a min_sdk_version (it's optional for non-updatable apexes), and that its // min_sdk_version value is lower than the one to override with. - minApiLevel := minSdkVersionFromValue(ctx, proptools.String(a.overridableProperties.Min_sdk_version)) + minApiLevel := minSdkVersionFromValue(ctx, proptools.String(a.properties.Min_sdk_version)) if minApiLevel.IsNone() { return "" } @@ -3534,8 +3534,8 @@ func convertWithBp2build(a *apexBundle, ctx android.TopDownMutatorContext) (baze // TODO(b/219503907) this would need to be set to a.MinSdkVersionValue(ctx) but // given it's coming via config, we probably don't want to put it in here. var minSdkVersion bazel.StringAttribute - if a.overridableProperties.Min_sdk_version != nil { - minSdkVersion.SetValue(*a.overridableProperties.Min_sdk_version) + if a.properties.Min_sdk_version != nil { + minSdkVersion.SetValue(*a.properties.Min_sdk_version) } if props, ok := productVariableProps[minSdkVersionPropName]; ok { for c, p := range props { From b95998be731406209f18fab764b96421a17ab4c9 Mon Sep 17 00:00:00 2001 From: Jiakai Zhang Date: Thu, 11 May 2023 16:39:27 +0100 Subject: [PATCH 0132/1460] Prepare tests for dexpreopt changes. After this change, there is a clear separation between tests that are related to dexpreopt and tests that are not. The former uses PrepareForTestWithDexpreopt, while the latter uses PrepareForTestWithJavaDefaultModules. The benefit is that the latter will no longer affected by any dexpreopt changes. Bug: 280776428 Test: m nothing Change-Id: Ib957765b9287d51c082e0a33cee17a6bb56daeef --- apex/Android.bp | 1 + apex/apex_test.go | 2 +- apex/bootclasspath_fragment_test.go | 2 +- {java => apex}/dexpreopt_bootjars_test.go | 11 ++++++----- apex/metadata_test.go | 2 +- apex/platform_bootclasspath_test.go | 8 ++++---- java/Android.bp | 1 - java/dexpreopt_test.go | 8 ++++---- java/hiddenapi_singleton_test.go | 4 +++- java/java_test.go | 3 +-- java/platform_bootclasspath_test.go | 2 -- java/testing.go | 18 ++++++++++++------ sdk/bootclasspath_fragment_sdk_test.go | 2 +- 13 files changed, 35 insertions(+), 29 deletions(-) rename {java => apex}/dexpreopt_bootjars_test.go (94%) diff --git a/apex/Android.bp b/apex/Android.bp index 7ffca0ea51..61d7fb2e2e 100644 --- a/apex/Android.bp +++ b/apex/Android.bp @@ -38,6 +38,7 @@ bootstrap_go_package { "apex_test.go", "bootclasspath_fragment_test.go", "classpath_element_test.go", + "dexpreopt_bootjars_test.go", "metadata_test.go", "platform_bootclasspath_test.go", "systemserver_classpath_fragment_test.go", diff --git a/apex/apex_test.go b/apex/apex_test.go index bf9c71b2e8..a83b94db0f 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -147,7 +147,7 @@ var prepareForApexTest = android.GroupFixturePreparers( android.PrepareForTestWithAndroidBuildComponents, bpf.PrepareForTestWithBpf, cc.PrepareForTestWithCcBuildComponents, - java.PrepareForTestWithJavaDefaultModules, + java.PrepareForTestWithDexpreopt, prebuilt_etc.PrepareForTestWithPrebuiltEtc, rust.PrepareForTestWithRustDefaultModules, sh.PrepareForTestWithShBuildComponents, diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index 3e55ccc117..0a72116a6e 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -1250,7 +1250,7 @@ func TestBootclasspathFragment_AndroidNonUpdatable(t *testing.T) { func TestBootclasspathFragment_AndroidNonUpdatable_AlwaysUsePrebuiltSdks(t *testing.T) { result := android.GroupFixturePreparers( prepareForTestWithBootclasspathFragment, - java.PrepareForTestWithJavaDefaultModules, + java.PrepareForTestWithDexpreopt, prepareForTestWithArtApex, prepareForTestWithMyapex, // Configure bootclasspath jars to ensure that hidden API encoding is performed on them. diff --git a/java/dexpreopt_bootjars_test.go b/apex/dexpreopt_bootjars_test.go similarity index 94% rename from java/dexpreopt_bootjars_test.go rename to apex/dexpreopt_bootjars_test.go index 908380813d..ef66807f0f 100644 --- a/java/dexpreopt_bootjars_test.go +++ b/apex/dexpreopt_bootjars_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package java +package apex import ( "path/filepath" @@ -20,6 +20,7 @@ import ( "testing" "android/soong/android" + "android/soong/java" ) func testDexpreoptBoot(t *testing.T, ruleFile string, expectedInputs, expectedOutputs []string) { @@ -48,10 +49,10 @@ func testDexpreoptBoot(t *testing.T, ruleFile string, expectedInputs, expectedOu ` result := android.GroupFixturePreparers( - prepareForJavaTest, - PrepareForTestWithJavaSdkLibraryFiles, - FixtureWithLastReleaseApis("foo"), - FixtureConfigureBootJars("platform:foo", "system_ext:bar", "platform:baz"), + java.PrepareForTestWithDexpreopt, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.FixtureWithLastReleaseApis("foo"), + java.FixtureConfigureBootJars("platform:foo", "system_ext:bar", "platform:baz"), ).RunTestWithBp(t, bp) platformBootclasspath := result.ModuleForTests("platform-bootclasspath", "android_common") diff --git a/apex/metadata_test.go b/apex/metadata_test.go index f6ead424ea..fed5beae7a 100644 --- a/apex/metadata_test.go +++ b/apex/metadata_test.go @@ -27,7 +27,7 @@ import ( func TestModulesSingleton(t *testing.T) { result := android.GroupFixturePreparers( PrepareForTestWithApexMultitreeSingleton, - java.PrepareForTestWithDexpreopt, + java.PrepareForTestWithJavaDefaultModules, PrepareForTestWithApexBuildComponents, java.FixtureConfigureApexBootJars("myapex:foo"), java.PrepareForTestWithJavaSdkLibraryFiles, diff --git a/apex/platform_bootclasspath_test.go b/apex/platform_bootclasspath_test.go index 4b48da8e27..05bb136894 100644 --- a/apex/platform_bootclasspath_test.go +++ b/apex/platform_bootclasspath_test.go @@ -20,6 +20,7 @@ import ( "testing" "android/soong/android" + "android/soong/dexpreopt" "android/soong/java" "github.com/google/blueprint" @@ -30,7 +31,7 @@ import ( // apexes. var prepareForTestWithPlatformBootclasspath = android.GroupFixturePreparers( - java.PrepareForTestWithDexpreopt, + java.PrepareForTestWithJavaDefaultModules, PrepareForTestWithApexBuildComponents, ) @@ -249,6 +250,8 @@ func TestPlatformBootclasspathDependencies(t *testing.T) { java.FixtureConfigureApexBootJars("myapex:bar"), java.PrepareForTestWithJavaSdkLibraryFiles, java.FixtureWithLastReleaseApis("foo"), + java.PrepareForTestWithDexpreopt, + dexpreopt.FixtureDisableDexpreoptBootImages(false), ).RunTestWithBp(t, ` apex { name: "com.android.art", @@ -539,9 +542,6 @@ func TestPlatformBootclasspath_AlwaysUsePrebuiltSdks(t *testing.T) { // Not a prebuilt as no prebuilt existed when it was added. "platform:legacy.core.platform.api.stubs", - // Needed for generating the boot image. - "platform:dex2oatd", - // The platform_bootclasspath intentionally adds dependencies on both source and prebuilt // modules when available as it does not know which one will be preferred. "myapex:foo", diff --git a/java/Android.bp b/java/Android.bp index 27a0a3853b..4af2a14eb4 100644 --- a/java/Android.bp +++ b/java/Android.bp @@ -87,7 +87,6 @@ bootstrap_go_package { "device_host_converter_test.go", "dex_test.go", "dexpreopt_test.go", - "dexpreopt_bootjars_test.go", "dexpreopt_config_test.go", "droiddoc_test.go", "droidstubs_test.go", diff --git a/java/dexpreopt_test.go b/java/dexpreopt_test.go index f91ac5cc3b..fedd5640ef 100644 --- a/java/dexpreopt_test.go +++ b/java/dexpreopt_test.go @@ -212,7 +212,7 @@ func TestDexpreoptEnabled(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { preparers := android.GroupFixturePreparers( - PrepareForTestWithJavaDefaultModules, + PrepareForTestWithDexpreopt, PrepareForTestWithFakeApexMutator, dexpreopt.FixtureSetApexSystemServerJars("com.android.apex1:service-foo"), ) @@ -257,7 +257,7 @@ func TestDex2oatToolDeps(t *testing.T) { preparers := android.GroupFixturePreparers( cc.PrepareForTestWithCcDefaultModules, - PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd, + PrepareForTestWithDexpreoptWithoutFakeDex2oatd, dexpreopt.PrepareForTestByEnablingDexpreopt) testDex2oatToolDep := func(sourceEnabled, prebuiltEnabled, prebuiltPreferred bool, @@ -299,7 +299,7 @@ func TestDex2oatToolDeps(t *testing.T) { func TestDexpreoptBuiltInstalledForApex(t *testing.T) { preparers := android.GroupFixturePreparers( - PrepareForTestWithJavaDefaultModules, + PrepareForTestWithDexpreopt, PrepareForTestWithFakeApexMutator, dexpreopt.FixtureSetApexSystemServerJars("com.android.apex1:service-foo"), ) @@ -386,7 +386,7 @@ func verifyEntries(t *testing.T, message string, expectedModule string, func TestAndroidMkEntriesForApex(t *testing.T) { preparers := android.GroupFixturePreparers( - PrepareForTestWithJavaDefaultModules, + PrepareForTestWithDexpreopt, PrepareForTestWithFakeApexMutator, dexpreopt.FixtureSetApexSystemServerJars("com.android.apex1:service-foo"), ) diff --git a/java/hiddenapi_singleton_test.go b/java/hiddenapi_singleton_test.go index 75b7bb7c84..ef792f9703 100644 --- a/java/hiddenapi_singleton_test.go +++ b/java/hiddenapi_singleton_test.go @@ -39,7 +39,9 @@ var prepareForTestWithDefaultPlatformBootclasspath = android.FixtureAddTextFile( `) var hiddenApiFixtureFactory = android.GroupFixturePreparers( - prepareForJavaTest, PrepareForTestWithHiddenApiBuildComponents) + PrepareForTestWithJavaDefaultModules, + PrepareForTestWithHiddenApiBuildComponents, +) func TestHiddenAPISingleton(t *testing.T) { result := android.GroupFixturePreparers( diff --git a/java/java_test.go b/java/java_test.go index 553b762eed..2a4913ecd3 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -46,12 +46,11 @@ var prepareForJavaTest = android.GroupFixturePreparers( // Get the CC build components but not default modules. cc.PrepareForTestWithCcBuildComponents, // Include all the default java modules. - PrepareForTestWithJavaDefaultModules, + PrepareForTestWithDexpreopt, PrepareForTestWithOverlayBuildComponents, android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { ctx.RegisterPreSingletonType("sdk_versions", sdkPreSingletonFactory) }), - PrepareForTestWithDexpreopt, ) func TestMain(m *testing.M) { diff --git a/java/platform_bootclasspath_test.go b/java/platform_bootclasspath_test.go index 10c9187151..ff2da4bb2a 100644 --- a/java/platform_bootclasspath_test.go +++ b/java/platform_bootclasspath_test.go @@ -18,14 +18,12 @@ import ( "testing" "android/soong/android" - "android/soong/dexpreopt" ) // Contains some simple tests for platform_bootclasspath. var prepareForTestWithPlatformBootclasspath = android.GroupFixturePreparers( PrepareForTestWithJavaDefaultModules, - dexpreopt.PrepareForTestByEnablingDexpreopt, ) func TestPlatformBootclasspath(t *testing.T) { diff --git a/java/testing.go b/java/testing.go index f68e12ff46..4ff2aa497b 100644 --- a/java/testing.go +++ b/java/testing.go @@ -80,9 +80,7 @@ var prepareForTestWithFrameworkDeps = android.GroupFixturePreparers( }.AddToFixture(), ) -// Test fixture preparer that will define all default java modules except the -// fake_tool_binary for dex2oatd. -var PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd = android.GroupFixturePreparers( +var prepareForTestWithJavaDefaultModulesBase = android.GroupFixturePreparers( // Make sure that all the module types used in the defaults are registered. PrepareForTestWithJavaBuildComponents, prepareForTestWithFrameworkDeps, @@ -92,13 +90,21 @@ var PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd = android.GroupFixtu // Test fixture preparer that will define default java modules, e.g. standard prebuilt modules. var PrepareForTestWithJavaDefaultModules = android.GroupFixturePreparers( - PrepareForTestWithJavaDefaultModulesWithoutFakeDex2oatd, - dexpreopt.PrepareForTestWithFakeDex2oatd, + prepareForTestWithJavaDefaultModulesBase, + dexpreopt.FixtureDisableDexpreoptBootImages(true), + dexpreopt.FixtureDisableDexpreopt(true), ) // Provides everything needed by dexpreopt. var PrepareForTestWithDexpreopt = android.GroupFixturePreparers( - PrepareForTestWithJavaDefaultModules, + prepareForTestWithJavaDefaultModulesBase, + dexpreopt.PrepareForTestWithFakeDex2oatd, + dexpreopt.PrepareForTestByEnablingDexpreopt, +) + +// Provides everything needed by dexpreopt except the fake_tool_binary for dex2oatd. +var PrepareForTestWithDexpreoptWithoutFakeDex2oatd = android.GroupFixturePreparers( + prepareForTestWithJavaDefaultModulesBase, dexpreopt.PrepareForTestByEnablingDexpreopt, ) diff --git a/sdk/bootclasspath_fragment_sdk_test.go b/sdk/bootclasspath_fragment_sdk_test.go index 0d6496dd35..bef82d6014 100644 --- a/sdk/bootclasspath_fragment_sdk_test.go +++ b/sdk/bootclasspath_fragment_sdk_test.go @@ -74,7 +74,7 @@ func fixtureAddPrebuiltApexForBootclasspathFragment(apex, fragment string) andro func TestSnapshotWithBootclasspathFragment_ImageName(t *testing.T) { result := android.GroupFixturePreparers( prepareForSdkTestWithJava, - java.PrepareForTestWithJavaDefaultModules, + java.PrepareForTestWithDexpreopt, prepareForSdkTestWithApex, // Some additional files needed for the art apex. From b47caccbc037b65462f757c462a99732ad6dd51c Mon Sep 17 00:00:00 2001 From: Jiakai Zhang Date: Wed, 10 May 2023 16:40:18 +0100 Subject: [PATCH 0133/1460] Remove the ability to install boot images by bootclasspath_fragments. We don't need this anymore because we are going to compile ART jars and framework jars together. Bug: 280776428 Test: m Change-Id: I070157530449a1bb5779e25984c367df3dde7b36 --- apex/apex.go | 18 +--- apex/bootclasspath_fragment_test.go | 161 +--------------------------- apex/prebuilt.go | 5 +- java/bootclasspath_fragment.go | 132 ++--------------------- java/dexpreopt_bootjars.go | 24 ++--- 5 files changed, 17 insertions(+), 323 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index 6a64ad6cd4..6d3fdaa1aa 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -2287,16 +2287,13 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, ctx.PropertyErrorf("sh_binaries", "%q is not a sh_binary module", depName) } case bcpfTag: - bcpfModule, ok := child.(*java.BootclasspathFragmentModule) + _, ok := child.(*java.BootclasspathFragmentModule) if !ok { ctx.PropertyErrorf("bootclasspath_fragments", "%q is not a bootclasspath_fragment module", depName) return false } vctx.filesInfo = append(vctx.filesInfo, apexBootclasspathFragmentFiles(ctx, child)...) - for _, makeModuleName := range bcpfModule.BootImageDeviceInstallMakeModules() { - a.makeModulesToInstall = append(a.makeModulesToInstall, makeModuleName) - } return true case sscpfTag: if _, ok := child.(*java.SystemServerClasspathModule); !ok { @@ -2662,19 +2659,6 @@ func apexBootclasspathFragmentFiles(ctx android.ModuleContext, module blueprint. bootclasspathFragmentInfo := ctx.OtherModuleProvider(module, java.BootclasspathFragmentApexContentInfoProvider).(java.BootclasspathFragmentApexContentInfo) var filesToAdd []apexFile - // Add the boot image files, e.g. .art, .oat and .vdex files. - if bootclasspathFragmentInfo.ShouldInstallBootImageInApex() { - for arch, files := range bootclasspathFragmentInfo.AndroidBootImageFilesByArchType() { - dirInApex := filepath.Join("javalib", arch.String()) - for _, f := range files { - androidMkModuleName := "javalib_" + arch.String() + "_" + filepath.Base(f.String()) - // TODO(b/177892522) - consider passing in the bootclasspath fragment module here instead of nil - af := newApexFile(ctx, f, androidMkModuleName, dirInApex, etc, nil) - filesToAdd = append(filesToAdd, af) - } - } - } - // Add classpaths.proto config. if af := apexClasspathFragmentProtoFile(ctx, module); af != nil { filesToAdd = append(filesToAdd, *af) diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index 0a72116a6e..1b52886961 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -115,20 +115,7 @@ func TestBootclasspathFragments(t *testing.T) { // Make sure that the art-bootclasspath-fragment is using the correct configuration. checkBootclasspathFragment(t, result, "art-bootclasspath-fragment", "android_common_apex10000", - "com.android.art:baz,com.android.art:quuz", ` -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.art -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.oat -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-quuz.vdex -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.art -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.oat -dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-quuz.vdex -`) + "com.android.art:baz,com.android.art:quuz") } func TestBootclasspathFragments_FragmentDependency(t *testing.T) { @@ -261,7 +248,7 @@ func TestBootclasspathFragments_FragmentDependency(t *testing.T) { checkAPIScopeStubs("other", otherInfo, java.CorePlatformHiddenAPIScope) } -func checkBootclasspathFragment(t *testing.T, result *android.TestResult, moduleName, variantName string, expectedConfiguredModules string, expectedBootclasspathFragmentFiles string) { +func checkBootclasspathFragment(t *testing.T, result *android.TestResult, moduleName, variantName string, expectedConfiguredModules string) { t.Helper() bootclasspathFragment := result.ModuleForTests(moduleName, variantName).Module().(*java.BootclasspathFragmentModule) @@ -269,19 +256,6 @@ func checkBootclasspathFragment(t *testing.T, result *android.TestResult, module bootclasspathFragmentInfo := result.ModuleProvider(bootclasspathFragment, java.BootclasspathFragmentApexContentInfoProvider).(java.BootclasspathFragmentApexContentInfo) modules := bootclasspathFragmentInfo.Modules() android.AssertStringEquals(t, "invalid modules for "+moduleName, expectedConfiguredModules, modules.String()) - - // Get a list of all the paths in the boot image sorted by arch type. - allPaths := []string{} - bootImageFilesByArchType := bootclasspathFragmentInfo.AndroidBootImageFilesByArchType() - for _, archType := range android.ArchTypeList() { - if paths, ok := bootImageFilesByArchType[archType]; ok { - for _, path := range paths { - allPaths = append(allPaths, android.NormalizePathForTesting(path)) - } - } - } - - android.AssertTrimmedStringEquals(t, "invalid paths for "+moduleName, expectedBootclasspathFragmentFiles, strings.Join(allPaths, "\n")) } func TestBootclasspathFragmentInArtApex(t *testing.T) { @@ -420,18 +394,6 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ "etc/boot-image.prof", "etc/classpaths/bootclasspath.pb", - "javalib/arm/boot.art", - "javalib/arm/boot.oat", - "javalib/arm/boot.vdex", - "javalib/arm/boot-bar.art", - "javalib/arm/boot-bar.oat", - "javalib/arm/boot-bar.vdex", - "javalib/arm64/boot.art", - "javalib/arm64/boot.oat", - "javalib/arm64/boot.vdex", - "javalib/arm64/boot-bar.art", - "javalib/arm64/boot-bar.oat", - "javalib/arm64/boot-bar.vdex", "javalib/bar.jar", "javalib/foo.jar", }) @@ -441,62 +403,12 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { `mybootclasspathfragment`, }) - // The boot images are installed in the APEX by Soong, so there shouldn't be any dexpreopt-related Make modules. - ensureDoesNotContainRequiredDeps(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ - "mybootclasspathfragment-dexpreopt-arm64-boot.art", - "mybootclasspathfragment-dexpreopt-arm64-boot.oat", - "mybootclasspathfragment-dexpreopt-arm64-boot.vdex", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.art", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.oat", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.vdex", - "mybootclasspathfragment-dexpreopt-arm-boot.art", - "mybootclasspathfragment-dexpreopt-arm-boot.oat", - "mybootclasspathfragment-dexpreopt-arm-boot.vdex", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.art", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.oat", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.vdex", - }) - // Make sure that the source bootclasspath_fragment copies its dex files to the predefined // locations for the art image. module := result.ModuleForTests("mybootclasspathfragment", "android_common_apex10000") checkCopiesToPredefinedLocationForArt(t, result.Config, module, "bar", "foo") }) - t.Run("boot image files from source no boot image in apex", func(t *testing.T) { - result := android.GroupFixturePreparers( - commonPreparer, - - // Configure some libraries in the art bootclasspath_fragment that match the source - // bootclasspath_fragment's contents property. - java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), - addSource("foo", "bar"), - java.FixtureSetBootImageInstallDirOnDevice("art", "system/framework"), - ).RunTest(t) - - ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ - "etc/boot-image.prof", - "etc/classpaths/bootclasspath.pb", - "javalib/bar.jar", - "javalib/foo.jar", - }) - - ensureContainsRequiredDeps(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ - "mybootclasspathfragment-dexpreopt-arm64-boot.art", - "mybootclasspathfragment-dexpreopt-arm64-boot.oat", - "mybootclasspathfragment-dexpreopt-arm64-boot.vdex", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.art", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.oat", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.vdex", - "mybootclasspathfragment-dexpreopt-arm-boot.art", - "mybootclasspathfragment-dexpreopt-arm-boot.oat", - "mybootclasspathfragment-dexpreopt-arm-boot.vdex", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.art", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.oat", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.vdex", - }) - }) - t.Run("generate boot image profile even if dexpreopt is disabled", func(t *testing.T) { result := android.GroupFixturePreparers( commonPreparer, @@ -552,18 +464,6 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { ensureExactDeapexedContents(t, result.TestContext, "com.android.art", "android_common", []string{ "etc/boot-image.prof", - "javalib/arm/boot.art", - "javalib/arm/boot.oat", - "javalib/arm/boot.vdex", - "javalib/arm/boot-bar.art", - "javalib/arm/boot-bar.oat", - "javalib/arm/boot-bar.vdex", - "javalib/arm64/boot.art", - "javalib/arm64/boot.oat", - "javalib/arm64/boot.vdex", - "javalib/arm64/boot-bar.art", - "javalib/arm64/boot-bar.oat", - "javalib/arm64/boot-bar.vdex", "javalib/bar.jar", "javalib/foo.jar", }) @@ -574,65 +474,12 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { `prebuilt_com.android.art`, }) - // The boot images are installed in the APEX by Soong, so there shouldn't be any dexpreopt-related Make modules. - ensureDoesNotContainRequiredDeps(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ - "mybootclasspathfragment-dexpreopt-arm64-boot.art", - "mybootclasspathfragment-dexpreopt-arm64-boot.oat", - "mybootclasspathfragment-dexpreopt-arm64-boot.vdex", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.art", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.oat", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.vdex", - "mybootclasspathfragment-dexpreopt-arm-boot.art", - "mybootclasspathfragment-dexpreopt-arm-boot.oat", - "mybootclasspathfragment-dexpreopt-arm-boot.vdex", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.art", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.oat", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.vdex", - }) - // Make sure that the prebuilt bootclasspath_fragment copies its dex files to the predefined // locations for the art image. module := result.ModuleForTests("prebuilt_mybootclasspathfragment", "android_common_com.android.art") checkCopiesToPredefinedLocationForArt(t, result.Config, module, "bar", "foo") }) - t.Run("boot image files from preferred prebuilt no boot image in apex", func(t *testing.T) { - result := android.GroupFixturePreparers( - commonPreparer, - - // Configure some libraries in the art bootclasspath_fragment that match the source - // bootclasspath_fragment's contents property. - java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), - addSource("foo", "bar"), - - // Make sure that a preferred prebuilt with consistent contents doesn't affect the apex. - addPrebuilt(true, "foo", "bar"), - - java.FixtureSetBootImageInstallDirOnDevice("art", "system/framework"), - ).RunTest(t) - - ensureExactDeapexedContents(t, result.TestContext, "com.android.art", "android_common", []string{ - "etc/boot-image.prof", - "javalib/bar.jar", - "javalib/foo.jar", - }) - - ensureContainsRequiredDeps(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ - "mybootclasspathfragment-dexpreopt-arm64-boot.art", - "mybootclasspathfragment-dexpreopt-arm64-boot.oat", - "mybootclasspathfragment-dexpreopt-arm64-boot.vdex", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.art", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.oat", - "mybootclasspathfragment-dexpreopt-arm64-boot-bar.vdex", - "mybootclasspathfragment-dexpreopt-arm-boot.art", - "mybootclasspathfragment-dexpreopt-arm-boot.oat", - "mybootclasspathfragment-dexpreopt-arm-boot.vdex", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.art", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.oat", - "mybootclasspathfragment-dexpreopt-arm-boot-bar.vdex", - }) - }) - t.Run("source with inconsistency between config and contents", func(t *testing.T) { android.GroupFixturePreparers( commonPreparer, @@ -782,10 +629,6 @@ func TestBootclasspathFragmentInPrebuiltArtApex(t *testing.T) { module := result.ModuleForTests("mybootclasspathfragment", "android_common_com.android.art") checkCopiesToPredefinedLocationForArt(t, result.Config, module, "bar", "foo") - - // Check that the right deapexer module was chosen for a boot image. - param := module.Output("out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art") - android.AssertStringDoesContain(t, "didn't find the expected deapexer in the input path", param.Input.String(), "/com.android.art.deapexer") }) t.Run("enabled alternative APEX", func(t *testing.T) { diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 31cecf1116..0d83830f16 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -199,14 +199,11 @@ func (p *prebuiltCommon) initApexFilesForAndroidMk(ctx android.ModuleContext) { p.apexFilesForAndroidMk = append(p.apexFilesForAndroidMk, af) } } else if tag == exportedBootclasspathFragmentTag { - bcpfModule, ok := child.(*java.PrebuiltBootclasspathFragmentModule) + _, ok := child.(*java.PrebuiltBootclasspathFragmentModule) if !ok { ctx.PropertyErrorf("exported_bootclasspath_fragments", "%q is not a prebuilt_bootclasspath_fragment module", name) return false } - for _, makeModuleName := range bcpfModule.BootImageDeviceInstallMakeModules() { - p.requiredModuleNames = append(p.requiredModuleNames, makeModuleName) - } // Visit the children of the bootclasspath_fragment. return true } else if tag == exportedSystemserverclasspathFragmentTag { diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index 6ccf5a3314..f451659433 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -240,11 +240,6 @@ type BootclasspathFragmentModule struct { // Collect the module directory for IDE info in java/jdeps.go. modulePaths []string - - // Installs for on-device boot image files. This list has entries only if the installs should be - // handled by Make (e.g., the boot image should be installed on the system partition, rather than - // in the APEX). - bootImageDeviceInstalls []dexpreopterInstall } // commonBootclasspathFragment defines the methods that are implemented by both source and prebuilt @@ -268,12 +263,6 @@ type commonBootclasspathFragment interface { var _ commonBootclasspathFragment = (*BootclasspathFragmentModule)(nil) -// bootImageFilesByArch is a map from android.ArchType to the paths to the boot image files. -// -// The paths include the .art, .oat and .vdex files, one for each of the modules from which the boot -// image is created. -type bootImageFilesByArch map[android.ArchType]android.Paths - func bootclasspathFragmentFactory() android.Module { m := &BootclasspathFragmentModule{} m.AddProperties(&m.properties, &m.sourceOnlyProperties) @@ -395,12 +384,6 @@ type BootclasspathFragmentApexContentInfo struct { // set image_name: "art". modules android.ConfiguredJarList - // Map from arch type to the boot image files. - bootImageFilesByArch bootImageFilesByArch - - // True if the boot image should be installed in the APEX. - shouldInstallBootImageInApex bool - // Map from the base module name (without prebuilt_ prefix) of a fragment's contents module to the // hidden API encoded dex jar path. contentModuleDexJarPaths bootDexJarByModule @@ -417,18 +400,6 @@ func (i BootclasspathFragmentApexContentInfo) Modules() android.ConfiguredJarLis return i.modules } -// Get a map from ArchType to the associated boot image's contents for Android. -// -// Extension boot images only return their own files, not the files of the boot images they extend. -func (i BootclasspathFragmentApexContentInfo) AndroidBootImageFilesByArchType() bootImageFilesByArch { - return i.bootImageFilesByArch -} - -// Return true if the boot image should be installed in the APEX. -func (i *BootclasspathFragmentApexContentInfo) ShouldInstallBootImageInApex() bool { - return i.shouldInstallBootImageInApex -} - // DexBootJarPathForContentModule returns the path to the dex boot jar for specified module. // // The dex boot jar is one which has had hidden API encoding performed on it. @@ -566,25 +537,6 @@ func (b *BootclasspathFragmentModule) GenerateAndroidBuildActions(ctx android.Mo // Copy the dex jars of this fragment's content modules to their predefined locations. copyBootJarsToPredefinedLocations(ctx, hiddenAPIOutput.EncodedBootDexFilesByModule, imageConfig.dexPathsByModule) } - - for _, variant := range bootImageFiles.variants { - archType := variant.config.target.Arch.ArchType - arch := archType.String() - for _, install := range variant.deviceInstalls { - // Remove the "/" prefix because the path should be relative to $ANDROID_PRODUCT_OUT. - installDir := strings.TrimPrefix(filepath.Dir(install.To), "/") - installBase := filepath.Base(install.To) - installPath := android.PathForModuleInPartitionInstall(ctx, "", installDir) - - b.bootImageDeviceInstalls = append(b.bootImageDeviceInstalls, dexpreopterInstall{ - name: arch + "-" + installBase, - moduleName: b.Name(), - outputPathOnHost: install.From, - installDirOnDevice: installPath, - installFileOnDevice: installBase, - }) - } - } } // A prebuilt fragment cannot contribute to an apex. @@ -643,12 +595,8 @@ func (b *BootclasspathFragmentModule) provideApexContentInfo(ctx android.ModuleC info.profilePathOnHost = bootImageFiles.profile info.profileInstallPathInApex = imageConfig.profileInstallPathInApex } - - info.shouldInstallBootImageInApex = imageConfig.shouldInstallInApex() } - info.bootImageFilesByArch = bootImageFiles.byArch - // Make the apex content info available for other modules. ctx.SetProvider(BootclasspathFragmentApexContentInfoProvider, info) } @@ -960,21 +908,9 @@ func (b *BootclasspathFragmentModule) AndroidMkEntries() []android.AndroidMkEntr }, }, }} - for _, install := range b.bootImageDeviceInstalls { - entriesList = append(entriesList, install.ToMakeEntries()) - } return entriesList } -// Returns the names of all Make modules that handle the installation of the boot image. -func (b *BootclasspathFragmentModule) BootImageDeviceInstallMakeModules() []string { - var makeModules []string - for _, install := range b.bootImageDeviceInstalls { - makeModules = append(makeModules, install.FullModuleName()) - } - return makeModules -} - // Collect information for opening IDE project files in java/jdeps.go. func (b *BootclasspathFragmentModule) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, b.properties.Contents...) @@ -1262,59 +1198,17 @@ func (module *PrebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx and // built without a profile as the prebuilt modules do not provide a profile. buildBootImageVariantsForBuildOs(ctx, imageConfig, profile) - if imageConfig.shouldInstallInApex() { - // If the boot image files for the android variants are in the prebuilt apex, we must use those - // rather than building new ones because those boot image files are going to be used on device. - files := bootImageFilesByArch{} - bootImageFiles := bootImageOutputs{ - byArch: files, - profile: profile, - } - for _, variant := range imageConfig.apexVariants() { - arch := variant.target.Arch.ArchType - bootImageFiles.variants = append(bootImageFiles.variants, bootImageVariantOutputs{ - variant, - // No device installs needed when installed in APEX. - nil, - }) - for _, toPath := range variant.imagesDeps { - apexRelativePath := apexRootRelativePathToBootImageFile(arch, toPath.Base()) - // Get the path to the file that the deapexer extracted from the prebuilt apex file. - fromPath := di.PrebuiltExportPath(apexRelativePath) - - // Return the toPath as the calling code expects the paths in the returned map to be the - // paths predefined in the bootImageConfig. - files[arch] = append(files[arch], toPath) - - // Copy the file to the predefined location. - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: fromPath, - Output: toPath, - }) - } - } - return bootImageFiles - } else { - if profile == nil && imageConfig.isProfileGuided() { - ctx.ModuleErrorf("Unable to produce boot image files: neither boot image files nor profiles exists in the prebuilt apex") - return bootImageOutputs{} - } - // Build boot image files for the android variants from the dex files provided by the contents - // of this module. - return buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) + if profile == nil && imageConfig.isProfileGuided() { + ctx.ModuleErrorf("Unable to produce boot image files: profiles not found in the prebuilt apex") + return bootImageOutputs{} } + // Build boot image files for the android variants from the dex files provided by the contents + // of this module. + return buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) } var _ commonBootclasspathFragment = (*PrebuiltBootclasspathFragmentModule)(nil) -// createBootImageTag creates the tag to uniquely identify the boot image file among all of the -// files that a module requires from the prebuilt .apex file. -func createBootImageTag(arch android.ArchType, baseName string) string { - tag := fmt.Sprintf(".bootimage-%s-%s", arch, baseName) - return tag -} - // RequiredFilesFromPrebuiltApex returns the list of all files the prebuilt_bootclasspath_fragment // requires from a prebuilt .apex file. // @@ -1328,25 +1222,11 @@ func (module *PrebuiltBootclasspathFragmentModule) RequiredFilesFromPrebuiltApex // Add the boot image profile. files = append(files, imageConfig.profileInstallPathInApex) } - if imageConfig.shouldInstallInApex() { - // Add the boot image files, e.g. .art, .oat and .vdex files. - for _, variant := range imageConfig.apexVariants() { - arch := variant.target.Arch.ArchType - for _, path := range variant.imagesDeps.Paths() { - base := path.Base() - files = append(files, apexRootRelativePathToBootImageFile(arch, base)) - } - } - } return files } return nil } -func apexRootRelativePathToBootImageFile(arch android.ArchType, base string) string { - return filepath.Join("javalib", arch.String(), base) -} - var _ android.RequiredFilesFromPrebuiltApex = (*PrebuiltBootclasspathFragmentModule)(nil) func prebuiltBootclasspathFragmentFactory() android.Module { diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 8e79674fb9..7b5c294a08 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -421,11 +421,6 @@ func (image *bootImageConfig) apexVariants() []*bootImageVariant { return variants } -// Returns true if the boot image should be installed in the APEX. -func (image *bootImageConfig) shouldInstallInApex() bool { - return strings.HasPrefix(image.installDirOnDevice, "apex/") -} - // Return boot image locations (as a list of symbolic paths). // // The image "location" is a symbolic path that, with multiarchitecture support, doesn't really @@ -596,6 +591,12 @@ func buildBootImageVariantsForBuildOs(ctx android.ModuleContext, image *bootImag buildBootImageForOsType(ctx, image, profile, ctx.Config().BuildOS) } +// bootImageFilesByArch is a map from android.ArchType to the paths to the boot image files. +// +// The paths include the .art, .oat and .vdex files, one for each of the modules from which the boot +// image is created. +type bootImageFilesByArch map[android.ArchType]android.Paths + // bootImageOutputs encapsulates information about boot images that were created/obtained by // commonBootclasspathFragment.produceBootImageFiles. type bootImageOutputs struct { @@ -656,8 +657,7 @@ func buildBootImageZipInPredefinedLocation(ctx android.ModuleContext, image *boo } type bootImageVariantOutputs struct { - config *bootImageVariant - deviceInstalls android.RuleBuilderInstalls + config *bootImageVariant } // Generate boot image build rules for a specific target. @@ -800,7 +800,6 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p var vdexInstalls android.RuleBuilderInstalls var unstrippedInstalls android.RuleBuilderInstalls - var deviceInstalls android.RuleBuilderInstalls for _, artOrOat := range image.moduleFiles(ctx, outputDir, ".art", ".oat") { cmd.ImplicitOutput(artOrOat) @@ -826,14 +825,6 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p android.RuleBuilderInstall{unstrippedOat, filepath.Join(installDir, unstrippedOat.Base())}) } - if image.installDirOnHost != image.installDirOnDevice && !image.shouldInstallInApex() && !ctx.Config().UnbundledBuild() { - installDirOnDevice := filepath.Join("/", image.installDirOnDevice, arch.String()) - for _, file := range image.moduleFiles(ctx, outputDir, ".art", ".oat", ".vdex") { - deviceInstalls = append(deviceInstalls, - android.RuleBuilderInstall{file, filepath.Join(installDirOnDevice, file.Base())}) - } - } - rule.Build(image.name+"JarsDexpreopt_"+image.target.String(), "dexpreopt "+image.name+" jars "+arch.String()) // save output and installed files for makevars @@ -849,7 +840,6 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p return bootImageVariantOutputs{ image, - deviceInstalls, } } From 09d88df0407263e846b01c226184b262f2e36678 Mon Sep 17 00:00:00 2001 From: Jiakai Zhang Date: Wed, 10 May 2023 17:04:53 +0100 Subject: [PATCH 0134/1460] Unify installDirOnHost and installDirOnDevice. These two fields never do what they are described to do. This CL unifies them to avoid the confusion. Bug: 280440941 Test: m Change-Id: I3652d73a50832a2e494d9f5cae750f5fc38293b4 --- java/dexpreopt_bootjars.go | 17 +++++++++-------- java/dexpreopt_config.go | 25 +++++++++++-------------- java/dexpreopt_config_testing.go | 29 ++++++++++++----------------- java/testing.go | 4 ++-- 4 files changed, 34 insertions(+), 41 deletions(-) diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 7b5c294a08..792fa63963 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -250,11 +250,12 @@ type bootImageConfig struct { // Output directory for the image files with debug symbols. symbolsDir android.OutputPath - // Subdirectory where the image files are installed. - installDirOnHost string - - // Subdirectory where the image files on device are installed. - installDirOnDevice string + // The relative location where the image files are installed. On host, the location is relative to + // $ANDROID_PRODUCT_OUT. + // + // Only the configs that are built by platform_bootclasspath are installable on device. On device, + // the location is relative to "/". + installDir string // Install path of the boot image profile if it needs to be installed in the APEX, or empty if not // needed. @@ -668,9 +669,9 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p arch := image.target.Arch.ArchType os := image.target.Os.String() // We need to distinguish host-x86 and device-x86. - symbolsDir := image.symbolsDir.Join(ctx, os, image.installDirOnHost, arch.String()) + symbolsDir := image.symbolsDir.Join(ctx, os, image.installDir, arch.String()) symbolsFile := symbolsDir.Join(ctx, image.stem+".oat") - outputDir := image.dir.Join(ctx, os, image.installDirOnHost, arch.String()) + outputDir := image.dir.Join(ctx, os, image.installDir, arch.String()) outputPath := outputDir.Join(ctx, image.stem+".oat") oatLocation := dexpreopt.PathToLocation(outputPath, arch) imagePath := outputPath.ReplaceExtension(ctx, "art") @@ -796,7 +797,7 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p cmd.Textf(`|| ( echo %s ; false )`, proptools.ShellEscape(failureMessage)) - installDir := filepath.Join("/", image.installDirOnHost, arch.String()) + installDir := filepath.Dir(image.imagePathOnDevice) var vdexInstalls android.RuleBuilderInstalls var unstrippedInstalls android.RuleBuilderInstalls diff --git a/java/dexpreopt_config.go b/java/dexpreopt_config.go index 8f732cf56b..5f168e70a6 100644 --- a/java/dexpreopt_config.go +++ b/java/dexpreopt_config.go @@ -62,8 +62,7 @@ func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig artCfg := bootImageConfig{ name: artBootImageName, stem: bootImageStem, - installDirOnHost: "apex/art_boot_images/javalib", - installDirOnDevice: frameworkSubdir, + installDir: "apex/art_boot_images/javalib", profileInstallPathInApex: "etc/boot-image.prof", modules: artModules, preloadedClassesFile: "art/build/boot/preloaded-classes", @@ -77,8 +76,7 @@ func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig extends: &artCfg, name: frameworkBootImageName, stem: bootImageStem, - installDirOnHost: frameworkSubdir, - installDirOnDevice: frameworkSubdir, + installDir: frameworkSubdir, modules: frameworkModules, preloadedClassesFile: "frameworks/base/config/preloaded-classes", compilerFilter: "speed-profile", @@ -86,14 +84,13 @@ func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig } mainlineCfg := bootImageConfig{ - extends: &frameworkCfg, - name: mainlineBootImageName, - stem: bootImageStem, - installDirOnHost: frameworkSubdir, - installDirOnDevice: frameworkSubdir, - modules: mainlineBcpModules, - compilerFilter: "verify", - singleImage: true, + extends: &frameworkCfg, + name: mainlineBootImageName, + stem: bootImageStem, + installDir: frameworkSubdir, + modules: mainlineBcpModules, + compilerFilter: "verify", + singleImage: true, } return map[string]*bootImageConfig{ @@ -132,12 +129,12 @@ func genBootImageConfigs(ctx android.PathContext) map[string]*bootImageConfig { // Create target-specific variants. for _, target := range targets { arch := target.Arch.ArchType - imageDir := c.dir.Join(ctx, target.Os.String(), c.installDirOnHost, arch.String()) + imageDir := c.dir.Join(ctx, target.Os.String(), c.installDir, arch.String()) variant := &bootImageVariant{ bootImageConfig: c, target: target, imagePathOnHost: imageDir.Join(ctx, imageName), - imagePathOnDevice: filepath.Join("/", c.installDirOnDevice, arch.String(), imageName), + imagePathOnDevice: filepath.Join("/", c.installDir, arch.String(), imageName), imagesDeps: c.moduleFiles(ctx, imageDir, ".art", ".oat", ".vdex"), dexLocations: c.modules.DevicePaths(ctx.Config(), target.Os), } diff --git a/java/dexpreopt_config_testing.go b/java/dexpreopt_config_testing.go index 6b98ca58c2..3ee3bd5538 100644 --- a/java/dexpreopt_config_testing.go +++ b/java/dexpreopt_config_testing.go @@ -147,8 +147,7 @@ type expectedConfig struct { stem string dir string symbolsDir string - installDirOnDevice string - installDirOnHost string + installDir string profileInstallPathInApex string modules android.ConfiguredJarList dexPaths []string @@ -209,8 +208,7 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b stem: "boot", dir: "out/soong/dexpreopt_arm64/dex_artjars", symbolsDir: "out/soong/dexpreopt_arm64/dex_artjars_unstripped", - installDirOnDevice: "system/framework", - installDirOnHost: "apex/art_boot_images/javalib", + installDir: "apex/art_boot_images/javalib", profileInstallPathInApex: "etc/boot-image.prof", modules: android.CreateTestConfiguredJarList([]string{"com.android.art:core1", "com.android.art:core2"}), dexPaths: []string{"out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar", "out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar"}, @@ -222,7 +220,7 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b dexLocations: []string{"/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar"}, dexLocationsDeps: []string{"/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar"}, imagePathOnHost: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - imagePathOnDevice: "/system/framework/arm64/boot.art", + imagePathOnDevice: "/apex/art_boot_images/javalib/arm64/boot.art", imagesDeps: []string{ "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", @@ -276,7 +274,7 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b dexLocations: []string{"/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar"}, dexLocationsDeps: []string{"/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar"}, imagePathOnHost: "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", - imagePathOnDevice: "/system/framework/arm/boot.art", + imagePathOnDevice: "/apex/art_boot_images/javalib/arm/boot.art", imagesDeps: []string{ "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", @@ -330,7 +328,7 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b dexLocations: []string{"host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar"}, dexLocationsDeps: []string{"host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar"}, imagePathOnHost: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", - imagePathOnDevice: "/system/framework/x86_64/boot.art", + imagePathOnDevice: "/apex/art_boot_images/javalib/x86_64/boot.art", imagesDeps: []string{ "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", @@ -382,7 +380,7 @@ func checkArtBootImageConfig(t *testing.T, result *android.TestResult, mutated b dexLocations: []string{"host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar"}, dexLocationsDeps: []string{"host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar"}, imagePathOnHost: "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", - imagePathOnDevice: "/system/framework/x86/boot.art", + imagePathOnDevice: "/apex/art_boot_images/javalib/x86/boot.art", imagesDeps: []string{ "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", @@ -462,8 +460,7 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut stem: "boot", dir: "out/soong/dexpreopt_arm64/dex_bootjars", symbolsDir: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped", - installDirOnDevice: "system/framework", - installDirOnHost: "system/framework", + installDir: "system/framework", profileInstallPathInApex: "", modules: android.CreateTestConfiguredJarList([]string{"platform:framework"}), dexPaths: []string{"out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar"}, @@ -693,8 +690,7 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { stem: "boot", dir: "out/soong/dexpreopt_arm64/dex_mainlinejars", symbolsDir: "out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped", - installDirOnDevice: "system/framework", - installDirOnHost: "system/framework", + installDir: "system/framework", profileInstallPathInApex: "", modules: android.CreateTestConfiguredJarList([]string{ "com.android.foo:framework-foo", @@ -999,8 +995,7 @@ func nestedCheckBootImageConfig(t *testing.T, imageConfig *bootImageConfig, expe android.AssertStringEquals(t, "stem", expected.stem, imageConfig.stem) android.AssertPathRelativeToTopEquals(t, "dir", expected.dir, imageConfig.dir) android.AssertPathRelativeToTopEquals(t, "symbolsDir", expected.symbolsDir, imageConfig.symbolsDir) - android.AssertStringEquals(t, "installDirOnDevice", expected.installDirOnDevice, imageConfig.installDirOnDevice) - android.AssertStringEquals(t, "installDirOnHost", expected.installDirOnHost, imageConfig.installDirOnHost) + android.AssertStringEquals(t, "installDir", expected.installDir, imageConfig.installDir) android.AssertStringEquals(t, "profileInstallPathInApex", expected.profileInstallPathInApex, imageConfig.profileInstallPathInApex) android.AssertDeepEquals(t, "modules", expected.modules, imageConfig.modules) android.AssertPathsRelativeToTopEquals(t, "dexPaths", expected.dexPaths, imageConfig.dexPaths.Paths()) @@ -1096,9 +1091,9 @@ DEXPREOPT_IMAGE_LICENSE_METADATA_mainline_arm=out/soong/.intermediates/framework DEXPREOPT_IMAGE_LICENSE_METADATA_mainline_arm64=out/soong/.intermediates/frameworks/base/boot/platform-bootclasspath/android_common/meta_lic DEXPREOPT_IMAGE_LICENSE_METADATA_mainline_host_x86=out/soong/.intermediates/frameworks/base/boot/platform-bootclasspath/android_common/meta_lic DEXPREOPT_IMAGE_LICENSE_METADATA_mainline_host_x86_64=out/soong/.intermediates/frameworks/base/boot/platform-bootclasspath/android_common/meta_lic -DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEart=/system/framework/boot.art -DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEboot=/system/framework/boot.art:/system/framework/boot-framework.art -DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEmainline=/system/framework/boot.art:/system/framework/boot-framework.art:/system/framework/boot-framework-foo.art +DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEart=/apex/art_boot_images/javalib/boot.art +DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEboot=/apex/art_boot_images/javalib/boot.art:/system/framework/boot-framework.art +DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEmainline=/apex/art_boot_images/javalib/boot.art:/system/framework/boot-framework.art:/system/framework/boot-framework-foo.art DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTart=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTboot=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art:out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/boot-framework.art DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTmainline=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art:out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/boot-framework.art:out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/boot-framework-foo.art diff --git a/java/testing.go b/java/testing.go index 4ff2aa497b..6671bf0c72 100644 --- a/java/testing.go +++ b/java/testing.go @@ -610,9 +610,9 @@ func FixtureModifyBootImageConfig(name string, configModifier func(*bootImageCon }) } -// Sets the value of `installDirOnDevice` of the boot image config with the given name. +// Sets the value of `installDir` of the boot image config with the given name. func FixtureSetBootImageInstallDirOnDevice(name string, installDir string) android.FixturePreparer { return FixtureModifyBootImageConfig(name, func(config *bootImageConfig) { - config.installDirOnDevice = installDir + config.installDir = installDir }) } From 379221f08633a14f7dcf72066aa4c05f31788e85 Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Thu, 30 Mar 2023 13:19:29 +0000 Subject: [PATCH 0135/1460] Move disable downloads to bazelrc from mixed build Keeps mixed builds and b consistent. Bug: 275724420 Test: presubmits, especially bp2build-incremental Change-Id: I89942111f259bf9a88c27e124adbdb73936c514c --- android/bazel_handler.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 5291ecae94..10cf60a0a7 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -29,6 +29,7 @@ import ( "android/soong/bazel/cquery" "android/soong/shared" "android/soong/starlark_fmt" + "github.com/google/blueprint" "github.com/google/blueprint/metrics" @@ -700,9 +701,6 @@ func (context *mixedBuildBazelContext) createBazelCommand(config Config, runName // We don't need to set --host_platforms because it's set in bazelrc files // that the bazel shell script wrapper passes - // Explicitly disable downloading rules (such as canonical C++ and Java rules) from the network. - "--experimental_repository_disable_download", - // Suppress noise "--ui_event_filters=-INFO", "--noshow_progress", From c08c162b5a82ddfc44e9944d798636b2241308de Mon Sep 17 00:00:00 2001 From: Jiakai Zhang Date: Wed, 10 May 2023 18:38:34 +0100 Subject: [PATCH 0136/1460] Dexpreopt ART jars and framework jars together. Bug: 280776428 Test: atest art_standalone_dexpreopt_tests Test: - 1. m 2. Check .invocation file (http://gpaste/6498044089466880) 3. Check files in $ANDROID_PRODUCT_OUT/system/framework/x86_64 Test: - 1. m dist 2. Check files in out/dist/boot.zip Test: - 1. art/tools/buildbot-build.sh --host 2. m test-art-host-gtest 3. art/test/testrunner/testrunner.py --host Test: m build-art-target-golem Change-Id: I89490252e56a05edab03fdddc6539fa4d7f79756 --- apex/apex_test.go | 14 +- apex/dexpreopt_bootjars_test.go | 193 +++++++++++++--- java/bootclasspath_fragment.go | 26 +++ java/dexpreopt_bootjars.go | 33 +++ java/dexpreopt_config.go | 4 +- java/dexpreopt_config_testing.go | 372 +++++++++++++++++++++---------- java/platform_bootclasspath.go | 48 +++- 7 files changed, 520 insertions(+), 170 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index a83b94db0f..80d118f92e 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -8341,8 +8341,8 @@ func TestNoUpdatableJarsInBootImage(t *testing.T) { testNoUpdatableJarsInBootImage(t, "", preparer, fragments...) }) - t.Run("updatable jar from ART apex in the framework boot image => error", func(t *testing.T) { - err := `module "some-art-lib" from updatable apexes \["com.android.art.debug"\] is not allowed in the framework boot image` + t.Run("updatable jar from ART apex in the platform bootclasspath => error", func(t *testing.T) { + err := `module "some-art-lib" from updatable apexes \["com.android.art.debug"\] is not allowed in the platform bootclasspath` // Update the dexpreopt BootJars directly. preparer := android.GroupFixturePreparers( prepareSetBootJars("com.android.art.debug:some-art-lib"), @@ -8365,8 +8365,8 @@ func TestNoUpdatableJarsInBootImage(t *testing.T) { testNoUpdatableJarsInBootImage(t, err, preparer) }) - t.Run("updatable jar from some other apex in the framework boot image => error", func(t *testing.T) { - err := `module "some-updatable-apex-lib" from updatable apexes \["some-updatable-apex"\] is not allowed in the framework boot image` + t.Run("updatable jar from some other apex in the platform bootclasspath => error", func(t *testing.T) { + err := `module "some-updatable-apex-lib" from updatable apexes \["some-updatable-apex"\] is not allowed in the platform bootclasspath` preparer := android.GroupFixturePreparers( java.FixtureConfigureBootJars("some-updatable-apex:some-updatable-apex-lib"), java.FixtureConfigureApexBootJars("some-non-updatable-apex:some-non-updatable-apex-lib"), @@ -8374,7 +8374,7 @@ func TestNoUpdatableJarsInBootImage(t *testing.T) { testNoUpdatableJarsInBootImage(t, err, preparer) }) - t.Run("non-updatable jar from some other apex in the framework boot image => ok", func(t *testing.T) { + t.Run("non-updatable jar from some other apex in the platform bootclasspath => ok", func(t *testing.T) { preparer := java.FixtureConfigureApexBootJars("some-non-updatable-apex:some-non-updatable-apex-lib") fragment := java.ApexVariantReference{ Apex: proptools.StringPtr("some-non-updatable-apex"), @@ -8389,7 +8389,7 @@ func TestNoUpdatableJarsInBootImage(t *testing.T) { testNoUpdatableJarsInBootImage(t, err, preparer) }) - t.Run("nonexistent jar in the framework boot image => error", func(t *testing.T) { + t.Run("nonexistent jar in the platform bootclasspath => error", func(t *testing.T) { err := `"platform-bootclasspath" depends on undefined module "nonexistent"` preparer := java.FixtureConfigureBootJars("platform:nonexistent") testNoUpdatableJarsInBootImage(t, err, preparer) @@ -8402,7 +8402,7 @@ func TestNoUpdatableJarsInBootImage(t *testing.T) { testNoUpdatableJarsInBootImage(t, err, preparer) }) - t.Run("platform jar in the framework boot image => ok", func(t *testing.T) { + t.Run("platform jar in the platform bootclasspath => ok", func(t *testing.T) { preparer := android.GroupFixturePreparers( java.FixtureConfigureBootJars("platform:some-platform-lib"), java.FixtureConfigureApexBootJars("some-non-updatable-apex:some-non-updatable-apex-lib"), diff --git a/apex/dexpreopt_bootjars_test.go b/apex/dexpreopt_bootjars_test.go index ef66807f0f..bba8bb6cb7 100644 --- a/apex/dexpreopt_bootjars_test.go +++ b/apex/dexpreopt_bootjars_test.go @@ -15,6 +15,7 @@ package apex import ( + "fmt" "path/filepath" "sort" "testing" @@ -23,8 +24,10 @@ import ( "android/soong/java" ) -func testDexpreoptBoot(t *testing.T, ruleFile string, expectedInputs, expectedOutputs []string) { +func testDexpreoptBoot(t *testing.T, ruleFile string, expectedInputs, expectedOutputs []string, preferPrebuilt bool) { bp := ` + // Platform. + java_sdk_library { name: "foo", srcs: ["a.java"], @@ -45,6 +48,84 @@ func testDexpreoptBoot(t *testing.T, ruleFile string, expectedInputs, expectedOu platform_bootclasspath { name: "platform-bootclasspath", + fragments: [ + { + apex: "com.android.art", + module: "art-bootclasspath-fragment", + }, + ], + } + + // Source ART APEX. + + java_library { + name: "core-oj", + srcs: ["core-oj.java"], + installable: true, + apex_available: [ + "com.android.art", + ], + } + + bootclasspath_fragment { + name: "art-bootclasspath-fragment", + image_name: "art", + contents: ["core-oj"], + apex_available: [ + "com.android.art", + ], + hidden_api: { + split_packages: ["*"], + }, + } + + apex_key { + name: "com.android.art.key", + public_key: "com.android.art.avbpubkey", + private_key: "com.android.art.pem", + } + + apex { + name: "com.android.art", + key: "com.android.art.key", + bootclasspath_fragments: ["art-bootclasspath-fragment"], + updatable: false, + } + + // Prebuilt ART APEX. + + java_import { + name: "core-oj", + prefer: %[1]t, + jars: ["core-oj.jar"], + apex_available: [ + "com.android.art", + ], + } + + prebuilt_bootclasspath_fragment { + name: "art-bootclasspath-fragment", + prefer: %[1]t, + image_name: "art", + contents: ["core-oj"], + hidden_api: { + annotation_flags: "my-bootclasspath-fragment/annotation-flags.csv", + metadata: "my-bootclasspath-fragment/metadata.csv", + index: "my-bootclasspath-fragment/index.csv", + stub_flags: "my-bootclasspath-fragment/stub-flags.csv", + all_flags: "my-bootclasspath-fragment/all-flags.csv", + }, + apex_available: [ + "com.android.art", + ], + } + + prebuilt_apex { + name: "com.android.art", + prefer: %[1]t, + apex_name: "com.android.art", + src: "com.android.art-arm.apex", + exported_bootclasspath_fragments: ["art-bootclasspath-fragment"], } ` @@ -52,20 +133,14 @@ func testDexpreoptBoot(t *testing.T, ruleFile string, expectedInputs, expectedOu java.PrepareForTestWithDexpreopt, java.PrepareForTestWithJavaSdkLibraryFiles, java.FixtureWithLastReleaseApis("foo"), - java.FixtureConfigureBootJars("platform:foo", "system_ext:bar", "platform:baz"), - ).RunTestWithBp(t, bp) + java.FixtureConfigureBootJars("com.android.art:core-oj", "platform:foo", "system_ext:bar", "platform:baz"), + PrepareForTestWithApexBuildComponents, + prepareForTestWithArtApex, + ).RunTestWithBp(t, fmt.Sprintf(bp, preferPrebuilt)) platformBootclasspath := result.ModuleForTests("platform-bootclasspath", "android_common") rule := platformBootclasspath.Output(ruleFile) - for i := range expectedInputs { - expectedInputs[i] = filepath.Join("out/soong/dexpreopt_arm64", expectedInputs[i]) - } - - for i := range expectedOutputs { - expectedOutputs[i] = filepath.Join("out/soong/dexpreopt_arm64", expectedOutputs[i]) - } - inputs := rule.Implicits.Strings() sort.Strings(inputs) sort.Strings(expectedInputs) @@ -79,33 +154,76 @@ func testDexpreoptBoot(t *testing.T, ruleFile string, expectedInputs, expectedOu android.AssertStringPathsRelativeToTopEquals(t, "outputs", result.Config, expectedOutputs, outputs) } -func TestDexpreoptBootJars(t *testing.T) { - ruleFile := "boot-foo.art" +func TestDexpreoptBootJarsWithSourceArtApex(t *testing.T) { + ruleFile := "boot.art" + + expectedInputs := []string{ + "out/soong/dexpreopt_arm64/dex_bootjars_input/core-oj.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/foo.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/bar.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/baz.jar", + "out/soong/dexpreopt_arm64/dex_artjars/boot.prof", + "out/soong/dexpreopt_arm64/dex_bootjars/boot.prof", + } + + expectedOutputs := []string{ + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.invocation", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-foo.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-bar.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-baz.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-foo.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-bar.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-baz.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-foo.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-bar.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-baz.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-foo.oat", + "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-bar.oat", + "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-baz.oat", + } + + testDexpreoptBoot(t, ruleFile, expectedInputs, expectedOutputs, false) +} + +// The only difference is that the ART profile should be deapexed from the prebuilt APEX. Other +// inputs and outputs should be the same as above. +func TestDexpreoptBootJarsWithPrebuiltArtApex(t *testing.T) { + ruleFile := "boot.art" expectedInputs := []string{ - "dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - "dex_bootjars_input/foo.jar", - "dex_bootjars_input/bar.jar", - "dex_bootjars_input/baz.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/core-oj.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/foo.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/bar.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/baz.jar", + "out/soong/.intermediates/com.android.art.deapexer/android_common/deapexer/etc/boot-image.prof", + "out/soong/dexpreopt_arm64/dex_bootjars/boot.prof", } expectedOutputs := []string{ - "dex_bootjars/android/system/framework/arm64/boot.invocation", - "dex_bootjars/android/system/framework/arm64/boot-foo.art", - "dex_bootjars/android/system/framework/arm64/boot-bar.art", - "dex_bootjars/android/system/framework/arm64/boot-baz.art", - "dex_bootjars/android/system/framework/arm64/boot-foo.oat", - "dex_bootjars/android/system/framework/arm64/boot-bar.oat", - "dex_bootjars/android/system/framework/arm64/boot-baz.oat", - "dex_bootjars/android/system/framework/arm64/boot-foo.vdex", - "dex_bootjars/android/system/framework/arm64/boot-bar.vdex", - "dex_bootjars/android/system/framework/arm64/boot-baz.vdex", - "dex_bootjars_unstripped/android/system/framework/arm64/boot-foo.oat", - "dex_bootjars_unstripped/android/system/framework/arm64/boot-bar.oat", - "dex_bootjars_unstripped/android/system/framework/arm64/boot-baz.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.invocation", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-foo.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-bar.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-baz.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-foo.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-bar.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-baz.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-foo.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-bar.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-baz.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-foo.oat", + "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-bar.oat", + "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-baz.oat", } - testDexpreoptBoot(t, ruleFile, expectedInputs, expectedOutputs) + testDexpreoptBoot(t, ruleFile, expectedInputs, expectedOutputs, true) } // Changes to the boot.zip structure may break the ART APK scanner. @@ -116,16 +234,21 @@ func TestDexpreoptBootZip(t *testing.T) { expectedInputs := []string{} for _, target := range ctx.Config().Targets[android.Android] { for _, ext := range []string{".art", ".oat", ".vdex"} { - for _, jar := range []string{"foo", "bar", "baz"} { + for _, suffix := range []string{"", "-foo", "-bar", "-baz"} { expectedInputs = append(expectedInputs, - filepath.Join("dex_bootjars", target.Os.String(), "system/framework", target.Arch.ArchType.String(), "boot-"+jar+ext)) + filepath.Join( + "out/soong/dexpreopt_arm64/dex_bootjars", + target.Os.String(), + "system/framework", + target.Arch.ArchType.String(), + "boot"+suffix+ext)) } } } expectedOutputs := []string{ - "dex_bootjars/boot.zip", + "out/soong/dexpreopt_arm64/dex_bootjars/boot.zip", } - testDexpreoptBoot(t, ruleFile, expectedInputs, expectedOutputs) + testDexpreoptBoot(t, ruleFile, expectedInputs, expectedOutputs, false) } diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index f451659433..108fdd4835 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -240,6 +240,9 @@ type BootclasspathFragmentModule struct { // Collect the module directory for IDE info in java/jdeps.go. modulePaths []string + + // Path to the boot image profile. + profilePath android.Path } // commonBootclasspathFragment defines the methods that are implemented by both source and prebuilt @@ -259,6 +262,12 @@ type commonBootclasspathFragment interface { // If it could not create the files then it will return nil. Otherwise, it will return a map from // android.ArchType to the predefined paths of the boot image files. produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig) bootImageOutputs + + // getImageName returns the `image_name` property of this fragment. + getImageName() *string + + // getProfilePath returns the path to the boot image profile. + getProfilePath() android.Path } var _ commonBootclasspathFragment = (*BootclasspathFragmentModule)(nil) @@ -528,6 +537,7 @@ func (b *BootclasspathFragmentModule) GenerateAndroidBuildActions(ctx android.Mo // Delegate the production of the boot image files to a module type specific method. common := ctx.Module().(commonBootclasspathFragment) bootImageFiles = common.produceBootImageFiles(ctx, imageConfig) + b.profilePath = bootImageFiles.profile if shouldCopyBootFilesToPredefinedLocations(ctx, imageConfig) { // Zip the boot image files up, if available. This will generate the zip file in a @@ -911,6 +921,14 @@ func (b *BootclasspathFragmentModule) AndroidMkEntries() []android.AndroidMkEntr return entriesList } +func (b *BootclasspathFragmentModule) getImageName() *string { + return b.properties.Image_name +} + +func (b *BootclasspathFragmentModule) getProfilePath() android.Path { + return b.profilePath +} + // Collect information for opening IDE project files in java/jdeps.go. func (b *BootclasspathFragmentModule) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, b.properties.Contents...) @@ -1207,6 +1225,14 @@ func (module *PrebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx and return buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) } +func (b *PrebuiltBootclasspathFragmentModule) getImageName() *string { + return b.properties.Image_name +} + +func (b *PrebuiltBootclasspathFragmentModule) getProfilePath() android.Path { + return b.profilePath +} + var _ commonBootclasspathFragment = (*PrebuiltBootclasspathFragmentModule)(nil) // RequiredFilesFromPrebuiltApex returns the list of all files the prebuilt_bootclasspath_fragment diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 792fa63963..f477f404e1 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -295,6 +295,11 @@ type bootImageConfig struct { // The "--single-image" argument. singleImage bool + + // Profiles imported from other boot image configs. Each element must represent a + // `bootclasspath_fragment` of an APEX (i.e., the `name` field of each element must refer to the + // `image_name` property of a `bootclasspath_fragment`). + profileImports []*bootImageConfig } // Target-dependent description of a boot image. @@ -711,6 +716,34 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p cmd.FlagWithInput("--profile-file=", profile) } + fragments := make(map[string]commonBootclasspathFragment) + ctx.VisitDirectDepsWithTag(bootclasspathFragmentDepTag, func(child android.Module) { + fragment := child.(commonBootclasspathFragment) + if fragment.getImageName() != nil && android.IsModulePreferred(child) { + fragments[*fragment.getImageName()] = fragment + } + }) + + for _, profileImport := range image.profileImports { + fragment := fragments[profileImport.name] + if fragment == nil { + ctx.ModuleErrorf("Boot image config '%[1]s' imports profile from '%[2]s', but a "+ + "bootclasspath_fragment with image name '%[2]s' doesn't exist or is not added as a "+ + "dependency of '%[1]s'", + image.name, + profileImport.name) + return bootImageVariantOutputs{} + } + if fragment.getProfilePath() == nil { + ctx.ModuleErrorf("Boot image config '%[1]s' imports profile from '%[2]s', but '%[2]s' "+ + "doesn't provide a profile", + image.name, + profileImport.name) + return bootImageVariantOutputs{} + } + cmd.FlagWithInput("--profile-file=", fragment.getProfilePath()) + } + dirtyImageFile := "frameworks/base/config/dirty-image-objects" dirtyImagePath := android.ExistentPathForSource(ctx, dirtyImageFile) if dirtyImagePath.Valid() { diff --git a/java/dexpreopt_config.go b/java/dexpreopt_config.go index 5f168e70a6..9100e87c6f 100644 --- a/java/dexpreopt_config.go +++ b/java/dexpreopt_config.go @@ -53,7 +53,7 @@ func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig global := dexpreopt.GetGlobalConfig(ctx) artModules := global.ArtApexJars - frameworkModules := global.BootJars.RemoveList(artModules) + frameworkModules := global.BootJars // This includes `artModules`. mainlineBcpModules := global.ApexBootJars frameworkSubdir := "system/framework" @@ -73,7 +73,6 @@ func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig // Framework config for the boot image extension. // It includes framework libraries and depends on the ART config. frameworkCfg := bootImageConfig{ - extends: &artCfg, name: frameworkBootImageName, stem: bootImageStem, installDir: frameworkSubdir, @@ -81,6 +80,7 @@ func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig preloadedClassesFile: "frameworks/base/config/preloaded-classes", compilerFilter: "speed-profile", singleImage: false, + profileImports: []*bootImageConfig{&artCfg}, } mainlineCfg := bootImageConfig{ diff --git a/java/dexpreopt_config_testing.go b/java/dexpreopt_config_testing.go index 3ee3bd5538..6f3aa2be8c 100644 --- a/java/dexpreopt_config_testing.go +++ b/java/dexpreopt_config_testing.go @@ -462,36 +462,65 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut symbolsDir: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped", installDir: "system/framework", profileInstallPathInApex: "", - modules: android.CreateTestConfiguredJarList([]string{"platform:framework"}), - dexPaths: []string{"out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar"}, - dexPathsDeps: []string{"out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar", "out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar", "out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar"}, - zip: "out/soong/dexpreopt_arm64/dex_bootjars/boot.zip", + modules: android.CreateTestConfiguredJarList([]string{ + "com.android.art:core1", + "com.android.art:core2", + "platform:framework", + }), + dexPaths: []string{ + "out/soong/dexpreopt_arm64/dex_bootjars_input/core1.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/core2.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar", + }, + dexPathsDeps: []string{ + "out/soong/dexpreopt_arm64/dex_bootjars_input/core1.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/core2.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar", + }, + zip: "out/soong/dexpreopt_arm64/dex_bootjars/boot.zip", variants: []*expectedVariant{ { - archType: android.Arm64, - dexLocations: []string{"/system/framework/framework.jar"}, + archType: android.Arm64, + dexLocations: []string{ + "/apex/com.android.art/javalib/core1.jar", + "/apex/com.android.art/javalib/core2.jar", + "/system/framework/framework.jar", + }, dexLocationsDeps: []string{ "/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar", "/system/framework/framework.jar", }, - imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", - imagePathOnDevice: "/system/framework/arm64/boot-framework.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art", + imagePathOnDevice: "/system/framework/arm64/boot.art", imagesDeps: []string{ + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.vdex", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", }, - baseImages: []string{"out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art"}, - baseImagesDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", - }, installs: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art", + to: "/system/framework/arm64/boot.art", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.oat", + to: "/system/framework/arm64/boot.oat", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.art", + to: "/system/framework/arm64/boot-core2.art", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.oat", + to: "/system/framework/arm64/boot-core2.oat", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", to: "/system/framework/arm64/boot-framework.art", @@ -502,12 +531,28 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut }, }, vdexInstalls: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.vdex", + to: "/system/framework/arm64/boot.vdex", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.vdex", + to: "/system/framework/arm64/boot-core2.vdex", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", to: "/system/framework/arm64/boot-framework.vdex", }, }, unstrippedInstalls: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot.oat", + to: "/system/framework/arm64/boot.oat", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-core2.oat", + to: "/system/framework/arm64/boot-core2.oat", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-framework.oat", to: "/system/framework/arm64/boot-framework.oat", @@ -516,30 +561,47 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut licenseMetadataFile: expectedLicenseMetadataFile, }, { - archType: android.Arm, - dexLocations: []string{"/system/framework/framework.jar"}, + archType: android.Arm, + dexLocations: []string{ + "/apex/com.android.art/javalib/core1.jar", + "/apex/com.android.art/javalib/core2.jar", + "/system/framework/framework.jar", + }, dexLocationsDeps: []string{ "/apex/com.android.art/javalib/core1.jar", "/apex/com.android.art/javalib/core2.jar", "/system/framework/framework.jar", }, - imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", - imagePathOnDevice: "/system/framework/arm/boot-framework.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.art", + imagePathOnDevice: "/system/framework/arm/boot.art", imagesDeps: []string{ + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.vdex", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex", }, - baseImages: []string{"out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art"}, - baseImagesDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", - }, installs: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.art", + to: "/system/framework/arm/boot.art", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.oat", + to: "/system/framework/arm/boot.oat", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.art", + to: "/system/framework/arm/boot-core2.art", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.oat", + to: "/system/framework/arm/boot-core2.oat", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", to: "/system/framework/arm/boot-framework.art", @@ -550,12 +612,28 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut }, }, vdexInstalls: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.vdex", + to: "/system/framework/arm/boot.vdex", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.vdex", + to: "/system/framework/arm/boot-core2.vdex", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex", to: "/system/framework/arm/boot-framework.vdex", }, }, unstrippedInstalls: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot.oat", + to: "/system/framework/arm/boot.oat", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot-core2.oat", + to: "/system/framework/arm/boot-core2.oat", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot-framework.oat", to: "/system/framework/arm/boot-framework.oat", @@ -564,30 +642,47 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut licenseMetadataFile: expectedLicenseMetadataFile, }, { - archType: android.X86_64, - dexLocations: []string{"host/linux-x86/system/framework/framework.jar"}, + archType: android.X86_64, + dexLocations: []string{ + "host/linux-x86/apex/com.android.art/javalib/core1.jar", + "host/linux-x86/apex/com.android.art/javalib/core2.jar", + "host/linux-x86/system/framework/framework.jar", + }, dexLocationsDeps: []string{ "host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar", "host/linux-x86/system/framework/framework.jar", }, - imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", - imagePathOnDevice: "/system/framework/x86_64/boot-framework.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.art", + imagePathOnDevice: "/system/framework/x86_64/boot.art", imagesDeps: []string{ + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.vdex", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", }, - baseImages: []string{"out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art"}, - baseImagesDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", - }, installs: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.art", + to: "/system/framework/x86_64/boot.art", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.oat", + to: "/system/framework/x86_64/boot.oat", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.art", + to: "/system/framework/x86_64/boot-core2.art", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.oat", + to: "/system/framework/x86_64/boot-core2.oat", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", to: "/system/framework/x86_64/boot-framework.art", @@ -598,12 +693,28 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut }, }, vdexInstalls: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.vdex", + to: "/system/framework/x86_64/boot.vdex", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.vdex", + to: "/system/framework/x86_64/boot-core2.vdex", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", to: "/system/framework/x86_64/boot-framework.vdex", }, }, unstrippedInstalls: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot.oat", + to: "/system/framework/x86_64/boot.oat", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-core2.oat", + to: "/system/framework/x86_64/boot-core2.oat", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-framework.oat", to: "/system/framework/x86_64/boot-framework.oat", @@ -612,30 +723,47 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut licenseMetadataFile: expectedLicenseMetadataFile, }, { - archType: android.X86, - dexLocations: []string{"host/linux-x86/system/framework/framework.jar"}, + archType: android.X86, + dexLocations: []string{ + "host/linux-x86/apex/com.android.art/javalib/core1.jar", + "host/linux-x86/apex/com.android.art/javalib/core2.jar", + "host/linux-x86/system/framework/framework.jar", + }, dexLocationsDeps: []string{ "host/linux-x86/apex/com.android.art/javalib/core1.jar", "host/linux-x86/apex/com.android.art/javalib/core2.jar", "host/linux-x86/system/framework/framework.jar", }, - imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", - imagePathOnDevice: "/system/framework/x86/boot-framework.art", + imagePathOnHost: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.art", + imagePathOnDevice: "/system/framework/x86/boot.art", imagesDeps: []string{ + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.vdex", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", }, - baseImages: []string{"out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art"}, - baseImagesDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", - }, installs: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.art", + to: "/system/framework/x86/boot.art", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.oat", + to: "/system/framework/x86/boot.oat", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.art", + to: "/system/framework/x86/boot-core2.art", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.oat", + to: "/system/framework/x86/boot-core2.oat", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", to: "/system/framework/x86/boot-framework.art", @@ -646,12 +774,28 @@ func checkFrameworkBootImageConfig(t *testing.T, result *android.TestResult, mut }, }, vdexInstalls: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.vdex", + to: "/system/framework/x86/boot.vdex", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.vdex", + to: "/system/framework/x86/boot-core2.vdex", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", to: "/system/framework/x86/boot-framework.vdex", }, }, unstrippedInstalls: []normalizedInstall{ + { + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot.oat", + to: "/system/framework/x86/boot.oat", + }, + { + from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-core2.oat", + to: "/system/framework/x86/boot-core2.oat", + }, { from: "out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-framework.oat", to: "/system/framework/x86/boot-framework.oat", @@ -701,8 +845,8 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "out/soong/dexpreopt_arm64/dex_mainlinejars_input/framework-bar.jar", }, dexPathsDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar", - "out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/core1.jar", + "out/soong/dexpreopt_arm64/dex_bootjars_input/core2.jar", "out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar", "out/soong/dexpreopt_arm64/dex_mainlinejars_input/framework-foo.jar", "out/soong/dexpreopt_arm64/dex_mainlinejars_input/framework-bar.jar", @@ -730,16 +874,15 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex", }, baseImages: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art", }, baseImagesDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.vdex", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex", @@ -789,16 +932,15 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex", }, baseImages: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", - "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.art", }, baseImagesDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.vdex", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat", - "out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.vdex", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat", "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex", @@ -848,16 +990,15 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86_64/boot-framework-foo.vdex", }, baseImages: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", - "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.art", }, baseImagesDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.vdex", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex", @@ -907,16 +1048,15 @@ func CheckMainlineBootImageConfig(t *testing.T, result *android.TestResult) { "out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex", }, baseImages: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", - "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.art", }, baseImagesDeps: []string{ - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat", - "out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.vdex", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.art", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.oat", + "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.vdex", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat", "out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex", @@ -1051,18 +1191,18 @@ func checkDexpreoptMakeVars(t *testing.T, result *android.TestResult, expectedLi fmt.Fprintf(out, "%s=%s\n", v.Name(), android.StringRelativeToTop(result.Config, v.Value())) } format := ` -DEXPREOPT_BOOTCLASSPATH_DEX_FILES=out/soong/dexpreopt_arm64/dex_artjars_input/core1.jar out/soong/dexpreopt_arm64/dex_artjars_input/core2.jar out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar +DEXPREOPT_BOOTCLASSPATH_DEX_FILES=out/soong/dexpreopt_arm64/dex_bootjars_input/core1.jar out/soong/dexpreopt_arm64/dex_bootjars_input/core2.jar out/soong/dexpreopt_arm64/dex_bootjars_input/framework.jar DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS=/apex/com.android.art/javalib/core1.jar /apex/com.android.art/javalib/core2.jar /system/framework/framework.jar -DEXPREOPT_BOOT_JARS_MODULES=platform:framework +DEXPREOPT_BOOT_JARS_MODULES=com.android.art:core1:com.android.art:core2:platform:framework DEXPREOPT_GEN=out/host/linux-x86/bin/dexpreopt_gen DEXPREOPT_IMAGE_BUILT_INSTALLED_art_arm=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.art:/apex/art_boot_images/javalib/arm/boot.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot.oat:/apex/art_boot_images/javalib/arm/boot.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.art:/apex/art_boot_images/javalib/arm/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm/boot-core2.oat:/apex/art_boot_images/javalib/arm/boot-core2.oat DEXPREOPT_IMAGE_BUILT_INSTALLED_art_arm64=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art:/apex/art_boot_images/javalib/arm64/boot.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat:/apex/art_boot_images/javalib/arm64/boot.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art:/apex/art_boot_images/javalib/arm64/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat:/apex/art_boot_images/javalib/arm64/boot-core2.oat DEXPREOPT_IMAGE_BUILT_INSTALLED_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art:/apex/art_boot_images/javalib/x86/boot.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat:/apex/art_boot_images/javalib/x86/boot.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art:/apex/art_boot_images/javalib/x86/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat:/apex/art_boot_images/javalib/x86/boot-core2.oat DEXPREOPT_IMAGE_BUILT_INSTALLED_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art:/apex/art_boot_images/javalib/x86_64/boot.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat:/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art:/apex/art_boot_images/javalib/x86_64/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat:/apex/art_boot_images/javalib/x86_64/boot-core2.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art:/system/framework/arm/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat:/system/framework/arm/boot-framework.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art:/system/framework/arm64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat:/system/framework/arm64/boot-framework.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art:/system/framework/x86/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat:/system/framework/x86/boot-framework.oat -DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art:/system/framework/x86_64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat:/system/framework/x86_64/boot-framework.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.art:/system/framework/arm/boot.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.oat:/system/framework/arm/boot.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.art:/system/framework/arm/boot-core2.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.oat:/system/framework/arm/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art:/system/framework/arm/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat:/system/framework/arm/boot-framework.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art:/system/framework/arm64/boot.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.oat:/system/framework/arm64/boot.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.art:/system/framework/arm64/boot-core2.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.oat:/system/framework/arm64/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art:/system/framework/arm64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat:/system/framework/arm64/boot-framework.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.art:/system/framework/x86/boot.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.oat:/system/framework/x86/boot.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.art:/system/framework/x86/boot-core2.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.oat:/system/framework/x86/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art:/system/framework/x86/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat:/system/framework/x86/boot-framework.oat +DEXPREOPT_IMAGE_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.art:/system/framework/x86_64/boot.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.oat:/system/framework/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.art:/system/framework/x86_64/boot-core2.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.oat:/system/framework/x86_64/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art:/system/framework/x86_64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat:/system/framework/x86_64/boot-framework.oat DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art:/system/framework/arm/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat:/system/framework/arm/boot-framework-foo.oat DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art:/system/framework/arm64/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat:/system/framework/arm64/boot-framework-foo.oat DEXPREOPT_IMAGE_BUILT_INSTALLED_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art:/system/framework/x86/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat:/system/framework/x86/boot-framework-foo.oat @@ -1071,10 +1211,10 @@ DEXPREOPT_IMAGE_DEPS_art_arm=out/soong/dexpreopt_arm64/dex_artjars/android/apex/ DEXPREOPT_IMAGE_DEPS_art_arm64=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.oat out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex DEXPREOPT_IMAGE_DEPS_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex DEXPREOPT_IMAGE_DEPS_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.art out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex -DEXPREOPT_IMAGE_DEPS_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex -DEXPREOPT_IMAGE_DEPS_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex -DEXPREOPT_IMAGE_DEPS_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex -DEXPREOPT_IMAGE_DEPS_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex +DEXPREOPT_IMAGE_DEPS_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.vdex out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.vdex out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex +DEXPREOPT_IMAGE_DEPS_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.vdex out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.vdex out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex +DEXPREOPT_IMAGE_DEPS_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.vdex out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.vdex out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex +DEXPREOPT_IMAGE_DEPS_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.vdex out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.vdex out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.oat out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex DEXPREOPT_IMAGE_DEPS_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.oat out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex DEXPREOPT_IMAGE_DEPS_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.oat out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex DEXPREOPT_IMAGE_DEPS_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.oat out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex @@ -1092,11 +1232,11 @@ DEXPREOPT_IMAGE_LICENSE_METADATA_mainline_arm64=out/soong/.intermediates/framewo DEXPREOPT_IMAGE_LICENSE_METADATA_mainline_host_x86=out/soong/.intermediates/frameworks/base/boot/platform-bootclasspath/android_common/meta_lic DEXPREOPT_IMAGE_LICENSE_METADATA_mainline_host_x86_64=out/soong/.intermediates/frameworks/base/boot/platform-bootclasspath/android_common/meta_lic DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEart=/apex/art_boot_images/javalib/boot.art -DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEboot=/apex/art_boot_images/javalib/boot.art:/system/framework/boot-framework.art -DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEmainline=/apex/art_boot_images/javalib/boot.art:/system/framework/boot-framework.art:/system/framework/boot-framework-foo.art +DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEboot=/system/framework/boot.art +DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICEmainline=/system/framework/boot.art:/system/framework/boot-framework-foo.art DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTart=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art -DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTboot=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art:out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/boot-framework.art -DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTmainline=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/boot.art:out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/boot-framework.art:out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/boot-framework-foo.art +DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTboot=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/boot.art +DEXPREOPT_IMAGE_LOCATIONS_ON_HOSTmainline=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/boot.art:out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/boot-framework-foo.art DEXPREOPT_IMAGE_NAMES=art boot mainline DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED=out/soong/dexpreopt_arm64/dex_bootjars/boot.bprof:/system/etc/boot-image.bprof out/soong/dexpreopt_arm64/dex_bootjars/boot.prof:/system/etc/boot-image.prof DEXPREOPT_IMAGE_PROFILE_LICENSE_METADATA=out/soong/.intermediates/frameworks/base/boot/platform-bootclasspath/android_common/meta_lic @@ -1104,10 +1244,10 @@ DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_arm=out/soong/dexpreopt_arm64/dex DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_arm64=out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot.oat:/apex/art_boot_images/javalib/arm64/boot.oat out/soong/dexpreopt_arm64/dex_artjars_unstripped/android/apex/art_boot_images/javalib/arm64/boot-core2.oat:/apex/art_boot_images/javalib/arm64/boot-core2.oat DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot.oat:/apex/art_boot_images/javalib/x86/boot.oat out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.oat:/apex/art_boot_images/javalib/x86/boot-core2.oat DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.oat:/apex/art_boot_images/javalib/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_artjars_unstripped/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.oat:/apex/art_boot_images/javalib/x86_64/boot-core2.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot-framework.oat:/system/framework/arm/boot-framework.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-framework.oat:/system/framework/arm64/boot-framework.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-framework.oat:/system/framework/x86/boot-framework.oat -DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-framework.oat:/system/framework/x86_64/boot-framework.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot.oat:/system/framework/arm/boot.oat out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot-core2.oat:/system/framework/arm/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm/boot-framework.oat:/system/framework/arm/boot-framework.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot.oat:/system/framework/arm64/boot.oat out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-core2.oat:/system/framework/arm64/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars_unstripped/android/system/framework/arm64/boot-framework.oat:/system/framework/arm64/boot-framework.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot.oat:/system/framework/x86/boot.oat out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-core2.oat:/system/framework/x86/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86/boot-framework.oat:/system/framework/x86/boot-framework.oat +DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot.oat:/system/framework/x86_64/boot.oat out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-core2.oat:/system/framework/x86_64/boot-core2.oat out/soong/dexpreopt_arm64/dex_bootjars_unstripped/linux_glibc/system/framework/x86_64/boot-framework.oat:/system/framework/x86_64/boot-framework.oat DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/android/system/framework/arm/boot-framework-foo.oat:/system/framework/arm/boot-framework-foo.oat DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/android/system/framework/arm64/boot-framework-foo.oat:/system/framework/arm64/boot-framework-foo.oat DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars_unstripped/linux_glibc/system/framework/x86/boot-framework-foo.oat:/system/framework/x86/boot-framework-foo.oat @@ -1116,10 +1256,10 @@ DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_arm=out/soong/dexpreopt_arm64/dex_artja DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_arm64=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.vdex:/apex/art_boot_images/javalib/arm64/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot-core2.vdex:/apex/art_boot_images/javalib/arm64/boot-core2.vdex DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.vdex:/apex/art_boot_images/javalib/x86/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot-core2.vdex:/apex/art_boot_images/javalib/x86/boot-core2.vdex DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.vdex:/apex/art_boot_images/javalib/x86_64/boot.vdex out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot-core2.vdex:/apex/art_boot_images/javalib/x86_64/boot-core2.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex:/system/framework/arm/boot-framework.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex:/system/framework/arm64/boot-framework.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex:/system/framework/x86/boot-framework.vdex -DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex:/system/framework/x86_64/boot-framework.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.vdex:/system/framework/arm/boot.vdex out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-core2.vdex:/system/framework/arm/boot-core2.vdex out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.vdex:/system/framework/arm/boot-framework.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.vdex:/system/framework/arm64/boot.vdex out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-core2.vdex:/system/framework/arm64/boot-core2.vdex out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.vdex:/system/framework/arm64/boot-framework.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.vdex:/system/framework/x86/boot.vdex out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-core2.vdex:/system/framework/x86/boot-core2.vdex out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.vdex:/system/framework/x86/boot-framework.vdex +DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.vdex:/system/framework/x86_64/boot.vdex out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-core2.vdex:/system/framework/x86_64/boot-core2.vdex out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.vdex:/system/framework/x86_64/boot-framework.vdex DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.vdex:/system/framework/arm/boot-framework-foo.vdex DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.vdex:/system/framework/arm64/boot-framework-foo.vdex DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.vdex:/system/framework/x86/boot-framework-foo.vdex @@ -1131,10 +1271,10 @@ DEXPREOPT_IMAGE_art_arm=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_b DEXPREOPT_IMAGE_art_arm64=out/soong/dexpreopt_arm64/dex_artjars/android/apex/art_boot_images/javalib/arm64/boot.art DEXPREOPT_IMAGE_art_host_x86=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86/boot.art DEXPREOPT_IMAGE_art_host_x86_64=out/soong/dexpreopt_arm64/dex_artjars/linux_glibc/apex/art_boot_images/javalib/x86_64/boot.art -DEXPREOPT_IMAGE_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot-framework.art -DEXPREOPT_IMAGE_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot-framework.art -DEXPREOPT_IMAGE_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot-framework.art -DEXPREOPT_IMAGE_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot-framework.art +DEXPREOPT_IMAGE_boot_arm=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm/boot.art +DEXPREOPT_IMAGE_boot_arm64=out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art +DEXPREOPT_IMAGE_boot_host_x86=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86/boot.art +DEXPREOPT_IMAGE_boot_host_x86_64=out/soong/dexpreopt_arm64/dex_bootjars/linux_glibc/system/framework/x86_64/boot.art DEXPREOPT_IMAGE_mainline_arm=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm/boot-framework-foo.art DEXPREOPT_IMAGE_mainline_arm64=out/soong/dexpreopt_arm64/dex_mainlinejars/android/system/framework/arm64/boot-framework-foo.art DEXPREOPT_IMAGE_mainline_host_x86=out/soong/dexpreopt_arm64/dex_mainlinejars/linux_glibc/system/framework/x86/boot-framework-foo.art diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index d5779f75b1..07fb92cfe5 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -129,8 +129,7 @@ func (b *platformBootclasspathModule) BootclasspathDepsMutator(ctx android.Botto // Add dependencies on all the non-updatable module configured in the "boot" boot image. That does // not include modules configured in the "art" boot image. - bootImageConfig := defaultBootImageConfig(ctx) - addDependenciesOntoBootImageModules(ctx, bootImageConfig.modules, platformBootclasspathBootJarDepTag) + addDependenciesOntoBootImageModules(ctx, b.platformJars(ctx), platformBootclasspathBootJarDepTag) // Add dependencies on all the apex jars. apexJars := dexpreopt.GetGlobalConfig(ctx).ApexBootJars @@ -187,7 +186,8 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo bootDexJarByModule := b.generateHiddenAPIBuildActions(ctx, b.configuredModules, b.fragments) buildRuleForBootJarsPackageCheck(ctx, bootDexJarByModule) - b.generateBootImageBuildActions(ctx, platformModules, apexModules) + b.generateBootImageBuildActions(ctx) + b.copyApexBootJarsForAppsDexpreopt(ctx, apexModules) } // Generate classpaths.proto config @@ -200,7 +200,7 @@ func (b *platformBootclasspathModule) generateClasspathProtoBuildActions(ctx and func (b *platformBootclasspathModule) configuredJars(ctx android.ModuleContext) android.ConfiguredJarList { // Include all non APEX jars - jars := defaultBootImageConfig(ctx).modules + jars := b.platformJars(ctx) // Include jars from APEXes that don't populate their classpath proto config. remainingJars := dexpreopt.GetGlobalConfig(ctx).ApexBootJars @@ -217,6 +217,10 @@ func (b *platformBootclasspathModule) configuredJars(ctx android.ModuleContext) return jars } +func (b *platformBootclasspathModule) platformJars(ctx android.PathContext) android.ConfiguredJarList { + return defaultBootImageConfig(ctx).modules.RemoveList(artBootImageConfig(ctx).modules) +} + // checkPlatformModules ensures that the non-updatable modules supplied are not part of an // apex module. func (b *platformBootclasspathModule) checkPlatformModules(ctx android.ModuleContext, modules []android.Module) { @@ -226,7 +230,7 @@ func (b *platformBootclasspathModule) checkPlatformModules(ctx android.ModuleCon fromUpdatableApex := apexInfo.Updatable if fromUpdatableApex { // error: this jar is part of an updatable apex - ctx.ModuleErrorf("module %q from updatable apexes %q is not allowed in the framework boot image", ctx.OtherModuleName(m), apexInfo.InApexVariants) + ctx.ModuleErrorf("module %q from updatable apexes %q is not allowed in the platform bootclasspath", ctx.OtherModuleName(m), apexInfo.InApexVariants) } else { // ok: this jar is part of the platform or a non-updatable apex } @@ -396,7 +400,7 @@ func (b *platformBootclasspathModule) generateHiddenApiMakeVars(ctx android.Make } // generateBootImageBuildActions generates ninja rules related to the boot image creation. -func (b *platformBootclasspathModule) generateBootImageBuildActions(ctx android.ModuleContext, platformModules, apexModules []android.Module) { +func (b *platformBootclasspathModule) generateBootImageBuildActions(ctx android.ModuleContext) { // Force the GlobalSoongConfig to be created and cached for use by the dex_bootjars // GenerateSingletonBuildActions method as it cannot create it for itself. dexpreopt.GetGlobalSoongConfig(ctx) @@ -408,15 +412,16 @@ func (b *platformBootclasspathModule) generateBootImageBuildActions(ctx android. frameworkBootImageConfig := defaultBootImageConfig(ctx) bootFrameworkProfileRule(ctx, frameworkBootImageConfig) - b.generateBootImage(ctx, frameworkBootImageName, platformModules) - b.generateBootImage(ctx, mainlineBootImageName, apexModules) - b.copyApexBootJarsForAppsDexpreopt(ctx, apexModules) + b.generateBootImage(ctx, frameworkBootImageName) + b.generateBootImage(ctx, mainlineBootImageName) dumpOatRules(ctx, frameworkBootImageConfig) } -func (b *platformBootclasspathModule) generateBootImage(ctx android.ModuleContext, imageName string, modules []android.Module) { +func (b *platformBootclasspathModule) generateBootImage(ctx android.ModuleContext, imageName string) { imageConfig := genBootImageConfigs(ctx)[imageName] + modules := b.getModulesForImage(ctx, imageConfig) + // Copy module dex jars to their predefined locations. bootDexJarsByModule := extractEncodedDexJarsFromModules(ctx, modules) copyBootJarsToPredefinedLocations(ctx, bootDexJarsByModule, imageConfig.dexPathsByModule) @@ -446,3 +451,26 @@ func (b *platformBootclasspathModule) copyApexBootJarsForAppsDexpreopt(ctx andro apexBootDexJarsByModule := extractEncodedDexJarsFromModules(ctx, apexModules) copyBootJarsToPredefinedLocations(ctx, apexBootDexJarsByModule, config.dexPathsByModule) } + +func (b *platformBootclasspathModule) getModulesForImage(ctx android.ModuleContext, imageConfig *bootImageConfig) []android.Module { + modules := make([]android.Module, 0, imageConfig.modules.Len()) + for i := 0; i < imageConfig.modules.Len(); i++ { + found := false + for _, module := range b.configuredModules { + name := android.RemoveOptionalPrebuiltPrefix(module.Name()) + if name == imageConfig.modules.Jar(i) { + modules = append(modules, module) + found = true + break + } + } + if !found && !ctx.Config().AllowMissingDependencies() { + ctx.ModuleErrorf( + "Boot image '%s' module '%s' not added as a dependency of platform_bootclasspath", + imageConfig.name, + imageConfig.modules.Jar(i)) + return []android.Module{} + } + } + return modules +} From 6b82616c837278587514fca9782e716d5c83b3eb Mon Sep 17 00:00:00 2001 From: Chris Wailes Date: Thu, 11 May 2023 16:07:30 -0700 Subject: [PATCH 0137/1460] rustc-1.69.0 Build 10117114 Bug: https://issuetracker.google.com/issues/274972941 Test: m rust Change-Id: If5570ba26080bdff38bdcbd65f6f1d7dc9f5bffc --- rust/config/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/config/global.go b/rust/config/global.go index 2d1f0c1f94..748bb3d2f3 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.68.0" + RustDefaultVersion = "1.69.0" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ From 8bd3afe68c2670db7a39c7730a605d4b17c94122 Mon Sep 17 00:00:00 2001 From: Justin Yun Date: Fri, 12 May 2023 15:53:06 +0900 Subject: [PATCH 0138/1460] VSDK: etc snapshot may have sub_dir for relative_install_path prebuilt_etc may use sub_dir property to define a relative install path. Use SubDir() to read either sub_dir or relative_install_path property. Bug: 282080219 Test: m vendor-image Change-Id: I07a21c5aeb87b403854ed32976d26e70b046b8c1 --- etc/prebuilt_etc.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index 6817dcef0e..3e1bbded66 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -652,9 +652,7 @@ func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.Singleto prop := snapshot.SnapshotJsonFlags{} propOut := snapshotLibOut + ".json" prop.InitBaseSnapshotProps(m) - if m.subdirProperties.Relative_install_path != nil { - prop.RelativeInstallPath = *m.subdirProperties.Relative_install_path - } + prop.RelativeInstallPath = m.SubDir() if m.properties.Filename != nil { prop.Filename = *m.properties.Filename From b565f2f0dfbbda0e142a9fc1e67dce47d66cc90b Mon Sep 17 00:00:00 2001 From: Rico Wind Date: Fri, 12 May 2023 12:30:07 +0200 Subject: [PATCH 0139/1460] Ensure outputs correctly declared for proguard config in R8 Bug: 281081834 Test: manually inspect the file Change-Id: Iaa8a99f1f1ae62b853e2a8dad613c2c6c886ff48 --- java/dex.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/java/dex.go b/java/dex.go index f7c1361e24..7e7da00fd0 100644 --- a/java/dex.go +++ b/java/dex.go @@ -161,7 +161,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", "$r8Template": &remoteexec.REParams{ Labels: map[string]string{"type": "compile", "compiler": "r8"}, Inputs: []string{"$implicits", "${config.R8Jar}"}, - OutputFiles: []string{"${outUsage}"}, + OutputFiles: []string{"${outUsage}", "${outConfig}", "${outDict}"}, ExecStrategy: "${config.RER8ExecStrategy}", ToolchainInputs: []string{"${config.JavaCmd}"}, Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, @@ -399,13 +399,16 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam args["implicits"] = strings.Join(r8Deps.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ - Rule: rule, - Description: "r8", - Output: javalibJar, - ImplicitOutputs: android.WritablePaths{proguardDictionary, proguardUsageZip}, - Input: dexParams.classesJar, - Implicits: r8Deps, - Args: args, + Rule: rule, + Description: "r8", + Output: javalibJar, + ImplicitOutputs: android.WritablePaths{ + proguardDictionary, + proguardUsageZip, + proguardConfiguration}, + Input: dexParams.classesJar, + Implicits: r8Deps, + Args: args, }) } else { d8Flags, d8Deps := d8Flags(dexParams.flags) From e9850f81531f070ac5cd193fdcd767bc91605bfe Mon Sep 17 00:00:00 2001 From: Yi Kong Date: Fri, 12 May 2023 12:12:38 +0100 Subject: [PATCH 0140/1460] Revert "Turn off stack protector check for noreturn calls" clang-r487747c contains upstream fix for the excessive stack protactor checks (fc4494dffa54) and checks are no longer inserted for C only code or C++ code built without exeception. The Android platform does not enable C++ exception by default, thus turning on the check should not have significant performance impact. This reverts commit ecc18df6ff68853517749c49f411070a0b8d219f. Test: presubmit Bug: 277565884 Bug: 279955553 Change-Id: I2fc5826048ba5d00cabcc8401ac40a6f54812a44 --- cc/config/global.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cc/config/global.go b/cc/config/global.go index 530b79aefe..e5ce48ebef 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -111,9 +111,6 @@ var ( // Turn off FMA which got enabled by default in clang-r445002 (http://b/218805949) "-ffp-contract=off", - - // Turn off stack protector check for noreturn calls. (http://b/264965700) - "-mllvm -disable-check-noreturn-call", } commonGlobalConlyflags = []string{} @@ -150,9 +147,6 @@ var ( commonGlobalLldflags = []string{ "-fuse-ld=lld", "-Wl,--icf=safe", - - // Turn off stack protector check for noreturn calls. (http://b/264965700) - "-Wl,-mllvm,-disable-check-noreturn-call", } deviceGlobalCppflags = []string{ From e7a4712ec9f32acdf609d87b9721947c7e2dddcc Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Fri, 12 May 2023 08:44:51 -0400 Subject: [PATCH 0141/1460] Add missing package to allowlist Test: CI Change-Id: I15b95dd8a0acbde89f0a9f2033d8ea9c904b65fc --- android/allowlists/allowlists.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 71281a6e88..a9c846f0b0 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -203,23 +203,24 @@ var ( "frameworks/av/media/audioaidlconversion": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/codec2/components/aom": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/codecs": Bp2BuildDefaultTrueRecursively, - "frameworks/av/media/module/codecs": Bp2BuildDefaultTrueRecursively, - "frameworks/av/media/module/foundation": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/liberror": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/libmediahelper": Bp2BuildDefaultTrue, "frameworks/av/media/libshmem": Bp2BuildDefaultTrueRecursively, + "frameworks/av/media/module/codecs": Bp2BuildDefaultTrueRecursively, + "frameworks/av/media/module/foundation": Bp2BuildDefaultTrueRecursively, "frameworks/av/media/module/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/av/services/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/base/libs/androidfw": Bp2BuildDefaultTrue, "frameworks/base/media/tests/MediaDump": Bp2BuildDefaultTrue, - "frameworks/base/services/tests/servicestests/aidl": Bp2BuildDefaultTrue, "frameworks/base/proto": Bp2BuildDefaultTrue, + "frameworks/base/services/tests/servicestests/aidl": Bp2BuildDefaultTrue, "frameworks/base/startop/apps/test": Bp2BuildDefaultTrue, "frameworks/base/tests/appwidgets/AppWidgetHostTest": Bp2BuildDefaultTrueRecursively, "frameworks/base/tools/aapt2": Bp2BuildDefaultTrue, "frameworks/base/tools/codegen": Bp2BuildDefaultTrueRecursively, "frameworks/base/tools/streaming_proto": Bp2BuildDefaultTrueRecursively, "frameworks/hardware/interfaces/stats/aidl": Bp2BuildDefaultTrue, + "frameworks/libs/modules-utils/build": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/adbd_auth": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/arect": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/gui": Bp2BuildDefaultTrue, From 3c226ffbd2e9976cd9fa79b0cc906ac4dd421970 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Fri, 12 May 2023 08:49:54 -0400 Subject: [PATCH 0142/1460] Clear out allowlist to get CI green Test: 282042844 Change-Id: I9437b9c08402e913a0a6ee9c6ed453332fe04fa9 --- android/allowlists/allowlists.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 71281a6e88..d55763b3f2 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -1555,9 +1555,10 @@ var ( // also be built - do not add them to this list. StagingMixedBuildsEnabledList = []string{ // M13: media.swcodec launch - "com.android.media.swcodec", - "test_com.android.media.swcodec", - "libstagefright_foundation", + // TODO(b/282042844): reenable + // "com.android.media.swcodec", + // "test_com.android.media.swcodec", + // "libstagefright_foundation", } // These should be the libs that are included by the apexes in the ProdMixedBuildsEnabledList From a3ae007641d85106e3cfda196bc0b15d49f923a3 Mon Sep 17 00:00:00 2001 From: Chris Parsons Date: Wed, 10 May 2023 21:10:08 +0000 Subject: [PATCH 0143/1460] Touch soong_build output at the end of main This fixes an incrementality bug that resulted in superfluous build.ninja regeneration after a new glob definition is added. Fixes: 279674820 Test: Treehugger Test; New integration test in this CL Change-Id: Ifefe66a0eb1c125e9ad5373d60437a1cb1e6fdec --- cmd/soong_build/main.go | 15 ++++++++------- tests/bootstrap_test.sh | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 5de232664f..10b09d3eb9 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -540,10 +540,16 @@ func main() { } writeMetrics(configuration, ctx.EventHandler, metricsDir) } - writeUsedEnvironmentFile(configuration, finalOutputFile) + writeUsedEnvironmentFile(configuration) + + // Touch the output file so that it's the newest file created by soong_build. + // This is necessary because, if soong_build generated any files which + // are ninja inputs to the main output file, then ninja would superfluously + // rebuild this output file on the next build invocation. + touch(shared.JoinPath(topDir, finalOutputFile)) } -func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) { +func writeUsedEnvironmentFile(configuration android.Config) { if usedEnvFile == "" { return } @@ -562,11 +568,6 @@ func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile stri } err = os.WriteFile(path, data, 0666) maybeQuit(err, "error writing used environment file '%s'", usedEnvFile) - - // Touch the output file so that it's not older than the file we just - // wrote. We can't write the environment file earlier because one an access - // new environment variables while writing it. - touch(shared.JoinPath(topDir, finalOutputFile)) } func touch(path string) { diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index fda5ca0863..59352477cb 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -885,4 +885,37 @@ function test_queryview_null_build() { fi } +# This test verifies that adding a new glob to a blueprint file only +# causes build.ninja to be regenerated on the *next* build, and *not* +# the build after. (This is a regression test for a bug where globs +# resulted in two successive regenerations.) +function test_new_glob_incrementality { + setup + + run_soong nothing + local -r mtime1=$(stat -c "%y" out/soong/build.ninja) + + mkdir -p globdefpkg/ + cat > globdefpkg/Android.bp <<'EOF' +filegroup { + name: "fg_with_glob", + srcs: ["*.txt"], +} +EOF + + run_soong nothing + local -r mtime2=$(stat -c "%y" out/soong/build.ninja) + + if [[ "$mtime1" == "$mtime2" ]]; then + fail "Ninja file was not regenerated, despite a new bp file" + fi + + run_soong nothing + local -r mtime3=$(stat -c "%y" out/soong/build.ninja) + + if [[ "$mtime2" != "$mtime3" ]]; then + fail "Ninja file was regenerated despite no previous bp changes" + fi +} + scan_and_run_tests From 6974c7ec725a948a56cd8e83bff4a2cc9cf909e3 Mon Sep 17 00:00:00 2001 From: Yi Kong Date: Fri, 12 May 2023 15:03:29 +0100 Subject: [PATCH 0144/1460] Update clang version for Rust bindgen to clang-r487747c Test: presubmit Bug: 279955553 Change-Id: I9092263024b5b8dc7a1538b00f318ef0d5c29bb7 --- rust/bindgen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/bindgen.go b/rust/bindgen.go index 13fa81e668..96645b0756 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -29,7 +29,7 @@ var ( defaultBindgenFlags = []string{""} // bindgen should specify its own Clang revision so updating Clang isn't potentially blocked on bindgen failures. - bindgenClangVersion = "clang-r487747" + bindgenClangVersion = "clang-r487747c" _ = pctx.VariableFunc("bindgenClangVersion", func(ctx android.PackageVarContext) string { if override := ctx.Config().Getenv("LLVM_BINDGEN_PREBUILTS_VERSION"); override != "" { From 24870216e985226834b7b224e0b0d7bdf5d40dba Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Mon, 27 Feb 2023 19:00:26 +0000 Subject: [PATCH 0145/1460] Revert "Disable LTO and CFI for riscv64" This reverts commit ecf4e664e060597c6f66654c08787d4cd3262d30. Reason for revert: Not needed as emultated tls is now globally disabled https://android-review.git.corp.google.com/c/platform/build/soong/+/2431153 Change-Id: Idc3de2771afbbe2d5976f3c7974b3ffd15441b13 --- cc/lto.go | 5 ----- cc/sanitize.go | 6 ------ 2 files changed, 11 deletions(-) diff --git a/cc/lto.go b/cc/lto.go index 1afa1ddd9c..581856b46e 100644 --- a/cc/lto.go +++ b/cc/lto.go @@ -91,11 +91,6 @@ func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { return flags } - // TODO(b/254713216): LTO doesn't work on riscv64 yet. - if ctx.Arch().ArchType == android.Riscv64 { - return flags - } - if lto.LTO(ctx) { var ltoCFlag string var ltoLdFlag string diff --git a/cc/sanitize.go b/cc/sanitize.go index 7fddc1b836..df96b89c81 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -653,12 +653,6 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Integer_overflow = nil } - // TODO(b/254713216): CFI doesn't work for riscv64 yet because LTO doesn't work. - if ctx.Arch().ArchType == android.Riscv64 { - s.Cfi = nil - s.Diag.Cfi = nil - } - // Disable CFI for musl if ctx.toolchain().Musl() { s.Cfi = nil From ed885bb7dd15d8c3b099ba504ce050f6508f62c1 Mon Sep 17 00:00:00 2001 From: Gurpreet Singh Date: Fri, 21 Apr 2023 16:30:03 +0000 Subject: [PATCH 0146/1460] Add genrule to build *.latest.version build target. Build a new target *.latest.version which will contain a text file containing the last finalized version. Bug: 242316893, 282140551 Test: atest prebuilt_apis_test (cherry picked from https://android-review.googlesource.com/q/commit:daa314ac97d5ab8076113ee7f269b4e4cc3cf620) Merged-In: I41fa91c9ec273f342b7807c66c4d65ba13260124 Change-Id: I41fa91c9ec273f342b7807c66c4d65ba13260124 --- java/prebuilt_apis.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go index 206d995276..0740467eb8 100644 --- a/java/prebuilt_apis.go +++ b/java/prebuilt_apis.go @@ -135,6 +135,19 @@ func createApiModule(mctx android.LoadHookContext, name string, path string) { mctx.CreateModule(genrule.GenRuleFactory, &genruleProps) } +func createLatestApiModuleExtensionVersionFile(mctx android.LoadHookContext, name string, version string) { + genruleProps := struct { + Name *string + Srcs []string + Out []string + Cmd *string + }{} + genruleProps.Name = proptools.StringPtr(name) + genruleProps.Out = []string{name} + genruleProps.Cmd = proptools.StringPtr("echo " + version + " > $(out)") + mctx.CreateModule(genrule.GenRuleFactory, &genruleProps) +} + func createEmptyFile(mctx android.LoadHookContext, name string) { props := struct { Name *string @@ -233,9 +246,10 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { type latestApiInfo struct { module, scope, path string version int + isExtensionApiFile bool } - getLatest := func(files []string) map[string]latestApiInfo { + getLatest := func(files []string, isExtensionApiFile bool) map[string]latestApiInfo { m := make(map[string]latestApiInfo) for _, f := range files { module, version, scope := parseFinalizedPrebuiltPath(mctx, f) @@ -245,16 +259,16 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { key := module + "." + scope info, exists := m[key] if !exists || version > info.version { - m[key] = latestApiInfo{module, scope, f, version} + m[key] = latestApiInfo{module, scope, f, version, isExtensionApiFile} } } return m } - latest := getLatest(apiLevelFiles) + latest := getLatest(apiLevelFiles, false) if p.properties.Extensions_dir != nil { extensionApiFiles := globExtensionDirs(mctx, p, "api/*.txt") - for k, v := range getLatest(extensionApiFiles) { + for k, v := range getLatest(extensionApiFiles, true) { if _, exists := latest[k]; !exists { mctx.ModuleErrorf("Module %v finalized for extension %d but never during an API level; likely error", v.module, v.version) } @@ -267,6 +281,12 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { for _, k := range android.SortedKeys(latest) { info := latest[k] name := PrebuiltApiModuleName(info.module, info.scope, "latest") + latestExtensionVersionModuleName := PrebuiltApiModuleName(info.module, info.scope, "latest.extension_version") + if info.isExtensionApiFile { + createLatestApiModuleExtensionVersionFile(mctx, latestExtensionVersionModuleName, strconv.Itoa(info.version)) + } else { + createLatestApiModuleExtensionVersionFile(mctx, latestExtensionVersionModuleName, "-1") + } createApiModule(mctx, name, info.path) } From fee845a737667c026a1379f4c66f28d9520f311a Mon Sep 17 00:00:00 2001 From: Joe Onorato Date: Tue, 9 May 2023 08:14:14 -0700 Subject: [PATCH 0147/1460] Aconfig rules Test: m services_device_config (which runs sooong tests too) Change-Id: I432e914d01d2bff77ba68de65ae5baea527090f5 --- android/config.go | 10 ++ android/variable.go | 3 + device_config/Android.bp | 30 +++++ device_config/device_config_definitions.go | 150 +++++++++++++++++++++ device_config/device_config_test.go | 61 +++++++++ device_config/device_config_value_set.go | 92 +++++++++++++ device_config/device_config_values.go | 70 ++++++++++ device_config/init.go | 70 ++++++++++ device_config/testing.go | 19 +++ 9 files changed, 505 insertions(+) create mode 100644 device_config/Android.bp create mode 100644 device_config/device_config_definitions.go create mode 100644 device_config/device_config_test.go create mode 100644 device_config/device_config_value_set.go create mode 100644 device_config/device_config_values.go create mode 100644 device_config/init.go create mode 100644 device_config/testing.go diff --git a/android/config.go b/android/config.go index 9e94e05177..acadb3ff33 100644 --- a/android/config.go +++ b/android/config.go @@ -183,6 +183,16 @@ func (c Config) MaxPageSizeSupported() string { return String(c.config.productVariables.DeviceMaxPageSizeSupported) } +// The release version passed to aconfig, derived from RELEASE_VERSION +func (c Config) ReleaseVersion() string { + return c.config.productVariables.ReleaseVersion +} + +// The flag values files passed to aconfig, derived from RELEASE_VERSION +func (c Config) ReleaseDeviceConfigValueSets() []string { + return c.config.productVariables.ReleaseDeviceConfigValueSets +} + // A DeviceConfig object represents the configuration for a particular device // being built. For now there will only be one of these, but in the future there // may be multiple devices being built. diff --git a/android/variable.go b/android/variable.go index bf66135b55..97171e7b99 100644 --- a/android/variable.go +++ b/android/variable.go @@ -473,6 +473,9 @@ type productVariables struct { ProductManufacturer string `json:",omitempty"` ProductBrand string `json:",omitempty"` BuildVersionTags []string `json:",omitempty"` + + ReleaseVersion string `json:",omitempty"` + ReleaseDeviceConfigValueSets []string `json:",omitempty"` } func boolPtr(v bool) *bool { diff --git a/device_config/Android.bp b/device_config/Android.bp new file mode 100644 index 0000000000..360b3898a6 --- /dev/null +++ b/device_config/Android.bp @@ -0,0 +1,30 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-device_config", + pkgPath: "android/soong/device_config", + deps: [ + "blueprint", + "blueprint-pathtools", + "sbox_proto", + "soong", + "soong-android", + "soong-bazel", + "soong-shared", + ], + srcs: [ + "device_config_definitions.go", + "device_config_values.go", + "device_config_value_set.go", + "init.go", + //"testing.go" + ], + /* + testSrcs: [ + "device_config_test.go", + ], + */ + pluginFor: ["soong_build"], +} diff --git a/device_config/device_config_definitions.go b/device_config/device_config_definitions.go new file mode 100644 index 0000000000..c5657666c1 --- /dev/null +++ b/device_config/device_config_definitions.go @@ -0,0 +1,150 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package device_config + +import ( + "android/soong/android" + "fmt" + "github.com/google/blueprint" + "strings" +) + +type DefinitionsModule struct { + android.ModuleBase + android.DefaultableModuleBase + + // Properties for "device_config_definitions" + properties struct { + // aconfig files, relative to this Android.bp file + Srcs []string `android:"path"` + + // Release config flag namespace + Namespace string + + // Values from TARGET_RELEASE / RELEASE_DEVICE_CONFIG_VALUE_SETS + Values []string `blueprint:"mutated"` + } + + intermediatePath android.WritablePath + srcJarPath android.WritablePath +} + +func DefinitionsFactory() android.Module { + module := &DefinitionsModule{} + + android.InitAndroidModule(module) + android.InitDefaultableModule(module) + module.AddProperties(&module.properties) + // TODO: bp2build + //android.InitBazelModule(module) + + return module +} + +type implicitValuesTagType struct { + blueprint.BaseDependencyTag +} + +var implicitValuesTag = implicitValuesTagType{} + +func (module *DefinitionsModule) DepsMutator(ctx android.BottomUpMutatorContext) { + // Validate Properties + if len(module.properties.Srcs) == 0 { + ctx.PropertyErrorf("srcs", "missing source files") + return + } + if len(module.properties.Namespace) == 0 { + ctx.PropertyErrorf("namespace", "missing namespace property") + } + + // Add a dependency on the device_config_value_sets defined in + // RELEASE_DEVICE_CONFIG_VALUE_SETS, and add any device_config_values that + // match our namespace. + valuesFromConfig := ctx.Config().ReleaseDeviceConfigValueSets() + ctx.AddDependency(ctx.Module(), implicitValuesTag, valuesFromConfig...) +} + +func (module *DefinitionsModule) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case ".srcjar": + return []android.Path{module.srcJarPath}, nil + case "": + // The default output of this module is the intermediates format, which is + // not installable and in a private format that no other rules can handle + // correctly. + return []android.Path{module.intermediatePath}, nil + default: + return nil, fmt.Errorf("unsupported device_config_definitions module reference tag %q", tag) + } +} + +func joinAndPrefix(prefix string, values []string) string { + var sb strings.Builder + for _, v := range values { + sb.WriteString(prefix) + sb.WriteString(v) + } + return sb.String() +} + +func (module *DefinitionsModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Get the values that came from the global RELEASE_DEVICE_CONFIG_VALUE_SETS flag + ctx.VisitDirectDeps(func(dep android.Module) { + if !ctx.OtherModuleHasProvider(dep, valueSetProviderKey) { + // Other modules get injected as dependencies too, for example the license modules + return + } + depData := ctx.OtherModuleProvider(dep, valueSetProviderKey).(valueSetProviderData) + valuesFiles, ok := depData.AvailableNamespaces[module.properties.Namespace] + if ok { + for _, path := range valuesFiles { + module.properties.Values = append(module.properties.Values, path.String()) + } + } + }) + + // Intermediate format + inputFiles := android.PathsForModuleSrc(ctx, module.properties.Srcs) + module.intermediatePath = android.PathForModuleOut(ctx, "intermediate.json") + ctx.Build(pctx, android.BuildParams{ + Rule: aconfigRule, + Inputs: inputFiles, + Output: module.intermediatePath, + Description: "device_config_definitions", + Args: map[string]string{ + "release_version": ctx.Config().ReleaseVersion(), + "namespace": module.properties.Namespace, + "values": joinAndPrefix(" --values ", module.properties.Values), + }, + }) + + // Generated java inside a srcjar + module.srcJarPath = android.PathForModuleGen(ctx, ctx.ModuleName()+".srcjar") + ctx.Build(pctx, android.BuildParams{ + Rule: srcJarRule, + Input: module.intermediatePath, + Output: module.srcJarPath, + Description: "device_config.srcjar", + }) + + // TODO: C++ + + // Phony target for debugging convenience + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: android.PathForPhony(ctx, ctx.ModuleName()), + Inputs: []android.Path{module.srcJarPath}, // TODO: C++ + }) +} diff --git a/device_config/device_config_test.go b/device_config/device_config_test.go new file mode 100644 index 0000000000..91a06a781e --- /dev/null +++ b/device_config/device_config_test.go @@ -0,0 +1,61 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package device_config + +import ( + "os" + "testing" + + "android/soong/android" + "android/soong/java" +) + +func TestMain(m *testing.M) { + os.Exit(m.Run()) +} + +func test(t *testing.T, bp string) *android.TestResult { + t.Helper() + + mockFS := android.MockFS{ + "config.aconfig": nil, + } + + result := android.GroupFixturePreparers( + java.PrepareForTestWithJavaDefaultModules, + PrepareForTestWithSyspropBuildComponents, + // TODO: Consider values files, although maybe in its own test + // android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + // variables.ReleaseConfigValuesBasePaths = ... + //}) + mockFS.AddToFixture(), + android.FixtureWithRootAndroidBp(bp), + ).RunTest(t) + + return result +} + +func TestOutputs(t *testing.T) { + /*result := */ test(t, ` + device_config { + name: "my_device_config", + srcs: ["config.aconfig"], + } + `) + + // TODO: Make sure it exports a .srcjar, which is used by java libraries + // TODO: Make sure it exports an intermediates file + // TODO: Make sure the intermediates file is propagated to the Android.mk file +} diff --git a/device_config/device_config_value_set.go b/device_config/device_config_value_set.go new file mode 100644 index 0000000000..e406d20e0e --- /dev/null +++ b/device_config/device_config_value_set.go @@ -0,0 +1,92 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package device_config + +import ( + "android/soong/android" + "github.com/google/blueprint" +) + +// Properties for "device_config_value_set" +type ValueSetModule struct { + android.ModuleBase + android.DefaultableModuleBase + + properties struct { + // device_config_values modules + Values []string + } +} + +func ValueSetFactory() android.Module { + module := &ValueSetModule{} + + android.InitAndroidModule(module) + android.InitDefaultableModule(module) + module.AddProperties(&module.properties) + // TODO: bp2build + //android.InitBazelModule(module) + + return module +} + +// Dependency tag for values property +type valueSetType struct { + blueprint.BaseDependencyTag +} + +var valueSetTag = valueSetType{} + +// Provider published by device_config_value_set +type valueSetProviderData struct { + // The namespace of each of the + // (map of namespace --> device_config_module) + AvailableNamespaces map[string]android.Paths +} + +var valueSetProviderKey = blueprint.NewProvider(valueSetProviderData{}) + +func (module *ValueSetModule) DepsMutator(ctx android.BottomUpMutatorContext) { + deps := ctx.AddDependency(ctx.Module(), valueSetTag, module.properties.Values...) + for _, dep := range deps { + _, ok := dep.(*ValuesModule) + if !ok { + ctx.PropertyErrorf("values", "values must be a device_config_values module") + return + } + } +} + +func (module *ValueSetModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Accumulate the namespaces of the values modules listed, and set that as an + // valueSetProviderKey provider that device_config modules can read and use + // to append values to their aconfig actions. + namespaces := make(map[string]android.Paths) + ctx.VisitDirectDeps(func(dep android.Module) { + if !ctx.OtherModuleHasProvider(dep, valuesProviderKey) { + // Other modules get injected as dependencies too, for example the license modules + return + } + depData := ctx.OtherModuleProvider(dep, valuesProviderKey).(valuesProviderData) + + srcs := make([]android.Path, len(depData.Values)) + copy(srcs, depData.Values) + namespaces[depData.Namespace] = srcs + + }) + ctx.SetProvider(valueSetProviderKey, valueSetProviderData{ + AvailableNamespaces: namespaces, + }) +} diff --git a/device_config/device_config_values.go b/device_config/device_config_values.go new file mode 100644 index 0000000000..110f12a1cc --- /dev/null +++ b/device_config/device_config_values.go @@ -0,0 +1,70 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package device_config + +import ( + "android/soong/android" + "github.com/google/blueprint" +) + +// Properties for "device_config_value" +type ValuesModule struct { + android.ModuleBase + android.DefaultableModuleBase + + properties struct { + // aconfig files, relative to this Android.bp file + Srcs []string `android:"path"` + + // Release config flag namespace + Namespace string + } +} + +func ValuesFactory() android.Module { + module := &ValuesModule{} + + android.InitAndroidModule(module) + android.InitDefaultableModule(module) + module.AddProperties(&module.properties) + // TODO: bp2build + //android.InitBazelModule(module) + + return module +} + +// Provider published by device_config_value_set +type valuesProviderData struct { + // The namespace that this values module values + Namespace string + + // The values aconfig files, relative to the root of the tree + Values android.Paths +} + +var valuesProviderKey = blueprint.NewProvider(valuesProviderData{}) + +func (module *ValuesModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if len(module.properties.Namespace) == 0 { + ctx.PropertyErrorf("namespace", "missing namespace property") + } + + // Provide the our source files list to the device_config_value_set as a list of files + providerData := valuesProviderData{ + Namespace: module.properties.Namespace, + Values: android.PathsForModuleSrc(ctx, module.properties.Srcs), + } + ctx.SetProvider(valuesProviderKey, providerData) +} diff --git a/device_config/init.go b/device_config/init.go new file mode 100644 index 0000000000..0a4645b2be --- /dev/null +++ b/device_config/init.go @@ -0,0 +1,70 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package device_config + +import ( + "android/soong/android" + "github.com/google/blueprint" +) + +var ( + pctx = android.NewPackageContext("android/soong/device_config") + + // For device_config_definitions: Generate cache file + // TODO: Restat + aconfigRule = pctx.AndroidStaticRule("aconfig", + blueprint.RuleParams{ + Command: `${aconfig} create-cache` + + ` --namespace ${namespace}` + + ` --declarations ${in}` + + ` ${values}` + + ` --cache ${out}.tmp` + + ` && ( if cmp -s ${out}.tmp ; then rm ${out}.tmp ; else mv ${out}.tmp ${out} ; fi )`, + // ` --build-id ${release_version}` + + CommandDeps: []string{ + "${aconfig}", + }, + Restat: true, + }, "release_version", "namespace", "values") + + // For device_config_definitions: Generate java file + srcJarRule = pctx.AndroidStaticRule("aconfig_srcjar", + blueprint.RuleParams{ + Command: `rm -rf ${out}.tmp` + + ` && mkdir -p ${out}.tmp` + + ` && ${aconfig} create-java-lib` + + ` --cache ${in}` + + ` --out ${out}.tmp` + + ` && $soong_zip -write_if_changed -jar -o ${out} -C ${out}.tmp -D ${out}.tmp` + + ` && rm -rf ${out}.tmp`, + CommandDeps: []string{ + "$aconfig", + "$soong_zip", + }, + Restat: true, + }) +) + +func init() { + registerBuildComponents(android.InitRegistrationContext) +} + +func registerBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("device_config_definitions", DefinitionsFactory) + ctx.RegisterModuleType("device_config_values", ValuesFactory) + ctx.RegisterModuleType("device_config_value_set", ValueSetFactory) + pctx.HostBinToolVariable("aconfig", "aconfig") + pctx.HostBinToolVariable("soong_zip", "soong_zip") +} diff --git a/device_config/testing.go b/device_config/testing.go new file mode 100644 index 0000000000..f0544766b5 --- /dev/null +++ b/device_config/testing.go @@ -0,0 +1,19 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package device_config + +import "android/soong/android" + +var PrepareForTestWithSyspropBuildComponents = android.FixtureRegisterWithContext(registerBuildComponents) From 5f0c2ac5359561bc80b85d3a27268dec0bd64242 Mon Sep 17 00:00:00 2001 From: Trevor Radcliffe Date: Mon, 15 May 2023 18:00:59 +0000 Subject: [PATCH 0148/1460] Change bp2build to reflect arm_isa refactor We now rely on a `with_features` clause to effectively disable thumb when `arm_isa_arm` is enabled. Thus, we no longer need to add `-arm_isa_thumb` in bp2build. Bug: 261733820 Test: Unit tests Change-Id: I9da85834855b297a75097ce3a11e101f55bd53f4 --- bp2build/cc_binary_conversion_test.go | 5 +---- bp2build/cc_library_conversion_test.go | 5 +---- cc/bp2build.go | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/bp2build/cc_binary_conversion_test.go b/bp2build/cc_binary_conversion_test.go index 89eac8a000..6ec37031b8 100644 --- a/bp2build/cc_binary_conversion_test.go +++ b/bp2build/cc_binary_conversion_test.go @@ -644,10 +644,7 @@ func TestCcBinaryWithInstructionSet(t *testing.T) { targets: []testBazelTarget{ {"cc_binary", "foo", AttrNameToString{ "features": `select({ - "//build/bazel/platforms/arch:arm": [ - "arm_isa_arm", - "-arm_isa_thumb", - ], + "//build/bazel/platforms/arch:arm": ["arm_isa_arm"], "//conditions:default": [], })`, "local_includes": `["."]`, diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 1b681efb0c..41cd451b30 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -3219,10 +3219,7 @@ func TestCcLibraryWithInstructionSet(t *testing.T) { `, ExpectedBazelTargets: makeCcLibraryTargets("foo", AttrNameToString{ "features": `select({ - "//build/bazel/platforms/arch:arm": [ - "arm_isa_arm", - "-arm_isa_thumb", - ], + "//build/bazel/platforms/arch:arm": ["arm_isa_arm"], "//conditions:default": [], })`, "local_includes": `["."]`, diff --git a/cc/bp2build.go b/cc/bp2build.go index cf5f74d4d9..082e55f126 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -492,7 +492,7 @@ func (ca *compilerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversi instructionSet := proptools.StringDefault(props.Instruction_set, "") if instructionSet == "arm" { - ca.features.SetSelectValue(axis, config, []string{"arm_isa_arm", "-arm_isa_thumb"}) + ca.features.SetSelectValue(axis, config, []string{"arm_isa_arm"}) } else if instructionSet != "" && instructionSet != "thumb" { ctx.ModuleErrorf("Unknown value for instruction_set: %s", instructionSet) } From 367d89da7887825f74b0c9554b2e1b9619d94f08 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 0149/1460] Use aidl_library in cc libraries Introduce aidl.libs prop on cc libraries to pass in aidl_library. The goal is to eventually disallow aidl.include_dirs (a pattern for passing aidl headers dir for aidl compilation) and enforce aidl headers to be explicitly specified in Android.bp. Bug: 278704136 Test: go test Change-Id: Ia78bc11dfa12f47d2d1bb90dc65372ddb17f7e14 --- aidl_library/aidl_library.go | 4 + aidl_library/aidl_library_test.go | 4 - bp2build/cc_library_conversion_test.go | 142 +++++++++++++++++++------ cc/Android.bp | 2 +- cc/bp2build.go | 98 ++++++++++------- cc/cc.go | 27 +++++ cc/cc_test.go | 78 +++++++++++++- cc/compiler.go | 20 +++- cc/gen.go | 33 ++++-- cc/library.go | 2 +- 10 files changed, 322 insertions(+), 88 deletions(-) diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go index 9b5f0a8141..8a84e6bfa9 100644 --- a/aidl_library/aidl_library.go +++ b/aidl_library/aidl_library.go @@ -22,6 +22,10 @@ import ( "github.com/google/blueprint/proptools" ) +var PrepareForTestWithAidlLibrary = android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + registerAidlLibraryBuildComponents(ctx) +}) + func init() { registerAidlLibraryBuildComponents(android.InitRegistrationContext) } diff --git a/aidl_library/aidl_library_test.go b/aidl_library/aidl_library_test.go index 42fa5367e3..d9b410acca 100644 --- a/aidl_library/aidl_library_test.go +++ b/aidl_library/aidl_library_test.go @@ -19,10 +19,6 @@ import ( "testing" ) -var PrepareForTestWithAidlLibrary = android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { - registerAidlLibraryBuildComponents(ctx) -}) - func TestAidlLibrary(t *testing.T) { t.Parallel() ctx := android.GroupFixturePreparers( diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 1b681efb0c..e998ac8f45 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -18,6 +18,7 @@ import ( "fmt" "testing" + "android/soong/aidl_library" "android/soong/android" "android/soong/cc" ) @@ -63,6 +64,7 @@ func registerCcLibraryModuleTypes(ctx android.RegistrationContext) { ctx.RegisterModuleType("cc_library_static", cc.LibraryStaticFactory) ctx.RegisterModuleType("cc_prebuilt_library_static", cc.PrebuiltStaticLibraryFactory) ctx.RegisterModuleType("cc_library_headers", cc.LibraryHeaderFactory) + ctx.RegisterModuleType("aidl_library", aidl_library.AidlLibraryFactory) } func TestCcLibrarySimple(t *testing.T) { @@ -3315,6 +3317,46 @@ func TestCcLibraryArchVariantSuffix(t *testing.T) { }) } +func TestCcLibraryWithAidlLibrary(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library with aidl_library", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: ` +aidl_library { + name: "A_aidl", + srcs: ["aidl/A.aidl"], + hdrs: ["aidl/Header.aidl"], + strip_import_prefix: "aidl", +} +cc_library { + name: "foo", + aidl: { + libs: ["A_aidl"], + } +}`, + ExpectedBazelTargets: []string{ + MakeBazelTargetNoRestrictions("aidl_library", "A_aidl", AttrNameToString{ + "srcs": `["aidl/A.aidl"]`, + "hdrs": `["aidl/Header.aidl"]`, + "strip_import_prefix": `"aidl"`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ + "deps": `[":A_aidl"]`, + }), + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, + "local_includes": `["."]`, + }), + }, + }) +} + func TestCcLibraryWithAidlSrcs(t *testing.T) { runCcLibraryTestCase(t, Bp2buildTestCase{ Description: "cc_library with aidl srcs", @@ -3397,37 +3439,77 @@ cc_library { } func TestCcLibraryWithExportAidlHeaders(t *testing.T) { - runCcLibraryTestCase(t, Bp2buildTestCase{ - Description: "cc_library with export aidl headers", - ModuleTypeUnderTest: "cc_library", - ModuleTypeUnderTestFactory: cc.LibraryFactory, - Blueprint: ` -cc_library { - name: "foo", - srcs: [ - "Foo.aidl", - ], - aidl: { - export_aidl_headers: true, - } -}`, - ExpectedBazelTargets: []string{ - MakeBazelTarget("aidl_library", "foo_aidl_library", AttrNameToString{ - "srcs": `["Foo.aidl"]`, - }), - MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `[":foo_aidl_library"]`, - }), - MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ - "whole_archive_deps": `[":foo_cc_aidl_library"]`, - "local_includes": `["."]`, - }), - MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ - "whole_archive_deps": `[":foo_cc_aidl_library"]`, - "local_includes": `["."]`, - }), + t.Parallel() + + expectedBazelTargets := []string{ + MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ + "deps": `[":foo_aidl_library"]`, + }), + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "whole_archive_deps": `[":foo_cc_aidl_library"]`, + "local_includes": `["."]`, + }), + MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ + "whole_archive_deps": `[":foo_cc_aidl_library"]`, + "local_includes": `["."]`, + }), + } + testCases := []struct { + description string + bp string + expectedBazelTargets []string + }{ + { + description: "cc_library with aidl srcs and aidl.export_aidl_headers set", + bp: ` + cc_library { + name: "foo", + srcs: [ + "Foo.aidl", + ], + aidl: { + export_aidl_headers: true, + } + }`, + expectedBazelTargets: append( + expectedBazelTargets, + MakeBazelTarget("aidl_library", "foo_aidl_library", AttrNameToString{ + "srcs": `["Foo.aidl"]`, + })), }, - }) + { + description: "cc_library with aidl.libs and aidl.export_aidl_headers set", + bp: ` + aidl_library { + name: "foo_aidl_library", + srcs: ["Foo.aidl"], + } + cc_library { + name: "foo", + aidl: { + libs: ["foo_aidl_library"], + export_aidl_headers: true, + } + }`, + expectedBazelTargets: append( + expectedBazelTargets, + MakeBazelTargetNoRestrictions("aidl_library", "foo_aidl_library", AttrNameToString{ + "srcs": `["Foo.aidl"]`, + "tags": `["apex_available=//apex_available:anyapex"]`, + }), + ), + }, + } + + for _, testCase := range testCases { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library with export aidl headers", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: testCase.bp, + ExpectedBazelTargets: testCase.expectedBazelTargets, + }) + } } func TestCcLibraryWithTargetApex(t *testing.T) { diff --git a/cc/Android.bp b/cc/Android.bp index be2cc5a34c..f49dc1a9ec 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -9,6 +9,7 @@ bootstrap_go_package { "blueprint", "blueprint-pathtools", "soong", + "soong-aidl-library", "soong-android", "soong-bazel", "soong-cc-config", @@ -22,7 +23,6 @@ bootstrap_go_package { srcs: [ "afdo.go", "fdo_profile.go", - "androidmk.go", "api_level.go", "bp2build.go", diff --git a/cc/bp2build.go b/cc/bp2build.go index cf5f74d4d9..0f978a5cad 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -728,6 +728,8 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) compilerAttrs := compilerAttributes{} linkerAttrs := linkerAttributes{} + var aidlLibs bazel.LabelList + // Iterate through these axes in a deterministic order. This is required // because processing certain dependencies may result in concatenating // elements along other axes. (For example, processing NoConfig may result @@ -743,6 +745,7 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) compilerAttrs.lexopts.SetSelectValue(axis, cfg, baseCompilerProps.Lex.Flags) } (&compilerAttrs).bp2buildForAxisAndConfig(ctx, axis, cfg, baseCompilerProps) + aidlLibs.Append(android.BazelLabelForModuleDeps(ctx, baseCompilerProps.Aidl.Libs)) } var exportHdrs []string @@ -815,7 +818,14 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) (&linkerAttrs).wholeArchiveDeps.Add(protoDep.wholeStaticLib) (&linkerAttrs).implementationWholeArchiveDeps.Add(protoDep.implementationWholeStaticLib) - aidlDep := bp2buildCcAidlLibrary(ctx, module, compilerAttrs.aidlSrcs, linkerAttrs) + aidlDep := bp2buildCcAidlLibrary( + ctx, module, + compilerAttrs.aidlSrcs, + bazel.LabelListAttribute{ + Value: aidlLibs, + }, + linkerAttrs, + ) if aidlDep != nil { if lib, ok := module.linker.(*libraryDecorator); ok { if proptools.Bool(lib.Properties.Aidl.Export_aidl_headers) { @@ -912,11 +922,15 @@ func bp2buildFdoProfile( func bp2buildCcAidlLibrary( ctx android.Bp2buildMutatorContext, m *Module, - aidlLabelList bazel.LabelListAttribute, + aidlSrcs bazel.LabelListAttribute, + aidlLibs bazel.LabelListAttribute, linkerAttrs linkerAttributes, ) *bazel.LabelAttribute { - if !aidlLabelList.IsEmpty() { - aidlLibs, aidlSrcs := aidlLabelList.Partition(func(src bazel.Label) bool { + var aidlLibsFromSrcs, aidlFiles bazel.LabelListAttribute + apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) + + if !aidlSrcs.IsEmpty() { + aidlLibsFromSrcs, aidlFiles = aidlSrcs.Partition(func(src bazel.Label) bool { if fg, ok := android.ToFileGroupAsLibrary(ctx, src.OriginalModuleName); ok && fg.ShouldConvertToAidlLibrary(ctx) { return true @@ -924,55 +938,59 @@ func bp2buildCcAidlLibrary( return false }) - apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) - sdkAttrs := bp2BuildParseSdkAttributes(m) - - if !aidlSrcs.IsEmpty() { + if !aidlFiles.IsEmpty() { aidlLibName := m.Name() + "_aidl_library" ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: "aidl_library", Bzl_load_location: "//build/bazel/rules/aidl:aidl_library.bzl", }, - android.CommonAttributes{Name: aidlLibName}, - &aidlLibraryAttributes{ - Srcs: aidlSrcs, + android.CommonAttributes{ + Name: aidlLibName, Tags: apexAvailableTags, }, + &aidlLibraryAttributes{ + Srcs: aidlFiles, + }, ) - aidlLibs.Add(&bazel.LabelAttribute{Value: &bazel.Label{Label: ":" + aidlLibName}}) + aidlLibsFromSrcs.Add(&bazel.LabelAttribute{Value: &bazel.Label{Label: ":" + aidlLibName}}) } + } - if !aidlLibs.IsEmpty() { - ccAidlLibrarylabel := m.Name() + "_cc_aidl_library" - // Since parent cc_library already has these dependencies, we can add them as implementation - // deps so that they don't re-export - implementationDeps := linkerAttrs.deps.Clone() - implementationDeps.Append(linkerAttrs.implementationDeps) - implementationDynamicDeps := linkerAttrs.dynamicDeps.Clone() - implementationDynamicDeps.Append(linkerAttrs.implementationDynamicDeps) + allAidlLibs := aidlLibs.Clone() + allAidlLibs.Append(aidlLibsFromSrcs) - ctx.CreateBazelTargetModule( - bazel.BazelTargetModuleProperties{ - Rule_class: "cc_aidl_library", - Bzl_load_location: "//build/bazel/rules/cc:cc_aidl_library.bzl", - }, - android.CommonAttributes{Name: ccAidlLibrarylabel}, - &ccAidlLibraryAttributes{ - Deps: aidlLibs, - Implementation_deps: *implementationDeps, - Implementation_dynamic_deps: *implementationDynamicDeps, - Tags: apexAvailableTags, - sdkAttributes: sdkAttrs, - }, - ) - label := &bazel.LabelAttribute{ - Value: &bazel.Label{ - Label: ":" + ccAidlLibrarylabel, - }, - } - return label + if !allAidlLibs.IsEmpty() { + ccAidlLibrarylabel := m.Name() + "_cc_aidl_library" + // Since parent cc_library already has these dependencies, we can add them as implementation + // deps so that they don't re-export + implementationDeps := linkerAttrs.deps.Clone() + implementationDeps.Append(linkerAttrs.implementationDeps) + implementationDynamicDeps := linkerAttrs.dynamicDeps.Clone() + implementationDynamicDeps.Append(linkerAttrs.implementationDynamicDeps) + + sdkAttrs := bp2BuildParseSdkAttributes(m) + + ctx.CreateBazelTargetModule( + bazel.BazelTargetModuleProperties{ + Rule_class: "cc_aidl_library", + Bzl_load_location: "//build/bazel/rules/cc:cc_aidl_library.bzl", + }, + android.CommonAttributes{Name: ccAidlLibrarylabel}, + &ccAidlLibraryAttributes{ + Deps: *allAidlLibs, + Implementation_deps: *implementationDeps, + Implementation_dynamic_deps: *implementationDynamicDeps, + Tags: apexAvailableTags, + sdkAttributes: sdkAttrs, + }, + ) + label := &bazel.LabelAttribute{ + Value: &bazel.Label{ + Label: ":" + ccAidlLibrarylabel, + }, } + return label } return nil diff --git a/cc/cc.go b/cc/cc.go index 7237686e4c..f4b5655500 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -27,6 +27,7 @@ import ( "github.com/google/blueprint" "github.com/google/blueprint/proptools" + "android/soong/aidl_library" "android/soong/android" "android/soong/bazel/cquery" "android/soong/cc/config" @@ -110,6 +111,9 @@ type Deps struct { // Used by DepsMutator to pass system_shared_libs information to check_elf_file.py. SystemSharedLibs []string + // Used by DepMutator to pass aidl_library modules to aidl compiler + AidlLibs []string + // If true, statically link the unwinder into native libraries/binaries. StaticUnwinderIfLegacy bool @@ -182,6 +186,9 @@ type PathDeps struct { // For Darwin builds, the path to the second architecture's output that should // be combined with this architectures's output into a FAT MachO file. DarwinSecondArchOutput android.OptionalPath + + // Paths to direct srcs and transitive include dirs from direct aidl_library deps + AidlLibraryInfos []aidl_library.AidlLibraryInfo } // LocalOrGlobalFlags contains flags that need to have values set globally by the build system or locally by the module @@ -765,6 +772,7 @@ var ( stubImplDepTag = dependencyTag{name: "stub_impl"} JniFuzzLibTag = dependencyTag{name: "jni_fuzz_lib_tag"} FdoProfileTag = dependencyTag{name: "fdo_profile"} + aidlLibraryTag = dependencyTag{name: "aidl_library"} ) func IsSharedDepTag(depTag blueprint.DependencyTag) bool { @@ -2751,6 +2759,14 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { } } + if len(deps.AidlLibs) > 0 { + actx.AddDependency( + c, + aidlLibraryTag, + deps.AidlLibs..., + ) + } + updateImportedLibraryDependency(ctx) } @@ -3055,6 +3071,17 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { return } + if depTag == aidlLibraryTag { + if ctx.OtherModuleHasProvider(dep, aidl_library.AidlLibraryProvider) { + depPaths.AidlLibraryInfos = append( + depPaths.AidlLibraryInfos, + ctx.OtherModuleProvider( + dep, + aidl_library.AidlLibraryProvider).(aidl_library.AidlLibraryInfo), + ) + } + } + ccDep, ok := dep.(LinkableInterface) if !ok { diff --git a/cc/cc_test.go b/cc/cc_test.go index f9e661ff2b..d3d55e8ee8 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -24,6 +24,7 @@ import ( "strings" "testing" + "android/soong/aidl_library" "android/soong/android" "android/soong/bazel/cquery" ) @@ -4418,9 +4419,65 @@ func TestStubsLibReexportsHeaders(t *testing.T) { } } +func TestAidlLibraryWithHeader(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary, + android.MockFS{ + "package_bar/Android.bp": []byte(` + aidl_library { + name: "bar", + srcs: ["x/y/Bar.aidl"], + strip_import_prefix: "x", + } + `)}.AddToFixture(), + android.MockFS{ + "package_foo/Android.bp": []byte(` + aidl_library { + name: "foo", + srcs: ["a/b/Foo.aidl"], + strip_import_prefix: "a", + deps: ["bar"], + } + cc_library { + name: "libfoo", + aidl: { + libs: ["foo"], + } + } + `), + }.AddToFixture(), + ).RunTest(t).TestContext + + libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static") + manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl.sbox.textproto")) + aidlCommand := manifest.Commands[0].GetCommand() + + expectedAidlFlags := "-Ipackage_foo/a -Ipackage_bar/x" + if !strings.Contains(aidlCommand, expectedAidlFlags) { + t.Errorf("aidl command %q does not contain %q", aidlCommand, expectedAidlFlags) + } + + outputs := strings.Join(libfoo.AllOutputs(), " ") + + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BpFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BnFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/Foo.h") + android.AssertStringDoesContain(t, "aidl-generated cpp", outputs, "b/Foo.cpp") + // Confirm that the aidl header doesn't get compiled to cpp and h files + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BpBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BnBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/Bar.h") + android.AssertStringDoesNotContain(t, "aidl-generated cpp", outputs, "y/Bar.cpp") +} + func TestAidlFlagsPassedToTheAidlCompiler(t *testing.T) { t.Parallel() - ctx := testCc(t, ` + ctx := android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary, + ).RunTestWithBp(t, ` cc_library { name: "libfoo", srcs: ["a/Foo.aidl"], @@ -4705,7 +4762,15 @@ func TestIncludeDirsExporting(t *testing.T) { }) t.Run("ensure only aidl headers are exported", func(t *testing.T) { - ctx := testCc(t, genRuleModules+` + ctx := android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary, + ).RunTestWithBp(t, ` + aidl_library { + name: "libfoo_aidl", + srcs: ["x/y/Bar.aidl"], + strip_import_prefix: "x", + } cc_library_shared { name: "libfoo", srcs: [ @@ -4714,10 +4779,11 @@ func TestIncludeDirsExporting(t *testing.T) { "a.proto", ], aidl: { + libs: ["libfoo_aidl"], export_aidl_headers: true, } } - `) + `).TestContext foo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_shared").Module() checkIncludeDirs(t, ctx, foo, expectedIncludeDirs(` @@ -4728,11 +4794,17 @@ func TestIncludeDirsExporting(t *testing.T) { .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h `), expectedOrderOnlyDeps(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h `), ) }) diff --git a/cc/compiler.go b/cc/compiler.go index 88985b6f9c..5da745e027 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -120,6 +120,9 @@ type BaseCompilerProperties struct { Lex *LexProperties Aidl struct { + // List of aidl_library modules + Libs []string + // list of directories that will be added to the aidl include paths. Include_dirs []string @@ -272,6 +275,7 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { deps.GeneratedSources = append(deps.GeneratedSources, compiler.Properties.Generated_sources...) deps.GeneratedSources = removeListFromList(deps.GeneratedSources, compiler.Properties.Exclude_generated_sources) deps.GeneratedHeaders = append(deps.GeneratedHeaders, compiler.Properties.Generated_headers...) + deps.AidlLibs = append(deps.AidlLibs, compiler.Properties.Aidl.Libs...) android.ProtoDeps(ctx, &compiler.Proto) if compiler.hasSrcExt(".proto") { @@ -561,7 +565,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps "-I"+android.PathForModuleGen(ctx, "yacc", ctx.ModuleDir()).String()) } - if compiler.hasSrcExt(".aidl") { + if compiler.hasAidl(deps) { flags.aidlFlags = append(flags.aidlFlags, compiler.Properties.Aidl.Flags...) if len(compiler.Properties.Aidl.Local_include_dirs) > 0 { localAidlIncludeDirs := android.PathsForModuleSrc(ctx, compiler.Properties.Aidl.Local_include_dirs) @@ -572,6 +576,14 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps flags.aidlFlags = append(flags.aidlFlags, includeDirsToFlags(rootAidlIncludeDirs)) } + var rootAidlIncludeDirs android.Paths + for _, aidlLibraryInfo := range deps.AidlLibraryInfos { + rootAidlIncludeDirs = append(rootAidlIncludeDirs, aidlLibraryInfo.IncludeDirs.ToList()...) + } + if len(rootAidlIncludeDirs) > 0 { + flags.aidlFlags = append(flags.aidlFlags, includeDirsToFlags(rootAidlIncludeDirs)) + } + if proptools.BoolDefault(compiler.Properties.Aidl.Generate_traces, true) { flags.aidlFlags = append(flags.aidlFlags, "-t") } @@ -660,6 +672,10 @@ func ndkPathDeps(ctx ModuleContext) android.Paths { return nil } +func (compiler *baseCompiler) hasAidl(deps PathDeps) bool { + return len(deps.AidlLibraryInfos) > 0 || compiler.hasSrcExt(".aidl") +} + func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { pathDeps := deps.GeneratedDeps pathDeps = append(pathDeps, ndkPathDeps(ctx)...) @@ -668,7 +684,7 @@ func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathD srcs := append(android.Paths(nil), compiler.srcsBeforeGen...) - srcs, genDeps, info := genSources(ctx, srcs, buildFlags) + srcs, genDeps, info := genSources(ctx, deps.AidlLibraryInfos, srcs, buildFlags) pathDeps = append(pathDeps, genDeps...) compiler.pathDeps = pathDeps diff --git a/cc/gen.go b/cc/gen.go index dfbb177e67..dbb9560baf 100644 --- a/cc/gen.go +++ b/cc/gen.go @@ -18,7 +18,9 @@ import ( "path/filepath" "strings" + "android/soong/aidl_library" "android/soong/bazel" + "github.com/google/blueprint" "android/soong/android" @@ -124,11 +126,6 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr headerBn := outDir.Join(ctx, aidlPackage, "Bn"+shortName+".h") headerBp := outDir.Join(ctx, aidlPackage, "Bp"+shortName+".h") - baseDir := strings.TrimSuffix(aidlFile.String(), aidlFile.Rel()) - if baseDir != "" { - aidlFlags += " -I" + baseDir - } - cmd := rule.Command() cmd.BuiltTool("aidl-cpp"). FlagWithDepFile("-d", depFile). @@ -282,7 +279,10 @@ type generatedSourceInfo struct { syspropOrderOnlyDeps android.Paths } -func genSources(ctx android.ModuleContext, srcFiles android.Paths, +func genSources( + ctx android.ModuleContext, + aidlLibraryInfos []aidl_library.AidlLibraryInfo, + srcFiles android.Paths, buildFlags builderFlags) (android.Paths, android.Paths, generatedSourceInfo) { var info generatedSourceInfo @@ -330,7 +330,8 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths, aidlRule = android.NewRuleBuilder(pctx, ctx).Sbox(android.PathForModuleGen(ctx, "aidl"), android.PathForModuleGen(ctx, "aidl.sbox.textproto")) } - cppFile, aidlHeaders := genAidl(ctx, aidlRule, srcFile, buildFlags.aidlFlags) + baseDir := strings.TrimSuffix(srcFile.String(), srcFile.Rel()) + cppFile, aidlHeaders := genAidl(ctx, aidlRule, srcFile, buildFlags.aidlFlags+" -I"+baseDir) srcFiles[i] = cppFile info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) @@ -352,6 +353,24 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths, } } + for _, aidlLibraryInfo := range aidlLibraryInfos { + for _, aidlSrc := range aidlLibraryInfo.Srcs { + if aidlRule == nil { + // TODO(b/279960133): Sandbox inputs to ensure aidl headers are explicitly specified + aidlRule = android.NewRuleBuilder(pctx, ctx).Sbox(android.PathForModuleGen(ctx, "aidl"), + android.PathForModuleGen(ctx, "aidl.sbox.textproto")) + } + cppFile, aidlHeaders := genAidl(ctx, aidlRule, aidlSrc, buildFlags.aidlFlags) + + srcFiles = append(srcFiles, cppFile) + info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) + // Use the generated headers as order only deps to ensure that they are up to date when + // needed. + // TODO: Reduce the size of the ninja file by using one order only dep for the whole rule + info.aidlOrderOnlyDeps = append(info.aidlOrderOnlyDeps, aidlHeaders...) + } + } + if aidlRule != nil { aidlRule.Build("aidl", "gen aidl") } diff --git a/cc/library.go b/cc/library.go index 13b333a121..ad4688fe69 100644 --- a/cc/library.go +++ b/cc/library.go @@ -2114,7 +2114,7 @@ func (library *libraryDecorator) link(ctx ModuleContext, // Optionally export aidl headers. if Bool(library.Properties.Aidl.Export_aidl_headers) { - if library.baseCompiler.hasSrcExt(".aidl") { + if library.baseCompiler.hasAidl(deps) { dir := android.PathForModuleGen(ctx, "aidl") library.reexportDirs(dir) From 069010cc7442dbc8393f833a098a81be545b7c89 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 15 May 2023 17:21:47 -0400 Subject: [PATCH 0150/1460] don't require package_name for non-override android_apps For a non-override android_app, we can assume that the privapp_allowlist already contains the correct package_name, and so we don't need to overwrite it in this case. Bug: 242509786 Test: go test Ignore-AOSP-First: need to submit here to update PermissionController in sync with internal version Change-Id: I0f137e34cae3478dc8b9178d138121ff1d936f07 --- apex/apex_test.go | 3 +-- java/app.go | 10 +++++++++- java/app_test.go | 18 ++++++------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 6ca5afb84f..cd0c8a3baf 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6222,8 +6222,7 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, - privapp_allowlist: "perms.xml", - package_name: "com.android.AppFooPriv", + privapp_allowlist: "privapp_allowlist_com.android.AppFooPriv.xml", stl: "none", apex_available: [ "myapex" ], } diff --git a/java/app.go b/java/app.go index 3344647722..f9454edb69 100755 --- a/java/app.go +++ b/java/app.go @@ -620,13 +620,21 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } -func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) android.Path { if a.appProperties.Privapp_allowlist == nil { return nil } + + isOverrideApp := a.GetOverriddenBy() != "" + if !isOverrideApp { + // if this is not an override, we don't need to rewrite the existing privapp allowlist + return android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist) + } + if a.overridableAppProperties.Package_name == nil { ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") } + packageName := *a.overridableAppProperties.Package_name fileName := "privapp_allowlist_" + packageName + ".xml" outPath := android.PathForModuleOut(ctx, fileName).OutputPath diff --git a/java/app_test.go b/java/app_test.go index daff94ca9e..b250bc1b0c 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3555,9 +3555,8 @@ func TestPrivappAllowlist(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - privapp_allowlist: "perms.xml", + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", privileged: true, - package_name: "com.android.foo", sdk_version: "current", } override_android_app { @@ -3570,17 +3569,12 @@ func TestPrivappAllowlist(t *testing.T) { app := result.ModuleForTests("foo", "android_common") overrideApp := result.ModuleForTests("foo", "android_common_bar") - // verify that privapp allowlist is created - app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + // verify that privapp allowlist is created for override apps overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") - expectedAllowlist := "perms.xml" - actualAllowlist := app.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != actualAllowlist { - t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) - } - overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != overrideActualAllowlist { - t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + expectedAllowlistInput := "privapp_allowlist_com.android.foo.xml" + overrideActualAllowlistInput := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlistInput != overrideActualAllowlistInput { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlistInput, overrideActualAllowlistInput) } // verify that permissions are copied to device From 1281b99495b0cea56701750bd63b4c7cd8e074be Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Tue, 16 May 2023 01:15:24 +0000 Subject: [PATCH 0151/1460] Disable LTO and CFI for riscv64 This reverts commit 24870216e985226834b7b224e0b0d7bdf5d40dba. Reason for revert: b/282584808 Change-Id: I3808121b4437fdd01db5d305d8f0377e6ad7bd9e --- cc/lto.go | 5 +++++ cc/sanitize.go | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/cc/lto.go b/cc/lto.go index 581856b46e..1afa1ddd9c 100644 --- a/cc/lto.go +++ b/cc/lto.go @@ -91,6 +91,11 @@ func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { return flags } + // TODO(b/254713216): LTO doesn't work on riscv64 yet. + if ctx.Arch().ArchType == android.Riscv64 { + return flags + } + if lto.LTO(ctx) { var ltoCFlag string var ltoLdFlag string diff --git a/cc/sanitize.go b/cc/sanitize.go index df96b89c81..7fddc1b836 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -653,6 +653,12 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Integer_overflow = nil } + // TODO(b/254713216): CFI doesn't work for riscv64 yet because LTO doesn't work. + if ctx.Arch().ArchType == android.Riscv64 { + s.Cfi = nil + s.Diag.Cfi = nil + } + // Disable CFI for musl if ctx.toolchain().Musl() { s.Cfi = nil From 036ff9a5c190c1cc2d415bd306ebd8ba168779ae Mon Sep 17 00:00:00 2001 From: Nicolas Geoffray Date: Mon, 15 May 2023 10:46:38 +0100 Subject: [PATCH 0152/1460] Add support for missing public / private key files for apexes. Test: art/tools/buildbot-build.sh on a manifest that doesn't contain all dependencies. Change-Id: I8274bc0b616907032b51bc1b68c54888e7e58571 --- apex/apex.go | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index 026734eeec..ff7ee35c0a 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -404,7 +404,7 @@ type apexBundle struct { /////////////////////////////////////////////////////////////////////////////////////////// // Inputs - // Keys for apex_paylaod.img + // Keys for apex_payload.img publicKeyFile android.Path privateKeyFile android.Path @@ -2606,8 +2606,45 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.WalkDepsBlueprint(func(child, parent blueprint.Module) bool { return a.depVisitor(&vctx, ctx, child, parent) }) vctx.normalizeFileInfo(ctx) if a.privateKeyFile == nil { - ctx.PropertyErrorf("key", "private_key for %q could not be found", String(a.overridableProperties.Key)) - return + if ctx.Config().AllowMissingDependencies() { + // TODO(b/266099037): a better approach for slim manifests. + ctx.AddMissingDependencies([]string{String(a.overridableProperties.Key)}) + // Create placeholder paths for later stages that expect to see those paths, + // though they won't be used. + var unusedPath = android.PathForModuleOut(ctx, "nonexistentprivatekey") + ctx.Build(pctx, android.BuildParams{ + Rule: android.ErrorRule, + Output: unusedPath, + Args: map[string]string{ + "error": "Private key not available", + }, + }) + a.privateKeyFile = unusedPath + } else { + ctx.PropertyErrorf("key", "private_key for %q could not be found", String(a.overridableProperties.Key)) + return + } + } + + if a.publicKeyFile == nil { + if ctx.Config().AllowMissingDependencies() { + // TODO(b/266099037): a better approach for slim manifests. + ctx.AddMissingDependencies([]string{String(a.overridableProperties.Key)}) + // Create placeholder paths for later stages that expect to see those paths, + // though they won't be used. + var unusedPath = android.PathForModuleOut(ctx, "nonexistentpublickey") + ctx.Build(pctx, android.BuildParams{ + Rule: android.ErrorRule, + Output: unusedPath, + Args: map[string]string{ + "error": "Public key not available", + }, + }) + a.publicKeyFile = unusedPath + } else { + ctx.PropertyErrorf("key", "public_key for %q could not be found", String(a.overridableProperties.Key)) + return + } } //////////////////////////////////////////////////////////////////////////////////////////// From 5f108fa89577e88c41d5654f71ab04533f94bf98 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Thu, 11 May 2023 14:33:17 -0400 Subject: [PATCH 0153/1460] Update apex test to verify behavior Verify logic for transitive shared vs static deps to ensure in Bazel that these are handled similarly. Test: go test soong tests Change-Id: I015935fa4d7eb1ac40666d47a8de8e48150a043e --- apex/apex_test.go | 106 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index bf9c71b2e8..ed5eb2e9ad 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -1368,6 +1368,8 @@ func TestApexWithRuntimeLibsDependency(t *testing.T) { cc_library { name: "mylib", srcs: ["mylib.cpp"], + static_libs: ["libstatic"], + shared_libs: ["libshared"], runtime_libs: ["libfoo", "libbar"], system_shared_libs: [], stl: "none", @@ -1392,6 +1394,39 @@ func TestApexWithRuntimeLibsDependency(t *testing.T) { apex_available: [ "myapex" ], } + cc_library { + name: "libstatic", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + runtime_libs: ["libstatic_to_runtime"], + } + + cc_library { + name: "libshared", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + runtime_libs: ["libshared_to_runtime"], + } + + cc_library { + name: "libstatic_to_runtime", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } + + cc_library { + name: "libshared_to_runtime", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: [ "myapex" ], + } `) apexRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule") @@ -1405,11 +1440,14 @@ func TestApexWithRuntimeLibsDependency(t *testing.T) { // Ensure that runtime_libs dep in included ensureContains(t, copyCmds, "image.apex/lib64/libbar.so") + ensureContains(t, copyCmds, "image.apex/lib64/libshared.so") + ensureContains(t, copyCmds, "image.apex/lib64/libshared_to_runtime.so") + + ensureNotContains(t, copyCmds, "image.apex/lib64/libstatic_to_runtime.so") apexManifestRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexManifestRule") ensureListEmpty(t, names(apexManifestRule.Args["provideNativeLibs"])) ensureListContains(t, names(apexManifestRule.Args["requireNativeLibs"]), "libfoo.so") - } var prepareForTestOfRuntimeApexWithHwasan = android.GroupFixturePreparers( @@ -6533,6 +6571,72 @@ func TestApexAvailable_IndirectDep(t *testing.T) { }`) } +func TestApexAvailable_IndirectStaticDep(t *testing.T) { + testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + updatable: false, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + static_libs: ["libbar"], + system_shared_libs: [], + apex_available: ["myapex"], + } + + cc_library { + name: "libbar", + stl: "none", + shared_libs: ["libbaz"], + system_shared_libs: [], + apex_available: ["myapex"], + } + + cc_library { + name: "libbaz", + stl: "none", + system_shared_libs: [], + }`) + + testApexError(t, `requires "libbar" that doesn't list the APEX under 'apex_available'.`, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo"], + updatable: false, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "libfoo", + stl: "none", + static_libs: ["libbar"], + system_shared_libs: [], + apex_available: ["myapex"], + } + + cc_library { + name: "libbar", + stl: "none", + system_shared_libs: [], + }`) +} + func TestApexAvailable_InvalidApexName(t *testing.T) { testApexError(t, "\"otherapex\" is not a valid module name", ` apex { From d20214f6fb4b341fb91c27c908f80a4e13dcb79c Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Tue, 16 May 2023 00:43:18 +0000 Subject: [PATCH 0154/1460] apex: drop metadata gathering for multitree. This was only partially implemented, and is not needed. Test: presubmits Change-Id: Ia6534c62051cd484d46333fbbf8a0fa8ce8288e7 --- apex/Android.bp | 2 - apex/metadata.go | 99 ---------------------------------------- apex/metadata_test.go | 103 ------------------------------------------ 3 files changed, 204 deletions(-) delete mode 100644 apex/metadata.go delete mode 100644 apex/metadata_test.go diff --git a/apex/Android.bp b/apex/Android.bp index 61d7fb2e2e..0791497374 100644 --- a/apex/Android.bp +++ b/apex/Android.bp @@ -29,7 +29,6 @@ bootstrap_go_package { "bp2build.go", "deapexer.go", "key.go", - "metadata.go", "prebuilt.go", "testing.go", "vndk.go", @@ -39,7 +38,6 @@ bootstrap_go_package { "bootclasspath_fragment_test.go", "classpath_element_test.go", "dexpreopt_bootjars_test.go", - "metadata_test.go", "platform_bootclasspath_test.go", "systemserver_classpath_fragment_test.go", "vndk_test.go", diff --git a/apex/metadata.go b/apex/metadata.go deleted file mode 100644 index b1dff3e3f5..0000000000 --- a/apex/metadata.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package apex - -import ( - "encoding/json" - - "github.com/google/blueprint" - - "android/soong/android" -) - -var ( - mtctx = android.NewPackageContext("android/soong/multitree_apex") -) - -func init() { - RegisterModulesSingleton(android.InitRegistrationContext) -} - -func RegisterModulesSingleton(ctx android.RegistrationContext) { - ctx.RegisterSingletonType("apex_multitree_singleton", multitreeAnalysisSingletonFactory) -} - -var PrepareForTestWithApexMultitreeSingleton = android.FixtureRegisterWithContext(RegisterModulesSingleton) - -func multitreeAnalysisSingletonFactory() android.Singleton { - return &multitreeAnalysisSingleton{} -} - -type multitreeAnalysisSingleton struct { - multitreeApexMetadataPath android.OutputPath -} - -type ApexMultitreeMetadataEntry struct { - // The name of the apex. - Name string - - // TODO: Add other properties as needed. -} - -type ApexMultitreeMetadata struct { - // Information about the installable apexes. - Apexes map[string]ApexMultitreeMetadataEntry -} - -func (p *multitreeAnalysisSingleton) GenerateBuildActions(context android.SingletonContext) { - data := ApexMultitreeMetadata{ - Apexes: make(map[string]ApexMultitreeMetadataEntry, 0), - } - context.VisitAllModules(func(module android.Module) { - // If this module is not being installed, ignore it. - if !module.Enabled() || module.IsSkipInstall() { - return - } - // Actual apexes provide ApexBundleInfoProvider. - if _, ok := context.ModuleProvider(module, ApexBundleInfoProvider).(ApexBundleInfo); !ok { - return - } - bundle, ok := module.(*apexBundle) - if ok && !bundle.testApex && !bundle.vndkApex && bundle.primaryApexType { - name := module.Name() - entry := ApexMultitreeMetadataEntry{ - Name: name, - } - data.Apexes[name] = entry - } - }) - p.multitreeApexMetadataPath = android.PathForOutput(context, "multitree_apex_metadata.json") - - jsonStr, err := json.Marshal(data) - if err != nil { - context.Errorf(err.Error()) - } - android.WriteFileRule(context, p.multitreeApexMetadataPath, string(jsonStr)) - // This seems cleaner, but doesn't emit the phony rule in testing. - // context.Phony("multitree_apex_metadata", p.multitreeApexMetadataPath) - - context.Build(mtctx, android.BuildParams{ - Rule: blueprint.Phony, - Description: "phony rule for multitree_apex_metadata", - Inputs: []android.Path{p.multitreeApexMetadataPath}, - Output: android.PathForPhony(context, "multitree_apex_metadata"), - }) -} diff --git a/apex/metadata_test.go b/apex/metadata_test.go deleted file mode 100644 index fed5beae7a..0000000000 --- a/apex/metadata_test.go +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package apex - -import ( - "strings" - "testing" - - "android/soong/android" - "android/soong/java" -) - -func TestModulesSingleton(t *testing.T) { - result := android.GroupFixturePreparers( - PrepareForTestWithApexMultitreeSingleton, - java.PrepareForTestWithJavaDefaultModules, - PrepareForTestWithApexBuildComponents, - java.FixtureConfigureApexBootJars("myapex:foo"), - java.PrepareForTestWithJavaSdkLibraryFiles, - ).RunTestWithBp(t, ` - prebuilt_apex { - name: "myapex", - src: "myapex.apex", - exported_bootclasspath_fragments: ["mybootclasspath-fragment"], - } - - // A prebuilt java_sdk_library_import that is not preferred by default but will be preferred - // because AlwaysUsePrebuiltSdks() is true. - java_sdk_library_import { - name: "foo", - prefer: false, - shared_library: false, - permitted_packages: ["foo"], - public: { - jars: ["sdk_library/public/foo-stubs.jar"], - stub_srcs: ["sdk_library/public/foo_stub_sources"], - current_api: "sdk_library/public/foo.txt", - removed_api: "sdk_library/public/foo-removed.txt", - sdk_version: "current", - }, - apex_available: ["myapex"], - } - - prebuilt_bootclasspath_fragment { - name: "mybootclasspath-fragment", - apex_available: [ - "myapex", - ], - contents: [ - "foo", - ], - hidden_api: { - stub_flags: "prebuilt-stub-flags.csv", - annotation_flags: "prebuilt-annotation-flags.csv", - metadata: "prebuilt-metadata.csv", - index: "prebuilt-index.csv", - all_flags: "prebuilt-all-flags.csv", - }, - } - - platform_bootclasspath { - name: "myplatform-bootclasspath", - fragments: [ - { - apex: "myapex", - module:"mybootclasspath-fragment", - }, - ], - } -`, - ) - - outputs := result.SingletonForTests("apex_multitree_singleton").AllOutputs() - for _, output := range outputs { - testingBuildParam := result.SingletonForTests("apex_multitree_singleton").Output(output) - switch { - case strings.Contains(output, "soong/multitree_apex_metadata.json"): - android.AssertStringEquals(t, "Invalid build rule", "android/soong/android.writeFile", testingBuildParam.Rule.String()) - android.AssertIntEquals(t, "Invalid input", len(testingBuildParam.Inputs), 0) - android.AssertStringDoesContain(t, "Invalid output path", output, "soong/multitree_apex_metadata.json") - - case strings.HasSuffix(output, "multitree_apex_metadata"): - android.AssertStringEquals(t, "Invalid build rule", ":phony", testingBuildParam.Rule.String()) - android.AssertStringEquals(t, "Invalid input", testingBuildParam.Inputs[0].String(), "out/soong/multitree_apex_metadata.json") - android.AssertStringEquals(t, "Invalid output path", output, "multitree_apex_metadata") - android.AssertIntEquals(t, "Invalid args", len(testingBuildParam.Args), 0) - } - } -} From e684294fb2f0407cd2e0cbfd8663542d3185b570 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 0155/1460] Pass includes attrs to cc_aidl_library https://source.android.com/docs/core/architecture/aidl/aidl-backends#custom-parcelables Some aidl files have custom parcelables with references to cpp headers. Hence, we need to pass the include dirs from cc libs to cc_aidl_library so that custom cpp headers are findable by the generated cpp/h files. Test: go test Bug: 278704136 Change-Id: I4ede1d23fbc04bdbfb69823d955c6df6b0a7b4ee --- bp2build/cc_library_conversion_test.go | 19 ++++-- cc/bp2build.go | 12 ++++ cc/cc_test.go | 1 + cc/library.go | 93 +++++++++++++------------- 4 files changed, 75 insertions(+), 50 deletions(-) diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 8c090d881a..5f3ff6217d 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -3330,7 +3330,8 @@ cc_library { name: "foo", aidl: { libs: ["A_aidl"], - } + }, + export_include_dirs: ["include"], }`, ExpectedBazelTargets: []string{ MakeBazelTargetNoRestrictions("aidl_library", "A_aidl", AttrNameToString{ @@ -3340,15 +3341,19 @@ cc_library { "tags": `["apex_available=//apex_available:anyapex"]`, }), MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `[":A_aidl"]`, + "deps": `[":A_aidl"]`, + "local_includes": `["."]`, + "export_includes": `["include"]`, }), MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, "local_includes": `["."]`, + "export_includes": `["include"]`, }), MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, "local_includes": `["."]`, + "export_includes": `["include"]`, }), }, }) @@ -3382,6 +3387,7 @@ cc_library { "srcs": `["B.aidl"]`, }), MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ + "local_includes": `["."]`, "deps": `[ ":A_aidl", ":foo_aidl_library", @@ -3421,7 +3427,8 @@ cc_library { }`, ExpectedBazelTargets: []string{ MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `["//path/to/A:A_aidl"]`, + "local_includes": `["."]`, + "deps": `["//path/to/A:A_aidl"]`, }), MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ "implementation_whole_archive_deps": `[":foo_cc_aidl_library"]`, @@ -3440,7 +3447,8 @@ func TestCcLibraryWithExportAidlHeaders(t *testing.T) { expectedBazelTargets := []string{ MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `[":foo_aidl_library"]`, + "local_includes": `["."]`, + "deps": `[":foo_aidl_library"]`, }), MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ "whole_archive_deps": `[":foo_cc_aidl_library"]`, @@ -3721,7 +3729,8 @@ cc_library_static { "srcs": `["Foo.aidl"]`, }), MakeBazelTarget("cc_aidl_library", "foo_cc_aidl_library", AttrNameToString{ - "deps": `[":foo_aidl_library"]`, + "local_includes": `["."]`, + "deps": `[":foo_aidl_library"]`, "implementation_deps": `[ ":baz-static", ":bar-static", diff --git a/cc/bp2build.go b/cc/bp2build.go index 1980ab9713..202eebce16 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -844,6 +844,7 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) Value: aidlLibs, }, linkerAttrs, + compilerAttrs, ) if aidlDep != nil { if lib, ok := module.linker.(*libraryDecorator); ok { @@ -992,6 +993,7 @@ func bp2buildCcAidlLibrary( aidlSrcs bazel.LabelListAttribute, aidlLibs bazel.LabelListAttribute, linkerAttrs linkerAttributes, + compilerAttrs compilerAttributes, ) *bazel.LabelAttribute { var aidlLibsFromSrcs, aidlFiles bazel.LabelListAttribute apexAvailableTags := android.ApexAvailableTagsWithoutTestApexes(ctx.(android.TopDownMutatorContext), ctx.Module()) @@ -1038,6 +1040,15 @@ func bp2buildCcAidlLibrary( sdkAttrs := bp2BuildParseSdkAttributes(m) + exportedIncludes := bp2BuildParseExportedIncludes(ctx, m, &compilerAttrs.includes) + includeAttrs := includesAttributes{ + Export_includes: exportedIncludes.Includes, + Export_absolute_includes: exportedIncludes.AbsoluteIncludes, + Export_system_includes: exportedIncludes.SystemIncludes, + Local_includes: compilerAttrs.localIncludes, + Absolute_includes: compilerAttrs.absoluteIncludes, + } + ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: "cc_aidl_library", @@ -1050,6 +1061,7 @@ func bp2buildCcAidlLibrary( Implementation_dynamic_deps: *implementationDynamicDeps, Tags: apexAvailableTags, sdkAttributes: sdkAttrs, + includesAttributes: includeAttrs, }, ) label := &bazel.LabelAttribute{ diff --git a/cc/cc_test.go b/cc/cc_test.go index d3d55e8ee8..422df73106 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -39,6 +39,7 @@ func TestMain(m *testing.M) { var prepareForCcTest = android.GroupFixturePreparers( PrepareForTestWithCcIncludeVndk, + aidl_library.PrepareForTestWithAidlLibrary, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { variables.DeviceVndkVersion = StringPtr("current") variables.ProductVndkVersion = StringPtr("current") diff --git a/cc/library.go b/cc/library.go index ad4688fe69..09a7253abb 100644 --- a/cc/library.go +++ b/cc/library.go @@ -271,7 +271,9 @@ type ccAidlLibraryAttributes struct { Implementation_deps bazel.LabelListAttribute Implementation_dynamic_deps bazel.LabelListAttribute Tags bazel.StringListAttribute + sdkAttributes + includesAttributes } type stripAttributes struct { @@ -335,6 +337,14 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { Native_coverage: baseAttributes.Native_coverage, } + includeAttrs := includesAttributes{ + Export_includes: exportedIncludes.Includes, + Export_absolute_includes: exportedIncludes.AbsoluteIncludes, + Export_system_includes: exportedIncludes.SystemIncludes, + Local_includes: compilerAttrs.localIncludes, + Absolute_includes: compilerAttrs.absoluteIncludes, + } + sharedCommonAttrs := staticOrSharedAttributes{ Srcs: *srcs.Clone().Append(sharedAttrs.Srcs), Srcs_c: *compilerAttrs.cSrcs.Clone().Append(sharedAttrs.Srcs_c), @@ -356,41 +366,34 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { staticTargetAttrs := &bazelCcLibraryStaticAttributes{ staticOrSharedAttributes: staticCommonAttrs, + includesAttributes: includeAttrs, Cppflags: compilerAttrs.cppFlags, Conlyflags: compilerAttrs.conlyFlags, Asflags: asFlags, - Export_includes: exportedIncludes.Includes, - Export_absolute_includes: exportedIncludes.AbsoluteIncludes, - Export_system_includes: exportedIncludes.SystemIncludes, - Local_includes: compilerAttrs.localIncludes, - Absolute_includes: compilerAttrs.absoluteIncludes, - Rtti: compilerAttrs.rtti, - Stl: compilerAttrs.stl, - Cpp_std: compilerAttrs.cppStd, - C_std: compilerAttrs.cStd, + Rtti: compilerAttrs.rtti, + Stl: compilerAttrs.stl, + Cpp_std: compilerAttrs.cppStd, + C_std: compilerAttrs.cStd, Features: *staticFeatures, } sharedTargetAttrs := &bazelCcLibrarySharedAttributes{ staticOrSharedAttributes: sharedCommonAttrs, - Cppflags: compilerAttrs.cppFlags, - Conlyflags: compilerAttrs.conlyFlags, - Asflags: asFlags, + includesAttributes: includeAttrs, - Export_includes: exportedIncludes.Includes, - Export_absolute_includes: exportedIncludes.AbsoluteIncludes, - Export_system_includes: exportedIncludes.SystemIncludes, - Local_includes: compilerAttrs.localIncludes, - Absolute_includes: compilerAttrs.absoluteIncludes, - Linkopts: linkerAttrs.linkopts, - Rtti: compilerAttrs.rtti, - Stl: compilerAttrs.stl, - Cpp_std: compilerAttrs.cppStd, - C_std: compilerAttrs.cStd, - Use_version_lib: linkerAttrs.useVersionLib, + Cppflags: compilerAttrs.cppFlags, + Conlyflags: compilerAttrs.conlyFlags, + Asflags: asFlags, + + Linkopts: linkerAttrs.linkopts, + Rtti: compilerAttrs.rtti, + Stl: compilerAttrs.stl, + Cpp_std: compilerAttrs.cppStd, + C_std: compilerAttrs.cStd, + Use_version_lib: linkerAttrs.useVersionLib, Additional_linker_inputs: linkerAttrs.additionalLinkerInputs, @@ -2880,6 +2883,13 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo linkerAttrs := baseAttributes.linkerAttributes exportedIncludes := bp2BuildParseExportedIncludes(ctx, module, &compilerAttrs.includes) + includeAttrs := includesAttributes{ + Export_includes: exportedIncludes.Includes, + Export_absolute_includes: exportedIncludes.AbsoluteIncludes, + Export_system_includes: exportedIncludes.SystemIncludes, + Local_includes: compilerAttrs.localIncludes, + Absolute_includes: compilerAttrs.absoluteIncludes, + } // Append shared/static{} stanza properties. These won't be specified on // cc_library_* itself, but may be specified in cc_defaults that this module @@ -2937,11 +2947,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Cpp_std: compilerAttrs.cppStd, C_std: compilerAttrs.cStd, - Export_includes: exportedIncludes.Includes, - Export_absolute_includes: exportedIncludes.AbsoluteIncludes, - Export_system_includes: exportedIncludes.SystemIncludes, - Local_includes: compilerAttrs.localIncludes, - Absolute_includes: compilerAttrs.absoluteIncludes, + includesAttributes: includeAttrs, Cppflags: compilerAttrs.cppFlags, Conlyflags: compilerAttrs.conlyFlags, @@ -2967,11 +2973,8 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Cpp_std: compilerAttrs.cppStd, C_std: compilerAttrs.cStd, - Export_includes: exportedIncludes.Includes, - Export_absolute_includes: exportedIncludes.AbsoluteIncludes, - Export_system_includes: exportedIncludes.SystemIncludes, - Local_includes: compilerAttrs.localIncludes, - Absolute_includes: compilerAttrs.absoluteIncludes, + includesAttributes: includeAttrs, + Additional_linker_inputs: linkerAttrs.additionalLinkerInputs, Strip: stripAttrsFromLinkerAttrs(&linkerAttrs), @@ -3007,9 +3010,18 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name(), Tags: tags}, attrs) } +type includesAttributes struct { + Export_includes bazel.StringListAttribute + Export_absolute_includes bazel.StringListAttribute + Export_system_includes bazel.StringListAttribute + Local_includes bazel.StringListAttribute + Absolute_includes bazel.StringListAttribute +} + // TODO(b/199902614): Can this be factored to share with the other Attributes? type bazelCcLibraryStaticAttributes struct { staticOrSharedAttributes + includesAttributes Use_version_lib bazel.BoolAttribute Rtti bazel.BoolAttribute @@ -3017,12 +3029,7 @@ type bazelCcLibraryStaticAttributes struct { Cpp_std *string C_std *string - Export_includes bazel.StringListAttribute - Export_absolute_includes bazel.StringListAttribute - Export_system_includes bazel.StringListAttribute - Local_includes bazel.StringListAttribute - Absolute_includes bazel.StringListAttribute - Hdrs bazel.LabelListAttribute + Hdrs bazel.LabelListAttribute Cppflags bazel.StringListAttribute Conlyflags bazel.StringListAttribute @@ -3034,6 +3041,7 @@ type bazelCcLibraryStaticAttributes struct { // TODO(b/199902614): Can this be factored to share with the other Attributes? type bazelCcLibrarySharedAttributes struct { staticOrSharedAttributes + includesAttributes Linkopts bazel.StringListAttribute Use_version_lib bazel.BoolAttribute @@ -3043,12 +3051,7 @@ type bazelCcLibrarySharedAttributes struct { Cpp_std *string C_std *string - Export_includes bazel.StringListAttribute - Export_absolute_includes bazel.StringListAttribute - Export_system_includes bazel.StringListAttribute - Local_includes bazel.StringListAttribute - Absolute_includes bazel.StringListAttribute - Hdrs bazel.LabelListAttribute + Hdrs bazel.LabelListAttribute Strip stripAttributes Additional_linker_inputs bazel.LabelListAttribute From 320aecfbc81a545d3a7c5f84b74118cef29b5cbc Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Fri, 28 Apr 2023 11:21:25 -0400 Subject: [PATCH 0156/1460] Allowlist libservices Test: CI Change-Id: I6e7a969228ca491344520bba0d66159d5060e996 --- android/allowlists/allowlists.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 5b7f117894..704eda43e7 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -211,6 +211,7 @@ var ( "frameworks/av/media/module/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/av/services/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/base/libs/androidfw": Bp2BuildDefaultTrue, + "frameworks/base/libs/services": Bp2BuildDefaultTrue, "frameworks/base/media/tests/MediaDump": Bp2BuildDefaultTrue, "frameworks/base/proto": Bp2BuildDefaultTrue, "frameworks/base/services/tests/servicestests/aidl": Bp2BuildDefaultTrue, @@ -570,6 +571,9 @@ var ( //external/fec "libfec_rs", + //frameworks/base/core/java + "IDropBoxManagerService_aidl", + //system/core/libsparse "libsparse", From fb04c41299fd1751da2e5c066f8a08130b9e0e7a Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Mon, 15 May 2023 18:35:36 +0000 Subject: [PATCH 0157/1460] bp2build support for host_ldlibs host_ldlibs are used during linking. Set these in `linkopts` for now. Note that this CL does not do the `CheckBadHostLdLibs` validation of Soong. There are some different ways to do this, and these are being discussed in b/216626461. It is likely that we will need to create a new property `host_ldlibs` to do the validation. But for now, re-use `linkopts`. Bug: 216626461 Test: bp2build unit tests Change-Id: Id4c77e4460fb1fb003fa58ea27bab5b50ea8cefe --- bazel/configurability.go | 24 +++++++------- bp2build/cc_library_conversion_test.go | 46 ++++++++++++++++++++++++++ cc/bp2build.go | 3 ++ 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/bazel/configurability.go b/bazel/configurability.go index d042fe8b8e..8f63ec45bf 100644 --- a/bazel/configurability.go +++ b/bazel/configurability.go @@ -31,11 +31,11 @@ const ( // OsType names in arch.go OsAndroid = "android" - osDarwin = "darwin" - osLinux = "linux_glibc" + OsDarwin = "darwin" + OsLinux = "linux_glibc" osLinuxMusl = "linux_musl" osLinuxBionic = "linux_bionic" - osWindows = "windows" + OsWindows = "windows" // Targets in arch.go osArchAndroidArm = "android_arm" @@ -156,11 +156,11 @@ var ( // constraint_value for the @platforms//os:os constraint_setting platformOsMap = map[string]string{ OsAndroid: "//build/bazel/platforms/os:android", - osDarwin: "//build/bazel/platforms/os:darwin", - osLinux: "//build/bazel/platforms/os:linux_glibc", + OsDarwin: "//build/bazel/platforms/os:darwin", + OsLinux: "//build/bazel/platforms/os:linux_glibc", osLinuxMusl: "//build/bazel/platforms/os:linux_musl", osLinuxBionic: "//build/bazel/platforms/os:linux_bionic", - osWindows: "//build/bazel/platforms/os:windows", + OsWindows: "//build/bazel/platforms/os:windows", ConditionsDefaultConfigKey: ConditionsDefaultSelectKey, // The default condition of an os select map. } @@ -192,22 +192,22 @@ var ( // in a cyclic dependency. osToArchMap = map[string][]string{ OsAndroid: {archArm, archArm64, archRiscv64, archX86, archX86_64}, - osLinux: {archX86, archX86_64}, + OsLinux: {archX86, archX86_64}, osLinuxMusl: {archX86, archX86_64}, - osDarwin: {archArm64, archX86_64}, + OsDarwin: {archArm64, archX86_64}, osLinuxBionic: {archArm64, archX86_64}, // TODO(cparsons): According to arch.go, this should contain archArm, archArm64, as well. - osWindows: {archX86, archX86_64}, + OsWindows: {archX86, archX86_64}, } osAndInApexMap = map[string]string{ AndroidAndInApex: "//build/bazel/rules/apex:android-in_apex", AndroidPlatform: "//build/bazel/rules/apex:system", - osDarwin: "//build/bazel/platforms/os:darwin", - osLinux: "//build/bazel/platforms/os:linux_glibc", + OsDarwin: "//build/bazel/platforms/os:darwin", + OsLinux: "//build/bazel/platforms/os:linux_glibc", osLinuxMusl: "//build/bazel/platforms/os:linux_musl", osLinuxBionic: "//build/bazel/platforms/os:linux_bionic", - osWindows: "//build/bazel/platforms/os:windows", + OsWindows: "//build/bazel/platforms/os:windows", ConditionsDefaultConfigKey: ConditionsDefaultSelectKey, } diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 17b1608610..7cccec1a44 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -4657,3 +4657,49 @@ cc_library { }, }) } + +func TestCcLibraryHostLdLibs(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_binary linker flags for host_ldlibs", + ModuleTypeUnderTest: "cc_binary", + ModuleTypeUnderTestFactory: cc.BinaryFactory, + Blueprint: soongCcLibraryPreamble + `cc_binary { + name: "a", + host_supported: true, + ldflags: ["-lcommon"], + target: { + linux: { + host_ldlibs: [ + "-llinux", + ], + }, + darwin: { + ldflags: ["-ldarwinadditional"], + host_ldlibs: [ + "-ldarwin", + ], + }, + windows: { + host_ldlibs: [ + "-lwindows", + ], + }, + }, +} +`, + ExpectedBazelTargets: []string{ + MakeBazelTargetNoRestrictions("cc_binary", "a", AttrNameToString{ + "linkopts": `["-lcommon"] + select({ + "//build/bazel/platforms/os:darwin": [ + "-ldarwinadditional", + "-ldarwin", + ], + "//build/bazel/platforms/os:linux_glibc": ["-llinux"], + "//build/bazel/platforms/os:windows": ["-lwindows"], + "//conditions:default": [], + })`, + "local_includes": `["."]`, + }), + }, + }) +} diff --git a/cc/bp2build.go b/cc/bp2build.go index 29e8494773..5029541aae 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1254,6 +1254,9 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion } la.additionalLinkerInputs.SetSelectValue(axis, config, additionalLinkerInputs) + if axis == bazel.OsConfigurationAxis && (config == bazel.OsDarwin || config == bazel.OsLinux || config == bazel.OsWindows) { + linkerFlags = append(linkerFlags, props.Host_ldlibs...) + } la.linkopts.SetSelectValue(axis, config, linkerFlags) if axisFeatures != nil { From b3d5c48cade01a9f8be8dcdb75875ef8e380efb7 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Mon, 15 May 2023 18:49:23 +0000 Subject: [PATCH 0158/1460] Allowlist python dependency of adb targets Bug: 216626461 Test: TH Change-Id: I4f16ce0037dd7afdf194e210ce63bfc60bd73cc1 --- android/allowlists/allowlists.go | 1 + 1 file changed, 1 insertion(+) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 5b7f117894..17785609f7 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -85,6 +85,7 @@ var ( "development/apps/DevelopmentSettings": Bp2BuildDefaultTrue, "development/apps/Fallback": Bp2BuildDefaultTrue, "development/apps/WidgetPreview": Bp2BuildDefaultTrue, + "development/python-packages/adb": Bp2BuildDefaultTrueRecursively, "development/samples/BasicGLSurfaceView": Bp2BuildDefaultTrue, "development/samples/BluetoothChat": Bp2BuildDefaultTrue, "development/samples/BrokenKeyDerivation": Bp2BuildDefaultTrue, From 795319fadb84b4e65c9fb3defbaf25b612acd53b Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Wed, 17 May 2023 00:23:38 +0000 Subject: [PATCH 0159/1460] Disallow missing api source files for java_api_library module The purpose of this change is to enable filegroups to be taken as inputs for `java_api_library` module `api_files` property. However, since android.MaybeExistentPathForSource() does not support this, it needs to be replaced with android.PathForModuleSrc(), which checks for the files' existence. Bug: 283006953 Test: go test ./java Change-Id: I8a7d7f200f900219cc17243194a4c26071329ee6 --- java/java.go | 6 ++---- java/java_test.go | 23 +++++++++++++++++++++++ java/testing.go | 7 ++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/java/java.go b/java/java.go index a98762cf11..44ff715b46 100644 --- a/java/java.go +++ b/java/java.go @@ -1651,7 +1651,7 @@ type JavaApiLibraryProperties struct { // list of api.txt files relative to this directory that contribute to the // API surface. // This is a list of relative paths - Api_files []string + Api_files []string `android:"path"` // List of flags to be passed to the javac compiler to generate jar file Javacflags []string @@ -1833,9 +1833,7 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Add the api_files inputs for _, api := range al.properties.Api_files { - // Use MaybeExistentPathForSource since the api file might not exist during analysis. - // This will be provided by the orchestrator in the combined execution. - srcFiles = append(srcFiles, android.MaybeExistentPathForSource(ctx, ctx.ModuleDir(), api)) + srcFiles = append(srcFiles, android.PathForModuleSrc(ctx, api)) } if srcFiles == nil { diff --git a/java/java_test.go b/java/java_test.go index 2a4913ecd3..ea89e6eb8f 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -2252,6 +2252,29 @@ func TestJavaApiLibraryDepApiSrcs(t *testing.T) { android.AssertStringDoesContain(t, "Command expected to contain output files list text file flag", manifestCommand, "--out __SBOX_SANDBOX_DIR__/out/sources.txt") } +func TestJavaApiLibraryFilegroupInput(t *testing.T) { + ctx, _ := testJavaWithFS(t, ` + filegroup { + name: "default_current.txt", + srcs: ["current.txt"], + } + + java_api_library { + name: "foo", + api_files: [":default_current.txt"], + } + `, + map[string][]byte{ + "current.txt": nil, + }) + + m := ctx.ModuleForTests("foo", "android_common") + outputs := fmt.Sprint(m.AllOutputs()) + if !strings.Contains(outputs, "foo/foo.jar") { + t.Errorf("Module output does not contain expected jar %s", "foo/foo.jar") + } +} + func TestTradefedOptions(t *testing.T) { result := PrepareForTestWithJavaBuildComponents.RunTestWithBp(t, ` java_test_host { diff --git a/java/testing.go b/java/testing.go index 6671bf0c72..ffc3a086f2 100644 --- a/java/testing.go +++ b/java/testing.go @@ -71,7 +71,12 @@ var prepareForTestWithFrameworkDeps = android.GroupFixturePreparers( // Needed for framework defaultJavaDir + "/framework/aidl": nil, // Needed for various deps defined in GatherRequiredDepsForTest() - defaultJavaDir + "/a.java": nil, + defaultJavaDir + "/a.java": nil, + defaultJavaDir + "/api/current.txt": nil, + defaultJavaDir + "/api/system-current.txt": nil, + defaultJavaDir + "/api/test-current.txt": nil, + defaultJavaDir + "/api/module-lib-current.txt": nil, + defaultJavaDir + "/api/system-server-current.txt": nil, // Needed for R8 rules on apps "build/make/core/proguard.flags": nil, From efc36678a94785754153a3524d1dfc8a84ffccc7 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Tue, 16 May 2023 21:26:45 +0000 Subject: [PATCH 0160/1460] Enable LTO for riscv64 This partially reverts commit 1281b99495b0cea56701750bd63b4c7cd8e074be. Bug: b/282584808, b/254713216 Reason for revert: LTO can be enabled as emultated tls is now globally disabled https://android-review.git.corp.google.com/c/platform/build/soong/+/2431153 Change-Id: I66c83b1116550179043490a11f66b8630c0e6b1b --- cc/lto.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cc/lto.go b/cc/lto.go index 1afa1ddd9c..581856b46e 100644 --- a/cc/lto.go +++ b/cc/lto.go @@ -91,11 +91,6 @@ func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { return flags } - // TODO(b/254713216): LTO doesn't work on riscv64 yet. - if ctx.Arch().ArchType == android.Riscv64 { - return flags - } - if lto.LTO(ctx) { var ltoCFlag string var ltoLdFlag string From eaf7f5ecb98ec34b1615ed289e369cef9f5bec2c Mon Sep 17 00:00:00 2001 From: Cory Barker Date: Fri, 3 Feb 2023 00:20:52 +0000 Subject: [PATCH 0161/1460] Fix Soong code, remove dead code, and clean up code for Java fuzz (cherry picked from commit fdf043a7b109250447f7c89bbeb29a1726a4d602) Test: built fuzz targets locally and tested them with Jazzer Change-Id: I3ff487ba4e34289e0f53f2077463f524f4bf4f11 --- java/androidmk.go | 10 ++- java/fuzz.go | 206 +++++++++++++--------------------------------- java/fuzz_test.go | 10 +-- java/java.go | 1 - 4 files changed, 71 insertions(+), 156 deletions(-) diff --git a/java/androidmk.go b/java/androidmk.go index 148d7c2769..9c21633fb1 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -133,13 +133,19 @@ func (library *Library) AndroidMkEntries() []android.AndroidMkEntries { return entriesList } -func (j *JavaFuzzLibrary) AndroidMkEntries() []android.AndroidMkEntries { +func (j *JavaFuzzTest) AndroidMkEntries() []android.AndroidMkEntries { entriesList := j.Library.AndroidMkEntries() entries := &entriesList[0] entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", "null-suite") - androidMkWriteTestData(j.jniFilePaths, entries) androidMkWriteTestData(android.Paths{j.implementationJarFile}, entries) + androidMkWriteTestData(j.jniFilePaths, entries) + if j.fuzzPackagedModule.Corpus != nil { + androidMkWriteTestData(j.fuzzPackagedModule.Corpus, entries) + } + if j.fuzzPackagedModule.Dictionary != nil { + androidMkWriteTestData(android.Paths{j.fuzzPackagedModule.Dictionary}, entries) + } }) return entriesList } diff --git a/java/fuzz.go b/java/fuzz.go index 9a0c908a03..4aa6dbffdd 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -19,12 +19,12 @@ import ( "sort" "strings" - "github.com/google/blueprint" - "github.com/google/blueprint/proptools" - "android/soong/android" "android/soong/cc" "android/soong/fuzz" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" ) const ( @@ -32,63 +32,63 @@ const ( targetString = "target" ) -type jniProperties struct { - // list of jni libs - Jni_libs []string - - // sanitization - Sanitizers []string -} - func init() { RegisterJavaFuzzBuildComponents(android.InitRegistrationContext) } func RegisterJavaFuzzBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("java_fuzz", JavaFuzzFactory) - ctx.RegisterModuleType("java_fuzz_host", JavaFuzzHostFactory) - ctx.RegisterSingletonType("java_fuzz_host_packaging", javaFuzzHostPackagingFactory) - ctx.RegisterSingletonType("java_fuzz_device_packaging", javaFuzzDevicePackagingFactory) + ctx.RegisterSingletonType("java_fuzz_packaging", javaFuzzPackagingFactory) } -type JavaFuzzLibrary struct { - Library +type JavaFuzzTest struct { + Test fuzzPackagedModule fuzz.FuzzPackagedModule - jniProperties jniProperties jniFilePaths android.Paths } -// IsSanitizerEnabledForJni implemented to make JavaFuzzLibrary implement -// cc.JniSanitizeable. It returns a bool for whether a cc dependency should be -// sanitized for the given sanitizer or not. -func (j *JavaFuzzLibrary) IsSanitizerEnabledForJni(ctx android.BaseModuleContext, sanitizerName string) bool { - // TODO: once b/231370928 is resolved, please uncomment the loop - // for _, s := range j.jniProperties.Sanitizers { - // if sanitizerName == s { - // return true - // } - // } - return false +// java_fuzz builds and links sources into a `.jar` file for the device. +// This generates .class files in a jar which can then be instrumented before +// fuzzing in Android Runtime (ART: Android OS on emulator or device) +func JavaFuzzFactory() android.Module { + module := &JavaFuzzTest{} + + module.addHostAndDeviceProperties() + module.AddProperties(&module.testProperties) + module.AddProperties(&module.fuzzPackagedModule.FuzzProperties) + + module.Module.properties.Installable = proptools.BoolPtr(true) + module.Module.dexpreopter.isTest = true + module.Module.linter.properties.Lint.Test = proptools.BoolPtr(true) + + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + disableLinuxBionic := struct { + Target struct { + Linux_bionic struct { + Enabled *bool + } + } + }{} + disableLinuxBionic.Target.Linux_bionic.Enabled = proptools.BoolPtr(false) + ctx.AppendProperties(&disableLinuxBionic) + }) + + InitJavaModuleMultiTargets(module, android.HostAndDeviceSupported) + return module } -func (j *JavaFuzzLibrary) DepsMutator(mctx android.BottomUpMutatorContext) { - if len(j.jniProperties.Jni_libs) > 0 { - if j.fuzzPackagedModule.FuzzProperties.Fuzz_config == nil { - config := &fuzz.FuzzConfig{} - j.fuzzPackagedModule.FuzzProperties.Fuzz_config = config - } - // this will be used by the ingestion pipeline to determine the version - // of jazzer to add to the fuzzer package - j.fuzzPackagedModule.FuzzProperties.Fuzz_config.IsJni = proptools.BoolPtr(true) - for _, target := range mctx.MultiTargets() { +func (j *JavaFuzzTest) DepsMutator(ctx android.BottomUpMutatorContext) { + if len(j.testProperties.Jni_libs) > 0 { + for _, target := range ctx.MultiTargets() { sharedLibVariations := append(target.Variations(), blueprint.Variation{Mutator: "link", Variation: "shared"}) - mctx.AddFarVariationDependencies(sharedLibVariations, cc.JniFuzzLibTag, j.jniProperties.Jni_libs...) + ctx.AddFarVariationDependencies(sharedLibVariations, jniLibTag, j.testProperties.Jni_libs...) } } - j.Library.DepsMutator(mctx) + + j.deps(ctx) } -func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { +func (j *JavaFuzzTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { if j.fuzzPackagedModule.FuzzProperties.Corpus != nil { j.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, j.fuzzPackagedModule.FuzzProperties.Corpus) } @@ -105,21 +105,15 @@ func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) } _, sharedDeps := cc.CollectAllSharedDependencies(ctx) - for _, dep := range sharedDeps { sharedLibInfo := ctx.OtherModuleProvider(dep, cc.SharedLibraryInfoProvider).(cc.SharedLibraryInfo) if sharedLibInfo.SharedLibrary != nil { - // The .class jars are output in slightly different locations - // relative to the jni libs. Therefore, for consistency across - // host and device fuzzers of jni lib location, we save it in a - // native_libs directory. - var relPath string + arch := "lib" if sharedLibInfo.Target.Arch.ArchType.Multilib == "lib64" { - relPath = filepath.Join("lib64", sharedLibInfo.SharedLibrary.Base()) - } else { - relPath = filepath.Join("lib", sharedLibInfo.SharedLibrary.Base()) + arch = "lib64" } - libPath := android.PathForModuleOut(ctx, relPath) + + libPath := android.PathForModuleOut(ctx, filepath.Join(arch, sharedLibInfo.SharedLibrary.Base())) ctx.Build(pctx, android.BuildParams{ Rule: android.Cp, Input: sharedLibInfo.SharedLibrary, @@ -129,104 +123,35 @@ func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) } else { ctx.PropertyErrorf("jni_libs", "%q of type %q is not supported", dep.Name(), ctx.OtherModuleType(dep)) } - } - - j.Library.GenerateAndroidBuildActions(ctx) -} - -// java_fuzz_host builds and links sources into a `.jar` file for the host. -// -// By default, a java_fuzz produces a `.jar` file containing `.class` files. -// This jar is not suitable for installing on a device. -func JavaFuzzHostFactory() android.Module { - module := &JavaFuzzLibrary{} - module.addHostProperties() - module.AddProperties(&module.jniProperties) - module.Module.properties.Installable = proptools.BoolPtr(true) - module.AddProperties(&module.fuzzPackagedModule.FuzzProperties) - - // java_fuzz packaging rules collide when both linux_glibc and linux_bionic are enabled, disable the linux_bionic variants. - android.AddLoadHook(module, func(ctx android.LoadHookContext) { - disableLinuxBionic := struct { - Target struct { - Linux_bionic struct { - Enabled *bool - } - } - }{} - disableLinuxBionic.Target.Linux_bionic.Enabled = proptools.BoolPtr(false) - ctx.AppendProperties(&disableLinuxBionic) - }) - InitJavaModuleMultiTargets(module, android.HostSupportedNoCross) - return module -} - -// java_fuzz builds and links sources into a `.jar` file for the device. -// This generates .class files in a jar which can then be instrumented before -// fuzzing in Android Runtime (ART: Android OS on emulator or device) -func JavaFuzzFactory() android.Module { - module := &JavaFuzzLibrary{} - module.addHostAndDeviceProperties() - module.AddProperties(&module.jniProperties) - module.Module.properties.Installable = proptools.BoolPtr(true) - module.AddProperties(&module.fuzzPackagedModule.FuzzProperties) - module.Module.dexpreopter.isTest = true - module.Module.linter.properties.Lint.Test = proptools.BoolPtr(true) - InitJavaModuleMultiTargets(module, android.DeviceSupported) - return module -} + } -// Responsible for generating rules that package host fuzz targets into -// a zip file. -type javaFuzzHostPackager struct { - fuzz.FuzzPackager + j.Test.GenerateAndroidBuildActions(ctx) } -// Responsible for generating rules that package device fuzz targets into -// a zip file. -type javaFuzzDevicePackager struct { +type javaFuzzPackager struct { fuzz.FuzzPackager } -func javaFuzzHostPackagingFactory() android.Singleton { - return &javaFuzzHostPackager{} -} - -func javaFuzzDevicePackagingFactory() android.Singleton { - return &javaFuzzDevicePackager{} +func javaFuzzPackagingFactory() android.Singleton { + return &javaFuzzPackager{} } -func (s *javaFuzzHostPackager) GenerateBuildActions(ctx android.SingletonContext) { - generateBuildActions(&s.FuzzPackager, hostString, ctx) -} - -func (s *javaFuzzDevicePackager) GenerateBuildActions(ctx android.SingletonContext) { - generateBuildActions(&s.FuzzPackager, targetString, ctx) -} - -func generateBuildActions(s *fuzz.FuzzPackager, hostOrTargetString string, ctx android.SingletonContext) { +func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { // Map between each architecture + host/device combination. archDirs := make(map[fuzz.ArchOs][]fuzz.FileToZip) - // List of individual fuzz targets. s.FuzzTargets = make(map[string]bool) - ctx.VisitAllModules(func(module android.Module) { // Discard non-fuzz targets. - javaFuzzModule, ok := module.(*JavaFuzzLibrary) + javaFuzzModule, ok := module.(*JavaFuzzTest) if !ok { return } - if hostOrTargetString == hostString { - if !javaFuzzModule.Host() { - return - } - } else if hostOrTargetString == targetString { - if javaFuzzModule.Host() || javaFuzzModule.Target().HostCross { - return - } + hostOrTargetString := "target" + if javaFuzzModule.Host() { + hostOrTargetString = "host" } fuzzModuleValidator := fuzz.FuzzModule{ @@ -246,7 +171,7 @@ func generateBuildActions(s *fuzz.FuzzPackager, hostOrTargetString string, ctx a var files []fuzz.FileToZip builder := android.NewRuleBuilder(pctx, ctx) - // Package the artifacts (data, corpus, config and dictionary into a zipfile. + // Package the artifacts (data, corpus, config and dictionary) into a zipfile. files = s.PackageArtifacts(ctx, module, javaFuzzModule.fuzzPackagedModule, archDir, builder) // Add .jar @@ -261,27 +186,14 @@ func generateBuildActions(s *fuzz.FuzzPackager, hostOrTargetString string, ctx a if !ok { return } - }) s.CreateFuzzPackage(ctx, archDirs, fuzz.Java, pctx) } -func (s *javaFuzzHostPackager) MakeVars(ctx android.MakeVarsContext) { +func (s *javaFuzzPackager) MakeVars(ctx android.MakeVarsContext) { packages := s.Packages.Strings() sort.Strings(packages) - - ctx.Strict("SOONG_JAVA_FUZZ_HOST_PACKAGING_ARCH_MODULES", strings.Join(packages, " ")) - - // Preallocate the slice of fuzz targets to minimize memory allocations. - s.PreallocateSlice(ctx, "ALL_JAVA_FUZZ_HOST_TARGETS") -} - -func (s *javaFuzzDevicePackager) MakeVars(ctx android.MakeVarsContext) { - packages := s.Packages.Strings() - sort.Strings(packages) - - ctx.Strict("SOONG_JAVA_FUZZ_DEVICE_PACKAGING_ARCH_MODULES", strings.Join(packages, " ")) - + ctx.Strict("SOONG_JAVA_FUZZ_PACKAGING_ARCH_MODULES", strings.Join(packages, " ")) // Preallocate the slice of fuzz targets to minimize memory allocations. - s.PreallocateSlice(ctx, "ALL_JAVA_FUZZ_DEVICE_TARGETS") + s.PreallocateSlice(ctx, "ALL_JAVA_FUZZ_TARGETS") } diff --git a/java/fuzz_test.go b/java/fuzz_test.go index 186c3aa957..dd1e96b3ed 100644 --- a/java/fuzz_test.go +++ b/java/fuzz_test.go @@ -31,17 +31,15 @@ var prepForJavaFuzzTest = android.GroupFixturePreparers( func TestJavaFuzz(t *testing.T) { result := prepForJavaFuzzTest.RunTestWithBp(t, ` - java_fuzz_host { + java_fuzz { name: "foo", srcs: ["a.java"], + host_supported: true, + device_supported: false, libs: ["bar"], static_libs: ["baz"], jni_libs: [ "libjni", - ], - sanitizers: [ - "address", - "fuzzer", ], } @@ -84,7 +82,7 @@ func TestJavaFuzz(t *testing.T) { } ctx := result.TestContext - foo := ctx.ModuleForTests("foo", osCommonTarget).Module().(*JavaFuzzLibrary) + foo := ctx.ModuleForTests("foo", osCommonTarget).Module().(*JavaFuzzTest) expected := "lib64/libjni.so" if runtime.GOOS == "darwin" { diff --git a/java/java.go b/java/java.go index a98762cf11..65bfce088c 100644 --- a/java/java.go +++ b/java/java.go @@ -1156,7 +1156,6 @@ func (j *TestHost) DepsMutator(ctx android.BottomUpMutatorContext) { } j.addDataDeviceBinsDeps(ctx) - j.deps(ctx) } From 64a4a5f5cd5ecb245f814c1e4070bed97ad5ee48 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 16 May 2023 17:54:27 -0700 Subject: [PATCH 0162/1460] Separate host and host cross fuzz packaging Prevent collisions between host and host cross fuzz packaging when both are supported as fuzz targets by using a different value for hostOrTargetString. Test: builds with HOST_CROSS_OS := linux_musl Change-Id: I0e82bab2987899cece94c89eb8398d1733ebbd15 --- cc/fuzz.go | 4 +++- java/fuzz.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cc/fuzz.go b/cc/fuzz.go index dfefc11f22..0f709c94fb 100644 --- a/cc/fuzz.go +++ b/cc/fuzz.go @@ -398,7 +398,9 @@ func (s *ccRustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) } hostOrTargetString := "target" - if ccModule.Host() { + if ccModule.Target().HostCross { + hostOrTargetString = "host_cross" + } else if ccModule.Host() { hostOrTargetString = "host" } diff --git a/java/fuzz.go b/java/fuzz.go index 4aa6dbffdd..e24594ab74 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -150,7 +150,9 @@ func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { } hostOrTargetString := "target" - if javaFuzzModule.Host() { + if javaFuzzModule.Target().HostCross { + hostOrTargetString = "host_cross" + } else if javaFuzzModule.Host() { hostOrTargetString = "host" } From 390fc746d0eab743c18033d9e851f5b4ec435a3d Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 2 May 2023 13:02:51 -0700 Subject: [PATCH 0163/1460] Disable sanitizers that use runtimes for linux_musl arm64 We don't built clang sanitizer runtimes for host arm64 yet, disable sanitizer that need them when building for linux_musl arm64. Test: builds Change-Id: Id4fcb7f9743ac30121157b2294eb60216eaff7b3 --- cc/sanitize.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cc/sanitize.go b/cc/sanitize.go index 7fddc1b836..6e732b617b 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -665,6 +665,21 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Diag.Cfi = nil } + // TODO(b/280478629): runtimes don't exist for musl arm64 yet. + if ctx.toolchain().Musl() && ctx.Arch().ArchType == android.Arm64 { + s.Address = nil + s.Hwaddress = nil + s.Thread = nil + s.Scudo = nil + s.Fuzzer = nil + s.Cfi = nil + s.Diag.Cfi = nil + s.Misc_undefined = nil + s.Undefined = nil + s.All_undefined = nil + s.Integer_overflow = nil + } + // Also disable CFI for VNDK variants of components if ctx.isVndk() && ctx.useVndk() { s.Cfi = nil From 6550358a0598140f61a31eb1939a82e227569a2e Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Wed, 17 May 2023 22:19:53 +0000 Subject: [PATCH 0164/1460] Export host available libraries to soong_injection This information will be used to validate linkopts for host targets. Test: m bp2build Bug: 216626461 Change-Id: Ie7ac46268e6177cf0688ca0389bdaa92b67f9a58 --- cc/config/toolchain.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cc/config/toolchain.go b/cc/config/toolchain.go index 6a10e14743..a0ef575955 100644 --- a/cc/config/toolchain.go +++ b/cc/config/toolchain.go @@ -20,6 +20,12 @@ import ( "android/soong/android" ) +func init() { + exportedVars.ExportStringListStaticVariable("DarwinAvailableLibraries", darwinAvailableLibraries) + exportedVars.ExportStringListStaticVariable("LinuxAvailableLibraries", linuxAvailableLibraries) + exportedVars.ExportStringListStaticVariable("WindowsAvailableLibraries", windowsAvailableLibraries) +} + type toolchainFactory func(arch android.Arch) Toolchain var toolchainFactories = make(map[android.OsType]map[android.ArchType]toolchainFactory) From fddeaff4a639c806e640283f9d53b42686524f2e Mon Sep 17 00:00:00 2001 From: Chris Parsons Date: Wed, 17 May 2023 21:53:34 +0000 Subject: [PATCH 0165/1460] Revert "Turn on HINT_FROM_SOONG by default" This reverts commit f5c872f36b129d97c222df61760381c1449558f5. Revert reason: The original commit seems to have a performance regression for a number of benchmarked performance CUJs. Bug: 283143307 Test: Benchmarking results. See attached bug. Change-Id: Ib9463c8d30a5ba61640993424696a84e2e03040a --- ui/build/config.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index 6b6ce7154f..8ec96800f3 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -316,9 +316,8 @@ func UploadOnlyConfig(ctx Context, args ...string) Config { func NewConfig(ctx Context, args ...string) Config { ret := &configImpl{ - environ: OsEnvironment(), - sandboxConfig: &SandboxConfig{}, - ninjaWeightListSource: HINT_FROM_SOONG, + environ: OsEnvironment(), + sandboxConfig: &SandboxConfig{}, } // Default matching ninja From b20a12553ef65a6401dfe0f67ab8b882da0f89db Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Wed, 17 May 2023 22:51:43 +0000 Subject: [PATCH 0166/1460] Revert "Restrict plugins to an existing allowlist" This reverts commit 09ddb3a73e14823d51d8227ecaf1997fa010bc14. Reason for revert: breaks branches without vendor Change-Id: I2de95d7b99e4e5fd3d696f51d5b4d6a68a298365 --- android/Android.bp | 1 - android/config.go | 4 -- android/plugin.go | 139 ---------------------------------------- android/variable.go | 1 - tests/bootstrap_test.sh | 24 +++---- 5 files changed, 12 insertions(+), 157 deletions(-) delete mode 100644 android/plugin.go diff --git a/android/Android.bp b/android/Android.bp index 94d2c04f6b..118087db7b 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -77,7 +77,6 @@ bootstrap_go_package { "path_properties.go", "paths.go", "phony.go", - "plugin.go", "prebuilt.go", "prebuilt_build_tool.go", "proto.go", diff --git a/android/config.go b/android/config.go index e0cf173f0e..7141e54c3d 100644 --- a/android/config.go +++ b/android/config.go @@ -1819,10 +1819,6 @@ func (c *deviceConfig) ShippingApiLevel() ApiLevel { return uncheckedFinalApiLevel(apiLevel) } -func (c *deviceConfig) BuildBrokenPluginValidation() []string { - return c.config.productVariables.BuildBrokenPluginValidation -} - func (c *deviceConfig) BuildBrokenClangAsFlags() bool { return c.config.productVariables.BuildBrokenClangAsFlags } diff --git a/android/plugin.go b/android/plugin.go deleted file mode 100644 index 5be3d3ad86..0000000000 --- a/android/plugin.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2022 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "strings" - - "github.com/google/blueprint" -) - -func init() { - RegisterPluginSingletonBuildComponents(InitRegistrationContext) -} - -func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { - ctx.RegisterSingletonType("plugins", pluginSingletonFactory) -} - -// pluginSingleton is a singleton to handle allowlisting of the final Android-.mk file -// output. -func pluginSingletonFactory() Singleton { - return &pluginSingleton{} -} - -type pluginSingleton struct{} - -var allowedPluginsByName = map[string]bool{ - "aidl-soong-rules": true, - "arm_compute_library_nn_driver": true, - "cuttlefish-soong-rules": true, - "gki-soong-rules": true, - "hidl-soong-rules": true, - "kernel-config-soong-rules": true, - "soong-angle-codegen": true, - "soong-api": true, - "soong-art": true, - "soong-ca-certificates": true, - "soong-clang": true, - "soong-clang-prebuilts": true, - "soong-csuite": true, - "soong-fluoride": true, - "soong-fs_config": true, - "soong-icu": true, - "soong-java-config-error_prone": true, - "soong-libchrome": true, - "soong-llvm": true, - "soong-robolectric": true, - "soong-rust-prebuilts": true, - "soong-selinux": true, - "soong-wayland-protocol-codegen": true, - "treble_report_app": true, - "treble_report_local": true, - "treble_report_module": true, - "vintf-compatibility-matrix-soong-rules": true, - "xsdc-soong-rules": true, -} - -const ( - internalPluginsPath = "vendor/google/build/soong/internal_plugins.json" -) - -type pluginProvider interface { - IsPluginFor(string) bool -} - -func maybeAddInternalPluginsToAllowlist(ctx SingletonContext) { - if path := ExistentPathForSource(ctx, internalPluginsPath); path.Valid() { - ctx.AddNinjaFileDeps(path.String()) - absPath := absolutePath(path.String()) - var moreAllowed map[string]bool - data, err := ioutil.ReadFile(absPath) - if err != nil { - ctx.Errorf("Failed to open internal plugins path %q %q", internalPluginsPath, err) - } - if err := json.Unmarshal(data, &moreAllowed); err != nil { - fmt.Fprintf(os.Stderr, "Internal plugins file %q did not parse correctly: %q", data, err) - } - for k, v := range moreAllowed { - allowedPluginsByName[k] = v - } - } -} - -func (p *pluginSingleton) GenerateBuildActions(ctx SingletonContext) { - for _, p := range ctx.DeviceConfig().BuildBrokenPluginValidation() { - allowedPluginsByName[p] = true - } - maybeAddInternalPluginsToAllowlist(ctx) - - disallowedPlugins := map[string]bool{} - ctx.VisitAllModulesBlueprint(func(module blueprint.Module) { - if ctx.ModuleType(module) != "bootstrap_go_package" { - return - } - - p, ok := module.(pluginProvider) - if !ok || !p.IsPluginFor("soong_build") { - return - } - - name := ctx.ModuleName(module) - if _, ok := allowedPluginsByName[name]; ok { - return - } - - dir := ctx.ModuleDir(module) - - // allow use of plugins within Soong to not allowlist everything - if strings.HasPrefix(dir, "build/soong") { - return - } - - // allow third party users outside of external to create new plugins, i.e. non-google paths - // under vendor or hardware - if !strings.HasPrefix(dir, "external/") && IsThirdPartyPath(dir) { - return - } - disallowedPlugins[name] = true - }) - if len(disallowedPlugins) > 0 { - ctx.Errorf("New plugins are not supported; however %q were found. Please reach out to the build team or use BUILD_BROKEN_PLUGIN_VALIDATION (see Changes.md for more info).", SortedStringKeys(disallowedPlugins)) - } -} diff --git a/android/variable.go b/android/variable.go index b5d9666683..249d53b07a 100644 --- a/android/variable.go +++ b/android/variable.go @@ -437,7 +437,6 @@ type productVariables struct { ShippingApiLevel *string `json:",omitempty"` - BuildBrokenPluginValidation []string `json:",omitempty"` BuildBrokenClangAsFlags bool `json:",omitempty"` BuildBrokenClangCFlags bool `json:",omitempty"` BuildBrokenClangProperty bool `json:",omitempty"` diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index c5a6336a84..fda5ca0863 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -207,8 +207,8 @@ EOF function test_soong_build_rerun_iff_environment_changes() { setup - mkdir -p build/soong/cherry - cat > build/soong/cherry/Android.bp <<'EOF' + mkdir -p cherry + cat > cherry/Android.bp <<'EOF' bootstrap_go_package { name: "cherry", pkgPath: "android/soong/cherry", @@ -224,7 +224,7 @@ bootstrap_go_package { } EOF - cat > build/soong/cherry/cherry.go <<'EOF' + cat > cherry/cherry.go <<'EOF' package cherry import ( @@ -317,8 +317,8 @@ function test_add_file_to_soong_build() { run_soong local -r mtime1=$(stat -c "%y" out/soong/build.ninja) - mkdir -p vendor/foo/picard - cat > vendor/foo/picard/Android.bp <<'EOF' + mkdir -p a + cat > a/Android.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -334,7 +334,7 @@ bootstrap_go_package { } EOF - cat > vendor/foo/picard/picard.go <<'EOF' + cat > a/picard.go <<'EOF' package picard import ( @@ -390,11 +390,11 @@ EOF function test_glob_during_bootstrapping() { setup - mkdir -p build/soong/picard - cat > build/soong/picard/Android.bp <<'EOF' + mkdir -p a + cat > a/Android.bp <<'EOF' build=["foo*.bp"] EOF - cat > build/soong/picard/fooa.bp <<'EOF' + cat > a/fooa.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -410,7 +410,7 @@ bootstrap_go_package { } EOF - cat > build/soong/picard/picard.go <<'EOF' + cat > a/picard.go <<'EOF' package picard import ( @@ -459,7 +459,7 @@ EOF grep -q "Make it so" out/soong/build.ninja || fail "Original action not present" - cat > build/soong/picard/foob.bp <<'EOF' + cat > a/foob.bp <<'EOF' bootstrap_go_package { name: "worf-soong-rules", pkgPath: "android/soong/worf", @@ -476,7 +476,7 @@ bootstrap_go_package { } EOF - cat > build/soong/picard/worf.go <<'EOF' + cat > a/worf.go <<'EOF' package worf import "android/soong/picard" From 3402c0ad4453cb7910d706818112e516bd0c9bd9 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Wed, 17 May 2023 16:32:15 -0400 Subject: [PATCH 0167/1460] re-add swcodec payload to mixed staging allowlist Bug: 279756270 Test: m Test: forrest Change-Id: I3305c1bd9388b326fdecb5d333c07cf653d310d8 --- android/allowlists/allowlists.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 9700f8bb93..e22eec5645 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -1565,10 +1565,10 @@ var ( // also be built - do not add them to this list. StagingMixedBuildsEnabledList = []string{ // M13: media.swcodec launch - // TODO(b/282042844): reenable - // "com.android.media.swcodec", - // "test_com.android.media.swcodec", - // "libstagefright_foundation", + "com.android.media.swcodec", + "test_com.android.media.swcodec", + "libstagefright_foundation", + "libcodec2_hidl@1.0", } // These should be the libs that are included by the apexes in the ProdMixedBuildsEnabledList From 09a8d459d5af56f238a38c86cd4b8237b4819bdc Mon Sep 17 00:00:00 2001 From: Yu Liu Date: Fri, 12 May 2023 17:22:30 -0700 Subject: [PATCH 0168/1460] Test dcla libs for b build. Bug: 273862399 Test: local Change-Id: Ic72239779d63d64b68a902de6f5b3c6567385878 --- tests/dcla_apex_comparison_test.sh | 36 +++++++++++++++++++++++++----- tests/run_integration_tests.sh | 4 ++-- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/dcla_apex_comparison_test.sh b/tests/dcla_apex_comparison_test.sh index 97ae97ecbb..667dde0b75 100755 --- a/tests/dcla_apex_comparison_test.sh +++ b/tests/dcla_apex_comparison_test.sh @@ -45,6 +45,11 @@ MODULES=( com.android.tethering ) +BAZEL_TARGETS=( + //packages/modules/adb/apex:com.android.adbd + //frameworks/av/apex:com.android.media.swcodec +) + DCLA_LIBS=( libbase.so libc++.so @@ -76,6 +81,10 @@ DEAPEXER="${DEAPEXER} --debugfs_path=${DEBUGFS}" ############ OUTPUT_DIR="$(mktemp -d tmp.XXXXXX)" +function call_bazel() { + build/bazel/bin/bazel $@ +} + function cleanup { rm -rf "${OUTPUT_DIR}" } @@ -87,7 +96,9 @@ trap cleanup EXIT function extract_dcla_libs() { local product=$1; shift - for module in "${MODULES[@]}"; do + local modules=("$@"); shift + + for module in "${modules[@]}"; do local apex="${OUTPUT_DIR}/${product}/${module}.apex" local extract_dir="${OUTPUT_DIR}/${product}/${module}/extract" @@ -97,11 +108,12 @@ function extract_dcla_libs() { function compare_dcla_libs() { local product=$1; shift + local modules=("$@"); shift for lib in "${DCLA_LIBS[@]}"; do for arch in lib lib64; do local prev_sha="" - for module in "${MODULES[@]}"; do + for module in "${modules[@]}"; do local file="${OUTPUT_DIR}/${product}/${module}/extract/${arch}/${lib}" if [[ ! -f "${file}" ]]; then # not all libs are present in a module @@ -112,7 +124,7 @@ function compare_dcla_libs() { sha="${sha% *}" if [ "${prev_sha}" == "" ]; then prev_sha="${sha}" - elif [ "${sha}" != "${prev_sha}" ] && { [ "${lib}" != "libcrypto.so" ] || [ "${module}" != "com.android.tethering" ]; }; then + elif [ "${sha}" != "${prev_sha}" ] && { [ "${lib}" != "libcrypto.so" ] || [[ "${module}" != *"com.android.tethering" ]]; }; then echo "Test failed, ${lib} has different hash value" exit 1 fi @@ -131,8 +143,22 @@ for product in "${TARGET_PRODUCTS[@]}"; do --product "${product}" \ --dist_dir "${OUTPUT_DIR}/${product}" - extract_dcla_libs "${product}" - compare_dcla_libs "${product}" + bazel_apexes=() + if [[ -n ${TEST_BAZEL+x} ]] && [ "${TEST_BAZEL}" = true ]; then + export TARGET_PRODUCT="${product/module/aosp}" + call_bazel build --config=bp2build --config=ci --config=android "${BAZEL_TARGETS[@]}" + for target in "${BAZEL_TARGETS[@]}"; do + apex_path="$(realpath $(call_bazel cquery --config=bp2build --config=android --config=ci --output=files $target))" + mkdir -p ${OUTPUT_DIR}/${product} + bazel_apex="bazel_$(basename $apex_path)" + mv $apex_path ${OUTPUT_DIR}/${product}/${bazel_apex} + bazel_apexes+=(${bazel_apex%".apex"}) + done + fi + + all_modeuls=(${MODULES[@]} ${bazel_apexes[@]}) + extract_dcla_libs "${product}" "${all_modeuls[@]}" + compare_dcla_libs "${product}" "${all_modeuls[@]}" done echo "Test passed" diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh index e1aa70c738..db14bd442f 100755 --- a/tests/run_integration_tests.sh +++ b/tests/run_integration_tests.sh @@ -15,8 +15,8 @@ TOP="$(readlink -f "$(dirname "$0")"/../../..)" # mock client. "$TOP/build/soong/tests/apex_comparison_tests.sh" "$TOP/build/soong/tests/apex_comparison_tests.sh" "module_arm64only" -extra_build_params=--bazel-mode-staging "$TOP/build/soong/tests/dcla_apex_comparison_test.sh" -BUILD_BROKEN_DISABLE_BAZEL=true "$TOP/build/soong/tests/dcla_apex_comparison_test.sh" +TEST_BAZEL=true extra_build_params=--bazel-mode-staging "$TOP/build/soong/tests/dcla_apex_comparison_test.sh" +#BUILD_BROKEN_DISABLE_BAZEL=true "$TOP/build/soong/tests/dcla_apex_comparison_test.sh" "$TOP/build/soong/tests/apex_cc_module_arch_variant_tests.sh" "$TOP/build/soong/tests/apex_cc_module_arch_variant_tests.sh" "aosp_arm" "armv7-a" "$TOP/build/soong/tests/apex_cc_module_arch_variant_tests.sh" "aosp_cf_arm64_phone" "armv8-a" "cortex-a53" From b7550aa09af04e92ce24e5bf2594b312d8bc51f7 Mon Sep 17 00:00:00 2001 From: Zyan Wu Date: Thu, 18 May 2023 15:46:31 +0800 Subject: [PATCH 0169/1460] Don't panic if no certificates found Getting the first certificate will panic if there are no certificates, which can happen when AllowMissingDependencies is set and the certificate property is a module reference to a missing module. Only get the first certificate if the list is not nil. Use mainCertificate since it handles the logic already. Test: TestAppMissingCertificateAllowMissingDependencies Bug: 283102635 Change-Id: I8b27f65aa7d071041171ad45ac52bf47fa31bf2b --- java/app.go | 27 ++++++++++++++------------- java/app_test.go | 8 ++++++++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/java/app.go b/java/app.go index 3344647722..e09509299c 100755 --- a/java/app.go +++ b/java/app.go @@ -585,19 +585,6 @@ func processMainCert(m android.ModuleBase, certPropValue string, certificates [] certificates = append([]Certificate{mainCert}, certificates...) } - if !m.Platform() { - certPath := certificates[0].Pem.String() - systemCertPath := ctx.Config().DefaultAppCertificateDir(ctx).String() - if strings.HasPrefix(certPath, systemCertPath) { - enforceSystemCert := ctx.Config().EnforceSystemCertificate() - allowed := ctx.Config().EnforceSystemCertificateAllowList() - - if enforceSystemCert && !inList(m.Name(), allowed) { - ctx.PropertyErrorf("certificate", "The module in product partition cannot be signed with certificate in system.") - } - } - } - if len(certificates) > 0 { mainCertificate = certificates[0] } else { @@ -613,6 +600,20 @@ func processMainCert(m android.ModuleBase, certPropValue string, certificates [] } } + if !m.Platform() { + certPath := mainCertificate.Pem.String() + systemCertPath := ctx.Config().DefaultAppCertificateDir(ctx).String() + if strings.HasPrefix(certPath, systemCertPath) { + enforceSystemCert := ctx.Config().EnforceSystemCertificate() + allowed := ctx.Config().EnforceSystemCertificateAllowList() + + if enforceSystemCert && !inList(m.Name(), allowed) { + ctx.PropertyErrorf("certificate", "The module in product partition cannot be signed with certificate in system.") + } + } + } + + return mainCertificate, certificates } diff --git a/java/app_test.go b/java/app_test.go index 8a69a03e72..c485478eb1 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3382,6 +3382,14 @@ func TestAppMissingCertificateAllowMissingDependencies(t *testing.T) { srcs: ["a.java"], certificate: ":missing_certificate", sdk_version: "current", + } + + android_app { + name: "bar", + srcs: ["a.java"], + certificate: ":missing_certificate", + product_specific: true, + sdk_version: "current", }`) foo := result.ModuleForTests("foo", "android_common") From dfe4a9e4dd088f42ba5e0a3c4f7777c253d378d3 Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Thu, 18 May 2023 11:54:46 +0000 Subject: [PATCH 0170/1460] Revert "Revert "Restrict plugins to an existing allowlist"" This reverts commit b20a12553ef65a6401dfe0f67ab8b882da0f89db. Reason for revert: Rollforward with fix soong-ca-certificates-apex was missing from allowlist Test: verify fixes CI breakages Change-Id: I00687bd2962dbdaaf55189ca4aa82183264eb8d7 --- android/Android.bp | 1 + android/config.go | 4 ++ android/plugin.go | 140 ++++++++++++++++++++++++++++++++++++++++ android/variable.go | 1 + tests/bootstrap_test.sh | 24 +++---- 5 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 android/plugin.go diff --git a/android/Android.bp b/android/Android.bp index 118087db7b..94d2c04f6b 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -77,6 +77,7 @@ bootstrap_go_package { "path_properties.go", "paths.go", "phony.go", + "plugin.go", "prebuilt.go", "prebuilt_build_tool.go", "proto.go", diff --git a/android/config.go b/android/config.go index 7141e54c3d..e0cf173f0e 100644 --- a/android/config.go +++ b/android/config.go @@ -1819,6 +1819,10 @@ func (c *deviceConfig) ShippingApiLevel() ApiLevel { return uncheckedFinalApiLevel(apiLevel) } +func (c *deviceConfig) BuildBrokenPluginValidation() []string { + return c.config.productVariables.BuildBrokenPluginValidation +} + func (c *deviceConfig) BuildBrokenClangAsFlags() bool { return c.config.productVariables.BuildBrokenClangAsFlags } diff --git a/android/plugin.go b/android/plugin.go new file mode 100644 index 0000000000..c9d1338f81 --- /dev/null +++ b/android/plugin.go @@ -0,0 +1,140 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/google/blueprint" +) + +func init() { + RegisterPluginSingletonBuildComponents(InitRegistrationContext) +} + +func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { + ctx.RegisterSingletonType("plugins", pluginSingletonFactory) +} + +// pluginSingleton is a singleton to handle allowlisting of the final Android-.mk file +// output. +func pluginSingletonFactory() Singleton { + return &pluginSingleton{} +} + +type pluginSingleton struct{} + +var allowedPluginsByName = map[string]bool{ + "aidl-soong-rules": true, + "arm_compute_library_nn_driver": true, + "cuttlefish-soong-rules": true, + "gki-soong-rules": true, + "hidl-soong-rules": true, + "kernel-config-soong-rules": true, + "soong-angle-codegen": true, + "soong-api": true, + "soong-art": true, + "soong-ca-certificates": true, + "soong-ca-certificates-apex": true, + "soong-clang": true, + "soong-clang-prebuilts": true, + "soong-csuite": true, + "soong-fluoride": true, + "soong-fs_config": true, + "soong-icu": true, + "soong-java-config-error_prone": true, + "soong-libchrome": true, + "soong-llvm": true, + "soong-robolectric": true, + "soong-rust-prebuilts": true, + "soong-selinux": true, + "soong-wayland-protocol-codegen": true, + "treble_report_app": true, + "treble_report_local": true, + "treble_report_module": true, + "vintf-compatibility-matrix-soong-rules": true, + "xsdc-soong-rules": true, +} + +const ( + internalPluginsPath = "vendor/google/build/soong/internal_plugins.json" +) + +type pluginProvider interface { + IsPluginFor(string) bool +} + +func maybeAddInternalPluginsToAllowlist(ctx SingletonContext) { + if path := ExistentPathForSource(ctx, internalPluginsPath); path.Valid() { + ctx.AddNinjaFileDeps(path.String()) + absPath := absolutePath(path.String()) + var moreAllowed map[string]bool + data, err := ioutil.ReadFile(absPath) + if err != nil { + ctx.Errorf("Failed to open internal plugins path %q %q", internalPluginsPath, err) + } + if err := json.Unmarshal(data, &moreAllowed); err != nil { + fmt.Fprintf(os.Stderr, "Internal plugins file %q did not parse correctly: %q", data, err) + } + for k, v := range moreAllowed { + allowedPluginsByName[k] = v + } + } +} + +func (p *pluginSingleton) GenerateBuildActions(ctx SingletonContext) { + for _, p := range ctx.DeviceConfig().BuildBrokenPluginValidation() { + allowedPluginsByName[p] = true + } + maybeAddInternalPluginsToAllowlist(ctx) + + disallowedPlugins := map[string]bool{} + ctx.VisitAllModulesBlueprint(func(module blueprint.Module) { + if ctx.ModuleType(module) != "bootstrap_go_package" { + return + } + + p, ok := module.(pluginProvider) + if !ok || !p.IsPluginFor("soong_build") { + return + } + + name := ctx.ModuleName(module) + if _, ok := allowedPluginsByName[name]; ok { + return + } + + dir := ctx.ModuleDir(module) + + // allow use of plugins within Soong to not allowlist everything + if strings.HasPrefix(dir, "build/soong") { + return + } + + // allow third party users outside of external to create new plugins, i.e. non-google paths + // under vendor or hardware + if !strings.HasPrefix(dir, "external/") && IsThirdPartyPath(dir) { + return + } + disallowedPlugins[name] = true + }) + if len(disallowedPlugins) > 0 { + ctx.Errorf("New plugins are not supported; however %q were found. Please reach out to the build team or use BUILD_BROKEN_PLUGIN_VALIDATION (see Changes.md for more info).", SortedStringKeys(disallowedPlugins)) + } +} diff --git a/android/variable.go b/android/variable.go index 249d53b07a..b5d9666683 100644 --- a/android/variable.go +++ b/android/variable.go @@ -437,6 +437,7 @@ type productVariables struct { ShippingApiLevel *string `json:",omitempty"` + BuildBrokenPluginValidation []string `json:",omitempty"` BuildBrokenClangAsFlags bool `json:",omitempty"` BuildBrokenClangCFlags bool `json:",omitempty"` BuildBrokenClangProperty bool `json:",omitempty"` diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index fda5ca0863..c5a6336a84 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -207,8 +207,8 @@ EOF function test_soong_build_rerun_iff_environment_changes() { setup - mkdir -p cherry - cat > cherry/Android.bp <<'EOF' + mkdir -p build/soong/cherry + cat > build/soong/cherry/Android.bp <<'EOF' bootstrap_go_package { name: "cherry", pkgPath: "android/soong/cherry", @@ -224,7 +224,7 @@ bootstrap_go_package { } EOF - cat > cherry/cherry.go <<'EOF' + cat > build/soong/cherry/cherry.go <<'EOF' package cherry import ( @@ -317,8 +317,8 @@ function test_add_file_to_soong_build() { run_soong local -r mtime1=$(stat -c "%y" out/soong/build.ninja) - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p vendor/foo/picard + cat > vendor/foo/picard/Android.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -334,7 +334,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > vendor/foo/picard/picard.go <<'EOF' package picard import ( @@ -390,11 +390,11 @@ EOF function test_glob_during_bootstrapping() { setup - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p build/soong/picard + cat > build/soong/picard/Android.bp <<'EOF' build=["foo*.bp"] EOF - cat > a/fooa.bp <<'EOF' + cat > build/soong/picard/fooa.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -410,7 +410,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > build/soong/picard/picard.go <<'EOF' package picard import ( @@ -459,7 +459,7 @@ EOF grep -q "Make it so" out/soong/build.ninja || fail "Original action not present" - cat > a/foob.bp <<'EOF' + cat > build/soong/picard/foob.bp <<'EOF' bootstrap_go_package { name: "worf-soong-rules", pkgPath: "android/soong/worf", @@ -476,7 +476,7 @@ bootstrap_go_package { } EOF - cat > a/worf.go <<'EOF' + cat > build/soong/picard/worf.go <<'EOF' package worf import "android/soong/picard" From 33cc80eb1f84d607da1a70b76d8dc0370b51a89a Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Thu, 18 May 2023 18:20:28 +0000 Subject: [PATCH 0171/1460] Revert "Revert "Restrict plugins to an existing allowlist"" This reverts commit b20a12553ef65a6401dfe0f67ab8b882da0f89db. Reason for revert: Rollforward with fix soong-ca-certificates-apex was missing from allowlist Test: verify fixes CI breakages (cherry picked from https://googleplex-android-review.googlesource.com/q/commit:dfe4a9e4dd088f42ba5e0a3c4f7777c253d378d3) Merged-In: I00687bd2962dbdaaf55189ca4aa82183264eb8d7 Change-Id: I00687bd2962dbdaaf55189ca4aa82183264eb8d7 --- android/Android.bp | 1 + android/config.go | 4 ++ android/plugin.go | 140 ++++++++++++++++++++++++++++++++++++++++ android/variable.go | 1 + tests/bootstrap_test.sh | 24 +++---- 5 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 android/plugin.go diff --git a/android/Android.bp b/android/Android.bp index 118087db7b..94d2c04f6b 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -77,6 +77,7 @@ bootstrap_go_package { "path_properties.go", "paths.go", "phony.go", + "plugin.go", "prebuilt.go", "prebuilt_build_tool.go", "proto.go", diff --git a/android/config.go b/android/config.go index acadb3ff33..01b8cfaa32 100644 --- a/android/config.go +++ b/android/config.go @@ -1850,6 +1850,10 @@ func (c *deviceConfig) ShippingApiLevel() ApiLevel { return uncheckedFinalApiLevel(apiLevel) } +func (c *deviceConfig) BuildBrokenPluginValidation() []string { + return c.config.productVariables.BuildBrokenPluginValidation +} + func (c *deviceConfig) BuildBrokenClangAsFlags() bool { return c.config.productVariables.BuildBrokenClangAsFlags } diff --git a/android/plugin.go b/android/plugin.go new file mode 100644 index 0000000000..c9d1338f81 --- /dev/null +++ b/android/plugin.go @@ -0,0 +1,140 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/google/blueprint" +) + +func init() { + RegisterPluginSingletonBuildComponents(InitRegistrationContext) +} + +func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { + ctx.RegisterSingletonType("plugins", pluginSingletonFactory) +} + +// pluginSingleton is a singleton to handle allowlisting of the final Android-.mk file +// output. +func pluginSingletonFactory() Singleton { + return &pluginSingleton{} +} + +type pluginSingleton struct{} + +var allowedPluginsByName = map[string]bool{ + "aidl-soong-rules": true, + "arm_compute_library_nn_driver": true, + "cuttlefish-soong-rules": true, + "gki-soong-rules": true, + "hidl-soong-rules": true, + "kernel-config-soong-rules": true, + "soong-angle-codegen": true, + "soong-api": true, + "soong-art": true, + "soong-ca-certificates": true, + "soong-ca-certificates-apex": true, + "soong-clang": true, + "soong-clang-prebuilts": true, + "soong-csuite": true, + "soong-fluoride": true, + "soong-fs_config": true, + "soong-icu": true, + "soong-java-config-error_prone": true, + "soong-libchrome": true, + "soong-llvm": true, + "soong-robolectric": true, + "soong-rust-prebuilts": true, + "soong-selinux": true, + "soong-wayland-protocol-codegen": true, + "treble_report_app": true, + "treble_report_local": true, + "treble_report_module": true, + "vintf-compatibility-matrix-soong-rules": true, + "xsdc-soong-rules": true, +} + +const ( + internalPluginsPath = "vendor/google/build/soong/internal_plugins.json" +) + +type pluginProvider interface { + IsPluginFor(string) bool +} + +func maybeAddInternalPluginsToAllowlist(ctx SingletonContext) { + if path := ExistentPathForSource(ctx, internalPluginsPath); path.Valid() { + ctx.AddNinjaFileDeps(path.String()) + absPath := absolutePath(path.String()) + var moreAllowed map[string]bool + data, err := ioutil.ReadFile(absPath) + if err != nil { + ctx.Errorf("Failed to open internal plugins path %q %q", internalPluginsPath, err) + } + if err := json.Unmarshal(data, &moreAllowed); err != nil { + fmt.Fprintf(os.Stderr, "Internal plugins file %q did not parse correctly: %q", data, err) + } + for k, v := range moreAllowed { + allowedPluginsByName[k] = v + } + } +} + +func (p *pluginSingleton) GenerateBuildActions(ctx SingletonContext) { + for _, p := range ctx.DeviceConfig().BuildBrokenPluginValidation() { + allowedPluginsByName[p] = true + } + maybeAddInternalPluginsToAllowlist(ctx) + + disallowedPlugins := map[string]bool{} + ctx.VisitAllModulesBlueprint(func(module blueprint.Module) { + if ctx.ModuleType(module) != "bootstrap_go_package" { + return + } + + p, ok := module.(pluginProvider) + if !ok || !p.IsPluginFor("soong_build") { + return + } + + name := ctx.ModuleName(module) + if _, ok := allowedPluginsByName[name]; ok { + return + } + + dir := ctx.ModuleDir(module) + + // allow use of plugins within Soong to not allowlist everything + if strings.HasPrefix(dir, "build/soong") { + return + } + + // allow third party users outside of external to create new plugins, i.e. non-google paths + // under vendor or hardware + if !strings.HasPrefix(dir, "external/") && IsThirdPartyPath(dir) { + return + } + disallowedPlugins[name] = true + }) + if len(disallowedPlugins) > 0 { + ctx.Errorf("New plugins are not supported; however %q were found. Please reach out to the build team or use BUILD_BROKEN_PLUGIN_VALIDATION (see Changes.md for more info).", SortedStringKeys(disallowedPlugins)) + } +} diff --git a/android/variable.go b/android/variable.go index 97171e7b99..40eb00675d 100644 --- a/android/variable.go +++ b/android/variable.go @@ -440,6 +440,7 @@ type productVariables struct { ShippingApiLevel *string `json:",omitempty"` + BuildBrokenPluginValidation []string `json:",omitempty"` BuildBrokenClangAsFlags bool `json:",omitempty"` BuildBrokenClangCFlags bool `json:",omitempty"` BuildBrokenClangProperty bool `json:",omitempty"` diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index 59352477cb..5fc05f8b10 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -207,8 +207,8 @@ EOF function test_soong_build_rerun_iff_environment_changes() { setup - mkdir -p cherry - cat > cherry/Android.bp <<'EOF' + mkdir -p build/soong/cherry + cat > build/soong/cherry/Android.bp <<'EOF' bootstrap_go_package { name: "cherry", pkgPath: "android/soong/cherry", @@ -224,7 +224,7 @@ bootstrap_go_package { } EOF - cat > cherry/cherry.go <<'EOF' + cat > build/soong/cherry/cherry.go <<'EOF' package cherry import ( @@ -317,8 +317,8 @@ function test_add_file_to_soong_build() { run_soong local -r mtime1=$(stat -c "%y" out/soong/build.ninja) - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p vendor/foo/picard + cat > vendor/foo/picard/Android.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -334,7 +334,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > vendor/foo/picard/picard.go <<'EOF' package picard import ( @@ -390,11 +390,11 @@ EOF function test_glob_during_bootstrapping() { setup - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p build/soong/picard + cat > build/soong/picard/Android.bp <<'EOF' build=["foo*.bp"] EOF - cat > a/fooa.bp <<'EOF' + cat > build/soong/picard/fooa.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -410,7 +410,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > build/soong/picard/picard.go <<'EOF' package picard import ( @@ -459,7 +459,7 @@ EOF grep -q "Make it so" out/soong/build.ninja || fail "Original action not present" - cat > a/foob.bp <<'EOF' + cat > build/soong/picard/foob.bp <<'EOF' bootstrap_go_package { name: "worf-soong-rules", pkgPath: "android/soong/worf", @@ -476,7 +476,7 @@ bootstrap_go_package { } EOF - cat > a/worf.go <<'EOF' + cat > build/soong/picard/worf.go <<'EOF' package worf import "android/soong/picard" From e8c70c5bd541b4f236885fe46db3a06443ac68d0 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 18 May 2023 11:51:56 -0700 Subject: [PATCH 0172/1460] Enable host cross python test builds Ic37c8db918873ddf324c86b12b5412952b0f2be2 converted python_binary_host and python_library_host from HostSupportedNoCross to HostSupported, but left python_test_host. Do the same for python_test_host. Fixes dependencies between java_test_host modules that create host cross variants and python_test_host modules that were missing host cross variants. Bug: 282918027 Test: builds Change-Id: Iae687aa3aa0f0b005f2dd27469f631145247a008 --- python/test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/test.go b/python/test.go index 31da17e614..7de7af2def 100644 --- a/python/test.go +++ b/python/test.go @@ -39,7 +39,7 @@ func NewTest(hod android.HostOrDeviceSupported) *PythonTestModule { } func PythonTestHostFactory() android.Module { - return NewTest(android.HostSupportedNoCross).init() + return NewTest(android.HostSupported).init() } func PythonTestFactory() android.Module { @@ -98,7 +98,7 @@ func (p *PythonTestModule) init() android.Module { android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) android.InitBazelModule(p) - if p.hod == android.HostSupportedNoCross && p.testProperties.Test_options.Unit_test == nil { + if p.hod == android.HostSupported && p.testProperties.Test_options.Unit_test == nil { p.testProperties.Test_options.Unit_test = proptools.BoolPtr(true) } return p From f2ecf7659e6c594da85037b46daea67a5debd663 Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Fri, 19 May 2023 14:03:45 +0900 Subject: [PATCH 0173/1460] Enable NINJA_LOG as weight list source Bug: 273947040 Test: m Change-Id: I1a803f5c1da75235395fc594e4f9d17f48c55db8 --- ui/build/config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index 8ec96800f3..b924aa0fa1 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -316,8 +316,9 @@ func UploadOnlyConfig(ctx Context, args ...string) Config { func NewConfig(ctx Context, args ...string) Config { ret := &configImpl{ - environ: OsEnvironment(), - sandboxConfig: &SandboxConfig{}, + environ: OsEnvironment(), + sandboxConfig: &SandboxConfig{}, + ninjaWeightListSource: NINJA_LOG, } // Default matching ninja From 1c92c3e09262d24947a6d314fe3be77329127a76 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Thu, 23 Mar 2023 17:44:51 +0000 Subject: [PATCH 0174/1460] Generate java_api_library from java_sdk_library This change enables java_sdk_library to generate java_api_library modules per api surface, so that from-text stubs can be generated per api domain scope. This module is only created when `--build-from-text-stub` flag is passed during build. Test: enable disabled modules in java/core-libraries/TxtStubLibraries.bp then m art.module.public.api.stubs.from-text --build-from-text-stub Bug: 276957733 Change-Id: Ic1ead15b3d0bcb921ca8d31bcaeeb4cd9ee8715c Merged-In: Ic1ead15b3d0bcb921ca8d31bcaeeb4cd9ee8715c --- android/config.go | 42 ++++++++++++++++++++++ java/sdk_library.go | 78 +++++++++++++++++++++++++++++++++++++++- java/sdk_library_test.go | 63 ++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 1 deletion(-) diff --git a/android/config.go b/android/config.go index acadb3ff33..4f0a64dfee 100644 --- a/android/config.go +++ b/android/config.go @@ -302,6 +302,9 @@ type config struct { // modules that aren't mixed-built for at least one variant will cause a build // failure ensureAllowlistIntegrity bool + + // List of Api libraries that contribute to Api surfaces. + apiLibraries map[string]struct{} } type deviceConfig struct { @@ -622,6 +625,33 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) config.BazelContext, err = NewBazelContext(config) config.Bp2buildPackageConfig = GetBp2BuildAllowList() + // TODO(b/276958307): Replace the hardcoded list to a sdk_library local prop. + config.apiLibraries = map[string]struct{}{ + "android.net.ipsec.ike": {}, + "art.module.public.api": {}, + "conscrypt.module.public.api": {}, + "framework-adservices": {}, + "framework-appsearch": {}, + "framework-bluetooth": {}, + "framework-connectivity": {}, + "framework-connectivity-t": {}, + "framework-graphics": {}, + "framework-media": {}, + "framework-mediaprovider": {}, + "framework-ondevicepersonalization": {}, + "framework-permission": {}, + "framework-permission-s": {}, + "framework-scheduling": {}, + "framework-sdkextensions": {}, + "framework-statsd": {}, + "framework-sdksandbox": {}, + "framework-tethering": {}, + "framework-uwb": {}, + "framework-virtualization": {}, + "framework-wifi": {}, + "i18n.module.public.api": {}, + } + return Config{config}, err } @@ -1979,8 +2009,20 @@ func (c *config) BuildFromTextStub() bool { func (c *config) SetBuildFromTextStub(b bool) { c.buildFromTextStub = b } + func (c *config) AddForceEnabledModules(forceEnabled []string) { for _, forceEnabledModule := range forceEnabled { c.bazelForceEnabledModules[forceEnabledModule] = struct{}{} } } + +func (c *config) SetApiLibraries(libs []string) { + c.apiLibraries = make(map[string]struct{}) + for _, lib := range libs { + c.apiLibraries[lib] = struct{}{} + } +} + +func (c *config) GetApiLibraries() map[string]struct{} { + return c.apiLibraries +} diff --git a/java/sdk_library.go b/java/sdk_library.go index 103f1ace72..89da19a197 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -156,6 +156,9 @@ type apiScope struct { // Whether the api scope can be treated as unstable, and should skip compat checks. unstable bool + + // Represents the SDK kind of this scope. + kind android.SdkKind } // Initialize a scope, creating and adding appropriate dependency tags @@ -229,6 +232,10 @@ func (scope *apiScope) stubsLibraryModuleNameSuffix() string { return ".stubs" + scope.moduleSuffix } +func (scope *apiScope) apiLibraryModuleName(baseName string) string { + return scope.stubsLibraryModuleName(baseName) + ".from-text" +} + func (scope *apiScope) stubsLibraryModuleName(baseName string) string { return baseName + scope.stubsLibraryModuleNameSuffix() } @@ -289,6 +296,7 @@ var ( return &module.sdkLibraryProperties.Public }, sdkVersion: "current", + kind: android.SdkPublic, }) apiScopeSystem = initApiScope(&apiScope{ name: "system", @@ -301,6 +309,7 @@ var ( moduleSuffix: ".system", sdkVersion: "system_current", annotation: "android.annotation.SystemApi(client=android.annotation.SystemApi.Client.PRIVILEGED_APPS)", + kind: android.SdkSystem, }) apiScopeTest = initApiScope(&apiScope{ name: "test", @@ -314,6 +323,7 @@ var ( sdkVersion: "test_current", annotation: "android.annotation.TestApi", unstable: true, + kind: android.SdkTest, }) apiScopeModuleLib = initApiScope(&apiScope{ name: "module-lib", @@ -331,6 +341,7 @@ var ( moduleSuffix: ".module_lib", sdkVersion: "module_current", annotation: "android.annotation.SystemApi(client=android.annotation.SystemApi.Client.MODULE_LIBRARIES)", + kind: android.SdkModule, }) apiScopeSystemServer = initApiScope(&apiScope{ name: "system-server", @@ -361,6 +372,7 @@ var ( // com.android.* classes are okay in this interface" "--hide", "InternalClasses", }, + kind: android.SdkSystemServer, }) allApiScopes = apiScopes{ apiScopePublic, @@ -842,6 +854,13 @@ func (c *commonToSdkLibraryAndImport) stubsSourceModuleName(apiScope *apiScope) return c.namingScheme.stubsSourceModuleName(apiScope, baseName) } +// Name of the java_api_library module that generates the from-text stubs source +// and compiles to a jar file. +func (c *commonToSdkLibraryAndImport) apiLibraryModuleName(apiScope *apiScope) string { + baseName := c.module.BaseModuleName() + return c.namingScheme.apiLibraryModuleName(apiScope, baseName) +} + // The component names for different outputs of the java_sdk_library. // // They are similar to the names used for the child modules it creates @@ -1269,7 +1288,9 @@ func (module *SdkLibrary) ComponentDepsMutator(ctx android.BottomUpMutatorContex // Add dependencies to the stubs library stubModuleName := module.stubsLibraryModuleName(apiScope) // Use JavaApiLibraryName function to be redirected to stubs generated from .txt if applicable - stubModuleName = android.JavaApiLibraryName(ctx.Config(), stubModuleName) + if module.contributesToApiSurface(ctx.Config()) { + stubModuleName = android.JavaApiLibraryName(ctx.Config(), stubModuleName) + } ctx.AddVariationDependencies(nil, apiScope.stubsTag, stubModuleName) // Add a dependency on the stubs source in order to access both stubs source and api information. @@ -1477,6 +1498,11 @@ func (module *SdkLibrary) latestIncompatibilitiesModuleName(apiScope *apiScope) return latestPrebuiltApiModuleName(module.distStem()+"-incompatibilities", apiScope) } +func (module *SdkLibrary) contributesToApiSurface(c android.Config) bool { + _, exists := c.GetApiLibraries()[module.Name()] + return exists +} + func childModuleVisibility(childVisibility []string) []string { if childVisibility == nil { // No child visibility set. The child will use the visibility of the sdk_library. @@ -1758,6 +1784,46 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC mctx.CreateModule(DroidstubsFactory, &props).(*Droidstubs).CallHookIfAvailable(mctx) } +func (module *SdkLibrary) createApiLibrary(mctx android.DefaultableHookContext, apiScope *apiScope) { + props := struct { + Name *string + Visibility []string + Api_contributions []string + Libs []string + Static_libs []string + Dep_api_srcs *string + }{} + + props.Name = proptools.StringPtr(module.apiLibraryModuleName(apiScope)) + props.Visibility = childModuleVisibility(module.sdkLibraryProperties.Stubs_library_visibility) + + apiContributions := []string{} + + // Api surfaces are not independent of each other, but have subset relationships, + // and so does the api files. To generate from-text stubs for api surfaces other than public, + // all subset api domains' api_contriubtions must be added as well. + scope := apiScope + for scope != nil { + apiContributions = append(apiContributions, module.stubsSourceModuleName(scope)+".api.contribution") + scope = scope.extends + } + + props.Api_contributions = apiContributions + props.Libs = module.properties.Libs + props.Libs = append(props.Libs, module.sdkLibraryProperties.Stub_only_libs...) + props.Libs = append(props.Libs, "stub-annotations") + props.Static_libs = module.sdkLibraryProperties.Stub_only_static_libs + props.Dep_api_srcs = proptools.StringPtr(apiScope.kind.DefaultJavaLibraryName() + ".from-text") + + // android_module_lib_stubs_current.from-text only comprises api contributions from art, conscrypt and i18n. + // Thus, replace with android_module_lib_stubs_current_full.from-text, which comprises every api domains. + if apiScope.kind == android.SdkModule { + props.Dep_api_srcs = proptools.StringPtr(apiScope.kind.DefaultJavaLibraryName() + "_full.from-text") + } + + mctx.CreateModule(ApiLibraryFactory, &props) +} + func (module *SdkLibrary) compareAgainstLatestApi(apiScope *apiScope) bool { return !(apiScope.unstable || module.sdkLibraryProperties.Unsafe_ignore_missing_latest_api) } @@ -1954,6 +2020,10 @@ func (module *SdkLibrary) CreateInternalModules(mctx android.DefaultableHookCont module.createStubsSourcesAndApi(mctx, scope, module.stubsSourceModuleName(scope), scope.droidstubsArgs) module.createStubsLibrary(mctx, scope) + + if module.contributesToApiSurface(mctx.Config()) { + module.createApiLibrary(mctx, scope) + } } if module.requiresRuntimeImplementationLibrary() { @@ -2006,6 +2076,8 @@ type sdkLibraryComponentNamingScheme interface { stubsLibraryModuleName(scope *apiScope, baseName string) string stubsSourceModuleName(scope *apiScope, baseName string) string + + apiLibraryModuleName(scope *apiScope, baseName string) string } type defaultNamingScheme struct { @@ -2019,6 +2091,10 @@ func (s *defaultNamingScheme) stubsSourceModuleName(scope *apiScope, baseName st return scope.stubsSourceModuleName(baseName) } +func (s *defaultNamingScheme) apiLibraryModuleName(scope *apiScope, baseName string) string { + return scope.apiLibraryModuleName(baseName) +} + var _ sdkLibraryComponentNamingScheme = (*defaultNamingScheme)(nil) func moduleStubLinkType(name string) (stub bool, ret sdkLinkType) { diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 1d0c13d4bc..141e16bf7a 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -35,6 +35,9 @@ func TestJavaSdkLibrary(t *testing.T) { "29": {"foo"}, "30": {"bar", "barney", "baz", "betty", "foo", "fred", "quuz", "wilma"}, }), + android.FixtureModifyConfig(func(config android.Config) { + config.SetApiLibraries([]string{"foo"}) + }), ).RunTestWithBp(t, ` droiddoc_exported_dir { name: "droiddoc-templates-sdk", @@ -121,6 +124,7 @@ func TestJavaSdkLibrary(t *testing.T) { result.ModuleForTests(apiScopeSystem.stubsSourceModuleName("foo"), "android_common") result.ModuleForTests(apiScopeTest.stubsSourceModuleName("foo"), "android_common") result.ModuleForTests(apiScopePublic.stubsSourceModuleName("foo")+".api.contribution", "") + result.ModuleForTests(apiScopePublic.apiLibraryModuleName("foo"), "android_common") result.ModuleForTests("foo"+sdkXmlFileSuffix, "android_common") result.ModuleForTests("foo.api.public.28", "") result.ModuleForTests("foo.api.system.28", "") @@ -1412,3 +1416,62 @@ func TestJavaSdkLibrary_StubOnlyLibs_PassedToDroidstubs(t *testing.T) { fooStubsSources := result.ModuleForTests("foo.stubs.source", "android_common").Module().(*Droidstubs) android.AssertStringListContains(t, "foo stubs should depend on bar-lib", fooStubsSources.Javadoc.properties.Libs, "bar-lib") } + +func TestJavaSdkLibrary_ApiLibrary(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForJavaTest, + PrepareForTestWithJavaSdkLibraryFiles, + FixtureWithLastReleaseApis("foo"), + android.FixtureModifyConfig(func(config android.Config) { + config.SetApiLibraries([]string{"foo"}) + }), + ).RunTestWithBp(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java", "b.java"], + api_packages: ["foo"], + system: { + enabled: true, + }, + module_lib: { + enabled: true, + }, + test: { + enabled: true, + }, + } + `) + + testCases := []struct { + scope *apiScope + apiContributions []string + depApiSrcs string + }{ + { + scope: apiScopePublic, + apiContributions: []string{"foo.stubs.source.api.contribution"}, + depApiSrcs: "android_stubs_current.from-text", + }, + { + scope: apiScopeSystem, + apiContributions: []string{"foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_system_stubs_current.from-text", + }, + { + scope: apiScopeTest, + apiContributions: []string{"foo.stubs.source.test.api.contribution", "foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_test_stubs_current.from-text", + }, + { + scope: apiScopeModuleLib, + apiContributions: []string{"foo.stubs.source.module_lib.api.contribution", "foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_module_lib_stubs_current_full.from-text", + }, + } + + for _, c := range testCases { + m := result.ModuleForTests(c.scope.apiLibraryModuleName("foo"), "android_common").Module().(*ApiLibrary) + android.AssertArrayString(t, "Module expected to contain api contributions", c.apiContributions, m.properties.Api_contributions) + android.AssertStringEquals(t, "Module expected to contain full api surface api library", c.depApiSrcs, *m.properties.Dep_api_srcs) + } +} From 480943ff647fdab582ddadc7a435bbc54ef53301 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Fri, 19 May 2023 06:02:08 +0000 Subject: [PATCH 0175/1460] Generate java_api_library from java_sdk_library This change enables java_sdk_library to generate java_api_library modules per api surface, so that from-text stubs can be generated per api domain scope. This module is only created when `--build-from-text-stub` flag is passed during build. Ignore-AOSP-First: config.apiLibraries list differ between AOSP and internal master Test: enable disabled modules in java/core-libraries/TxtStubLibraries.bp then m art.module.public.api.stubs.from-text --build-from-text-stub Bug: 276957733 Change-Id: Ic1ead15b3d0bcb921ca8d31bcaeeb4cd9ee8715c --- android/config.go | 45 +++++++++++++++++++++++ java/sdk_library.go | 78 +++++++++++++++++++++++++++++++++++++++- java/sdk_library_test.go | 63 ++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) diff --git a/android/config.go b/android/config.go index 01b8cfaa32..a60ceedf29 100644 --- a/android/config.go +++ b/android/config.go @@ -302,6 +302,9 @@ type config struct { // modules that aren't mixed-built for at least one variant will cause a build // failure ensureAllowlistIntegrity bool + + // List of Api libraries that contribute to Api surfaces. + apiLibraries map[string]struct{} } type deviceConfig struct { @@ -622,6 +625,36 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) config.BazelContext, err = NewBazelContext(config) config.Bp2buildPackageConfig = GetBp2BuildAllowList() + // TODO(b/276958307): Replace the hardcoded list to a sdk_library local prop. + config.apiLibraries = map[string]struct{}{ + "android.net.ipsec.ike": {}, + "art.module.public.api": {}, + "conscrypt.module.public.api": {}, + "framework-adservices": {}, + "framework-appsearch": {}, + "framework-bluetooth": {}, + "framework-configinfrastructure": {}, + "framework-connectivity": {}, + "framework-connectivity-t": {}, + "framework-devicelock": {}, + "framework-graphics": {}, + "framework-healthfitness": {}, + "framework-media": {}, + "framework-mediaprovider": {}, + "framework-ondevicepersonalization": {}, + "framework-permission": {}, + "framework-permission-s": {}, + "framework-scheduling": {}, + "framework-sdkextensions": {}, + "framework-statsd": {}, + "framework-sdksandbox": {}, + "framework-tethering": {}, + "framework-uwb": {}, + "framework-virtualization": {}, + "framework-wifi": {}, + "i18n.module.public.api": {}, + } + return Config{config}, err } @@ -1983,8 +2016,20 @@ func (c *config) BuildFromTextStub() bool { func (c *config) SetBuildFromTextStub(b bool) { c.buildFromTextStub = b } + func (c *config) AddForceEnabledModules(forceEnabled []string) { for _, forceEnabledModule := range forceEnabled { c.bazelForceEnabledModules[forceEnabledModule] = struct{}{} } } + +func (c *config) SetApiLibraries(libs []string) { + c.apiLibraries = make(map[string]struct{}) + for _, lib := range libs { + c.apiLibraries[lib] = struct{}{} + } +} + +func (c *config) GetApiLibraries() map[string]struct{} { + return c.apiLibraries +} diff --git a/java/sdk_library.go b/java/sdk_library.go index 103f1ace72..89da19a197 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -156,6 +156,9 @@ type apiScope struct { // Whether the api scope can be treated as unstable, and should skip compat checks. unstable bool + + // Represents the SDK kind of this scope. + kind android.SdkKind } // Initialize a scope, creating and adding appropriate dependency tags @@ -229,6 +232,10 @@ func (scope *apiScope) stubsLibraryModuleNameSuffix() string { return ".stubs" + scope.moduleSuffix } +func (scope *apiScope) apiLibraryModuleName(baseName string) string { + return scope.stubsLibraryModuleName(baseName) + ".from-text" +} + func (scope *apiScope) stubsLibraryModuleName(baseName string) string { return baseName + scope.stubsLibraryModuleNameSuffix() } @@ -289,6 +296,7 @@ var ( return &module.sdkLibraryProperties.Public }, sdkVersion: "current", + kind: android.SdkPublic, }) apiScopeSystem = initApiScope(&apiScope{ name: "system", @@ -301,6 +309,7 @@ var ( moduleSuffix: ".system", sdkVersion: "system_current", annotation: "android.annotation.SystemApi(client=android.annotation.SystemApi.Client.PRIVILEGED_APPS)", + kind: android.SdkSystem, }) apiScopeTest = initApiScope(&apiScope{ name: "test", @@ -314,6 +323,7 @@ var ( sdkVersion: "test_current", annotation: "android.annotation.TestApi", unstable: true, + kind: android.SdkTest, }) apiScopeModuleLib = initApiScope(&apiScope{ name: "module-lib", @@ -331,6 +341,7 @@ var ( moduleSuffix: ".module_lib", sdkVersion: "module_current", annotation: "android.annotation.SystemApi(client=android.annotation.SystemApi.Client.MODULE_LIBRARIES)", + kind: android.SdkModule, }) apiScopeSystemServer = initApiScope(&apiScope{ name: "system-server", @@ -361,6 +372,7 @@ var ( // com.android.* classes are okay in this interface" "--hide", "InternalClasses", }, + kind: android.SdkSystemServer, }) allApiScopes = apiScopes{ apiScopePublic, @@ -842,6 +854,13 @@ func (c *commonToSdkLibraryAndImport) stubsSourceModuleName(apiScope *apiScope) return c.namingScheme.stubsSourceModuleName(apiScope, baseName) } +// Name of the java_api_library module that generates the from-text stubs source +// and compiles to a jar file. +func (c *commonToSdkLibraryAndImport) apiLibraryModuleName(apiScope *apiScope) string { + baseName := c.module.BaseModuleName() + return c.namingScheme.apiLibraryModuleName(apiScope, baseName) +} + // The component names for different outputs of the java_sdk_library. // // They are similar to the names used for the child modules it creates @@ -1269,7 +1288,9 @@ func (module *SdkLibrary) ComponentDepsMutator(ctx android.BottomUpMutatorContex // Add dependencies to the stubs library stubModuleName := module.stubsLibraryModuleName(apiScope) // Use JavaApiLibraryName function to be redirected to stubs generated from .txt if applicable - stubModuleName = android.JavaApiLibraryName(ctx.Config(), stubModuleName) + if module.contributesToApiSurface(ctx.Config()) { + stubModuleName = android.JavaApiLibraryName(ctx.Config(), stubModuleName) + } ctx.AddVariationDependencies(nil, apiScope.stubsTag, stubModuleName) // Add a dependency on the stubs source in order to access both stubs source and api information. @@ -1477,6 +1498,11 @@ func (module *SdkLibrary) latestIncompatibilitiesModuleName(apiScope *apiScope) return latestPrebuiltApiModuleName(module.distStem()+"-incompatibilities", apiScope) } +func (module *SdkLibrary) contributesToApiSurface(c android.Config) bool { + _, exists := c.GetApiLibraries()[module.Name()] + return exists +} + func childModuleVisibility(childVisibility []string) []string { if childVisibility == nil { // No child visibility set. The child will use the visibility of the sdk_library. @@ -1758,6 +1784,46 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC mctx.CreateModule(DroidstubsFactory, &props).(*Droidstubs).CallHookIfAvailable(mctx) } +func (module *SdkLibrary) createApiLibrary(mctx android.DefaultableHookContext, apiScope *apiScope) { + props := struct { + Name *string + Visibility []string + Api_contributions []string + Libs []string + Static_libs []string + Dep_api_srcs *string + }{} + + props.Name = proptools.StringPtr(module.apiLibraryModuleName(apiScope)) + props.Visibility = childModuleVisibility(module.sdkLibraryProperties.Stubs_library_visibility) + + apiContributions := []string{} + + // Api surfaces are not independent of each other, but have subset relationships, + // and so does the api files. To generate from-text stubs for api surfaces other than public, + // all subset api domains' api_contriubtions must be added as well. + scope := apiScope + for scope != nil { + apiContributions = append(apiContributions, module.stubsSourceModuleName(scope)+".api.contribution") + scope = scope.extends + } + + props.Api_contributions = apiContributions + props.Libs = module.properties.Libs + props.Libs = append(props.Libs, module.sdkLibraryProperties.Stub_only_libs...) + props.Libs = append(props.Libs, "stub-annotations") + props.Static_libs = module.sdkLibraryProperties.Stub_only_static_libs + props.Dep_api_srcs = proptools.StringPtr(apiScope.kind.DefaultJavaLibraryName() + ".from-text") + + // android_module_lib_stubs_current.from-text only comprises api contributions from art, conscrypt and i18n. + // Thus, replace with android_module_lib_stubs_current_full.from-text, which comprises every api domains. + if apiScope.kind == android.SdkModule { + props.Dep_api_srcs = proptools.StringPtr(apiScope.kind.DefaultJavaLibraryName() + "_full.from-text") + } + + mctx.CreateModule(ApiLibraryFactory, &props) +} + func (module *SdkLibrary) compareAgainstLatestApi(apiScope *apiScope) bool { return !(apiScope.unstable || module.sdkLibraryProperties.Unsafe_ignore_missing_latest_api) } @@ -1954,6 +2020,10 @@ func (module *SdkLibrary) CreateInternalModules(mctx android.DefaultableHookCont module.createStubsSourcesAndApi(mctx, scope, module.stubsSourceModuleName(scope), scope.droidstubsArgs) module.createStubsLibrary(mctx, scope) + + if module.contributesToApiSurface(mctx.Config()) { + module.createApiLibrary(mctx, scope) + } } if module.requiresRuntimeImplementationLibrary() { @@ -2006,6 +2076,8 @@ type sdkLibraryComponentNamingScheme interface { stubsLibraryModuleName(scope *apiScope, baseName string) string stubsSourceModuleName(scope *apiScope, baseName string) string + + apiLibraryModuleName(scope *apiScope, baseName string) string } type defaultNamingScheme struct { @@ -2019,6 +2091,10 @@ func (s *defaultNamingScheme) stubsSourceModuleName(scope *apiScope, baseName st return scope.stubsSourceModuleName(baseName) } +func (s *defaultNamingScheme) apiLibraryModuleName(scope *apiScope, baseName string) string { + return scope.apiLibraryModuleName(baseName) +} + var _ sdkLibraryComponentNamingScheme = (*defaultNamingScheme)(nil) func moduleStubLinkType(name string) (stub bool, ret sdkLinkType) { diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 1d0c13d4bc..141e16bf7a 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -35,6 +35,9 @@ func TestJavaSdkLibrary(t *testing.T) { "29": {"foo"}, "30": {"bar", "barney", "baz", "betty", "foo", "fred", "quuz", "wilma"}, }), + android.FixtureModifyConfig(func(config android.Config) { + config.SetApiLibraries([]string{"foo"}) + }), ).RunTestWithBp(t, ` droiddoc_exported_dir { name: "droiddoc-templates-sdk", @@ -121,6 +124,7 @@ func TestJavaSdkLibrary(t *testing.T) { result.ModuleForTests(apiScopeSystem.stubsSourceModuleName("foo"), "android_common") result.ModuleForTests(apiScopeTest.stubsSourceModuleName("foo"), "android_common") result.ModuleForTests(apiScopePublic.stubsSourceModuleName("foo")+".api.contribution", "") + result.ModuleForTests(apiScopePublic.apiLibraryModuleName("foo"), "android_common") result.ModuleForTests("foo"+sdkXmlFileSuffix, "android_common") result.ModuleForTests("foo.api.public.28", "") result.ModuleForTests("foo.api.system.28", "") @@ -1412,3 +1416,62 @@ func TestJavaSdkLibrary_StubOnlyLibs_PassedToDroidstubs(t *testing.T) { fooStubsSources := result.ModuleForTests("foo.stubs.source", "android_common").Module().(*Droidstubs) android.AssertStringListContains(t, "foo stubs should depend on bar-lib", fooStubsSources.Javadoc.properties.Libs, "bar-lib") } + +func TestJavaSdkLibrary_ApiLibrary(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForJavaTest, + PrepareForTestWithJavaSdkLibraryFiles, + FixtureWithLastReleaseApis("foo"), + android.FixtureModifyConfig(func(config android.Config) { + config.SetApiLibraries([]string{"foo"}) + }), + ).RunTestWithBp(t, ` + java_sdk_library { + name: "foo", + srcs: ["a.java", "b.java"], + api_packages: ["foo"], + system: { + enabled: true, + }, + module_lib: { + enabled: true, + }, + test: { + enabled: true, + }, + } + `) + + testCases := []struct { + scope *apiScope + apiContributions []string + depApiSrcs string + }{ + { + scope: apiScopePublic, + apiContributions: []string{"foo.stubs.source.api.contribution"}, + depApiSrcs: "android_stubs_current.from-text", + }, + { + scope: apiScopeSystem, + apiContributions: []string{"foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_system_stubs_current.from-text", + }, + { + scope: apiScopeTest, + apiContributions: []string{"foo.stubs.source.test.api.contribution", "foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_test_stubs_current.from-text", + }, + { + scope: apiScopeModuleLib, + apiContributions: []string{"foo.stubs.source.module_lib.api.contribution", "foo.stubs.source.system.api.contribution", "foo.stubs.source.api.contribution"}, + depApiSrcs: "android_module_lib_stubs_current_full.from-text", + }, + } + + for _, c := range testCases { + m := result.ModuleForTests(c.scope.apiLibraryModuleName("foo"), "android_common").Module().(*ApiLibrary) + android.AssertArrayString(t, "Module expected to contain api contributions", c.apiContributions, m.properties.Api_contributions) + android.AssertStringEquals(t, "Module expected to contain full api surface api library", c.depApiSrcs, *m.properties.Dep_api_srcs) + } +} From f06dd914200a4a231595d65d32317d3123b7d423 Mon Sep 17 00:00:00 2001 From: Trevor Radcliffe Date: Fri, 19 May 2023 14:51:41 +0000 Subject: [PATCH 0176/1460] CFI Versionscript changes in bp2build Bug: 261733820 Test: Unit Tests Change-Id: I0bf2f42944738c0fefb10e59e859d2af44402792 --- bp2build/cc_binary_conversion_test.go | 2 ++ bp2build/cc_library_conversion_test.go | 13 +++++++++++-- bp2build/cc_library_shared_conversion_test.go | 3 +++ cc/bp2build.go | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/bp2build/cc_binary_conversion_test.go b/bp2build/cc_binary_conversion_test.go index 6ec37031b8..8a83cc0d5c 100644 --- a/bp2build/cc_binary_conversion_test.go +++ b/bp2build/cc_binary_conversion_test.go @@ -222,6 +222,7 @@ func TestCcBinaryVersionScriptAndDynamicList(t *testing.T) { "-Wl,--version-script,$(location vs)", "-Wl,--dynamic-list,$(location dynamic.list)", ]`, + "features": `["android_cfi_exports_map"]`, }, }, }, @@ -249,6 +250,7 @@ func TestCcBinaryLdflagsSplitBySpaceExceptSoongAdded(t *testing.T) { "version_script", "dynamic.list", ]`, + "features": `["android_cfi_exports_map"]`, "linkopts": `[ "--nospace_flag", "-z", diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 0e8705b54d..3dd9373195 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -900,7 +900,8 @@ cc_library { "-Wl,--version-script,$(location v.map)", "-Wl,--dynamic-list,$(location dynamic.list)", ]`, - "srcs": `["a.cpp"]`, + "srcs": `["a.cpp"]`, + "features": `["android_cfi_exports_map"]`, }), }, ) @@ -958,6 +959,11 @@ cc_library { "//conditions:default": [], })`, "srcs": `["a.cpp"]`, + "features": `select({ + "//build/bazel/platforms/arch:arm": ["android_cfi_exports_map"], + "//build/bazel/platforms/arch:arm64": ["android_cfi_exports_map"], + "//conditions:default": [], + })`, }), }, ) @@ -985,12 +991,15 @@ cc_library { } `, ExpectedBazelTargets: []string{ - MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{}), + MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ + "features": `["android_cfi_exports_map"]`, + }), MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ "additional_linker_inputs": `[ "version_script", "dynamic.list", ]`, + "features": `["android_cfi_exports_map"]`, "linkopts": `[ "--nospace_flag", "-z", diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go index 2ee9c99f9c..6c9f9a18f0 100644 --- a/bp2build/cc_library_shared_conversion_test.go +++ b/bp2build/cc_library_shared_conversion_test.go @@ -362,6 +362,7 @@ cc_library_shared { "-Wl,--version-script,$(location version_script)", "-Wl,--dynamic-list,$(location dynamic.list)", ]`, + "features": `["android_cfi_exports_map"]`, }), }, }) @@ -398,6 +399,7 @@ cc_library_shared { "-Wl,--version-script,$(location version_script)", "-Wl,--dynamic-list,$(location dynamic.list)", ]`, + "features": `["android_cfi_exports_map"]`, }), }, }) @@ -913,6 +915,7 @@ func TestCCLibraryFlagSpaceSplitting(t *testing.T) { "header.h", ]`, "linkopts": `["-Wl,--version-script,$(location version_script)"]`, + "features": `["android_cfi_exports_map"]`, }), }, }) diff --git a/cc/bp2build.go b/cc/bp2build.go index fa98df4cd6..259ba397a4 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -1257,6 +1257,7 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion label := android.BazelLabelForModuleSrcSingle(ctx, *props.Version_script) additionalLinkerInputs.Add(&label) linkerFlags = append(linkerFlags, fmt.Sprintf("-Wl,--version-script,$(location %s)", label.Label)) + axisFeatures = append(axisFeatures, "android_cfi_exports_map") } if props.Dynamic_list != nil { From d55347133f894fcab4ff3bc33e5916d08f37bcd1 Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Sat, 20 May 2023 00:49:39 +0900 Subject: [PATCH 0177/1460] NINJA_LOG option uses usesninjalogasweightlist As-is: Even with NINJA_LOG option, ninja weight list is copied from ninja log To-be: uses usesninjalogasweightlist to uses ninja log file directly If weight list is large(more than 100,000 lines), reading the whole list spends a few seconds(1-3s). It is ignorable in full build, but it might be considerable in small build or m nothing scenario. If weight list comes from ninja log, we can just directly read ninja log data instead of writing weight list from ninja log outside, and read the weight list. It doesn't spend additional time because ninja log is loaded by default. Bug: 271527305 Test: build with NINJA_LOG Change-Id: Id7a4cca95898ce439d0a682f9bfd954b462f1849 --- ui/build/ninja.go | 44 +------------------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/ui/build/ninja.go b/ui/build/ninja.go index 5d56531b29..61aaad86b2 100644 --- a/ui/build/ninja.go +++ b/ui/build/ninja.go @@ -35,48 +35,6 @@ const ( ninjaWeightListFileName = ".ninja_weight_list" ) -func useNinjaBuildLog(ctx Context, config Config, cmd *Cmd) { - ninjaLogFile := filepath.Join(config.OutDir(), ninjaLogFileName) - data, err := os.ReadFile(ninjaLogFile) - var outputBuilder strings.Builder - if err == nil { - lines := strings.Split(strings.TrimSpace(string(data)), "\n") - // ninja log: - // ninja weight list: , - for _, line := range lines { - if strings.HasPrefix(line, "#") { - continue - } - fields := strings.Split(line, "\t") - path := fields[3] - start, err := strconv.Atoi(fields[0]) - if err != nil { - continue - } - end, err := strconv.Atoi(fields[1]) - if err != nil { - continue - } - outputBuilder.WriteString(path) - outputBuilder.WriteString(",") - outputBuilder.WriteString(strconv.Itoa(end-start+1) + "\n") - } - } else { - // If there is no ninja log file, just pass empty ninja weight list. - // Because it is still efficient with critical path calculation logic even without weight. - ctx.Verbosef("There is an error during reading ninja log, so ninja will use empty weight list: %s", err) - } - - weightListFile := filepath.Join(config.OutDir(), ninjaWeightListFileName) - - err = os.WriteFile(weightListFile, []byte(outputBuilder.String()), 0644) - if err == nil { - cmd.Args = append(cmd.Args, "-o", "usesweightlist="+weightListFile) - } else { - ctx.Panicf("Could not write ninja weight list file %s", err) - } -} - // Constructs and runs the Ninja command line with a restricted set of // environment variables. It's important to restrict the environment Ninja runs // for hermeticity reasons, and to avoid spurious rebuilds. @@ -131,7 +89,7 @@ func runNinjaForBuild(ctx Context, config Config) { switch config.NinjaWeightListSource() { case NINJA_LOG: - useNinjaBuildLog(ctx, config, cmd) + cmd.Args = append(cmd.Args, "-o", "usesninjalogasweightlist=yes") case EVENLY_DISTRIBUTED: // pass empty weight list means ninja considers every tasks's weight as 1(default value). cmd.Args = append(cmd.Args, "-o", "usesweightlist=/dev/null") From 6e5a5b18ee3c7244876871ff041959a04b2ee5d5 Mon Sep 17 00:00:00 2001 From: Usta Shrestha Date: Fri, 19 May 2023 10:30:52 -0400 Subject: [PATCH 0178/1460] bp2build allowlist expanded Bug: 282160334 Test: ran bp2build.sh Change-Id: I354ded2cd4d2e9c89dc67ca8b9fdddd1c660ffa2 --- android/allowlists/allowlists.go | 1 + 1 file changed, 1 insertion(+) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index e22eec5645..751a4cb830 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -223,6 +223,7 @@ var ( "frameworks/base/tools/streaming_proto": Bp2BuildDefaultTrueRecursively, "frameworks/hardware/interfaces/stats/aidl": Bp2BuildDefaultTrue, "frameworks/libs/modules-utils/build": Bp2BuildDefaultTrueRecursively, + "frameworks/libs/net/common/native": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/adbd_auth": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/arect": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/gui": Bp2BuildDefaultTrue, From f521efae48f3167f6ce40fa69c1752d07220f8fe Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 19 May 2023 09:47:09 -0700 Subject: [PATCH 0179/1460] Support missing instrumented_for dependency when ALLOW_MISSING_DEPENDENCIES=true Don't panic when the instrumented_for dependency is missing or when instrumentedApp or files generated from instrumentedApp are nil. Fixes unbundled tradefed build. Fixes: 283451533 Test: tapas google-tradefed-all && BUILD_BROKEN_DISABLE_BAZEL=true nothing on tradefed branch Change-Id: Ibcd0da75fc55c7c0a697e98ae6bdd9fd3662259c --- java/robolectric.go | 66 +++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/java/robolectric.go b/java/robolectric.go index 008b8b1c92..6bbe872bbb 100644 --- a/java/robolectric.go +++ b/java/robolectric.go @@ -144,29 +144,37 @@ func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) roboTestConfig := android.PathForModuleGen(ctx, "robolectric"). Join(ctx, "com/android/tools/test_config.properties") + var ok bool + var instrumentedApp *AndroidApp + // TODO: this inserts paths to built files into the test, it should really be inserting the contents. instrumented := ctx.GetDirectDepsWithTag(instrumentationForTag) - if len(instrumented) != 1 { + if len(instrumented) == 1 { + instrumentedApp, ok = instrumented[0].(*AndroidApp) + if !ok { + ctx.PropertyErrorf("instrumentation_for", "dependency must be an android_app") + } + } else if !ctx.Config().AllowMissingDependencies() { panic(fmt.Errorf("expected exactly 1 instrumented dependency, got %d", len(instrumented))) } - instrumentedApp, ok := instrumented[0].(*AndroidApp) - if !ok { - ctx.PropertyErrorf("instrumentation_for", "dependency must be an android_app") - } - - r.manifest = instrumentedApp.mergedManifestFile - r.resourceApk = instrumentedApp.outputFile + if instrumentedApp != nil { + r.manifest = instrumentedApp.mergedManifestFile + r.resourceApk = instrumentedApp.outputFile - generateRoboTestConfig(ctx, roboTestConfig, instrumentedApp) - r.extraResources = android.Paths{roboTestConfig} + generateRoboTestConfig(ctx, roboTestConfig, instrumentedApp) + r.extraResources = android.Paths{roboTestConfig} + } r.Library.GenerateAndroidBuildActions(ctx) roboSrcJar := android.PathForModuleGen(ctx, "robolectric", ctx.ModuleName()+".srcjar") - r.generateRoboSrcJar(ctx, roboSrcJar, instrumentedApp) - r.roboSrcJar = roboSrcJar + + if instrumentedApp != nil { + r.generateRoboSrcJar(ctx, roboSrcJar, instrumentedApp) + r.roboSrcJar = roboSrcJar + } roboTestConfigJar := android.PathForModuleOut(ctx, "robolectric_samedir", "samedir_config.jar") generateSameDirRoboTestConfigJar(ctx, roboTestConfigJar) @@ -177,7 +185,10 @@ func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) // once the Make test runner is removed. roboTestConfigJar, r.outputFile, - instrumentedApp.implementationAndResourcesJar, + } + + if instrumentedApp != nil { + combinedJarJars = append(combinedJarJars, instrumentedApp.implementationAndResourcesJar) } handleLibDeps := func(dep android.Module) { @@ -213,21 +224,28 @@ func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) r.tests = append(r.tests, s) } - r.data = append(r.data, r.manifest, r.resourceApk) - - runtimes := ctx.GetDirectDepWithTag("robolectric-android-all-prebuilts", roboRuntimesTag) - installPath := android.PathForModuleInstall(ctx, r.BaseModuleName()) + var installDeps android.Paths - installedResourceApk := ctx.InstallFile(installPath, ctx.ModuleName()+".apk", r.resourceApk) - installedManifest := ctx.InstallFile(installPath, ctx.ModuleName()+"-AndroidManifest.xml", r.manifest) - installedConfig := ctx.InstallFile(installPath, ctx.ModuleName()+".config", r.testConfig) + if r.manifest != nil { + r.data = append(r.data, r.manifest) + installedManifest := ctx.InstallFile(installPath, ctx.ModuleName()+"-AndroidManifest.xml", r.manifest) + installDeps = append(installDeps, installedManifest) + } - var installDeps android.Paths + if r.resourceApk != nil { + r.data = append(r.data, r.resourceApk) + installedResourceApk := ctx.InstallFile(installPath, ctx.ModuleName()+".apk", r.resourceApk) + installDeps = append(installDeps, installedResourceApk) + } + + runtimes := ctx.GetDirectDepWithTag("robolectric-android-all-prebuilts", roboRuntimesTag) for _, runtime := range runtimes.(*robolectricRuntimes).runtimes { installDeps = append(installDeps, runtime) } - installDeps = append(installDeps, installedResourceApk, installedManifest, installedConfig) + + installedConfig := ctx.InstallFile(installPath, ctx.ModuleName()+".config", r.testConfig) + installDeps = append(installDeps, installedConfig) for _, data := range android.PathsForModuleSrc(ctx, r.testProperties.Data) { installedData := ctx.InstallFile(installPath, data.Rel(), data) @@ -340,7 +358,9 @@ func (r *robolectricTest) writeTestRunner(w io.Writer, module, name string, test fmt.Fprintln(w, "LOCAL_MODULE :=", name) android.AndroidMkEmitAssignList(w, "LOCAL_JAVA_LIBRARIES", []string{module}, r.libs) fmt.Fprintln(w, "LOCAL_TEST_PACKAGE :=", String(r.robolectricProperties.Instrumentation_for)) - fmt.Fprintln(w, "LOCAL_INSTRUMENT_SRCJARS :=", r.roboSrcJar.String()) + if r.roboSrcJar != nil { + fmt.Fprintln(w, "LOCAL_INSTRUMENT_SRCJARS :=", r.roboSrcJar.String()) + } android.AndroidMkEmitAssignList(w, "LOCAL_ROBOTEST_FILES", tests) if t := r.robolectricProperties.Test_options.Timeout; t != nil { fmt.Fprintln(w, "LOCAL_ROBOTEST_TIMEOUT :=", *t) From bbb1b74cbff2ac20e87977315d58f3ac71ab83ed Mon Sep 17 00:00:00 2001 From: Zi Wang Date: Mon, 8 May 2023 11:07:26 -0700 Subject: [PATCH 0180/1460] Add data_device_bins_both properties for python_test_host Test: presubmits and manual verification Bug: 274930471 Change-Id: Iafc85526afdb8264526e8a5a33319fa33a23c66b --- python/test.go | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/python/test.go b/python/test.go index 7de7af2def..6e23a447f4 100644 --- a/python/test.go +++ b/python/test.go @@ -66,6 +66,10 @@ type TestProperties struct { // Test options. Test_options TestOptions + + // list of device binary modules that should be installed alongside the test + // This property adds 64bit AND 32bit variants of the dependency + Data_device_bins_both []string `android:"arch_variant"` } type TestOptions struct { @@ -98,12 +102,48 @@ func (p *PythonTestModule) init() android.Module { android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) android.InitBazelModule(p) - if p.hod == android.HostSupported && p.testProperties.Test_options.Unit_test == nil { + if p.isTestHost() && p.testProperties.Test_options.Unit_test == nil { p.testProperties.Test_options.Unit_test = proptools.BoolPtr(true) } return p } +func (p *PythonTestModule) isTestHost() bool { + return p.hod == android.HostSupported +} + +var dataDeviceBinsTag = dependencyTag{name: "dataDeviceBins"} + +// python_test_host DepsMutator uses this method to add multilib dependencies of +// data_device_bin_both +func (p *PythonTestModule) addDataDeviceBinsDeps(ctx android.BottomUpMutatorContext, filter string) { + if len(p.testProperties.Data_device_bins_both) < 1 { + return + } + + var maybeAndroidTarget *android.Target + androidTargetList := android.FirstTarget(ctx.Config().Targets[android.Android], filter) + if len(androidTargetList) > 0 { + maybeAndroidTarget = &androidTargetList[0] + } + + if maybeAndroidTarget != nil { + ctx.AddFarVariationDependencies( + maybeAndroidTarget.Variations(), + dataDeviceBinsTag, + p.testProperties.Data_device_bins_both..., + ) + } +} + +func (p *PythonTestModule) DepsMutator(ctx android.BottomUpMutatorContext) { + p.PythonBinaryModule.DepsMutator(ctx) + if p.isTestHost() { + p.addDataDeviceBinsDeps(ctx, "lib32") + p.addDataDeviceBinsDeps(ctx, "lib64") + } +} + func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { // We inherit from only the library's GenerateAndroidBuildActions, and then // just use buildBinary() so that the binary is not installed into the location @@ -153,6 +193,12 @@ func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext p.data = append(p.data, android.DataPath{SrcPath: dataSrcPath}) } + if p.isTestHost() && len(p.testProperties.Data_device_bins_both) > 0 { + ctx.VisitDirectDepsWithTag(dataDeviceBinsTag, func(dep android.Module) { + p.data = append(p.data, android.DataPath{SrcPath: android.OutputFileForModule(ctx, dep, "")}) + }) + } + // Emulate the data property for java_data dependencies. for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) { for _, javaDataSrcPath := range android.OutputFilesForModule(ctx, javaData, "") { From e59c0db5360b850b444c3bed9b92b0fd30b15e1e Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Mon, 15 May 2023 21:50:29 +0000 Subject: [PATCH 0181/1460] android: Allow running some singletons in parallel. Many of the singletons are trivial and can be run in parallel, improving the performance during analysis. Bug: 281536768 Test: manual, presubmit Change-Id: I989333e2ff3fe71783601f27bf5e0732a1b4ea61 --- android/register.go | 52 ++++++++++++++++++++++++++++++++++----------- android/testing.go | 12 ++++++++++- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/android/register.go b/android/register.go index 1a3db9d903..c57964c181 100644 --- a/android/register.go +++ b/android/register.go @@ -65,16 +65,19 @@ type singleton struct { // True if this should be registered as a pre-singleton, false otherwise. pre bool + // True if this should be registered as a parallel singleton. + parallel bool + name string factory SingletonFactory } -func newSingleton(name string, factory SingletonFactory) singleton { - return singleton{false, name, factory} +func newSingleton(name string, factory SingletonFactory, parallel bool) singleton { + return singleton{pre: false, parallel: parallel, name: name, factory: factory} } func newPreSingleton(name string, factory SingletonFactory) singleton { - return singleton{true, name, factory} + return singleton{pre: true, parallel: false, name: name, factory: factory} } func (s singleton) componentName() string { @@ -86,7 +89,7 @@ func (s singleton) register(ctx *Context) { if s.pre { ctx.RegisterPreSingletonType(s.name, adaptor) } else { - ctx.RegisterSingletonType(s.name, adaptor) + ctx.RegisterSingletonType(s.name, adaptor, s.parallel) } } @@ -145,8 +148,16 @@ func RegisterModuleTypeForDocs(name string, factory reflect.Value) { moduleTypeByFactory[factory] = name } +func registerSingletonType(name string, factory SingletonFactory, parallel bool) { + singletons = append(singletons, newSingleton(name, factory, parallel)) +} + func RegisterSingletonType(name string, factory SingletonFactory) { - singletons = append(singletons, newSingleton(name, factory)) + registerSingletonType(name, factory, false) +} + +func RegisterParallelSingletonType(name string, factory SingletonFactory) { + registerSingletonType(name, factory, true) } func RegisterPreSingletonType(name string, factory SingletonFactory) { @@ -220,17 +231,17 @@ func (ctx *Context) registerSingletonMakeVarsProvider(makevars SingletonMakeVars func collateGloballyRegisteredSingletons() sortableComponents { allSingletons := append(sortableComponents(nil), singletons...) allSingletons = append(allSingletons, - singleton{false, "bazeldeps", BazelSingleton}, + singleton{pre: false, parallel: false, name: "bazeldeps", factory: BazelSingleton}, // Register phony just before makevars so it can write out its phony rules as Make rules - singleton{false, "phony", phonySingletonFactory}, + singleton{pre: false, parallel: false, name: "phony", factory: phonySingletonFactory}, // Register makevars after other singletons so they can export values through makevars - singleton{false, "makevars", makeVarsSingletonFunc}, + singleton{pre: false, parallel: false, name: "makevars", factory: makeVarsSingletonFunc}, // Register env and ninjadeps last so that they can track all used environment variables and // Ninja file dependencies stored in the config. - singleton{false, "ninjadeps", ninjaDepsSingletonFactory}, + singleton{pre: false, parallel: false, name: "ninjadeps", factory: ninjaDepsSingletonFactory}, ) return allSingletons @@ -259,7 +270,9 @@ func ModuleTypeByFactory() map[reflect.Value]string { type RegistrationContext interface { RegisterModuleType(name string, factory ModuleFactory) RegisterSingletonModuleType(name string, factory SingletonModuleFactory) + RegisterParallelSingletonModuleType(name string, factory SingletonModuleFactory) RegisterPreSingletonType(name string, factory SingletonFactory) + RegisterParallelSingletonType(name string, factory SingletonFactory) RegisterSingletonType(name string, factory SingletonFactory) PreArchMutators(f RegisterMutatorFunc) @@ -315,8 +328,15 @@ func (ctx *initRegistrationContext) RegisterModuleType(name string, factory Modu } func (ctx *initRegistrationContext) RegisterSingletonModuleType(name string, factory SingletonModuleFactory) { + ctx.registerSingletonModuleType(name, factory, false) +} +func (ctx *initRegistrationContext) RegisterParallelSingletonModuleType(name string, factory SingletonModuleFactory) { + ctx.registerSingletonModuleType(name, factory, true) +} + +func (ctx *initRegistrationContext) registerSingletonModuleType(name string, factory SingletonModuleFactory, parallel bool) { s, m := SingletonModuleFactoryAdaptor(name, factory) - ctx.RegisterSingletonType(name, s) + ctx.registerSingletonType(name, s, parallel) ctx.RegisterModuleType(name, m) // Overwrite moduleTypesForDocs with the original factory instead of the lambda returned by // SingletonModuleFactoryAdaptor so that docs can find the module type documentation on the @@ -324,12 +344,20 @@ func (ctx *initRegistrationContext) RegisterSingletonModuleType(name string, fac RegisterModuleTypeForDocs(name, reflect.ValueOf(factory)) } -func (ctx *initRegistrationContext) RegisterSingletonType(name string, factory SingletonFactory) { +func (ctx *initRegistrationContext) registerSingletonType(name string, factory SingletonFactory, parallel bool) { if _, present := ctx.singletonTypes[name]; present { panic(fmt.Sprintf("singleton type %q is already registered", name)) } ctx.singletonTypes[name] = factory - RegisterSingletonType(name, factory) + registerSingletonType(name, factory, parallel) +} + +func (ctx *initRegistrationContext) RegisterSingletonType(name string, factory SingletonFactory) { + ctx.registerSingletonType(name, factory, false) +} + +func (ctx *initRegistrationContext) RegisterParallelSingletonType(name string, factory SingletonFactory) { + ctx.registerSingletonType(name, factory, true) } func (ctx *initRegistrationContext) RegisterPreSingletonType(name string, factory SingletonFactory) { diff --git a/android/testing.go b/android/testing.go index 2a9c6584e0..5ad7ad07ff 100644 --- a/android/testing.go +++ b/android/testing.go @@ -495,8 +495,18 @@ func (ctx *TestContext) RegisterSingletonModuleType(name string, factory Singlet ctx.RegisterModuleType(name, m) } +func (ctx *TestContext) RegisterParallelSingletonModuleType(name string, factory SingletonModuleFactory) { + s, m := SingletonModuleFactoryAdaptor(name, factory) + ctx.RegisterParallelSingletonType(name, s) + ctx.RegisterModuleType(name, m) +} + func (ctx *TestContext) RegisterSingletonType(name string, factory SingletonFactory) { - ctx.singletons = append(ctx.singletons, newSingleton(name, factory)) + ctx.singletons = append(ctx.singletons, newSingleton(name, factory, false)) +} + +func (ctx *TestContext) RegisterParallelSingletonType(name string, factory SingletonFactory) { + ctx.singletons = append(ctx.singletons, newSingleton(name, factory, true)) } func (ctx *TestContext) RegisterPreSingletonType(name string, factory SingletonFactory) { From 34b080ddcf7757bba3dcb938df46aa660b2564a0 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Thu, 30 Mar 2023 16:40:24 -0700 Subject: [PATCH 0182/1460] Enable referencing of aapt-generated proguard flags Some targets may implement classes declared elsewhere, e.g., system server implements services defined in framework-res.apk's manifest. Allow depending on the aapt-generated proguard flags for a given target to support this. Bug: 272495195 Test: m + reference ":framework-res{.aapt.proguardOptionsFile}" Change-Id: I2a16632ed6e5a9bddbe326bdb7ab0dd79b45e587 --- java/app.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/app.go b/java/app.go index da9c6f3437..29a22877ce 100755 --- a/java/app.go +++ b/java/app.go @@ -976,6 +976,10 @@ func (a *AndroidApp) DepIsInSameApex(ctx android.BaseModuleContext, dep android. // For OutputFileProducer interface func (a *AndroidApp) OutputFiles(tag string) (android.Paths, error) { switch tag { + // In some instances, it can be useful to reference the aapt-generated flags from another + // target, e.g., system server implements services declared in the framework-res manifest. + case ".aapt.proguardOptionsFile": + return []android.Path{a.proguardOptionsFile}, nil case ".aapt.srcjar": return []android.Path{a.aaptSrcJar}, nil case ".export-package.apk": From 0c10e4dcc027e341655e8eedc20bd77f9539a22b Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Tue, 16 May 2023 00:58:37 +0000 Subject: [PATCH 0183/1460] Parallelize singleton execution Bug: 281536768 Test: manual, presubmits Change-Id: I57fdc76ba6b277e88e196b506af87127a530fd37 --- android/androidmk.go | 2 +- android/api_levels.go | 2 +- android/buildinfo_prop.go | 2 +- android/gen_notice.go | 2 +- android/metrics.go | 2 +- android/module.go | 2 +- android/register.go | 2 +- android/test_suites.go | 2 +- apex/apex_singleton.go | 2 +- apex/key.go | 2 +- bloaty/bloaty.go | 2 +- cc/cc.go | 2 +- cc/ccdeps.go | 2 +- cc/cmakelists.go | 2 +- cc/compdb.go | 2 +- cc/fuzz.go | 2 +- cc/ndk_abi.go | 4 ++-- cc/ndk_sysroot.go | 2 +- cc/stub_library.go | 2 +- cc/tidy.go | 2 +- cc/vndk.go | 14 +++++++------- dexpreopt/config.go | 2 +- java/dexpreopt_bootjars.go | 2 +- java/dexpreopt_check.go | 2 +- java/fuzz.go | 2 +- java/hiddenapi_singleton.go | 2 +- java/java.go | 4 ++-- java/jdeps.go | 2 +- java/lint.go | 2 +- java/platform_bootclasspath.go | 2 +- java/platform_compat_config.go | 2 +- java/sdk.go | 2 +- multitree/metadata.go | 2 +- provenance/provenance_singleton.go | 2 +- rust/doc.go | 2 +- rust/project_json.go | 2 +- rust/rust.go | 2 +- rust/testing.go | 4 ++-- snapshot/host_fake_snapshot.go | 2 +- snapshot/recovery_snapshot.go | 2 +- snapshot/vendor_snapshot.go | 4 ++-- 41 files changed, 51 insertions(+), 51 deletions(-) diff --git a/android/androidmk.go b/android/androidmk.go index aa411d1161..62f82f247e 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -42,7 +42,7 @@ func init() { } func RegisterAndroidMkBuildComponents(ctx RegistrationContext) { - ctx.RegisterSingletonType("androidmk", AndroidMkSingleton) + ctx.RegisterParallelSingletonType("androidmk", AndroidMkSingleton) } // Enable androidmk support. diff --git a/android/api_levels.go b/android/api_levels.go index fa919fda74..2391e6cc26 100644 --- a/android/api_levels.go +++ b/android/api_levels.go @@ -22,7 +22,7 @@ import ( ) func init() { - RegisterSingletonType("api_levels", ApiLevelsSingleton) + RegisterParallelSingletonType("api_levels", ApiLevelsSingleton) } const previewAPILevelBase = 9000 diff --git a/android/buildinfo_prop.go b/android/buildinfo_prop.go index 46f6488020..8e19ad5f4e 100644 --- a/android/buildinfo_prop.go +++ b/android/buildinfo_prop.go @@ -23,7 +23,7 @@ import ( func init() { ctx := InitRegistrationContext - ctx.RegisterSingletonModuleType("buildinfo_prop", buildinfoPropFactory) + ctx.RegisterParallelSingletonModuleType("buildinfo_prop", buildinfoPropFactory) } type buildinfoPropProperties struct { diff --git a/android/gen_notice.go b/android/gen_notice.go index 091345b9fa..1acc638e88 100644 --- a/android/gen_notice.go +++ b/android/gen_notice.go @@ -28,7 +28,7 @@ func init() { // Register the gen_notice module type. func RegisterGenNoticeBuildComponents(ctx RegistrationContext) { - ctx.RegisterSingletonType("gen_notice_build_rules", GenNoticeBuildRulesFactory) + ctx.RegisterParallelSingletonType("gen_notice_build_rules", GenNoticeBuildRulesFactory) ctx.RegisterModuleType("gen_notice", GenNoticeFactory) } diff --git a/android/metrics.go b/android/metrics.go index 3d41a1d6cb..63c72cd97b 100644 --- a/android/metrics.go +++ b/android/metrics.go @@ -42,7 +42,7 @@ func readSoongMetrics(config Config) (SoongMetrics, bool) { } func init() { - RegisterSingletonType("soong_metrics", soongMetricsSingletonFactory) + RegisterParallelSingletonType("soong_metrics", soongMetricsSingletonFactory) } func soongMetricsSingletonFactory() Singleton { return soongMetricsSingleton{} } diff --git a/android/module.go b/android/module.go index db602a0aa3..9024896e52 100644 --- a/android/module.go +++ b/android/module.go @@ -3724,7 +3724,7 @@ func (m *moduleContext) TargetRequiredModuleNames() []string { } func init() { - RegisterSingletonType("buildtarget", BuildTargetSingleton) + RegisterParallelSingletonType("buildtarget", BuildTargetSingleton) } func BuildTargetSingleton() Singleton { diff --git a/android/register.go b/android/register.go index c57964c181..64b0207e71 100644 --- a/android/register.go +++ b/android/register.go @@ -231,7 +231,7 @@ func (ctx *Context) registerSingletonMakeVarsProvider(makevars SingletonMakeVars func collateGloballyRegisteredSingletons() sortableComponents { allSingletons := append(sortableComponents(nil), singletons...) allSingletons = append(allSingletons, - singleton{pre: false, parallel: false, name: "bazeldeps", factory: BazelSingleton}, + singleton{pre: false, parallel: true, name: "bazeldeps", factory: BazelSingleton}, // Register phony just before makevars so it can write out its phony rules as Make rules singleton{pre: false, parallel: false, name: "phony", factory: phonySingletonFactory}, diff --git a/android/test_suites.go b/android/test_suites.go index b570b2383f..b48d71af64 100644 --- a/android/test_suites.go +++ b/android/test_suites.go @@ -15,7 +15,7 @@ package android func init() { - RegisterSingletonType("testsuites", testSuiteFilesFactory) + RegisterParallelSingletonType("testsuites", testSuiteFilesFactory) } func testSuiteFilesFactory() Singleton { diff --git a/apex/apex_singleton.go b/apex/apex_singleton.go index ebc35cf5c7..a63344fc10 100644 --- a/apex/apex_singleton.go +++ b/apex/apex_singleton.go @@ -27,7 +27,7 @@ func init() { } func registerApexDepsInfoComponents(ctx android.RegistrationContext) { - ctx.RegisterSingletonType("apex_depsinfo_singleton", apexDepsInfoSingletonFactory) + ctx.RegisterParallelSingletonType("apex_depsinfo_singleton", apexDepsInfoSingletonFactory) } type apexDepsInfoSingleton struct { diff --git a/apex/key.go b/apex/key.go index 0a7e80f8fc..3010d76be4 100644 --- a/apex/key.go +++ b/apex/key.go @@ -33,7 +33,7 @@ func init() { func registerApexKeyBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("apex_key", ApexKeyFactory) - ctx.RegisterSingletonType("apex_keys_text", apexKeysTextFactory) + ctx.RegisterParallelSingletonType("apex_keys_text", apexKeysTextFactory) } type apexKey struct { diff --git a/bloaty/bloaty.go b/bloaty/bloaty.go index 50f241f44c..3cff60fe6d 100644 --- a/bloaty/bloaty.go +++ b/bloaty/bloaty.go @@ -51,7 +51,7 @@ func init() { pctx.VariableConfigMethod("hostPrebuiltTag", android.Config.PrebuiltOS) pctx.SourcePathVariable("bloaty", "prebuilts/build-tools/${hostPrebuiltTag}/bin/bloaty") pctx.HostBinToolVariable("bloatyMerger", "bloaty_merger") - android.RegisterSingletonType("file_metrics", fileSizesSingleton) + android.RegisterParallelSingletonType("file_metrics", fileSizesSingleton) fileSizeMeasurerKey = blueprint.NewProvider(measuredFiles{}) } diff --git a/cc/cc.go b/cc/cc.go index f4b5655500..0e7f6c8d50 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -83,7 +83,7 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) { ctx.TopDown("sabi_deps", sabiDepsMutator) }) - ctx.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory) + ctx.RegisterParallelSingletonType("kythe_extract_all", kytheExtractAllFactory) } // Deps is a struct containing module names of dependencies, separated by the kind of dependency. diff --git a/cc/ccdeps.go b/cc/ccdeps.go index 75e1faf0b9..d30abbab79 100644 --- a/cc/ccdeps.go +++ b/cc/ccdeps.go @@ -30,7 +30,7 @@ import ( // The info file is generated in $OUT/module_bp_cc_depend.json. func init() { - android.RegisterSingletonType("ccdeps_generator", ccDepsGeneratorSingleton) + android.RegisterParallelSingletonType("ccdeps_generator", ccDepsGeneratorSingleton) } func ccDepsGeneratorSingleton() android.Singleton { diff --git a/cc/cmakelists.go b/cc/cmakelists.go index ad130baaa7..0f3f02da54 100644 --- a/cc/cmakelists.go +++ b/cc/cmakelists.go @@ -29,7 +29,7 @@ import ( // structure (see variable CLionOutputProjectsDirectory for root). func init() { - android.RegisterSingletonType("cmakelists_generator", cMakeListsGeneratorSingleton) + android.RegisterParallelSingletonType("cmakelists_generator", cMakeListsGeneratorSingleton) } func cMakeListsGeneratorSingleton() android.Singleton { diff --git a/cc/compdb.go b/cc/compdb.go index ea124438e3..617be1a194 100644 --- a/cc/compdb.go +++ b/cc/compdb.go @@ -32,7 +32,7 @@ import ( // make SOONG_GEN_COMPDB=1 nothing to get all targets. func init() { - android.RegisterSingletonType("compdb_generator", compDBGeneratorSingleton) + android.RegisterParallelSingletonType("compdb_generator", compDBGeneratorSingleton) } func compDBGeneratorSingleton() android.Singleton { diff --git a/cc/fuzz.go b/cc/fuzz.go index dfefc11f22..f55fd5cbc2 100644 --- a/cc/fuzz.go +++ b/cc/fuzz.go @@ -28,7 +28,7 @@ import ( func init() { android.RegisterModuleType("cc_fuzz", LibFuzzFactory) - android.RegisterSingletonType("cc_fuzz_packaging", fuzzPackagingFactory) + android.RegisterParallelSingletonType("cc_fuzz_packaging", fuzzPackagingFactory) } type FuzzProperties struct { diff --git a/cc/ndk_abi.go b/cc/ndk_abi.go index 3456c32bb2..86166dcbb2 100644 --- a/cc/ndk_abi.go +++ b/cc/ndk_abi.go @@ -19,8 +19,8 @@ import ( ) func init() { - android.RegisterSingletonType("ndk_abi_dump", NdkAbiDumpSingleton) - android.RegisterSingletonType("ndk_abi_diff", NdkAbiDiffSingleton) + android.RegisterParallelSingletonType("ndk_abi_dump", NdkAbiDumpSingleton) + android.RegisterParallelSingletonType("ndk_abi_diff", NdkAbiDiffSingleton) } func getNdkAbiDumpInstallBase(ctx android.PathContext) android.OutputPath { diff --git a/cc/ndk_sysroot.go b/cc/ndk_sysroot.go index dffc6c6141..0cf21b65a7 100644 --- a/cc/ndk_sysroot.go +++ b/cc/ndk_sysroot.go @@ -66,7 +66,7 @@ func RegisterNdkModuleTypes(ctx android.RegistrationContext) { ctx.RegisterModuleType("ndk_library", NdkLibraryFactory) ctx.RegisterModuleType("versioned_ndk_headers", versionedNdkHeadersFactory) ctx.RegisterModuleType("preprocessed_ndk_headers", preprocessedNdkHeadersFactory) - ctx.RegisterSingletonType("ndk", NdkSingleton) + ctx.RegisterParallelSingletonType("ndk", NdkSingleton) } func getNdkInstallBase(ctx android.PathContext) android.InstallPath { diff --git a/cc/stub_library.go b/cc/stub_library.go index f324dcc9bc..3a6d0aeaa4 100644 --- a/cc/stub_library.go +++ b/cc/stub_library.go @@ -23,7 +23,7 @@ import ( func init() { // Use singleton type to gather all generated soong modules. - android.RegisterSingletonType("stublibraries", stubLibrariesSingleton) + android.RegisterParallelSingletonType("stublibraries", stubLibrariesSingleton) } type stubLibraries struct { diff --git a/cc/tidy.go b/cc/tidy.go index bbcaece24c..7b123cb57c 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -201,7 +201,7 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { } func init() { - android.RegisterSingletonType("tidy_phony_targets", TidyPhonySingleton) + android.RegisterParallelSingletonType("tidy_phony_targets", TidyPhonySingleton) } // This TidyPhonySingleton generates both tidy-* and obj-* phony targets for C/C++ files. diff --git a/cc/vndk.go b/cc/vndk.go index 9b70004c52..7a2286eb12 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -417,16 +417,16 @@ func VndkMutator(mctx android.BottomUpMutatorContext) { func init() { RegisterVndkLibraryTxtTypes(android.InitRegistrationContext) - android.RegisterSingletonType("vndk-snapshot", VndkSnapshotSingleton) + android.RegisterParallelSingletonType("vndk-snapshot", VndkSnapshotSingleton) } func RegisterVndkLibraryTxtTypes(ctx android.RegistrationContext) { - ctx.RegisterSingletonModuleType("llndk_libraries_txt", llndkLibrariesTxtFactory) - ctx.RegisterSingletonModuleType("vndksp_libraries_txt", vndkSPLibrariesTxtFactory) - ctx.RegisterSingletonModuleType("vndkcore_libraries_txt", vndkCoreLibrariesTxtFactory) - ctx.RegisterSingletonModuleType("vndkprivate_libraries_txt", vndkPrivateLibrariesTxtFactory) - ctx.RegisterSingletonModuleType("vndkproduct_libraries_txt", vndkProductLibrariesTxtFactory) - ctx.RegisterSingletonModuleType("vndkcorevariant_libraries_txt", vndkUsingCoreVariantLibrariesTxtFactory) + ctx.RegisterParallelSingletonModuleType("llndk_libraries_txt", llndkLibrariesTxtFactory) + ctx.RegisterParallelSingletonModuleType("vndksp_libraries_txt", vndkSPLibrariesTxtFactory) + ctx.RegisterParallelSingletonModuleType("vndkcore_libraries_txt", vndkCoreLibrariesTxtFactory) + ctx.RegisterParallelSingletonModuleType("vndkprivate_libraries_txt", vndkPrivateLibrariesTxtFactory) + ctx.RegisterParallelSingletonModuleType("vndkproduct_libraries_txt", vndkProductLibrariesTxtFactory) + ctx.RegisterParallelSingletonModuleType("vndkcorevariant_libraries_txt", vndkUsingCoreVariantLibrariesTxtFactory) } type vndkLibrariesTxt struct { diff --git a/dexpreopt/config.go b/dexpreopt/config.go index 0cc3bd63bc..e61ebe6242 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -197,7 +197,7 @@ var pctx = android.NewPackageContext("android/soong/dexpreopt") func init() { pctx.Import("android/soong/android") - android.RegisterSingletonType("dexpreopt-soong-config", func() android.Singleton { + android.RegisterParallelSingletonType("dexpreopt-soong-config", func() android.Singleton { return &globalSoongConfigSingleton{} }) } diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index f477f404e1..116c833e00 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -465,7 +465,7 @@ func dexpreoptBootJarsFactory() android.SingletonModule { } func RegisterDexpreoptBootJarsComponents(ctx android.RegistrationContext) { - ctx.RegisterSingletonModuleType("dex_bootjars", dexpreoptBootJarsFactory) + ctx.RegisterParallelSingletonModuleType("dex_bootjars", dexpreoptBootJarsFactory) } func SkipDexpreoptBootJars(ctx android.PathContext) bool { diff --git a/java/dexpreopt_check.go b/java/dexpreopt_check.go index 83c088cd4c..7499481de4 100644 --- a/java/dexpreopt_check.go +++ b/java/dexpreopt_check.go @@ -28,7 +28,7 @@ func init() { } func RegisterDexpreoptCheckBuildComponents(ctx android.RegistrationContext) { - ctx.RegisterSingletonModuleType("dexpreopt_systemserver_check", dexpreoptSystemserverCheckFactory) + ctx.RegisterParallelSingletonModuleType("dexpreopt_systemserver_check", dexpreoptSystemserverCheckFactory) } // A build-time check to verify if all compilation artifacts of system server jars are installed diff --git a/java/fuzz.go b/java/fuzz.go index 4aa6dbffdd..e44669bce3 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -38,7 +38,7 @@ func init() { func RegisterJavaFuzzBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("java_fuzz", JavaFuzzFactory) - ctx.RegisterSingletonType("java_fuzz_packaging", javaFuzzPackagingFactory) + ctx.RegisterParallelSingletonType("java_fuzz_packaging", javaFuzzPackagingFactory) } type JavaFuzzTest struct { diff --git a/java/hiddenapi_singleton.go b/java/hiddenapi_singleton.go index 52934a3272..d4ee4fc9f4 100644 --- a/java/hiddenapi_singleton.go +++ b/java/hiddenapi_singleton.go @@ -25,7 +25,7 @@ func init() { } func RegisterHiddenApiSingletonComponents(ctx android.RegistrationContext) { - ctx.RegisterSingletonType("hiddenapi", hiddenAPISingletonFactory) + ctx.RegisterParallelSingletonType("hiddenapi", hiddenAPISingletonFactory) } var PrepareForTestWithHiddenApiBuildComponents = android.FixtureRegisterWithContext(RegisterHiddenApiSingletonComponents) diff --git a/java/java.go b/java/java.go index bb8fe62aa7..33846bec6b 100644 --- a/java/java.go +++ b/java/java.go @@ -73,8 +73,8 @@ func registerJavaBuildComponents(ctx android.RegistrationContext) { ctx.BottomUp("jacoco_deps", jacocoDepsMutator).Parallel() }) - ctx.RegisterSingletonType("logtags", LogtagsSingleton) - ctx.RegisterSingletonType("kythe_java_extract", kytheExtractJavaFactory) + ctx.RegisterParallelSingletonType("logtags", LogtagsSingleton) + ctx.RegisterParallelSingletonType("kythe_java_extract", kytheExtractJavaFactory) } func RegisterJavaSdkMemberTypes() { diff --git a/java/jdeps.go b/java/jdeps.go index a52b86708e..4c8c11c5da 100644 --- a/java/jdeps.go +++ b/java/jdeps.go @@ -26,7 +26,7 @@ import ( // called. Dependency info file is generated in $OUT/module_bp_java_depend.json. func init() { - android.RegisterSingletonType("jdeps_generator", jDepsGeneratorSingleton) + android.RegisterParallelSingletonType("jdeps_generator", jDepsGeneratorSingleton) } func jDepsGeneratorSingleton() android.Singleton { diff --git a/java/lint.go b/java/lint.go index 40ef484167..a0f99707a6 100644 --- a/java/lint.go +++ b/java/lint.go @@ -705,7 +705,7 @@ func (l *lintSingleton) MakeVars(ctx android.MakeVarsContext) { var _ android.SingletonMakeVarsProvider = (*lintSingleton)(nil) func init() { - android.RegisterSingletonType("lint", + android.RegisterParallelSingletonType("lint", func() android.Singleton { return &lintSingleton{} }) registerLintBuildComponents(android.InitRegistrationContext) diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index 07fb92cfe5..0d4db7ca15 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -26,7 +26,7 @@ func init() { } func registerPlatformBootclasspathBuildComponents(ctx android.RegistrationContext) { - ctx.RegisterSingletonModuleType("platform_bootclasspath", platformBootclasspathFactory) + ctx.RegisterParallelSingletonModuleType("platform_bootclasspath", platformBootclasspathFactory) } // The tags used for the dependencies between the platform bootclasspath and any configured boot diff --git a/java/platform_compat_config.go b/java/platform_compat_config.go index d417291509..2197304a53 100644 --- a/java/platform_compat_config.go +++ b/java/platform_compat_config.go @@ -36,7 +36,7 @@ var CompatConfigSdkMemberType = &compatConfigMemberType{ } func registerPlatformCompatConfigBuildComponents(ctx android.RegistrationContext) { - ctx.RegisterSingletonType("platform_compat_config_singleton", platformCompatConfigSingletonFactory) + ctx.RegisterParallelSingletonType("platform_compat_config_singleton", platformCompatConfigSingletonFactory) ctx.RegisterModuleType("platform_compat_config", PlatformCompatConfigFactory) ctx.RegisterModuleType("prebuilt_platform_compat_config", prebuiltCompatConfigFactory) ctx.RegisterModuleType("global_compat_config", globalCompatConfigFactory) diff --git a/java/sdk.go b/java/sdk.go index 8b4918add2..7fa604fdbb 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -28,7 +28,7 @@ import ( func init() { android.RegisterPreSingletonType("sdk_versions", sdkPreSingletonFactory) - android.RegisterSingletonType("sdk", sdkSingletonFactory) + android.RegisterParallelSingletonType("sdk", sdkSingletonFactory) android.RegisterMakeVarsProvider(pctx, sdkMakeVars) } diff --git a/multitree/metadata.go b/multitree/metadata.go index 3fd721599c..0eb0efc95a 100644 --- a/multitree/metadata.go +++ b/multitree/metadata.go @@ -20,7 +20,7 @@ import ( ) func init() { - android.RegisterSingletonType("update-meta", UpdateMetaSingleton) + android.RegisterParallelSingletonType("update-meta", UpdateMetaSingleton) } func UpdateMetaSingleton() android.Singleton { diff --git a/provenance/provenance_singleton.go b/provenance/provenance_singleton.go index fbb6212ffa..5d27c0cf45 100644 --- a/provenance/provenance_singleton.go +++ b/provenance/provenance_singleton.go @@ -51,7 +51,7 @@ func init() { } func RegisterProvenanceSingleton(ctx android.RegistrationContext) { - ctx.RegisterSingletonType("provenance_metadata_singleton", provenanceInfoSingletonFactory) + ctx.RegisterParallelSingletonType("provenance_metadata_singleton", provenanceInfoSingletonFactory) } var PrepareForTestWithProvenanceSingleton = android.FixtureRegisterWithContext(RegisterProvenanceSingleton) diff --git a/rust/doc.go b/rust/doc.go index fe3581b424..6970d7979f 100644 --- a/rust/doc.go +++ b/rust/doc.go @@ -19,7 +19,7 @@ import ( ) func init() { - android.RegisterSingletonType("rustdoc", RustdocSingleton) + android.RegisterParallelSingletonType("rustdoc", RustdocSingleton) } func RustdocSingleton() android.Singleton { diff --git a/rust/project_json.go b/rust/project_json.go index fe259d6c28..40aa7c7362 100644 --- a/rust/project_json.go +++ b/rust/project_json.go @@ -74,7 +74,7 @@ func rustProjectGeneratorSingleton() android.Singleton { } func init() { - android.RegisterSingletonType("rust_project_generator", rustProjectGeneratorSingleton) + android.RegisterParallelSingletonType("rust_project_generator", rustProjectGeneratorSingleton) } // sourceProviderVariantSource returns the path to the source file if this diff --git a/rust/rust.go b/rust/rust.go index dc53cc0280..4324ecbb9c 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -45,7 +45,7 @@ func init() { }) pctx.Import("android/soong/rust/config") pctx.ImportAs("cc_config", "android/soong/cc/config") - android.InitRegistrationContext.RegisterSingletonType("kythe_rust_extract", kytheExtractRustFactory) + android.InitRegistrationContext.RegisterParallelSingletonType("kythe_rust_extract", kytheExtractRustFactory) } type Flags struct { diff --git a/rust/testing.go b/rust/testing.go index 0a6a870d8f..7f3056954f 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -200,8 +200,8 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) { ctx.BottomUp("rust_stdlinkage", LibstdMutator).Parallel() ctx.BottomUp("rust_begin", BeginMutator).Parallel() }) - ctx.RegisterSingletonType("rust_project_generator", rustProjectGeneratorSingleton) - ctx.RegisterSingletonType("kythe_rust_extract", kytheExtractRustFactory) + ctx.RegisterParallelSingletonType("rust_project_generator", rustProjectGeneratorSingleton) + ctx.RegisterParallelSingletonType("kythe_rust_extract", kytheExtractRustFactory) ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { ctx.BottomUp("rust_sanitizers", rustSanitizerRuntimeMutator).Parallel() }) diff --git a/snapshot/host_fake_snapshot.go b/snapshot/host_fake_snapshot.go index b04657d3da..c4cfbb5ce2 100644 --- a/snapshot/host_fake_snapshot.go +++ b/snapshot/host_fake_snapshot.go @@ -75,7 +75,7 @@ type hostSnapshotFakeJsonFlags struct { } func registerHostSnapshotComponents(ctx android.RegistrationContext) { - ctx.RegisterSingletonType("host-fake-snapshot", HostToolsFakeAndroidSingleton) + ctx.RegisterParallelSingletonType("host-fake-snapshot", HostToolsFakeAndroidSingleton) } type hostFakeSingleton struct { diff --git a/snapshot/recovery_snapshot.go b/snapshot/recovery_snapshot.go index ac002be9ca..8ff59cbcfe 100644 --- a/snapshot/recovery_snapshot.go +++ b/snapshot/recovery_snapshot.go @@ -68,7 +68,7 @@ var RecoverySnapshotImageName = "recovery" type RecoverySnapshotImage struct{} func (RecoverySnapshotImage) Init(ctx android.RegistrationContext) { - ctx.RegisterSingletonType("recovery-snapshot", RecoverySnapshotSingleton) + ctx.RegisterParallelSingletonType("recovery-snapshot", RecoverySnapshotSingleton) } func (RecoverySnapshotImage) RegisterAdditionalModule(ctx android.RegistrationContext, name string, factory android.ModuleFactory) { diff --git a/snapshot/vendor_snapshot.go b/snapshot/vendor_snapshot.go index 8f7b8c2153..4484c855da 100644 --- a/snapshot/vendor_snapshot.go +++ b/snapshot/vendor_snapshot.go @@ -78,8 +78,8 @@ var VendorSnapshotImageName = "vendor" type VendorSnapshotImage struct{} func (VendorSnapshotImage) Init(ctx android.RegistrationContext) { - ctx.RegisterSingletonType("vendor-snapshot", VendorSnapshotSingleton) - ctx.RegisterSingletonType("vendor-fake-snapshot", VendorFakeSnapshotSingleton) + ctx.RegisterParallelSingletonType("vendor-snapshot", VendorSnapshotSingleton) + ctx.RegisterParallelSingletonType("vendor-fake-snapshot", VendorFakeSnapshotSingleton) } func (VendorSnapshotImage) RegisterAdditionalModule(ctx android.RegistrationContext, name string, factory android.ModuleFactory) { From 4e49bbde93bf6c2db4f6c70646d6d29a6c60a993 Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Wed, 26 Apr 2023 21:06:24 +0900 Subject: [PATCH 0184/1460] BUILD_NUMBER and BUILD_HOSTNAME doesn't affect kati regeneration * Extract BUILD_NUMBER, BUILD_HOSTNAME to file to avoid kati change * Handle FILE_NAME_TAG_PLACEHOLDER string in dist in build/make/packaging/distdir.mk Test: check if kati isn't invoked even though BUILD_NUMBER, BUILD_HOSTNAME is changed Test: m && m, and check if the second m is no-op Bug: 278060169 Change-Id: I65eefc6bb86d4076098a1bf8b317b4cb88201499 --- android_sdk/sdk_repo_host.go | 2 +- ui/build/kati.go | 60 +++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/android_sdk/sdk_repo_host.go b/android_sdk/sdk_repo_host.go index 61058df098..9623a8bc46 100644 --- a/android_sdk/sdk_repo_host.go +++ b/android_sdk/sdk_repo_host.go @@ -242,7 +242,7 @@ func (s *sdkRepoHost) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, ".PHONY:", name, "sdk_repo", "sdk-repo-"+name) fmt.Fprintln(w, "sdk_repo", "sdk-repo-"+name+":", strings.Join(s.FilesToInstall().Strings(), " ")) - fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-$(FILE_NAME_TAG).zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) + fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-FILE_NAME_TAG_PLACEHOLDER.zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) }, } } diff --git a/ui/build/kati.go b/ui/build/kati.go index dad68fac2f..4297378b37 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -22,6 +22,7 @@ import ( "os/user" "path/filepath" "strings" + "time" "android/soong/ui/metrics" "android/soong/ui/status" @@ -66,6 +67,21 @@ func genKatiSuffix(ctx Context, config Config) { } } +func writeValueIfChanged(ctx Context, config Config, dir string, filename string, value string) { + filePath := filepath.Join(dir, filename) + previousValue := "" + rawPreviousValue, err := ioutil.ReadFile(filePath) + if err == nil { + previousValue = string(rawPreviousValue) + } + + if previousValue != value { + if err = ioutil.WriteFile(filePath, []byte(value), 0666); err != nil { + ctx.Fatalf("Failed to write: %v", err) + } + } +} + // Base function to construct and run the Kati command line with additional // arguments, and a custom function closure to mutate the environment Kati runs // in. @@ -157,28 +173,57 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF } cmd.Stderr = cmd.Stdout - // Apply the caller's function closure to mutate the environment variables. - envFunc(cmd.Environment) - + var username string // Pass on various build environment metadata to Kati. - if _, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { - username := "unknown" + if usernameFromEnv, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { + username = "unknown" if u, err := user.Current(); err == nil { username = u.Username } else { ctx.Println("Failed to get current user:", err) } cmd.Environment.Set("BUILD_USERNAME", username) + } else { + username = usernameFromEnv } - if _, ok := cmd.Environment.Get("BUILD_HOSTNAME"); !ok { - hostname, err := os.Hostname() + hostname, ok := cmd.Environment.Get("BUILD_HOSTNAME") + if !ok { + hostname, err = os.Hostname() if err != nil { ctx.Println("Failed to read hostname:", err) hostname = "unknown" } cmd.Environment.Set("BUILD_HOSTNAME", hostname) } + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_hostname.txt", hostname) + + // BUILD_NUMBER should be set to the source control value that + // represents the current state of the source code. E.g., a + // perforce changelist number or a git hash. Can be an arbitrary string + // (to allow for source control that uses something other than numbers), + // but must be a single word and a valid file name. + // + // If no BUILD_NUMBER is set, create a useful "I am an engineering build + // from this date/time" value. Make it start with a non-digit so that + // anyone trying to parse it as an integer will probably get "0". + cmd.Environment.Unset("HAS_BUILD_NUMBER") + buildNumber, ok := cmd.Environment.Get("BUILD_NUMBER") + if ok { + cmd.Environment.Set("HAS_BUILD_NUMBER", "true") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", buildNumber) + } else { + buildNumber = fmt.Sprintf("eng.%.6s.%s", username, time.Now().Format("20060102.150405" /* YYYYMMDD.HHMMSS */)) + cmd.Environment.Set("HAS_BUILD_NUMBER", "false") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", username) + } + // Write the build number to a file so it can be read back in + // without changing the command line every time. Avoids rebuilds + // when using ninja. + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_number.txt", buildNumber) + + // Apply the caller's function closure to mutate the environment variables. + envFunc(cmd.Environment) cmd.StartOrFatal() // Set up the ToolStatus command line reader for Kati for a consistent UI @@ -336,6 +381,7 @@ func runKatiPackage(ctx Context, config Config) { "ANDROID_BUILD_SHELL", "DIST_DIR", "OUT_DIR", + "FILE_NAME_TAG", }...) if config.Dist() { From d0f9520414c41f3c78a0147c1b56b86672151686 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Mon, 24 Apr 2023 17:54:19 +0000 Subject: [PATCH 0185/1460] Add additional java_api_library testing modules This change adds android-non-updatable.stub.* java_api_library modules to java testing. Test: none Change-Id: I10110a21cc61905995803dfc1d27484358019807 --- java/testing.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/java/testing.go b/java/testing.go index ffc3a086f2..3a238d76f9 100644 --- a/java/testing.go +++ b/java/testing.go @@ -400,16 +400,20 @@ func gatherRequiredDepsForTest() string { } extraApiLibraryModules := map[string]string{ - "android_stubs_current.from-text": "api/current.txt", - "android_system_stubs_current.from-text": "api/system-current.txt", - "android_test_stubs_current.from-text": "api/test-current.txt", - "android_module_lib_stubs_current.from-text": "api/module-lib-current.txt", - "android_module_lib_stubs_current_full.from-text": "api/module-lib-current.txt", - "android_system_server_stubs_current.from-text": "api/system-server-current.txt", - "core.current.stubs.from-text": "api/current.txt", - "legacy.core.platform.api.stubs.from-text": "api/current.txt", - "stable.core.platform.api.stubs.from-text": "api/current.txt", - "core-lambda-stubs.from-text": "api/current.txt", + "android_stubs_current.from-text": "api/current.txt", + "android_system_stubs_current.from-text": "api/system-current.txt", + "android_test_stubs_current.from-text": "api/test-current.txt", + "android_module_lib_stubs_current.from-text": "api/module-lib-current.txt", + "android_module_lib_stubs_current_full.from-text": "api/module-lib-current.txt", + "android_system_server_stubs_current.from-text": "api/system-server-current.txt", + "core.current.stubs.from-text": "api/current.txt", + "legacy.core.platform.api.stubs.from-text": "api/current.txt", + "stable.core.platform.api.stubs.from-text": "api/current.txt", + "core-lambda-stubs.from-text": "api/current.txt", + "android-non-updatable.stubs.from-text": "api/current.txt", + "android-non-updatable.stubs.system.from-text": "api/system-current.txt", + "android-non-updatable.stubs.test.from-text": "api/test-current.txt", + "android-non-updatable.stubs.module_lib.from-text": "api/module-lib-current.txt", } for libName, apiFile := range extraApiLibraryModules { From 7e9e20e389d065e73c26f4b2dc54333292bc931c Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Mon, 24 Apr 2023 18:06:44 +0000 Subject: [PATCH 0186/1460] Fix bootclasspath_fragment build failure during from-text stub build This change enables bootclasspath_fragment module to be correctly built during from-text stub build by android-non-updatable module to be interpreted as its java_api_library equivalent. Test: go test ./java Bug: 279460171 Change-Id: Ia6b60edcb60782977cc59855f1cf5d3a46e9f682 --- java/bootclasspath_fragment_test.go | 36 +++++++++++++++++++++++++++++ java/hiddenapi_modular.go | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/java/bootclasspath_fragment_test.go b/java/bootclasspath_fragment_test.go index 2541f14ff7..9bdef749b2 100644 --- a/java/bootclasspath_fragment_test.go +++ b/java/bootclasspath_fragment_test.go @@ -432,3 +432,39 @@ func TestBootclasspathFragment_Test(t *testing.T) { fragment = result.Module("a_test_fragment", "android_common").(*BootclasspathFragmentModule) android.AssertBoolEquals(t, "is a test fragment by type", true, fragment.isTestFragment()) } + +func TestBootclassFragment_LinkTextStub(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForJavaTest, + prepareForTestWithBootclasspathFragment, + PrepareForTestWithJavaSdkLibraryFiles, + FixtureWithLastReleaseApis("mysdklibrary"), + android.FixtureModifyConfig(func(config android.Config) { + config.SetBuildFromTextStub(true) + }), + ).RunTestWithBp(t, ` + bootclasspath_fragment { + name: "myfragment", + contents: ["mysdklibrary"], + hidden_api: {split_packages: ["*"]}, + additional_stubs: [ + "android-non-updatable", + ], + } + java_sdk_library { + name: "mysdklibrary", + srcs: ["a.java"], + shared_library: false, + public: {enabled: true}, + system: {enabled: true}, + } + `) + + fragment := result.ModuleForTests("myfragment", "android_common") + ruleCommand := fragment.Rule("modularHiddenAPIStubFlagsFile").RuleParams.Command + android.AssertStringDoesContain(t, "Command expected to contain library as dependency stub dex", + ruleCommand, "--dependency-stub-dex=out/soong/.intermediates/default/java/android-non-updatable.stubs.module_lib.from-text/android_common/dex/android-non-updatable.stubs.module_lib.from-text.jar") + android.AssertStringDoesNotContain(t, + "Command not expected to contain multiple api_library as dependency stub dex", ruleCommand, + "--dependency-stub-dex=out/soong/.intermediates/default/java/android-non-updatable.stubs.from-text/android_common/dex/android-non-updatable.stubs.from-text.jar") +} diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index 96e084a205..e54275b882 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -647,7 +647,7 @@ func (s StubDexJarsByModule) addStubDexJar(ctx android.ModuleContext, module and // public version is provided by the art.module.public.api module. In those cases it is necessary // to treat all those modules as they were the same name, otherwise it will result in multiple // definitions of a single class being passed to hidden API processing which will cause an error. - if name == scope.nonUpdatablePrebuiltModule || name == scope.nonUpdatableSourceModule { + if name == scope.nonUpdatablePrebuiltModule || name == android.JavaApiLibraryName(ctx.Config(), scope.nonUpdatableSourceModule) { // Treat all *android-non-updatable* modules as if they were part of an android-non-updatable // java_sdk_library. // TODO(b/192067200): Remove once android-non-updatable is a java_sdk_library or equivalent. From 30bf7fe32d0ba644cd181645fab3c714a9a11b2f Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Mon, 22 May 2023 19:06:56 +0000 Subject: [PATCH 0187/1460] Run "plugins" singleton in parallel with the rest. This completes the changes in aosp/q/topic:"parallel-singletons". Ignore-AOSP-First: File only exists internally Bug: 281536768 Test: manual, presubmits Change-Id: I6529e957fbcca6c18441f823a00061db7556e303 --- android/plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/plugin.go b/android/plugin.go index c9d1338f81..4672453ee4 100644 --- a/android/plugin.go +++ b/android/plugin.go @@ -29,7 +29,7 @@ func init() { } func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { - ctx.RegisterSingletonType("plugins", pluginSingletonFactory) + ctx.RegisterParallelSingletonType("plugins", pluginSingletonFactory) } // pluginSingleton is a singleton to handle allowlisting of the final Android-.mk file From 19f380eb6ab5c33e788578eb17bb01110a2aeaab Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Thu, 18 May 2023 11:54:46 +0000 Subject: [PATCH 0188/1460] Revert "Revert "Restrict plugins to an existing allowlist"" This reverts commit b20a12553ef65a6401dfe0f67ab8b882da0f89db. Reason for revert: Rollforward with fix soong-ca-certificates-apex was missing from allowlist Test: verify fixes CI breakages (cherry picked from https://googleplex-android-review.googlesource.com/q/commit:dfe4a9e4dd088f42ba5e0a3c4f7777c253d378d3) Merged-In: I00687bd2962dbdaaf55189ca4aa82183264eb8d7 Change-Id: I00687bd2962dbdaaf55189ca4aa82183264eb8d7 --- android/Android.bp | 1 + android/config.go | 4 ++ android/plugin.go | 140 ++++++++++++++++++++++++++++++++++++++++ android/variable.go | 1 + tests/bootstrap_test.sh | 24 +++---- 5 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 android/plugin.go diff --git a/android/Android.bp b/android/Android.bp index 118087db7b..94d2c04f6b 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -77,6 +77,7 @@ bootstrap_go_package { "path_properties.go", "paths.go", "phony.go", + "plugin.go", "prebuilt.go", "prebuilt_build_tool.go", "proto.go", diff --git a/android/config.go b/android/config.go index 4f0a64dfee..d0f2ea44c2 100644 --- a/android/config.go +++ b/android/config.go @@ -1880,6 +1880,10 @@ func (c *deviceConfig) ShippingApiLevel() ApiLevel { return uncheckedFinalApiLevel(apiLevel) } +func (c *deviceConfig) BuildBrokenPluginValidation() []string { + return c.config.productVariables.BuildBrokenPluginValidation +} + func (c *deviceConfig) BuildBrokenClangAsFlags() bool { return c.config.productVariables.BuildBrokenClangAsFlags } diff --git a/android/plugin.go b/android/plugin.go new file mode 100644 index 0000000000..c9d1338f81 --- /dev/null +++ b/android/plugin.go @@ -0,0 +1,140 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/google/blueprint" +) + +func init() { + RegisterPluginSingletonBuildComponents(InitRegistrationContext) +} + +func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { + ctx.RegisterSingletonType("plugins", pluginSingletonFactory) +} + +// pluginSingleton is a singleton to handle allowlisting of the final Android-.mk file +// output. +func pluginSingletonFactory() Singleton { + return &pluginSingleton{} +} + +type pluginSingleton struct{} + +var allowedPluginsByName = map[string]bool{ + "aidl-soong-rules": true, + "arm_compute_library_nn_driver": true, + "cuttlefish-soong-rules": true, + "gki-soong-rules": true, + "hidl-soong-rules": true, + "kernel-config-soong-rules": true, + "soong-angle-codegen": true, + "soong-api": true, + "soong-art": true, + "soong-ca-certificates": true, + "soong-ca-certificates-apex": true, + "soong-clang": true, + "soong-clang-prebuilts": true, + "soong-csuite": true, + "soong-fluoride": true, + "soong-fs_config": true, + "soong-icu": true, + "soong-java-config-error_prone": true, + "soong-libchrome": true, + "soong-llvm": true, + "soong-robolectric": true, + "soong-rust-prebuilts": true, + "soong-selinux": true, + "soong-wayland-protocol-codegen": true, + "treble_report_app": true, + "treble_report_local": true, + "treble_report_module": true, + "vintf-compatibility-matrix-soong-rules": true, + "xsdc-soong-rules": true, +} + +const ( + internalPluginsPath = "vendor/google/build/soong/internal_plugins.json" +) + +type pluginProvider interface { + IsPluginFor(string) bool +} + +func maybeAddInternalPluginsToAllowlist(ctx SingletonContext) { + if path := ExistentPathForSource(ctx, internalPluginsPath); path.Valid() { + ctx.AddNinjaFileDeps(path.String()) + absPath := absolutePath(path.String()) + var moreAllowed map[string]bool + data, err := ioutil.ReadFile(absPath) + if err != nil { + ctx.Errorf("Failed to open internal plugins path %q %q", internalPluginsPath, err) + } + if err := json.Unmarshal(data, &moreAllowed); err != nil { + fmt.Fprintf(os.Stderr, "Internal plugins file %q did not parse correctly: %q", data, err) + } + for k, v := range moreAllowed { + allowedPluginsByName[k] = v + } + } +} + +func (p *pluginSingleton) GenerateBuildActions(ctx SingletonContext) { + for _, p := range ctx.DeviceConfig().BuildBrokenPluginValidation() { + allowedPluginsByName[p] = true + } + maybeAddInternalPluginsToAllowlist(ctx) + + disallowedPlugins := map[string]bool{} + ctx.VisitAllModulesBlueprint(func(module blueprint.Module) { + if ctx.ModuleType(module) != "bootstrap_go_package" { + return + } + + p, ok := module.(pluginProvider) + if !ok || !p.IsPluginFor("soong_build") { + return + } + + name := ctx.ModuleName(module) + if _, ok := allowedPluginsByName[name]; ok { + return + } + + dir := ctx.ModuleDir(module) + + // allow use of plugins within Soong to not allowlist everything + if strings.HasPrefix(dir, "build/soong") { + return + } + + // allow third party users outside of external to create new plugins, i.e. non-google paths + // under vendor or hardware + if !strings.HasPrefix(dir, "external/") && IsThirdPartyPath(dir) { + return + } + disallowedPlugins[name] = true + }) + if len(disallowedPlugins) > 0 { + ctx.Errorf("New plugins are not supported; however %q were found. Please reach out to the build team or use BUILD_BROKEN_PLUGIN_VALIDATION (see Changes.md for more info).", SortedStringKeys(disallowedPlugins)) + } +} diff --git a/android/variable.go b/android/variable.go index 97171e7b99..40eb00675d 100644 --- a/android/variable.go +++ b/android/variable.go @@ -440,6 +440,7 @@ type productVariables struct { ShippingApiLevel *string `json:",omitempty"` + BuildBrokenPluginValidation []string `json:",omitempty"` BuildBrokenClangAsFlags bool `json:",omitempty"` BuildBrokenClangCFlags bool `json:",omitempty"` BuildBrokenClangProperty bool `json:",omitempty"` diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index 59352477cb..5fc05f8b10 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -207,8 +207,8 @@ EOF function test_soong_build_rerun_iff_environment_changes() { setup - mkdir -p cherry - cat > cherry/Android.bp <<'EOF' + mkdir -p build/soong/cherry + cat > build/soong/cherry/Android.bp <<'EOF' bootstrap_go_package { name: "cherry", pkgPath: "android/soong/cherry", @@ -224,7 +224,7 @@ bootstrap_go_package { } EOF - cat > cherry/cherry.go <<'EOF' + cat > build/soong/cherry/cherry.go <<'EOF' package cherry import ( @@ -317,8 +317,8 @@ function test_add_file_to_soong_build() { run_soong local -r mtime1=$(stat -c "%y" out/soong/build.ninja) - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p vendor/foo/picard + cat > vendor/foo/picard/Android.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -334,7 +334,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > vendor/foo/picard/picard.go <<'EOF' package picard import ( @@ -390,11 +390,11 @@ EOF function test_glob_during_bootstrapping() { setup - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p build/soong/picard + cat > build/soong/picard/Android.bp <<'EOF' build=["foo*.bp"] EOF - cat > a/fooa.bp <<'EOF' + cat > build/soong/picard/fooa.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -410,7 +410,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > build/soong/picard/picard.go <<'EOF' package picard import ( @@ -459,7 +459,7 @@ EOF grep -q "Make it so" out/soong/build.ninja || fail "Original action not present" - cat > a/foob.bp <<'EOF' + cat > build/soong/picard/foob.bp <<'EOF' bootstrap_go_package { name: "worf-soong-rules", pkgPath: "android/soong/worf", @@ -476,7 +476,7 @@ bootstrap_go_package { } EOF - cat > a/worf.go <<'EOF' + cat > build/soong/picard/worf.go <<'EOF' package worf import "android/soong/picard" From 6a7940cc99c4cdafd7a33fe1677bdf4df2b28672 Mon Sep 17 00:00:00 2001 From: Yu Liu Date: Tue, 9 May 2023 17:12:22 -0700 Subject: [PATCH 0189/1460] Support sandboxing genrule Bug: 281067655 Test: local build Change-Id: I123b5e51eaf1476d1215e5208331b6540d45fd98 --- android/bazel_handler.go | 12 ++-- android/config.go | 4 +- android/variable.go | 2 +- genrule/Android.bp | 1 + genrule/allowlists.go | 147 +++++++++++++++++++++++++++++++++++++++ genrule/genrule.go | 30 +++++--- genrule/genrule_test.go | 75 ++++++++++---------- 7 files changed, 213 insertions(+), 58 deletions(-) create mode 100644 genrule/allowlists.go diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 10cf60a0a7..7e81792a1b 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -505,7 +505,7 @@ func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset { return []bazel.AqueryDepset{} } -func addToStringSet(set map[string]bool, items []string) { +func AddToStringSet(set map[string]bool, items []string) { for _, item := range items { set[item] = true } @@ -517,19 +517,19 @@ func GetBazelEnabledAndDisabledModules(buildMode SoongBuildMode, forceEnabled ma switch buildMode { case BazelProdMode: - addToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) + AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) for enabledAdHocModule := range forceEnabled { enabledModules[enabledAdHocModule] = true } case BazelStagingMode: // Staging mode includes all prod modules plus all staging modules. - addToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) - addToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList) + AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) + AddToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList) for enabledAdHocModule := range forceEnabled { enabledModules[enabledAdHocModule] = true } case BazelDevMode: - addToStringSet(disabledModules, allowlists.MixedBuildsDisabledList) + AddToStringSet(disabledModules, allowlists.MixedBuildsDisabledList) default: panic("Expected BazelProdMode, BazelStagingMode, or BazelDevMode") } @@ -609,7 +609,7 @@ func NewBazelContext(c *config) (BazelContext, error) { allowlists.StagingDclaMixedBuildsEnabledList...) } dclaEnabledModules := map[string]bool{} - addToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList) + AddToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList) return &mixedBuildBazelContext{ bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)}, paths: &paths, diff --git a/android/config.go b/android/config.go index acadb3ff33..58ad4c5c62 100644 --- a/android/config.go +++ b/android/config.go @@ -1886,8 +1886,8 @@ func (c *deviceConfig) BuildBrokenInputDir(name string) bool { return InList(name, c.config.productVariables.BuildBrokenInputDirModules) } -func (c *deviceConfig) BuildBrokenDepfile() bool { - return Bool(c.config.productVariables.BuildBrokenDepfile) +func (c *deviceConfig) GenruleSandboxing() bool { + return Bool(c.config.productVariables.GenruleSandboxing) } func (c *deviceConfig) RequiresInsecureExecmemForSwiftshader() bool { diff --git a/android/variable.go b/android/variable.go index 97171e7b99..a6822c6a43 100644 --- a/android/variable.go +++ b/android/variable.go @@ -443,7 +443,7 @@ type productVariables struct { BuildBrokenClangAsFlags bool `json:",omitempty"` BuildBrokenClangCFlags bool `json:",omitempty"` BuildBrokenClangProperty bool `json:",omitempty"` - BuildBrokenDepfile *bool `json:",omitempty"` + GenruleSandboxing *bool `json:",omitempty"` BuildBrokenEnforceSyspropOwner bool `json:",omitempty"` BuildBrokenTrebleSyspropNeverallow bool `json:",omitempty"` BuildBrokenUsesSoongPython2Modules bool `json:",omitempty"` diff --git a/genrule/Android.bp b/genrule/Android.bp index 8fb5c402cf..b201cae9a6 100644 --- a/genrule/Android.bp +++ b/genrule/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "soong-shared", ], srcs: [ + "allowlists.go", "genrule.go", "locations.go", ], diff --git a/genrule/allowlists.go b/genrule/allowlists.go new file mode 100644 index 0000000000..875dbabed6 --- /dev/null +++ b/genrule/allowlists.go @@ -0,0 +1,147 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package genrule + +import ( + "android/soong/android" +) + +var ( + DepfileAllowList = []string{ + "depfile_allowed_for_test", + "tflite_support_spm_config", + "tflite_support_spm_encoder_config", + "gen_uwb_core_proto", + "libtextclassifier_fbgen_utils_flatbuffers_flatbuffers_test", + "libtextclassifier_fbgen_utils_lua_utils_tests", + "libtextclassifier_fbgen_lang_id_common_flatbuffers_model", + "libtextclassifier_fbgen_lang_id_common_flatbuffers_embedding-network", + "libtextclassifier_fbgen_annotator_datetime_datetime", + "libtextclassifier_fbgen_annotator_model", + "libtextclassifier_fbgen_annotator_experimental_experimental", + "libtextclassifier_fbgen_annotator_entity-data", + "libtextclassifier_fbgen_annotator_person_name_person_name_model", + "libtextclassifier_fbgen_utils_tflite_text_encoder_config", + "libtextclassifier_fbgen_utils_codepoint-range", + "libtextclassifier_fbgen_utils_intents_intent-config", + "libtextclassifier_fbgen_utils_flatbuffers_flatbuffers", + "libtextclassifier_fbgen_utils_zlib_buffer", + "libtextclassifier_fbgen_utils_tokenizer", + "libtextclassifier_fbgen_utils_grammar_rules", + "libtextclassifier_fbgen_utils_grammar_semantics_expression", + "libtextclassifier_fbgen_utils_resources", + "libtextclassifier_fbgen_utils_i18n_language-tag", + "libtextclassifier_fbgen_utils_normalization", + "libtextclassifier_fbgen_utils_container_bit-vector", + "libtextclassifier_fbgen_actions_actions-entity-data", + "libtextclassifier_fbgen_actions_actions_model", + "libtextclassifier_fbgen_utils_grammar_testing_value", + } + + SandboxingDenyModuleList = []string{ + "framework-javastream-protos", + "RsBalls-rscript", + "CtsRsBlasTestCases-rscript", + "pvmfw_fdt_template_rs", + "RSTest_v14-rscript", + "com.android.apex.test.bar_stripped", + "com.android.apex.test.sharedlibs_secondary_generated", + "ImageProcessingJB-rscript", + "RSTest-rscript", + "BluetoothGeneratedDumpsysBinarySchema_bfbs", + "WmediumdServerProto_h", + "TracingVMProtoStub_h", + "FrontendStub_h", + "VehicleServerProtoStub_cc", + "AudioFocusControlProtoStub_cc", + "AudioFocusControlProtoStub_h", + "TracingVMProtoStub_cc", + "VehicleServerProtoStub_h", + "hidl2aidl_translate_cpp_test_gen_headers", + "hidl2aidl_translate_cpp_test_gen_src", + "hidl2aidl_translate_java_test_gen_src", + "hidl2aidl_translate_ndk_test_gen_headers", + "hidl2aidl_translate_ndk_test_gen_src", + "hidl_hash_test_gen", + "nos_app_avb_service_genc++", + "nos_app_avb_service_genc++_headers", + "nos_app_avb_service_genc++_mock", + "nos_app_identity_service_genc++", + "nos_app_keymaster_service_genc++", + "nos_generator_test_service_genc++_headers", + "nos_generator_test_service_genc++_mock", + "r8retrace-run-retrace", + "ltp_config_arm", + "ltp_config_arm_64_hwasan", + "ltp_config_arm_lowmem", + "ltp_config_arm_64", + "ltp_config_riscv_64", + "ltp_config_x86_64", + "vm-tests-tf-lib", + "hidl_cpp_impl_test_gen-headers", + "pandora_experimental-python-gen-src", + "framework-cppstream-protos", + "Refocus-rscript", + "RSTest_v11-rscript", + "RSTest_v16-rscript", + "ScriptGroupTest-rscript", + "ImageProcessing2-rscript", + "ImageProcessing-rscript", + "com.android.apex.test.pony_stripped", + "com.android.apex.test.baz_stripped", + "com.android.apex.test.foo_stripped", + "com.android.apex.test.sharedlibs_generated", + "CtsRenderscriptTestCases-rscript", + "BlueberryFacadeAndCertGeneratedStub_py", + "BlueberryFacadeGeneratedStub_cc", + "BlueberryFacadeGeneratedStub_h", + "BluetoothGeneratedDumpsysDataSchema_h", + "FrontendStub_cc", + "OpenwrtControlServerProto_cc", + "OpenwrtControlServerProto_h", + "WmediumdServerProto_cc", + "c2hal_test_genc++", + "c2hal_test_genc++_headers", + "hidl2aidl_test_gen_aidl", + "hidl_error_test_gen", + "hidl_export_test_gen-headers", + "hidl_format_test_diff", + "hidl_hash_version_gen", + "libbt_topshim_facade_py_proto", + "nos_app_identity_service_genc++_headers", + "nos_app_identity_service_genc++_mock", + "nos_app_keymaster_service_genc++_headers", + "nos_app_keymaster_service_genc++_mock", + "nos_app_weaver_service_genc++", + "nos_app_weaver_service_genc++_headers", + "nos_app_weaver_service_genc++_mock", + "nos_generator_test_service_genc++", + "pandora-python-gen-src", + } + + SandboxingDenyPathList = []string{ + "art/test", + "external/perfetto", + } +) +var DepfileAllowSet = map[string]bool{} +var SandboxingDenyModuleSet = map[string]bool{} +var SandboxingDenyPathSet = map[string]bool{} + +func init() { + android.AddToStringSet(DepfileAllowSet, DepfileAllowList) + android.AddToStringSet(SandboxingDenyModuleSet, append(DepfileAllowList, SandboxingDenyModuleList...)) + android.AddToStringSet(SandboxingDenyPathSet, SandboxingDenyPathList) +} diff --git a/genrule/genrule.go b/genrule/genrule.go index 00adb7025d..c830fcc436 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -452,7 +452,7 @@ func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { manifestPath := android.PathForModuleOut(ctx, manifestName) // Use a RuleBuilder to create a rule that runs the command inside an sbox sandbox. - rule := android.NewRuleBuilder(pctx, ctx).Sbox(task.genDir, manifestPath).SandboxTools() + rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(task.genDir, manifestPath)) cmd := rule.Command() for _, out := range task.out { @@ -594,14 +594,16 @@ func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Allowlist genrule to use depfile until we have a solution to remove it. // TODO(b/235582219): Remove allowlist for genrule - if ctx.ModuleType() == "gensrcs" && - !ctx.DeviceConfig().BuildBrokenDepfile() && - Bool(g.properties.Depfile) { - ctx.PropertyErrorf( - "depfile", - "Deprecated to ensure the module type is convertible to Bazel. "+ - "Try specifying the dependencies explicitly so that there is no need to use depfile. "+ - "If not possible, the escape hatch is to use BUILD_BROKEN_DEPFILE to bypass the error.") + if Bool(g.properties.Depfile) { + // TODO(b/283852474): Checking the GenruleSandboxing flag is temporary in + // order to pass the presubmit before internal master is updated. + if ctx.DeviceConfig().GenruleSandboxing() && !DepfileAllowSet[g.Name()] { + ctx.PropertyErrorf( + "depfile", + "Deprecated to ensure the module type is convertible to Bazel. "+ + "Try specifying the dependencies explicitly so that there is no need to use depfile. "+ + "If not possible, the escape hatch is to add the module to allowlists.go to bypass the error.") + } } g.generateCommonBuildActions(ctx) @@ -737,7 +739,7 @@ func NewGenSrcs() *Module { // TODO(ccross): this RuleBuilder is a hack to be able to call // rule.Command().PathForOutput. Replace this with passing the rule into the // generator. - rule := android.NewRuleBuilder(pctx, ctx).Sbox(genDir, nil).SandboxTools() + rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(genDir, nil)) for _, in := range shard { outFile := android.GenPathWithExt(ctx, finalSubDir, in, String(properties.Output_extension)) @@ -1020,3 +1022,11 @@ func DefaultsFactory(props ...interface{}) android.Module { return module } + +func getSandboxedRuleBuilder(ctx android.ModuleContext, r *android.RuleBuilder) *android.RuleBuilder { + if !ctx.DeviceConfig().GenruleSandboxing() || SandboxingDenyPathSet[ctx.ModuleDir()] || + SandboxingDenyModuleSet[ctx.ModuleName()] { + return r.SandboxTools() + } + return r.SandboxInputs() +} diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index 63f8fa9b82..370fe3d465 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -97,8 +97,9 @@ func testGenruleBp() string { func TestGenruleCmd(t *testing.T) { testcases := []struct { - name string - prop string + name string + moduleName string + prop string allowMissingDependencies bool @@ -285,7 +286,8 @@ func TestGenruleCmd(t *testing.T) { expect: "echo foo > __SBOX_SANDBOX_DIR__/out/out2", }, { - name: "depfile", + name: "depfile", + moduleName: "depfile_allowed_for_test", prop: ` out: ["out"], depfile: true, @@ -397,7 +399,8 @@ func TestGenruleCmd(t *testing.T) { err: "$(depfile) used without depfile property", }, { - name: "error no depfile", + name: "error no depfile", + moduleName: "depfile_allowed_for_test", prop: ` out: ["out"], depfile: true, @@ -440,11 +443,15 @@ func TestGenruleCmd(t *testing.T) { for _, test := range testcases { t.Run(test.name, func(t *testing.T) { - bp := "genrule {\n" - bp += "name: \"gen\",\n" - bp += test.prop - bp += "}\n" - + moduleName := "gen" + if test.moduleName != "" { + moduleName = test.moduleName + } + bp := fmt.Sprintf(` + genrule { + name: "%s", + %s + }`, moduleName, test.prop) var expectedErrors []string if test.err != "" { expectedErrors = append(expectedErrors, regexp.QuoteMeta(test.err)) @@ -455,6 +462,9 @@ func TestGenruleCmd(t *testing.T) { android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { variables.Allow_missing_dependencies = proptools.BoolPtr(test.allowMissingDependencies) }), + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.GenruleSandboxing = proptools.BoolPtr(true) + }), android.FixtureModifyContext(func(ctx *android.TestContext) { ctx.SetAllowMissingDependencies(test.allowMissingDependencies) }), @@ -466,7 +476,7 @@ func TestGenruleCmd(t *testing.T) { return } - gen := result.Module("gen", "").(*Module) + gen := result.Module(moduleName, "").(*Module) android.AssertStringEquals(t, "raw commands", test.expect, gen.rawCommands[0]) }) } @@ -627,53 +637,42 @@ func TestGenSrcs(t *testing.T) { } } -func TestGensrcsBuildBrokenDepfile(t *testing.T) { +func TestGenruleAllowlistingDepfile(t *testing.T) { tests := []struct { - name string - prop string - BuildBrokenDepfile *bool - err string + name string + prop string + err string + moduleName string }{ { - name: `error when BuildBrokenDepfile is set to false`, + name: `error when module is not allowlisted`, prop: ` depfile: true, cmd: "cat $(in) > $(out) && cat $(depfile)", `, - BuildBrokenDepfile: proptools.BoolPtr(false), - err: "depfile: Deprecated to ensure the module type is convertible to Bazel", + err: "depfile: Deprecated to ensure the module type is convertible to Bazel", }, { - name: `error when BuildBrokenDepfile is not set`, + name: `no error when module is allowlisted`, prop: ` depfile: true, cmd: "cat $(in) > $(out) && cat $(depfile)", `, - err: "depfile: Deprecated to ensure the module type is convertible to Bazel.", - }, - { - name: `no error when BuildBrokenDepfile is explicitly set to true`, - prop: ` - depfile: true, - cmd: "cat $(in) > $(out) && cat $(depfile)", - `, - BuildBrokenDepfile: proptools.BoolPtr(true), - }, - { - name: `no error if depfile is not set`, - prop: ` - cmd: "cat $(in) > $(out)", - `, + moduleName: `depfile_allowed_for_test`, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + moduleName := "foo" + if test.moduleName != "" { + moduleName = test.moduleName + } bp := fmt.Sprintf(` gensrcs { - name: "foo", + name: "%s", srcs: ["data.txt"], %s - }`, test.prop) + }`, moduleName, test.prop) var expectedErrors []string if test.err != "" { @@ -682,9 +681,7 @@ func TestGensrcsBuildBrokenDepfile(t *testing.T) { android.GroupFixturePreparers( prepareForGenRuleTest, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { - if test.BuildBrokenDepfile != nil { - variables.BuildBrokenDepfile = test.BuildBrokenDepfile - } + variables.GenruleSandboxing = proptools.BoolPtr(true) }), ). ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(expectedErrors)). From 2050c90e2f574f9962cddc3f5532e08119520169 Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Wed, 26 Apr 2023 21:06:24 +0900 Subject: [PATCH 0190/1460] BUILD_NUMBER and BUILD_HOSTNAME doesn't affect kati regeneration * Extract BUILD_NUMBER, BUILD_HOSTNAME to file to avoid kati change * Handle FILE_NAME_TAG_PLACEHOLDER string in dist in build/make/packaging/distdir.mk Ignore-AOSP-First: there are internal use cases of FILE_NAME_TAG Test: check if kati isn't invoked even though BUILD_NUMBER, BUILD_HOSTNAME is changed Test: m && m, and check if the second m is no-op Bug: 278060169 (cherry picked from https://googleplex-android-review.googlesource.com/q/commit:1fb4ece718bd34f9c47754c14281c7ad644bb174) Merged-In: I65eefc6bb86d4076098a1bf8b317b4cb88201499 Change-Id: I65eefc6bb86d4076098a1bf8b317b4cb88201499 --- android_sdk/sdk_repo_host.go | 2 +- ui/build/kati.go | 60 +++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/android_sdk/sdk_repo_host.go b/android_sdk/sdk_repo_host.go index 61058df098..9623a8bc46 100644 --- a/android_sdk/sdk_repo_host.go +++ b/android_sdk/sdk_repo_host.go @@ -242,7 +242,7 @@ func (s *sdkRepoHost) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, ".PHONY:", name, "sdk_repo", "sdk-repo-"+name) fmt.Fprintln(w, "sdk_repo", "sdk-repo-"+name+":", strings.Join(s.FilesToInstall().Strings(), " ")) - fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-$(FILE_NAME_TAG).zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) + fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-FILE_NAME_TAG_PLACEHOLDER.zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) }, } } diff --git a/ui/build/kati.go b/ui/build/kati.go index dad68fac2f..4297378b37 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -22,6 +22,7 @@ import ( "os/user" "path/filepath" "strings" + "time" "android/soong/ui/metrics" "android/soong/ui/status" @@ -66,6 +67,21 @@ func genKatiSuffix(ctx Context, config Config) { } } +func writeValueIfChanged(ctx Context, config Config, dir string, filename string, value string) { + filePath := filepath.Join(dir, filename) + previousValue := "" + rawPreviousValue, err := ioutil.ReadFile(filePath) + if err == nil { + previousValue = string(rawPreviousValue) + } + + if previousValue != value { + if err = ioutil.WriteFile(filePath, []byte(value), 0666); err != nil { + ctx.Fatalf("Failed to write: %v", err) + } + } +} + // Base function to construct and run the Kati command line with additional // arguments, and a custom function closure to mutate the environment Kati runs // in. @@ -157,28 +173,57 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF } cmd.Stderr = cmd.Stdout - // Apply the caller's function closure to mutate the environment variables. - envFunc(cmd.Environment) - + var username string // Pass on various build environment metadata to Kati. - if _, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { - username := "unknown" + if usernameFromEnv, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { + username = "unknown" if u, err := user.Current(); err == nil { username = u.Username } else { ctx.Println("Failed to get current user:", err) } cmd.Environment.Set("BUILD_USERNAME", username) + } else { + username = usernameFromEnv } - if _, ok := cmd.Environment.Get("BUILD_HOSTNAME"); !ok { - hostname, err := os.Hostname() + hostname, ok := cmd.Environment.Get("BUILD_HOSTNAME") + if !ok { + hostname, err = os.Hostname() if err != nil { ctx.Println("Failed to read hostname:", err) hostname = "unknown" } cmd.Environment.Set("BUILD_HOSTNAME", hostname) } + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_hostname.txt", hostname) + + // BUILD_NUMBER should be set to the source control value that + // represents the current state of the source code. E.g., a + // perforce changelist number or a git hash. Can be an arbitrary string + // (to allow for source control that uses something other than numbers), + // but must be a single word and a valid file name. + // + // If no BUILD_NUMBER is set, create a useful "I am an engineering build + // from this date/time" value. Make it start with a non-digit so that + // anyone trying to parse it as an integer will probably get "0". + cmd.Environment.Unset("HAS_BUILD_NUMBER") + buildNumber, ok := cmd.Environment.Get("BUILD_NUMBER") + if ok { + cmd.Environment.Set("HAS_BUILD_NUMBER", "true") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", buildNumber) + } else { + buildNumber = fmt.Sprintf("eng.%.6s.%s", username, time.Now().Format("20060102.150405" /* YYYYMMDD.HHMMSS */)) + cmd.Environment.Set("HAS_BUILD_NUMBER", "false") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", username) + } + // Write the build number to a file so it can be read back in + // without changing the command line every time. Avoids rebuilds + // when using ninja. + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_number.txt", buildNumber) + + // Apply the caller's function closure to mutate the environment variables. + envFunc(cmd.Environment) cmd.StartOrFatal() // Set up the ToolStatus command line reader for Kati for a consistent UI @@ -336,6 +381,7 @@ func runKatiPackage(ctx Context, config Config) { "ANDROID_BUILD_SHELL", "DIST_DIR", "OUT_DIR", + "FILE_NAME_TAG", }...) if config.Dist() { From f2af38d2cceb85f93180e666ca88b796410f69db Mon Sep 17 00:00:00 2001 From: Jeongik Cha Date: Tue, 23 May 2023 06:59:39 +0900 Subject: [PATCH 0191/1460] Unset BUILD_NUMBER/HOSTNAME in kati.go BUILD_NUMBER/HOSTNAME is passed by a file, not env variable. To avoid kati recreation, unset BUILD_NUMBER/HOSTNAME after writing it into a file Bug: 278060169 Test: BUILD_NUMBER=1 m && BUILD_NUMBER=2 m (check there is no kati re-run) Change-Id: I5a31461dcf1e4b0634974bcb48a0d7482d42852a --- ui/build/kati.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ui/build/kati.go b/ui/build/kati.go index 4297378b37..aea56d31a4 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -188,13 +188,14 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF } hostname, ok := cmd.Environment.Get("BUILD_HOSTNAME") + // Unset BUILD_HOSTNAME during kati run to avoid kati rerun, kati will use BUILD_HOSTNAME from a file. + cmd.Environment.Unset("BUILD_HOSTNAME") if !ok { hostname, err = os.Hostname() if err != nil { ctx.Println("Failed to read hostname:", err) hostname = "unknown" } - cmd.Environment.Set("BUILD_HOSTNAME", hostname) } writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_hostname.txt", hostname) @@ -209,6 +210,8 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF // anyone trying to parse it as an integer will probably get "0". cmd.Environment.Unset("HAS_BUILD_NUMBER") buildNumber, ok := cmd.Environment.Get("BUILD_NUMBER") + // Unset BUILD_NUMBER during kati run to avoid kati rerun, kati will use BUILD_NUMBER from a file. + cmd.Environment.Unset("BUILD_NUMBER") if ok { cmd.Environment.Set("HAS_BUILD_NUMBER", "true") writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", buildNumber) From f530696762418f2fa8704dfc545028b9fd852307 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Tue, 23 May 2023 06:57:38 +0000 Subject: [PATCH 0192/1460] Add missing dependency in create_mock_soong This change adds missing dependency of Soong modules as symlink directory in create_mock_soong so that a correct ninja graph can be created when invoking androidmk_test.sh. Test: build/soong/test/run_integration_tests.sh Bug: 283905797 Change-Id: I7dff0094fdc23b7c438c917dbc00a5143a4dc5ae --- tests/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/lib.sh b/tests/lib.sh index 715eac1430..7f3970eb7e 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -99,6 +99,7 @@ function create_mock_soong { symlink_directory external/python symlink_directory external/sqlite symlink_directory external/spdx-tools + symlink_directory libcore touch "$MOCK_TOP/Android.bp" } From b34bb067808deae66af79b5149d9210ca49349f2 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Mon, 8 May 2023 23:44:38 +0000 Subject: [PATCH 0193/1460] Enable java_api_library modules in TxtStubLibraries.bp Since the stubs generation from text file feature has been merged, enable the java_api_library modules in TxtStubLibraries.bp file. Test: m --build-from-text-stub Change-Id: Idc77d01882f136c4e255fb4284f9dc058fafe44d --- java/core-libraries/TxtStubLibraries.bp | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/java/core-libraries/TxtStubLibraries.bp b/java/core-libraries/TxtStubLibraries.bp index 813187e54d..0cf0f360b3 100644 --- a/java/core-libraries/TxtStubLibraries.bp +++ b/java/core-libraries/TxtStubLibraries.bp @@ -22,8 +22,6 @@ java_system_modules { libs: [ "core-current-stubs-for-system-modules-no-annotations.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -36,8 +34,6 @@ java_library { "core.current.stubs.from-text", "core-lambda-stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } // Same as core-module-lib-stubs-system-modules, but the stubs are generated from .txt files @@ -47,8 +43,6 @@ java_system_modules { libs: [ "core-module-lib-stubs-for-system-modules-no-annotations.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -61,8 +55,6 @@ java_library { "core.module_lib.stubs.from-text", "core-lambda-stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -79,8 +71,6 @@ java_library { sdk_version: "none", system_modules: "none", visibility: ["//visibility:private"], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } // Same as legacy-core-platform-api-stubs-system-modules, but the stubs are generated from .txt files @@ -91,8 +81,6 @@ java_system_modules { "legacy.core.platform.api.no.annotations.stubs.from-text", "core-lambda-stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -108,8 +96,6 @@ java_library { "legacy.core.platform.api.stubs.from-text", ], patch_module: "java.base", - // TODO: Enable after stub generation from .txt file is available - enabled: false, } // Same as stable-core-platform-api-stubs-system-modules, but the stubs are generated from .txt files @@ -120,8 +106,6 @@ java_system_modules { "stable.core.platform.api.no.annotations.stubs.from-text", "core-lambda-stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -137,8 +121,6 @@ java_library { "stable.core.platform.api.stubs.from-text", ], patch_module: "java.base", - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_api_library { @@ -151,6 +133,4 @@ java_api_library { // LambdaMetaFactory depends on CallSite etc. which is part of the Core API surface "core.current.stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } From c13f785061be5c3445cbdb3554b528d5483f2aae Mon Sep 17 00:00:00 2001 From: Liz Kammer Date: Wed, 17 May 2023 13:01:48 -0400 Subject: [PATCH 0194/1460] Handle missing dependencies in mixed builds Bazel will fail on queries with missing dependencies. Instead, we check for missing dependencies in mixed builds and we fall back to Soong when we are aware of missing dependencies in a module. Test: go test soong tests Change-Id: I4f83752704970d8b43650d5b55ff35799c7bc625 --- android/bazel.go | 22 ++++-- android/bazel_handler.go | 9 ++- android/bazel_test.go | 147 +++++++++++++++++++++++++++++++++++++++ android/module.go | 21 +++++- android/mutator.go | 20 ++++++ 5 files changed, 209 insertions(+), 10 deletions(-) diff --git a/android/bazel.go b/android/bazel.go index 114b1f5c2f..d326634834 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -46,6 +46,10 @@ const ( // that is not a platform incompatibility. Example: the module-type is not // enabled, or is not bp2build-converted. ModuleIncompatibility + + // Missing dependencies. We can't query Bazel for modules if it has missing dependencies, there + // will be failures. + ModuleMissingDeps ) // FileGroupAsLibrary describes a filegroup module that is converted to some library @@ -367,16 +371,26 @@ func GetBp2BuildAllowList() Bp2BuildConversionAllowlist { // As a side effect, calling this method will also log whether this module is // mixed build enabled for metrics reporting. func MixedBuildsEnabled(ctx BaseModuleContext) MixedBuildEnabledStatus { - module := ctx.Module() - apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) - withinApex := !apexInfo.IsForPlatform() - platformIncompatible := isPlatformIncompatible(ctx.Os(), ctx.Arch().ArchType) if platformIncompatible { ctx.Config().LogMixedBuild(ctx, false) return TechnicalIncompatibility } + if ctx.Config().AllowMissingDependencies() { + missingDeps := ctx.getMissingDependencies() + // If there are missing dependencies, querying Bazel will fail. Soong instead fails at execution + // time, not loading/analysis. disable mixed builds and fall back to Soong to maintain that + // behavior. + if len(missingDeps) > 0 { + ctx.Config().LogMixedBuild(ctx, false) + return ModuleMissingDeps + } + } + + module := ctx.Module() + apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) + withinApex := !apexInfo.IsForPlatform() mixedBuildEnabled := ctx.Config().IsMixedBuildsEnabled() && module.Enabled() && convertedToBazel(ctx, module) && diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 5291ecae94..a3c51c2901 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -29,6 +29,7 @@ import ( "android/soong/bazel/cquery" "android/soong/shared" "android/soong/starlark_fmt" + "github.com/google/blueprint" "github.com/google/blueprint/metrics" @@ -73,14 +74,12 @@ var ( } ) -func init() { - RegisterMixedBuildsMutator(InitRegistrationContext) +func registerMixedBuildsMutator(ctx RegisterMutatorsContext) { + ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() } func RegisterMixedBuildsMutator(ctx RegistrationContext) { - ctx.FinalDepsMutators(func(ctx RegisterMutatorsContext) { - ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() - }) + ctx.FinalDepsMutators(registerMixedBuildsMutator) } func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) { diff --git a/android/bazel_test.go b/android/bazel_test.go index 77e251575c..13fd408497 100644 --- a/android/bazel_test.go +++ b/android/bazel_test.go @@ -436,3 +436,150 @@ func TestShouldKeepExistingBuildFileForDir(t *testing.T) { } } } + +type mixedBuildModule struct { + ModuleBase + BazelModuleBase + props struct { + Deps []string + Mixed_build_incompatible *bool + QueuedBazelCall bool `blueprint:"mutated"` + } +} + +type mixedBuildModuleInfo struct { + QueuedBazelCall bool +} + +var mixedBuildModuleProvider = blueprint.NewProvider(mixedBuildModuleInfo{}) + +func mixedBuildModuleFactory() Module { + m := &mixedBuildModule{} + m.AddProperties(&m.props) + InitAndroidArchModule(m, HostAndDeviceDefault, MultilibBoth) + InitBazelModule(m) + + return m +} + +func (m *mixedBuildModule) ConvertWithBp2build(ctx TopDownMutatorContext) { +} + +func (m *mixedBuildModule) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddDependency(ctx.Module(), installDepTag{}, m.props.Deps...) +} + +func (m *mixedBuildModule) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +func (m *mixedBuildModule) IsMixedBuildSupported(ctx BaseModuleContext) bool { + return !proptools.Bool(m.props.Mixed_build_incompatible) +} + +func (m *mixedBuildModule) QueueBazelCall(ctx BaseModuleContext) { + m.props.QueuedBazelCall = true +} + +func (m *mixedBuildModule) ProcessBazelQueryResponse(ctx ModuleContext) { + ctx.SetProvider(mixedBuildModuleProvider, mixedBuildModuleInfo{ + QueuedBazelCall: m.props.QueuedBazelCall, + }) +} + +var prepareForMixedBuildTests = FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.RegisterModuleType("deps", mixedBuildModuleFactory) + RegisterMixedBuildsMutator(ctx) +}) + +func TestMixedBuildsEnabledForType(t *testing.T) { + baseBp := ` + deps { + name: "foo", + deps: ["bar"], + target: { windows: { enabled: true } }, + %s + } +` + depBp := ` + deps { + name: "bar", + target: { + windows: { + enabled: true, + }, + }, + } +` + testCases := []struct { + desc string + variant *string + missingDeps bool + extraBpInfo string + mixedBuildsEnabled bool + }{ + { + desc: "mixed builds works", + mixedBuildsEnabled: true, + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "missing deps", + missingDeps: true, + mixedBuildsEnabled: false, + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "windows no mixed builds", + mixedBuildsEnabled: false, + variant: proptools.StringPtr("windows_x86"), + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "mixed builds disabled by type", + mixedBuildsEnabled: false, + extraBpInfo: `mixed_build_incompatible: true, + bazel_module: { bp2build_available: true },`, + }, + { + desc: "mixed builds not bp2build available", + mixedBuildsEnabled: false, + extraBpInfo: `bazel_module: { bp2build_available: false },`, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + handlers := GroupFixturePreparers( + prepareForMixedBuildTests, + PrepareForTestWithArchMutator, + FixtureModifyConfig(func(config Config) { + config.BazelContext = MockBazelContext{ + OutputBaseDir: "base", + } + config.Targets[Windows] = []Target{ + {Windows, Arch{ArchType: X86_64}, NativeBridgeDisabled, "", "", true}, + {Windows, Arch{ArchType: X86}, NativeBridgeDisabled, "", "", true}, + } + }), + ) + bp := fmt.Sprintf(baseBp, tc.extraBpInfo) + if tc.missingDeps { + handlers = GroupFixturePreparers( + handlers, + PrepareForTestWithAllowMissingDependencies, + ) + } else { + bp += depBp + } + result := handlers.RunTestWithBp(t, bp) + + variant := proptools.StringDefault(tc.variant, "android_arm64_armv8-a") + + m := result.ModuleForTests("foo", variant) + mixedBuildModuleInfo := result.TestContext.ModuleProvider(m.Module(), mixedBuildModuleProvider).(mixedBuildModuleInfo) + if w, g := tc.mixedBuildsEnabled, mixedBuildModuleInfo.QueuedBazelCall; w != g { + t.Errorf("Expected mixed builds enabled %t, got mixed builds enabled %t", w, g) + } + }) + } +} diff --git a/android/module.go b/android/module.go index db602a0aa3..d8f4f77c59 100644 --- a/android/module.go +++ b/android/module.go @@ -354,6 +354,10 @@ type BaseModuleContext interface { AddMissingDependencies(missingDeps []string) + // getMissingDependencies returns the list of missing dependencies. + // Calling this function prevents adding new dependencies. + getMissingDependencies() []string + // AddUnconvertedBp2buildDep stores module name of a direct dependency that was not converted via bp2build AddUnconvertedBp2buildDep(dep string) @@ -939,7 +943,8 @@ type commonProperties struct { NamespaceExportedToMake bool `blueprint:"mutated"` - MissingDeps []string `blueprint:"mutated"` + MissingDeps []string `blueprint:"mutated"` + CheckedMissingDeps bool `blueprint:"mutated"` // Name and variant strings stored by mutators to enable Module.String() DebugName string `blueprint:"mutated"` @@ -2862,6 +2867,20 @@ func (b *baseModuleContext) AddMissingDependencies(deps []string) { } } +func (b *baseModuleContext) checkedMissingDeps() bool { + return b.Module().base().commonProperties.CheckedMissingDeps +} + +func (b *baseModuleContext) getMissingDependencies() []string { + checked := &b.Module().base().commonProperties.CheckedMissingDeps + *checked = true + var missingDeps []string + missingDeps = append(missingDeps, b.Module().base().commonProperties.MissingDeps...) + missingDeps = append(missingDeps, b.bp.EarlyGetMissingDependencies()...) + missingDeps = FirstUniqueStrings(missingDeps) + return missingDeps +} + type AllowDisabledModuleDependency interface { blueprint.DependencyTag AllowDisabledModuleDependency(target Module) bool diff --git a/android/mutator.go b/android/mutator.go index 0a091eb6fa..41853154f2 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -67,6 +67,8 @@ func registerMutatorsForBazelConversion(ctx *Context, bp2buildMutators []Registe // collateGloballyRegisteredMutators constructs the list of mutators that have been registered // with the InitRegistrationContext and will be used at runtime. func collateGloballyRegisteredMutators() sortableComponents { + // ensure mixed builds mutator is the last mutator + finalDeps = append(finalDeps, registerMixedBuildsMutator) return collateRegisteredMutators(preArch, preDeps, postDeps, finalDeps) } @@ -885,10 +887,16 @@ func (b *bottomUpMutatorContext) Rename(name string) { } func (b *bottomUpMutatorContext) AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddDependency(module, tag, name...) } func (b *bottomUpMutatorContext) AddReverseDependency(module blueprint.Module, tag blueprint.DependencyTag, name string) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.AddReverseDependency(module, tag, name) } @@ -938,11 +946,17 @@ func (b *bottomUpMutatorContext) SetDefaultDependencyVariation(variation *string func (b *bottomUpMutatorContext) AddVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddVariationDependencies(variations, tag, names...) } func (b *bottomUpMutatorContext) AddFarVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddFarVariationDependencies(variations, tag, names...) } @@ -952,10 +966,16 @@ func (b *bottomUpMutatorContext) AddInterVariantDependency(tag blueprint.Depende } func (b *bottomUpMutatorContext) ReplaceDependencies(name string) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.ReplaceDependencies(name) } func (b *bottomUpMutatorContext) ReplaceDependenciesIf(name string, predicate blueprint.ReplaceDependencyPredicate) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.ReplaceDependenciesIf(name, predicate) } From 3e65f2a53c6e5fd82ddad403360bb87b986ad0d8 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Tue, 23 May 2023 17:32:30 +0000 Subject: [PATCH 0195/1460] Increase R8 max heap size to 4GB The size was increased for D8 in an earlier change, but is more critical for R8. Match the D8 configuration. Bug: 241220562,283888124 Test: m Change-Id: I3b98bf6098f2196c87a97f6c8f7308d9299170d8 --- java/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/config/config.go b/java/config/config.go index b82a137e71..195dae16b1 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -98,7 +98,7 @@ func init() { "-JDcom.android.tools.r8.emitRecordAnnotationsExInDex", }, dexerJavaVmFlagsList...)) exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{ - "-JXmx2048M", + "-JXmx4096M", "-JDcom.android.tools.r8.emitRecordAnnotationsInDex", "-JDcom.android.tools.r8.emitPermittedSubclassesAnnotationsInDex", "-JDcom.android.tools.r8.emitRecordAnnotationsExInDex", From bd0fd20bdcef9e8e25047478d47179baa77d7239 Mon Sep 17 00:00:00 2001 From: Kun Niu Date: Tue, 23 May 2023 17:51:44 +0000 Subject: [PATCH 0196/1460] data_native_bins is only available in java_test_host, add java_test_host properties to java_defaults to make it's available in java_defaults as well. Test: local build Bug: 279622634 Change-Id: Iceaf960d587544ae934424a773a6ae3b17fb443e --- java/java.go | 1 + 1 file changed, 1 insertion(+) diff --git a/java/java.go b/java/java.go index 33846bec6b..a23526aee2 100644 --- a/java/java.go +++ b/java/java.go @@ -2606,6 +2606,7 @@ func DefaultsFactory() android.Module { &appProperties{}, &appTestProperties{}, &overridableAppProperties{}, + &hostTestProperties{}, &testProperties{}, &ImportProperties{}, &AARImportProperties{}, From 9b44c2a7254d4495a58491f6f37a5a634e10000f Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Mon, 22 May 2023 19:06:56 +0000 Subject: [PATCH 0197/1460] Run "plugins" singleton in parallel with the rest. This completes the changes in aosp/q/topic:"parallel-singletons". (cherry picked from commit 30bf7fe32d0ba644cd181645fab3c714a9a11b2f) Bug: 281536768 Test: manual, presubmits Change-Id: I6529e957fbcca6c18441f823a00061db7556e303 --- android/plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/plugin.go b/android/plugin.go index c9d1338f81..4672453ee4 100644 --- a/android/plugin.go +++ b/android/plugin.go @@ -29,7 +29,7 @@ func init() { } func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { - ctx.RegisterSingletonType("plugins", pluginSingletonFactory) + ctx.RegisterParallelSingletonType("plugins", pluginSingletonFactory) } // pluginSingleton is a singleton to handle allowlisting of the final Android-.mk file From 6b49888a17d4d00ea539d62ceb9b940f2d487f5f Mon Sep 17 00:00:00 2001 From: Chris Parsons Date: Tue, 23 May 2023 17:55:47 +0000 Subject: [PATCH 0198/1460] Remove GetPythonBinary request type This has been dead code for over a year. Fixes: 232976601 Test: m nothing Change-Id: Ida305fc8fa61d8ac83d74ea2cf007073a0019cec --- android/bazel_handler.go | 25 ------------------------- bazel/cquery/request_type.go | 28 ---------------------------- bazel/cquery/request_type_test.go | 28 ---------------------------- 3 files changed, 81 deletions(-) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 10cf60a0a7..3f645b2343 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -183,10 +183,6 @@ type BazelContext interface { // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order). GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) - // Returns the executable binary resultant from building together the python sources - // TODO(b/232976601): Remove. - GetPythonBinary(label string, cfgKey configKey) (string, error) - // Returns the results of the GetApexInfo query (including output files) GetApexInfo(label string, cfgkey configKey) (cquery.ApexInfo, error) @@ -315,14 +311,6 @@ func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcIn return result, nil } -func (m MockBazelContext) GetPythonBinary(label string, _ configKey) (string, error) { - result, ok := m.LabelToPythonBinary[label] - if !ok { - return "", fmt.Errorf("no target with label %q in LabelToPythonBinary", label) - } - return result, nil -} - func (m MockBazelContext) GetApexInfo(label string, _ configKey) (cquery.ApexInfo, error) { result, ok := m.LabelToApexInfo[label] if !ok { @@ -431,15 +419,6 @@ func (bazelCtx *mixedBuildBazelContext) GetCcInfo(label string, cfgKey configKey return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key) } -func (bazelCtx *mixedBuildBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { - key := makeCqueryKey(label, cquery.GetPythonBinary, cfgKey) - if rawString, ok := bazelCtx.results[key]; ok { - bazelOutput := strings.TrimSpace(rawString) - return cquery.GetPythonBinary.ParseResult(bazelOutput), nil - } - return "", fmt.Errorf("no bazel response found for %v", key) -} - func (bazelCtx *mixedBuildBazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexInfo, error) { key := makeCqueryKey(label, cquery.GetApexInfo, cfgKey) if rawString, ok := bazelCtx.results[key]; ok { @@ -468,10 +447,6 @@ func (n noopBazelContext) GetCcInfo(_ string, _ configKey) (cquery.CcInfo, error panic("unimplemented") } -func (n noopBazelContext) GetPythonBinary(_ string, _ configKey) (string, error) { - panic("unimplemented") -} - func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexInfo, error) { panic("unimplemented") } diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go index 6a3b3c82c7..ca96f23397 100644 --- a/bazel/cquery/request_type.go +++ b/bazel/cquery/request_type.go @@ -8,7 +8,6 @@ import ( var ( GetOutputFiles = &getOutputFilesRequestType{} - GetPythonBinary = &getPythonBinaryRequestType{} GetCcInfo = &getCcInfoType{} GetApexInfo = &getApexInfoType{} GetCcUnstrippedInfo = &getCcUnstrippedInfoType{} @@ -45,8 +44,6 @@ type CcInfo struct { type getOutputFilesRequestType struct{} -type getPythonBinaryRequestType struct{} - // Name returns a string name for this request type. Such request type names must be unique, // and must only consist of alphanumeric characters. func (g getOutputFilesRequestType) Name() string { @@ -72,31 +69,6 @@ func (g getOutputFilesRequestType) ParseResult(rawString string) []string { return splitOrEmpty(rawString, ", ") } -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getPythonBinaryRequestType) Name() string { - return "getPythonBinary" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. -// The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getPythonBinaryRequestType) StarlarkFunctionBody() string { - return "return providers(target)['FilesToRunProvider'].executable.path" -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getPythonBinaryRequestType) ParseResult(rawString string) string { - return rawString -} - type getCcInfoType struct{} // Name returns a string name for this request type. Such request type names must be unique, diff --git a/bazel/cquery/request_type_test.go b/bazel/cquery/request_type_test.go index 7003ce193d..e772bb7d60 100644 --- a/bazel/cquery/request_type_test.go +++ b/bazel/cquery/request_type_test.go @@ -40,34 +40,6 @@ func TestGetOutputFilesParseResults(t *testing.T) { } } -func TestGetPythonBinaryParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedOutput string - }{ - { - description: "no result", - input: "", - expectedOutput: "", - }, - { - description: "one result", - input: "test", - expectedOutput: "test", - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - actualOutput := GetPythonBinary.ParseResult(tc.input) - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - func TestGetCcInfoParseResults(t *testing.T) { t.Parallel() testCases := []struct { From 095819530af067884e1b131d9332c8c4ace31295 Mon Sep 17 00:00:00 2001 From: Vinh Tran Date: Tue, 16 May 2023 16:03:20 -0400 Subject: [PATCH 0199/1460] Sandbox inputs to aidl rule in cc Bug: 279960133 Test: go test Test: Remove hdrs prop from IDropBoxManagerService_aidl && run BUILD_BROKEN_DISABLE_BAZEL=true m libservices && Expect an error from aidl Change-Id: Ifdb260d8e2da9a5767f1e212393de4134b210616 --- aidl_library/aidl_library.go | 21 +++++-- aidl_library/aidl_library_test.go | 18 +++++- android/config.go | 8 +++ cc/cc_test.go | 94 ++++++++++++++++++++++++++----- cc/compiler.go | 15 ++++- cc/gen.go | 57 +++++++++++++++---- cc/library.go | 10 +++- 7 files changed, 188 insertions(+), 35 deletions(-) diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go index 8a84e6bfa9..5985103865 100644 --- a/aidl_library/aidl_library.go +++ b/aidl_library/aidl_library.go @@ -107,8 +107,10 @@ func (lib *AidlLibrary) ConvertWithBp2build(ctx android.TopDownMutatorContext) { type AidlLibraryInfo struct { // The direct aidl files of the module Srcs android.Paths - // The include dirs to the direct aidl files and those provided from aidl_library deps + // The include dirs to the direct aidl files and those provided from transitive aidl_library deps IncludeDirs android.DepSet + // The direct hdrs and hdrs from transitive deps + Hdrs android.DepSet } // AidlLibraryProvider provides the srcs and the transitive include dirs @@ -116,37 +118,48 @@ var AidlLibraryProvider = blueprint.NewProvider(AidlLibraryInfo{}) func (lib *AidlLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { includeDirsDepSetBuilder := android.NewDepSetBuilder(android.PREORDER) + hdrsDepSetBuilder := android.NewDepSetBuilder(android.PREORDER) if len(lib.properties.Srcs) == 0 && len(lib.properties.Hdrs) == 0 { ctx.ModuleErrorf("at least srcs or hdrs prop must be non-empty") } srcs := android.PathsForModuleSrc(ctx, lib.properties.Srcs) + hdrs := android.PathsForModuleSrc(ctx, lib.properties.Hdrs) + if lib.properties.Strip_import_prefix != nil { srcs = android.PathsWithModuleSrcSubDir( ctx, srcs, - android.String(lib.properties.Strip_import_prefix)) + android.String(lib.properties.Strip_import_prefix), + ) + + hdrs = android.PathsWithModuleSrcSubDir( + ctx, + hdrs, + android.String(lib.properties.Strip_import_prefix), + ) } + hdrsDepSetBuilder.Direct(hdrs...) includeDir := android.PathForModuleSrc( ctx, proptools.StringDefault(lib.properties.Strip_import_prefix, ""), ) - includeDirsDepSetBuilder.Direct(includeDir) for _, dep := range ctx.GetDirectDepsWithTag(aidlLibraryTag) { if ctx.OtherModuleHasProvider(dep, AidlLibraryProvider) { info := ctx.OtherModuleProvider(dep, AidlLibraryProvider).(AidlLibraryInfo) includeDirsDepSetBuilder.Transitive(&info.IncludeDirs) + hdrsDepSetBuilder.Transitive(&info.Hdrs) } } - // TODO(b/279960133) Propagate direct and transitive headers/srcs when aidl action sandboxes inputs ctx.SetProvider(AidlLibraryProvider, AidlLibraryInfo{ Srcs: srcs, IncludeDirs: *includeDirsDepSetBuilder.Build(), + Hdrs: *hdrsDepSetBuilder.Build(), }) } diff --git a/aidl_library/aidl_library_test.go b/aidl_library/aidl_library_test.go index d9b410acca..d9dd24515c 100644 --- a/aidl_library/aidl_library_test.go +++ b/aidl_library/aidl_library_test.go @@ -37,7 +37,7 @@ func TestAidlLibrary(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], - hdrs: ["Header.aidl"], + hdrs: ["a/Header.aidl"], strip_import_prefix: "a", deps: ["bar"], } @@ -61,6 +61,13 @@ func TestAidlLibrary(t *testing.T) { []string{"package_foo/a/b/Foo.aidl"}, actualInfo.Srcs, ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl hdrs paths", + []string{"package_foo/a/Header.aidl"}, + actualInfo.Hdrs.ToList(), + ) } func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { @@ -72,6 +79,7 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { aidl_library { name: "bar", srcs: ["x/y/Bar.aidl"], + hdrs: ["BarHeader.aidl"], } `), }.AddToFixture(), @@ -80,7 +88,6 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], - hdrs: ["Header.aidl"], deps: ["bar"], } `), @@ -103,6 +110,13 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { []string{"package_foo/a/b/Foo.aidl"}, actualInfo.Srcs, ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl hdrs paths", + []string{"package_bar/BarHeader.aidl"}, + actualInfo.Hdrs.ToList(), + ) } func TestAidlLibraryWithNoSrcsHdrsDeps(t *testing.T) { diff --git a/android/config.go b/android/config.go index acadb3ff33..4ed107d4fd 100644 --- a/android/config.go +++ b/android/config.go @@ -726,6 +726,14 @@ func (c *config) HostJavaToolPath(ctx PathContext, tool string) Path { return path } +func (c *config) HostCcSharedLibPath(ctx PathContext, lib string) Path { + libDir := "lib" + if ctx.Config().BuildArch.Multilib == "lib64" { + libDir = "lib64" + } + return pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, libDir, false, lib+".so") +} + // PrebuiltOS returns the name of the host OS used in prebuilts directories. func (c *config) PrebuiltOS() string { switch runtime.GOOS { diff --git a/cc/cc_test.go b/cc/cc_test.go index 422df73106..173911bc49 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -39,7 +39,6 @@ func TestMain(m *testing.M) { var prepareForCcTest = android.GroupFixturePreparers( PrepareForTestWithCcIncludeVndk, - aidl_library.PrepareForTestWithAidlLibrary, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { variables.DeviceVndkVersion = StringPtr("current") variables.ProductVndkVersion = StringPtr("current") @@ -4420,7 +4419,7 @@ func TestStubsLibReexportsHeaders(t *testing.T) { } } -func TestAidlLibraryWithHeader(t *testing.T) { +func TestAidlLibraryWithHeaders(t *testing.T) { t.Parallel() ctx := android.GroupFixturePreparers( prepareForCcTest, @@ -4430,6 +4429,7 @@ func TestAidlLibraryWithHeader(t *testing.T) { aidl_library { name: "bar", srcs: ["x/y/Bar.aidl"], + hdrs: ["x/HeaderBar.aidl"], strip_import_prefix: "x", } `)}.AddToFixture(), @@ -4438,6 +4438,7 @@ func TestAidlLibraryWithHeader(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], + hdrs: ["a/HeaderFoo.aidl"], strip_import_prefix: "a", deps: ["bar"], } @@ -4452,7 +4453,20 @@ func TestAidlLibraryWithHeader(t *testing.T) { ).RunTest(t).TestContext libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static") - manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl.sbox.textproto")) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl headers", + []string{ + "package_bar/x/HeaderBar.aidl", + "package_foo/a/HeaderFoo.aidl", + "package_foo/a/b/Foo.aidl", + "out/soong/.intermediates/package_foo/libfoo/android_arm64_armv8-a_static/gen/aidl_library.sbox.textproto", + }, + libfoo.Rule("aidl_library").Implicits, + ) + + manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl_library.sbox.textproto")) aidlCommand := manifest.Commands[0].GetCommand() expectedAidlFlags := "-Ipackage_foo/a -Ipackage_bar/x" @@ -4462,14 +4476,14 @@ func TestAidlLibraryWithHeader(t *testing.T) { outputs := strings.Join(libfoo.AllOutputs(), " ") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BpFoo.h") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BnFoo.h") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/Foo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/BpFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/BnFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/Foo.h") android.AssertStringDoesContain(t, "aidl-generated cpp", outputs, "b/Foo.cpp") // Confirm that the aidl header doesn't get compiled to cpp and h files - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BpBar.h") - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BnBar.h") - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/Bar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/BpBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/BnBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/Bar.h") android.AssertStringDoesNotContain(t, "aidl-generated cpp", outputs, "y/Bar.cpp") } @@ -4548,6 +4562,55 @@ func TestAidlFlagsWithMinSdkVersion(t *testing.T) { } } +func TestInvalidAidlProp(t *testing.T) { + t.Parallel() + + testCases := []struct { + description string + bp string + }{ + { + description: "Invalid use of aidl.libs and aidl.include_dirs", + bp: ` + cc_library { + name: "foo", + aidl: { + libs: ["foo_aidl"], + include_dirs: ["bar/include"], + } + } + `, + }, + { + description: "Invalid use of aidl.libs and aidl.local_include_dirs", + bp: ` + cc_library { + name: "foo", + aidl: { + libs: ["foo_aidl"], + local_include_dirs: ["include"], + } + } + `, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + bp := ` + aidl_library { + name: "foo_aidl", + srcs: ["Foo.aidl"], + } ` + testCase.bp + android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary. + ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern("For aidl headers, please only use aidl.libs prop")), + ).RunTestWithBp(t, bp) + }) + } +} + func TestMinSdkVersionInClangTriple(t *testing.T) { t.Parallel() ctx := testCc(t, ` @@ -4789,23 +4852,24 @@ func TestIncludeDirsExporting(t *testing.T) { checkIncludeDirs(t, ctx, foo, expectedIncludeDirs(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library `), expectedSystemIncludeDirs(``), expectedGeneratedHeaders(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BpBar.h `), expectedOrderOnlyDeps(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BpBar.h `), ) }) diff --git a/cc/compiler.go b/cc/compiler.go index 5da745e027..16f4a6ec0c 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -565,6 +565,11 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps "-I"+android.PathForModuleGen(ctx, "yacc", ctx.ModuleDir()).String()) } + if len(compiler.Properties.Aidl.Libs) > 0 && + (len(compiler.Properties.Aidl.Include_dirs) > 0 || len(compiler.Properties.Aidl.Local_include_dirs) > 0) { + ctx.ModuleErrorf("aidl.libs and (aidl.include_dirs or aidl.local_include_dirs) can't be set at the same time. For aidl headers, please only use aidl.libs prop") + } + if compiler.hasAidl(deps) { flags.aidlFlags = append(flags.aidlFlags, compiler.Properties.Aidl.Flags...) if len(compiler.Properties.Aidl.Local_include_dirs) > 0 { @@ -594,8 +599,14 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps } flags.aidlFlags = append(flags.aidlFlags, "--min_sdk_version="+aidlMinSdkVersion) - flags.Local.CommonFlags = append(flags.Local.CommonFlags, - "-I"+android.PathForModuleGen(ctx, "aidl").String()) + if compiler.hasSrcExt(".aidl") { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, + "-I"+android.PathForModuleGen(ctx, "aidl").String()) + } + if len(deps.AidlLibraryInfos) > 0 { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, + "-I"+android.PathForModuleGen(ctx, "aidl_library").String()) + } } if compiler.hasSrcExt(".rscript") || compiler.hasSrcExt(".fs") { diff --git a/cc/gen.go b/cc/gen.go index dbb9560baf..b15f164960 100644 --- a/cc/gen.go +++ b/cc/gen.go @@ -107,7 +107,14 @@ func genYacc(ctx android.ModuleContext, rule *android.RuleBuilder, yaccFile andr return ret } -func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile android.Path, aidlFlags string) (cppFile android.OutputPath, headerFiles android.Paths) { +func genAidl( + ctx android.ModuleContext, + rule *android.RuleBuilder, + outDirBase string, + aidlFile android.Path, + aidlHdrs android.Paths, + aidlFlags string, +) (cppFile android.OutputPath, headerFiles android.Paths) { aidlPackage := strings.TrimSuffix(aidlFile.Rel(), aidlFile.Base()) baseName := strings.TrimSuffix(aidlFile.Base(), aidlFile.Ext()) shortName := baseName @@ -119,7 +126,7 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr shortName = strings.TrimPrefix(baseName, "I") } - outDir := android.PathForModuleGen(ctx, "aidl") + outDir := android.PathForModuleGen(ctx, outDirBase) cppFile = outDir.Join(ctx, aidlPackage, baseName+".cpp") depFile := outDir.Join(ctx, aidlPackage, baseName+".cpp.d") headerI := outDir.Join(ctx, aidlPackage, baseName+".h") @@ -128,6 +135,8 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr cmd := rule.Command() cmd.BuiltTool("aidl-cpp"). + // libc++ is default stl for aidl-cpp (a cc_binary_host module) + ImplicitTool(ctx.Config().HostCcSharedLibPath(ctx, "libc++")). FlagWithDepFile("-d", depFile). Flag("--ninja"). Flag(aidlFlags). @@ -140,6 +149,10 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr headerBp, }) + if aidlHdrs != nil { + cmd.Implicits(aidlHdrs) + } + return cppFile, android.Paths{ headerI, headerBn, @@ -283,14 +296,19 @@ func genSources( ctx android.ModuleContext, aidlLibraryInfos []aidl_library.AidlLibraryInfo, srcFiles android.Paths, - buildFlags builderFlags) (android.Paths, android.Paths, generatedSourceInfo) { + buildFlags builderFlags, +) (android.Paths, android.Paths, generatedSourceInfo) { var info generatedSourceInfo var deps android.Paths var rsFiles android.Paths + // aidlRule supports compiling aidl files from srcs prop while aidlLibraryRule supports + // compiling aidl files from aidl_library modules specified in aidl.libs prop. + // The rules are separated so that they don't wipe out the other's outputDir var aidlRule *android.RuleBuilder + var aidlLibraryRule *android.RuleBuilder var yaccRule_ *android.RuleBuilder yaccRule := func() *android.RuleBuilder { @@ -331,7 +349,14 @@ func genSources( android.PathForModuleGen(ctx, "aidl.sbox.textproto")) } baseDir := strings.TrimSuffix(srcFile.String(), srcFile.Rel()) - cppFile, aidlHeaders := genAidl(ctx, aidlRule, srcFile, buildFlags.aidlFlags+" -I"+baseDir) + cppFile, aidlHeaders := genAidl( + ctx, + aidlRule, + "aidl", + srcFile, + nil, + buildFlags.aidlFlags+" -I"+baseDir, + ) srcFiles[i] = cppFile info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) @@ -354,13 +379,21 @@ func genSources( } for _, aidlLibraryInfo := range aidlLibraryInfos { + if aidlLibraryRule == nil { + aidlLibraryRule = android.NewRuleBuilder(pctx, ctx).Sbox( + android.PathForModuleGen(ctx, "aidl_library"), + android.PathForModuleGen(ctx, "aidl_library.sbox.textproto"), + ).SandboxInputs() + } for _, aidlSrc := range aidlLibraryInfo.Srcs { - if aidlRule == nil { - // TODO(b/279960133): Sandbox inputs to ensure aidl headers are explicitly specified - aidlRule = android.NewRuleBuilder(pctx, ctx).Sbox(android.PathForModuleGen(ctx, "aidl"), - android.PathForModuleGen(ctx, "aidl.sbox.textproto")) - } - cppFile, aidlHeaders := genAidl(ctx, aidlRule, aidlSrc, buildFlags.aidlFlags) + cppFile, aidlHeaders := genAidl( + ctx, + aidlLibraryRule, + "aidl_library", + aidlSrc, + aidlLibraryInfo.Hdrs.ToList(), + buildFlags.aidlFlags, + ) srcFiles = append(srcFiles, cppFile) info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) @@ -375,6 +408,10 @@ func genSources( aidlRule.Build("aidl", "gen aidl") } + if aidlLibraryRule != nil { + aidlLibraryRule.Build("aidl_library", "gen aidl_library") + } + if yaccRule_ != nil { yaccRule_.Build("yacc", "gen yacc") } diff --git a/cc/library.go b/cc/library.go index 09a7253abb..98096a8b69 100644 --- a/cc/library.go +++ b/cc/library.go @@ -2118,8 +2118,14 @@ func (library *libraryDecorator) link(ctx ModuleContext, // Optionally export aidl headers. if Bool(library.Properties.Aidl.Export_aidl_headers) { if library.baseCompiler.hasAidl(deps) { - dir := android.PathForModuleGen(ctx, "aidl") - library.reexportDirs(dir) + if library.baseCompiler.hasSrcExt(".aidl") { + dir := android.PathForModuleGen(ctx, "aidl") + library.reexportDirs(dir) + } + if len(deps.AidlLibraryInfos) > 0 { + dir := android.PathForModuleGen(ctx, "aidl_library") + library.reexportDirs(dir) + } library.reexportDeps(library.baseCompiler.aidlOrderOnlyDeps...) library.addExportedGeneratedHeaders(library.baseCompiler.aidlHeaders...) From 4c5da94fa4320aa868203ad2128fb49492274c73 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Thu, 30 Mar 2023 16:40:24 -0700 Subject: [PATCH 0200/1460] Enable referencing of aapt-generated proguard flags Some targets may implement classes declared elsewhere, e.g., system server implements services defined in framework-res.apk's manifest. Allow depending on the aapt-generated proguard flags for a given target to support this. Bug: 272495195 Test: m + reference ":framework-res{.aapt.proguardOptionsFile}" (cherry picked from https://android-review.googlesource.com/q/commit:34b080ddcf7757bba3dcb938df46aa660b2564a0) Merged-In: I2a16632ed6e5a9bddbe326bdb7ab0dd79b45e587 Change-Id: I2a16632ed6e5a9bddbe326bdb7ab0dd79b45e587 --- java/app.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/app.go b/java/app.go index 706f99a834..d333a89776 100755 --- a/java/app.go +++ b/java/app.go @@ -925,6 +925,10 @@ func (a *AndroidApp) DepIsInSameApex(ctx android.BaseModuleContext, dep android. // For OutputFileProducer interface func (a *AndroidApp) OutputFiles(tag string) (android.Paths, error) { switch tag { + // In some instances, it can be useful to reference the aapt-generated flags from another + // target, e.g., system server implements services declared in the framework-res manifest. + case ".aapt.proguardOptionsFile": + return []android.Path{a.proguardOptionsFile}, nil case ".aapt.srcjar": return []android.Path{a.aaptSrcJar}, nil case ".export-package.apk": From 6e1aa7830c9b3a37c22eb48736929227b39b7487 Mon Sep 17 00:00:00 2001 From: Usta Shrestha Date: Mon, 13 Feb 2023 18:54:42 -0500 Subject: [PATCH 0201/1460] simplify `m clean` `rm -rf out` used to fail because bazel would not set write permission on some files. Now it's been fixed and thus there is little reason to prefer `m clean` (also users are accustomed to `rm -rf out`) Bug: NA Test: verified `m clean` works;interestingly soong_metrics file is created at the end Change-Id: I000d16508613045811fc7792e5798f7c150dcc05 --- tests/soong_test.sh | 8 ++------ ui/build/cleanbuild.go | 29 ----------------------------- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/tests/soong_test.sh b/tests/soong_test.sh index f7bee4008f..6779d8a959 100755 --- a/tests/soong_test.sh +++ b/tests/soong_test.sh @@ -9,12 +9,8 @@ source "$(dirname "$0")/lib.sh" function test_m_clean_works { setup - # Create a directory with files that cannot be removed - mkdir -p out/bad_directory_permissions - touch out/bad_directory_permissions/unremovable_file - # File permissions are fine but directory permissions are bad - chmod a+rwx out/bad_directory_permissions/unremovable_file - chmod a-rwx out/bad_directory_permissions + mkdir -p out/some_directory + touch out/some_directory/some_file run_soong clean } diff --git a/ui/build/cleanbuild.go b/ui/build/cleanbuild.go index fd601772cc..ee53327352 100644 --- a/ui/build/cleanbuild.go +++ b/ui/build/cleanbuild.go @@ -17,7 +17,6 @@ package build import ( "bytes" "fmt" - "io/fs" "io/ioutil" "os" "path/filepath" @@ -59,37 +58,9 @@ const ( FILEMODE_USER_EXECUTE = FILEMODE_EXECUTE << FILEMODE_USER_SHIFT ) -// Ensures that files and directories in the out dir can be deleted. -// For example, Bazen can generate output directories where the write bit isn't set, causing 'm' clean' to fail. -func ensureOutDirRemovable(ctx Context, config Config) { - err := filepath.WalkDir(config.OutDir(), func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - info, err := d.Info() - if err != nil { - return err - } - // Equivalent to running chmod u+rwx on each directory - newMode := info.Mode() | FILEMODE_USER_READ | FILEMODE_USER_WRITE | FILEMODE_USER_EXECUTE - if err := os.Chmod(path, newMode); err != nil { - return err - } - } - // Continue walking the out dir... - return nil - }) - if err != nil && !os.IsNotExist(err) { - // Display the error, but don't crash. - ctx.Println(err.Error()) - } -} - // Remove everything under the out directory. Don't remove the out directory // itself in case it's a symlink. func clean(ctx Context, config Config) { - ensureOutDirRemovable(ctx, config) removeGlobs(ctx, filepath.Join(config.OutDir(), "*")) ctx.Println("Entire build directory removed.") } From 4d11d10ce276895f0e4de75b80129a3696819871 Mon Sep 17 00:00:00 2001 From: liulvping Date: Wed, 24 May 2023 10:27:44 +0800 Subject: [PATCH 0202/1460] build: Fix bp2build panic issue panic in top down mutator "bp2build_conversion_bp2build" for module "foo". reflect: call of reflect.Value.NumField on interface Value Test: go test ./bp2build Change-Id: I9ee460ca38a37c6a6c1a0787159fa5f38b852b55 --- android/variable.go | 3 ++ ...oong_config_module_type_conversion_test.go | 47 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/android/variable.go b/android/variable.go index 3832fcf946..77888e586f 100644 --- a/android/variable.go +++ b/android/variable.go @@ -848,6 +848,9 @@ func (productConfigProperties *ProductConfigProperties) AddSoongConfigProperties // indirections to extract the struct from the reflect.Value. if v, ok := maybeExtractConfigVarProp(variableStruct); ok { variableStruct = v + } else if !v.IsValid() { + // Skip invalid variables which may not used, else leads to panic + continue } for j := 0; j < variableStruct.NumField(); j++ { diff --git a/bp2build/soong_config_module_type_conversion_test.go b/bp2build/soong_config_module_type_conversion_test.go index ad07f68335..813773d000 100644 --- a/bp2build/soong_config_module_type_conversion_test.go +++ b/bp2build/soong_config_module_type_conversion_test.go @@ -200,6 +200,53 @@ custom_cc_library_static { )`}}) } +func TestSoongConfigModuleType_MultipleBoolVar_PartialUseNotPanic(t *testing.T) { + bp := ` +soong_config_bool_variable { + name: "feature1", +} + +soong_config_bool_variable { + name: "feature2", +} + +soong_config_module_type { + name: "custom_cc_library_static", + module_type: "cc_library_static", + config_namespace: "acme", + variables: ["feature1", "feature2",], + properties: ["cflags"], +} + +custom_cc_library_static { + name: "foo", + bazel_module: { bp2build_available: true }, + host_supported: true, + soong_config_variables: { + feature1: { + conditions_default: { + cflags: ["-DDEFAULT1"], + }, + cflags: ["-DFEATURE1"], + }, + }, +}` + + runSoongConfigModuleTypeTest(t, Bp2buildTestCase{ + Description: "soong config variables - used part of multiple bool variable do not panic", + ModuleTypeUnderTest: "cc_library_static", + ModuleTypeUnderTestFactory: cc.LibraryStaticFactory, + Blueprint: bp, + ExpectedBazelTargets: []string{`cc_library_static( + name = "foo", + copts = select({ + "//build/bazel/product_variables:acme__feature1": ["-DFEATURE1"], + "//conditions:default": ["-DDEFAULT1"], + }), + local_includes = ["."], +)`}}) +} + func TestSoongConfigModuleType_StringAndBoolVar(t *testing.T) { bp := ` soong_config_bool_variable { From d87d22372b34bcdeab6184fcb56391aeed06a84a Mon Sep 17 00:00:00 2001 From: Andrei Onea Date: Wed, 17 Aug 2022 16:50:48 +0000 Subject: [PATCH 0203/1460] Add script for modifying privapp permission allowlists This script will be used to change the package that is being allowed to use privileged permissions. Test: python scripts/modify_permissions_allowlist.py Bug: 242509786 (cherry picked from https://android-review.googlesource.com/q/commit:580f7b0094bd98d3769b6cdeaaab6a9748621253) Merged-In: I1d7cb3c906ad39aef42ad4394953251db40d1bf1 Change-Id: I1d7cb3c906ad39aef42ad4394953251db40d1bf1 --- scripts/Android.bp | 17 +++++ scripts/modify_permissions_allowlist.py | 70 ++++++++++++++++++ scripts/modify_permissions_allowlist_test.py | 76 ++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100755 scripts/modify_permissions_allowlist.py create mode 100755 scripts/modify_permissions_allowlist_test.py diff --git a/scripts/Android.bp b/scripts/Android.bp index 9367ff06aa..97f6ab424e 100644 --- a/scripts/Android.bp +++ b/scripts/Android.bp @@ -237,3 +237,20 @@ sh_binary_host { name: "jars-to-module-info-java", src: "jars-to-module-info-java.sh", } + +python_binary_host { + name: "modify_permissions_allowlist", + main: "modify_permissions_allowlist.py", + srcs: [ + "modify_permissions_allowlist.py", + ], +} + +python_test_host { + name: "modify_permissions_allowlist_test", + main: "modify_permissions_allowlist_test.py", + srcs: [ + "modify_permissions_allowlist_test.py", + "modify_permissions_allowlist.py", + ], +} diff --git a/scripts/modify_permissions_allowlist.py b/scripts/modify_permissions_allowlist.py new file mode 100755 index 0000000000..38ec7ec86e --- /dev/null +++ b/scripts/modify_permissions_allowlist.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for modifying privileged permission allowlists.""" + +from __future__ import print_function + +import argparse +import sys +from xml.dom import minidom + + +class InvalidRootNodeException(Exception): + pass + + +class InvalidNumberOfPrivappPermissionChildren(Exception): + pass + + +def modify_allowlist(allowlist_dom, package_name): + if allowlist_dom.documentElement.tagName != 'permissions': + raise InvalidRootNodeException + nodes = allowlist_dom.getElementsByTagName('privapp-permissions') + if nodes.length != 1: + raise InvalidNumberOfPrivappPermissionChildren + privapp_permissions = nodes[0] + privapp_permissions.setAttribute('package', package_name) + + +def parse_args(): + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument('input', help='input allowlist template file') + parser.add_argument( + 'package_name', help='package name to use in the allowlist' + ) + parser.add_argument('output', help='output allowlist file') + + return parser.parse_args() + + +def main(): + try: + args = parse_args() + doc = minidom.parse(args.input) + modify_allowlist(doc, args.package_name) + with open(args.output, 'w') as output_file: + doc.writexml(output_file, encoding='utf-8') + except Exception as err: + print('error: ' + str(err), file=sys.stderr) + sys.exit(-1) + + +if __name__ == '__main__': + main() diff --git a/scripts/modify_permissions_allowlist_test.py b/scripts/modify_permissions_allowlist_test.py new file mode 100755 index 0000000000..ee8b12cc83 --- /dev/null +++ b/scripts/modify_permissions_allowlist_test.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unit tests for modify_permissions_allowlist.py.""" + +from __future__ import print_function + +import unittest + +from xml.dom import minidom + +from modify_permissions_allowlist import InvalidRootNodeException, InvalidNumberOfPrivappPermissionChildren, modify_allowlist + + +class ModifyPermissionsAllowlistTest(unittest.TestCase): + + def test_invalid_root(self): + xml_data = '' + xml_dom = minidom.parseString(xml_data) + self.assertRaises(InvalidRootNodeException, modify_allowlist, xml_dom, 'x') + + def test_no_packages(self): + xml_data = '' + xml_dom = minidom.parseString(xml_data) + self.assertRaises( + InvalidNumberOfPrivappPermissionChildren, modify_allowlist, xml_dom, 'x' + ) + + def test_multiple_packages(self): + xml_data = ( + '' + ' ' + ' ' + '' + ) + xml_dom = minidom.parseString(xml_data) + self.assertRaises( + InvalidNumberOfPrivappPermissionChildren, modify_allowlist, xml_dom, 'x' + ) + + def test_modify_package_name(self): + xml_data = ( + '' + ' ' + ' ' + ' ' + '' + ) + xml_dom = minidom.parseString(xml_data) + modify_allowlist(xml_dom, 'bar.baz') + expected_data = ( + '' + '' + ' ' + ' ' + ' ' + '' + ) + self.assertEqual(expected_data, xml_dom.toxml()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From c53cfd54d937c34920d84c3ef2d6685a71dcd624 Mon Sep 17 00:00:00 2001 From: Andrei Onea Date: Wed, 17 Aug 2022 16:53:46 +0000 Subject: [PATCH 0204/1460] add privapp_allowlist property to android_app This change allows override_android_app to use the same privapp_allowlist as the non-override module so that they will always remain in sync. Test: go test ./java -v -run TestPrivappAllowlist Test: go test ./apex -v -run TestApexWithApps Test: m com.android.permission com.google.android.permission and verify manually that apex_payload.img contains correct privapp_allowlist Test: m com.android.permission before and after change && `diffoscope apex_payload_reference.img apex_payload_with_change.img` && verify that there are no semantic changes Bug: 242509786 (cherry picked from https://android-review.googlesource.com/q/commit:580636bdd23171f31bfedd773c065e0861dd5c4a) Merged-In: Ifdcb28af40763aed7a4aac9a7f681153554bc256 Change-Id: Ifdcb28af40763aed7a4aac9a7f681153554bc256 --- apex/apex.go | 19 +++++++++++---- apex/apex_test.go | 3 +++ java/app.go | 58 +++++++++++++++++++++++++++++++++++++++++++--- java/app_import.go | 4 ++++ java/app_test.go | 48 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 8 deletions(-) diff --git a/apex/apex.go b/apex/apex.go index c1c9e5c886..1773fcf23b 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1823,6 +1823,7 @@ type androidApp interface { Certificate() java.Certificate BaseModuleName() string LintDepSets() java.LintDepSets + PrivAppAllowlist() android.OptionalPath } var _ androidApp = (*java.AndroidApp)(nil) @@ -1843,7 +1844,7 @@ func sanitizedBuildIdForPath(ctx android.BaseModuleContext) string { return buildId } -func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexFile { +func apexFilesForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) []apexFile { appDir := "app" if aapp.Privileged() { appDir = "priv-app" @@ -1865,7 +1866,15 @@ func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexF }); ok { af.overriddenPackageName = app.OverriddenManifestPackageName() } - return af + apexFiles := []apexFile{af} + + if allowlist := aapp.PrivAppAllowlist(); allowlist.Valid() { + dirInApex := filepath.Join("etc", "permissions") + privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"privapp", dirInApex, etc, aapp) + apexFiles = append(apexFiles, privAppAllowlist) + } + + return apexFiles } func apexFileForRuntimeResourceOverlay(ctx android.BaseModuleContext, rro java.RuntimeResourceOverlayModule) apexFile { @@ -2310,12 +2319,12 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, case androidAppTag: switch ap := child.(type) { case *java.AndroidApp: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) return true // track transitive dependencies case *java.AndroidAppImport: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) case *java.AndroidTestHelperApp: - vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + vctx.filesInfo = append(vctx.filesInfo, apexFilesForAndroidApp(ctx, ap)...) case *java.AndroidAppSet: appDir := "app" if ap.Privileged() { diff --git a/apex/apex_test.go b/apex/apex_test.go index 139b77ef63..139d23d49b 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6165,6 +6165,8 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, + privapp_allowlist: "perms.xml", + package_name: "com.android.AppFooPriv", stl: "none", apex_available: [ "myapex" ], } @@ -6194,6 +6196,7 @@ func TestApexWithApps(t *testing.T) { ensureContains(t, copyCmds, "image.apex/app/AppFoo@TEST.BUILD_ID/AppFoo.apk") ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPriv@TEST.BUILD_ID/AppFooPriv.apk") + ensureContains(t, copyCmds, "image.apex/etc/permissions/privapp_allowlist_com.android.AppFooPriv.xml") appZipRule := ctx.ModuleForTests("AppFoo", "android_common_apex10000").Description("zip jni libs") // JNI libraries are uncompressed diff --git a/java/app.go b/java/app.go index 706f99a834..e4c5b706ad 100755 --- a/java/app.go +++ b/java/app.go @@ -33,8 +33,17 @@ import ( func init() { RegisterAppBuildComponents(android.InitRegistrationContext) + pctx.HostBinToolVariable("ModifyAllowlistCmd", "modify_permissions_allowlist") } +var ( + modifyAllowlist = pctx.AndroidStaticRule("modifyAllowlist", + blueprint.RuleParams{ + Command: "${ModifyAllowlistCmd} $in $packageName $out", + CommandDeps: []string{"${ModifyAllowlistCmd}"}, + }, "packageName") +) + func RegisterAppBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("android_app", AndroidAppFactory) ctx.RegisterModuleType("android_test", AndroidTestFactory) @@ -115,6 +124,9 @@ type appProperties struct { // Prefer using other specific properties if build behaviour must be changed; avoid using this // flag for anything but neverallow rules (unless the behaviour change is invisible to owners). Updatable *bool + + // Specifies the file that contains the allowlist for this app. + Privapp_allowlist *string `android:"path"` } // android_app properties that can be overridden by override_android_app @@ -179,6 +191,8 @@ type AndroidApp struct { android.ApexBundleDepsInfo javaApiUsedByOutputFile android.ModuleOutPath + + privAppAllowlist android.OptionalPath } func (a *AndroidApp) IsInstallable() bool { @@ -205,6 +219,10 @@ func (a *AndroidApp) JniCoverageOutputs() android.Paths { return a.jniCoverageOutputs } +func (a *AndroidApp) PrivAppAllowlist() android.OptionalPath { + return a.privAppAllowlist +} + var _ AndroidLibraryDependency = (*AndroidApp)(nil) type Certificate struct { @@ -269,6 +287,10 @@ func (a *AndroidApp) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato ctx.AddDependency(ctx.Module(), certificateTag, cert) } + if a.appProperties.Privapp_allowlist != nil && !Bool(a.appProperties.Privileged) { + ctx.PropertyErrorf("privapp_allowlist", "privileged must be set in order to use privapp_allowlist") + } + for _, cert := range a.appProperties.Additional_certificates { cert = android.SrcIsModule(cert) if cert != "" { @@ -599,6 +621,27 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { + if a.appProperties.Privapp_allowlist == nil { + return nil + } + if a.overridableAppProperties.Package_name == nil { + ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") + } + packageName := *a.overridableAppProperties.Package_name + fileName := "privapp_allowlist_" + packageName + ".xml" + outPath := android.PathForModuleOut(ctx, fileName).OutputPath + ctx.Build(pctx, android.BuildParams{ + Rule: modifyAllowlist, + Input: android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist), + Output: outPath, + Args: map[string]string{ + "packageName": packageName, + }, + }) + return &outPath +} + func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { var apkDeps android.Paths @@ -734,18 +777,27 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { BuildBundleModule(ctx, bundleFile, a.exportPackage, jniJarFile, dexJarFile) a.bundleFile = bundleFile + allowlist := a.createPrivappAllowlist(ctx) + if allowlist != nil { + a.privAppAllowlist = android.OptionalPathForPath(allowlist) + } + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) // Install the app package. - if (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && - !a.appProperties.PreventInstall { - + shouldInstallAppPackage := (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && !a.appProperties.PreventInstall + if shouldInstallAppPackage { var extraInstalledPaths android.Paths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) } ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile, extraInstalledPaths...) + + if a.privAppAllowlist.Valid() { + installPath := android.PathForModuleInstall(ctx, "etc", "permissions") + ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) + } } a.buildAppDependencyInfo(ctx) diff --git a/java/app_import.go b/java/app_import.go index 85b35ebaa9..3097d7fd5f 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -376,6 +376,10 @@ func (a *AndroidAppImport) ProvenanceMetaDataFile() android.OutputPath { return a.provenanceMetaDataFile } +func (a *AndroidAppImport) PrivAppAllowlist() android.OptionalPath { + return android.OptionalPath{} +} + var dpiVariantGroupType reflect.Type var archVariantGroupType reflect.Type var supportedDpis = []string{"ldpi", "mdpi", "hdpi", "xhdpi", "xxhdpi", "xxxhdpi"} diff --git a/java/app_test.go b/java/app_test.go index b154bc990d..f17284dd46 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3547,3 +3547,51 @@ func TestTargetSdkVersionMtsTests(t *testing.T) { android.AssertStringDoesContain(t, testCase.desc, manifestFixerArgs, "--targetSdkVersion "+testCase.targetSdkVersionExpected) } } + +func TestPrivappAllowlist(t *testing.T) { + testJavaError(t, "privileged must be set in order to use privapp_allowlist", ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "perms.xml", + } + `) + + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp( + t, + ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "perms.xml", + privileged: true, + package_name: "com.android.foo", + sdk_version: "current", + } + override_android_app { + name: "bar", + base: "foo", + package_name: "com.google.android.foo", + } + `, + ) + app := result.ModuleForTests("foo", "android_common") + overrideApp := result.ModuleForTests("foo", "android_common_bar") + + // verify that privapp allowlist is created + app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") + expectedAllowlist := "perms.xml" + actualAllowlist := app.Rule("modifyAllowlist").Input.String() + if expectedAllowlist != actualAllowlist { + t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) + } + overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlist != overrideActualAllowlist { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + } + + // verify that permissions are copied to device + app.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml") + overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml") +} From 502d807ae9ab60786108c38320c7618a5e1c65ec Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 15 May 2023 17:21:47 -0400 Subject: [PATCH 0205/1460] don't require package_name for non-override android_apps For a non-override android_app, we can assume that the privapp_allowlist already contains the correct package_name, and so we don't need to overwrite it in this case. Bug: 242509786 Test: go test Ignore-AOSP-First: need to submit here to update PermissionController in sync with internal version Merged-In: I0f137e34cae3478dc8b9178d138121ff1d936f07 Change-Id: I0f137e34cae3478dc8b9178d138121ff1d936f07 --- apex/apex_test.go | 3 +-- java/app.go | 10 +++++++++- java/app_test.go | 18 ++++++------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index 139d23d49b..6e440ffe20 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6165,8 +6165,7 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, - privapp_allowlist: "perms.xml", - package_name: "com.android.AppFooPriv", + privapp_allowlist: "privapp_allowlist_com.android.AppFooPriv.xml", stl: "none", apex_available: [ "myapex" ], } diff --git a/java/app.go b/java/app.go index e4c5b706ad..d2e9a4121c 100755 --- a/java/app.go +++ b/java/app.go @@ -621,13 +621,21 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } -func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) android.Path { if a.appProperties.Privapp_allowlist == nil { return nil } + + isOverrideApp := a.GetOverriddenBy() != "" + if !isOverrideApp { + // if this is not an override, we don't need to rewrite the existing privapp allowlist + return android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist) + } + if a.overridableAppProperties.Package_name == nil { ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") } + packageName := *a.overridableAppProperties.Package_name fileName := "privapp_allowlist_" + packageName + ".xml" outPath := android.PathForModuleOut(ctx, fileName).OutputPath diff --git a/java/app_test.go b/java/app_test.go index f17284dd46..6e5870f9e4 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3563,9 +3563,8 @@ func TestPrivappAllowlist(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - privapp_allowlist: "perms.xml", + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", privileged: true, - package_name: "com.android.foo", sdk_version: "current", } override_android_app { @@ -3578,17 +3577,12 @@ func TestPrivappAllowlist(t *testing.T) { app := result.ModuleForTests("foo", "android_common") overrideApp := result.ModuleForTests("foo", "android_common_bar") - // verify that privapp allowlist is created - app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + // verify that privapp allowlist is created for override apps overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") - expectedAllowlist := "perms.xml" - actualAllowlist := app.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != actualAllowlist { - t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) - } - overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != overrideActualAllowlist { - t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + expectedAllowlistInput := "privapp_allowlist_com.android.foo.xml" + overrideActualAllowlistInput := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlistInput != overrideActualAllowlistInput { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlistInput, overrideActualAllowlistInput) } // verify that permissions are copied to device From 15809f8101dc408512fc63215ac1d4131cea2feb Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 15 May 2023 17:21:47 -0400 Subject: [PATCH 0206/1460] don't require package_name for non-override android_apps For a non-override android_app, we can assume that the privapp_allowlist already contains the correct package_name, and so we don't need to overwrite it in this case. Bug: 242509786 Test: go test Merged-In: I0f137e34cae3478dc8b9178d138121ff1d936f07 Change-Id: I0f137e34cae3478dc8b9178d138121ff1d936f07 --- apex/apex_test.go | 3 +-- java/app.go | 10 +++++++++- java/app_test.go | 18 ++++++------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/apex/apex_test.go b/apex/apex_test.go index c1d80a39f2..2dad22b06a 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6260,8 +6260,7 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, - privapp_allowlist: "perms.xml", - package_name: "com.android.AppFooPriv", + privapp_allowlist: "privapp_allowlist_com.android.AppFooPriv.xml", stl: "none", apex_available: [ "myapex" ], } diff --git a/java/app.go b/java/app.go index 366005ce3f..d0bde6f22d 100755 --- a/java/app.go +++ b/java/app.go @@ -621,13 +621,21 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } -func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) android.Path { if a.appProperties.Privapp_allowlist == nil { return nil } + + isOverrideApp := a.GetOverriddenBy() != "" + if !isOverrideApp { + // if this is not an override, we don't need to rewrite the existing privapp allowlist + return android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist) + } + if a.overridableAppProperties.Package_name == nil { ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") } + packageName := *a.overridableAppProperties.Package_name fileName := "privapp_allowlist_" + packageName + ".xml" outPath := android.PathForModuleOut(ctx, fileName).OutputPath diff --git a/java/app_test.go b/java/app_test.go index c485478eb1..4b78c48a5b 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3563,9 +3563,8 @@ func TestPrivappAllowlist(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - privapp_allowlist: "perms.xml", + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", privileged: true, - package_name: "com.android.foo", sdk_version: "current", } override_android_app { @@ -3578,17 +3577,12 @@ func TestPrivappAllowlist(t *testing.T) { app := result.ModuleForTests("foo", "android_common") overrideApp := result.ModuleForTests("foo", "android_common_bar") - // verify that privapp allowlist is created - app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + // verify that privapp allowlist is created for override apps overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") - expectedAllowlist := "perms.xml" - actualAllowlist := app.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != actualAllowlist { - t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) - } - overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != overrideActualAllowlist { - t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + expectedAllowlistInput := "privapp_allowlist_com.android.foo.xml" + overrideActualAllowlistInput := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlistInput != overrideActualAllowlistInput { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlistInput, overrideActualAllowlistInput) } // verify that permissions are copied to device From 24e9eec8009e67a6a8a4a0360135c25ac0cc8b36 Mon Sep 17 00:00:00 2001 From: Chris Parsons Date: Wed, 24 May 2023 18:07:18 +0000 Subject: [PATCH 0207/1460] Open metrics visibility to script subpackages Test: N/A Change-Id: Iee750369efa9138d0a3ada74549dde239f87396f --- ui/metrics/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/metrics/BUILD.bazel b/ui/metrics/BUILD.bazel index 15ebb88649..2dc1ab633d 100644 --- a/ui/metrics/BUILD.bazel +++ b/ui/metrics/BUILD.bazel @@ -16,7 +16,7 @@ load("//build/bazel/rules/python:py_proto.bzl", "py_proto_library") py_proto_library( name = "metrics-py-proto", - visibility = ["//build/bazel/scripts/incremental_build:__pkg__"], + visibility = ["//build/bazel/scripts:__subpackages__"], deps = [":metrics-proto"], ) From b29a3cd2aa7d415f4d5f27a775cfb2e02cbe0ba0 Mon Sep 17 00:00:00 2001 From: Alix Date: Tue, 9 May 2023 20:37:49 +0000 Subject: [PATCH 0208/1460] bp2build java_resources that only contain a filegroup supports filegroup that specifies path property Bug: 280860624 Test: built libauto_value_plugin Change-Id: I9ed0b13e055beb92ba8090f6b5e88b9873c9ce61 --- android/allowlists/allowlists.go | 6 +++- android/filegroup.go | 13 ++++++++ bp2build/java_library_conversion_test.go | 40 ++++++++++++++++++++++++ java/java.go | 15 ++++++++- 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 751a4cb830..6405e9fe90 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -761,6 +761,11 @@ var ( // aidl "aidl", "libaidl-common", + + // java_resources containing only a single filegroup + "libauto_value_plugin", + "auto_value_plugin_resources", + "auto_value_extension", } Bp2buildModuleTypeAlwaysConvertList = []string{ @@ -835,7 +840,6 @@ var ( "libprotobuf-internal-python-srcs", // TODO(b/210751803), we don't handle path property for filegroups "libprotobuf-java-full", // TODO(b/210751803), we don't handle path property for filegroups "libprotobuf-java-util-full", // TODO(b/210751803), we don't handle path property for filegroups - "auto_value_plugin_resources", // TODO(b/210751803), we don't handle path property for filegroups // go deps: "analyze_bcpf", // depends on bpmodify a blueprint_go_binary. diff --git a/android/filegroup.go b/android/filegroup.go index f30ee51438..c042ff18ea 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -177,6 +177,17 @@ func (fg *fileGroup) ConvertWithBp2build(ctx TopDownMutatorContext) { } } +type FileGroupPath interface { + GetPath(ctx TopDownMutatorContext) string +} + +func (fg *fileGroup) GetPath(ctx TopDownMutatorContext) string { + if fg.properties.Path != nil { + return *fg.properties.Path + } + return "" +} + type fileGroupProperties struct { // srcs lists files that will be included in this filegroup Srcs []string `android:"path"` @@ -207,6 +218,7 @@ type fileGroup struct { BazelModuleBase DefaultableModuleBase FileGroupAsLibrary + FileGroupPath properties fileGroupProperties srcs Paths } @@ -214,6 +226,7 @@ type fileGroup struct { var _ MixedBuildBuildable = (*fileGroup)(nil) var _ SourceFileProducer = (*fileGroup)(nil) var _ FileGroupAsLibrary = (*fileGroup)(nil) +var _ FileGroupPath = (*fileGroup)(nil) // filegroup contains a list of files that are referenced by other modules // properties (such as "srcs") using the syntax ":". filegroup are diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go index 60766f42ad..fd92e95c65 100644 --- a/bp2build/java_library_conversion_test.go +++ b/bp2build/java_library_conversion_test.go @@ -826,3 +826,43 @@ func TestJavaLibraryArchVariantSrcsWithExcludes(t *testing.T) { }, }) } + +func TestJavaLibraryJavaResourcesSingleFilegroup(t *testing.T) { + runJavaLibraryTestCaseWithRegistrationCtxFunc(t, Bp2buildTestCase{ + Filesystem: map[string]string{ + "res/a.res": "", + "res/b.res": "", + "res/dir1/b.res": "", + }, + Description: "java_library", + Blueprint: `java_library { + name: "java-lib-1", + srcs: ["a.java"], + java_resources: [":filegroup1"], + bazel_module: { bp2build_available: true }, +} + +filegroup { + name: "filegroup1", + path: "foo", + srcs: ["foo/a", "foo/b"], +} + +`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("java_library", "java-lib-1", AttrNameToString{ + "srcs": `["a.java"]`, + "resources": `[":filegroup1"]`, + "resource_strip_prefix": `"foo"`, + }), + MakeNeverlinkDuplicateTarget("java_library", "java-lib-1"), + MakeBazelTargetNoRestrictions("filegroup", "filegroup1", AttrNameToString{ + "srcs": `[ + "foo/a", + "foo/b", + ]`}), + }, + }, func(ctx android.RegistrationContext) { + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + }) +} diff --git a/java/java.go b/java/java.go index a23526aee2..4dcc9b5822 100644 --- a/java/java.go +++ b/java/java.go @@ -2704,6 +2704,15 @@ type javaResourcesAttributes struct { Resource_strip_prefix *string } +func (m *Library) javaResourcesGetSingleFilegroupStripPrefix(ctx android.TopDownMutatorContext) (string, bool) { + if otherM, ok := ctx.ModuleFromName(m.properties.Java_resources[0]); ok && len(m.properties.Java_resources) == 1 { + if fg, isFilegroup := otherM.(android.FileGroupPath); isFilegroup { + return filepath.Join(ctx.OtherModuleDir(otherM), fg.GetPath(ctx)), true + } + } + return "", false +} + func (m *Library) convertJavaResourcesAttributes(ctx android.TopDownMutatorContext) *javaResourcesAttributes { var resources bazel.LabelList var resourceStripPrefix *string @@ -2713,8 +2722,12 @@ func (m *Library) convertJavaResourcesAttributes(ctx android.TopDownMutatorConte } if m.properties.Java_resources != nil { + if prefix, ok := m.javaResourcesGetSingleFilegroupStripPrefix(ctx); ok { + resourceStripPrefix = proptools.StringPtr(prefix) + } else { + resourceStripPrefix = proptools.StringPtr(ctx.ModuleDir()) + } resources.Append(android.BazelLabelForModuleSrc(ctx, m.properties.Java_resources)) - resourceStripPrefix = proptools.StringPtr(ctx.ModuleDir()) } //TODO(b/179889880) handle case where glob includes files outside package From 7f9d7cbb7547aa65e0ac1d2a3415112ca61a6649 Mon Sep 17 00:00:00 2001 From: Ivan Lozano Date: Wed, 24 May 2023 15:53:43 +0000 Subject: [PATCH 0209/1460] rust: Support sanitizers in rust_ffi modules Don't automatically assume that HWASAN or ASAN is enabled if Fuzzer is enabled. Bug: 178365482 Test: m bluetooth_stack_with_facade Test: SANITIZE_TARGET=fuzzer m bluetooth_stack_with_facade Test: SANITIZE_TARGET=address m bluetooth_stack_with_facade Test: SANITIZE_TARGET=hwaddress m bluetooth_stack_with_facade Test: SANITIZE_TARGET="fuzzer address" m bluetooth_stack_with_facade Test: SANITIZE_TARGET="fuzzer hwaddress" m bluetooth_stack_with_facade Change-Id: Ief8c0f899837c5889a8035782616025f1b0d54e7 --- rust/fuzz_test.go | 10 ++++------ rust/sanitize.go | 13 ------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/rust/fuzz_test.go b/rust/fuzz_test.go index 865665eef9..7fa9f5c8f0 100644 --- a/rust/fuzz_test.go +++ b/rust/fuzz_test.go @@ -46,18 +46,16 @@ func TestRustFuzz(t *testing.T) { // Check that compiler flags are set appropriately . fuzz_libtest := ctx.ModuleForTests("fuzz_libtest", "android_arm64_armv8-a_fuzzer").Rule("rustc") - if !strings.Contains(fuzz_libtest.Args["rustcFlags"], "-Z sanitizer=hwaddress") || - !strings.Contains(fuzz_libtest.Args["rustcFlags"], "-C passes='sancov-module'") || + if !strings.Contains(fuzz_libtest.Args["rustcFlags"], "-C passes='sancov-module'") || !strings.Contains(fuzz_libtest.Args["rustcFlags"], "--cfg fuzzing") { - t.Errorf("rust_fuzz module does not contain the expected flags (sancov-module, cfg fuzzing, hwaddress sanitizer).") + t.Errorf("rust_fuzz module does not contain the expected flags (sancov-module, cfg fuzzing).") } // Check that dependencies have 'fuzzer' variants produced for them as well. libtest_fuzzer := ctx.ModuleForTests("libtest_fuzzing", "android_arm64_armv8-a_rlib_rlib-std_fuzzer").Output("libtest_fuzzing.rlib") - if !strings.Contains(libtest_fuzzer.Args["rustcFlags"], "-Z sanitizer=hwaddress") || - !strings.Contains(libtest_fuzzer.Args["rustcFlags"], "-C passes='sancov-module'") || + if !strings.Contains(libtest_fuzzer.Args["rustcFlags"], "-C passes='sancov-module'") || !strings.Contains(libtest_fuzzer.Args["rustcFlags"], "--cfg fuzzing") { - t.Errorf("rust_fuzz dependent library does not contain the expected flags (sancov-module, cfg fuzzing, hwaddress sanitizer).") + t.Errorf("rust_fuzz dependent library does not contain the expected flags (sancov-module, cfg fuzzing).") } } diff --git a/rust/sanitize.go b/rust/sanitize.go index c68137ecd2..83cf055522 100644 --- a/rust/sanitize.go +++ b/rust/sanitize.go @@ -226,11 +226,6 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags, deps PathDeps) ( } if Bool(sanitize.Properties.Sanitize.Fuzzer) { flags.RustFlags = append(flags.RustFlags, fuzzerFlags...) - if ctx.Arch().ArchType == android.Arm64 && ctx.Os().Bionic() { - flags.RustFlags = append(flags.RustFlags, hwasanFlags...) - } else { - flags.RustFlags = append(flags.RustFlags, asanFlags...) - } } else if Bool(sanitize.Properties.Sanitize.Hwaddress) { flags.RustFlags = append(flags.RustFlags, hwasanFlags...) } else if Bool(sanitize.Properties.Sanitize.Address) { @@ -424,14 +419,6 @@ func (mod *Module) IsSanitizerExplicitlyDisabled(t cc.SanitizerType) bool { return true } - // TODO(b/178365482): Rust/CC interop doesn't work just yet; don't sanitize rust_ffi modules until - // linkage issues are resolved. - if lib, ok := mod.compiler.(libraryInterface); ok { - if lib.shared() || lib.static() { - return true - } - } - return mod.sanitize.isSanitizerExplicitlyDisabled(t) } From c29c37a2c014e819360113d32d26837902357741 Mon Sep 17 00:00:00 2001 From: Cory Barker Date: Wed, 24 May 2023 22:53:41 +0000 Subject: [PATCH 0210/1460] Add both classes jar and dexed jar to java_fuzz module packaged output for dist as we move towards instrumentation at runtime Test: Built locally and tested pipeline Bug: 283993246 Change-Id: I69eb98d2b7966d77ac8819348f7b99a7b18f40b6 --- java/fuzz.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/java/fuzz.go b/java/fuzz.go index 5c5f769a55..5dfaacfb30 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -177,7 +177,11 @@ func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { files = s.PackageArtifacts(ctx, module, javaFuzzModule.fuzzPackagedModule, archDir, builder) // Add .jar - files = append(files, fuzz.FileToZip{SourceFilePath: javaFuzzModule.implementationJarFile}) + if !javaFuzzModule.Host() { + files = append(files, fuzz.FileToZip{SourceFilePath: javaFuzzModule.implementationJarFile, DestinationPathPrefix: "classes"}) + } + + files = append(files, fuzz.FileToZip{SourceFilePath: javaFuzzModule.outputFile}) // Add jni .so files for _, fPath := range javaFuzzModule.jniFilePaths { From a37d9baa966bca57aef28c40cb7b02557122e175 Mon Sep 17 00:00:00 2001 From: Kiyoung Kim Date: Wed, 19 Apr 2023 13:13:45 +0900 Subject: [PATCH 0211/1460] Rename build.ninja with product name Current build.ninja does not contain any product name, while other ninja files (such as combined ninja) do. This change adds product name to the build.ninja so it can be separated over multiple lunch targets Bug: 277029044 Test: build succeeded and checked if out/soong/build.ninja has been renamed Change-Id: I16dc71f829fd76f01b98da0d509a8e0ef6f62fa9 --- android/config.go | 9 +++--- cmd/soong_build/main.go | 26 +++++++++++++++ ui/build/config.go | 21 +++++++++++-- ui/build/soong.go | 70 ++++++++++++++++++++++++----------------- 4 files changed, 91 insertions(+), 35 deletions(-) diff --git a/android/config.go b/android/config.go index 4f0a64dfee..ddf2d746cc 100644 --- a/android/config.go +++ b/android/config.go @@ -80,9 +80,10 @@ type SoongBuildMode int type CmdArgs struct { bootstrap.Args - RunGoTests bool - OutDir string - SoongOutDir string + RunGoTests bool + OutDir string + SoongOutDir string + SoongVariables string SymlinkForestMarker string Bp2buildMarker string @@ -491,7 +492,7 @@ func NullConfig(outDir, soongOutDir string) Config { func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) { // Make a config with default options. config := &config{ - ProductVariablesFileName: filepath.Join(cmdArgs.SoongOutDir, productVariablesFileName), + ProductVariablesFileName: cmdArgs.SoongVariables, env: availableEnv, diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index cf6c1c7b8a..e006c9d7e4 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -78,6 +78,7 @@ func init() { flag.StringVar(&cmdlineArgs.Bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit") flag.StringVar(&cmdlineArgs.SymlinkForestMarker, "symlink_forest_marker", "", "If set, create the bp2build symlink forest, touch the specified marker file, then exit") flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output") + flag.StringVar(&cmdlineArgs.SoongVariables, "soong_variables", "soong.variables", "the file contains all build variables") flag.StringVar(&cmdlineArgs.BazelForceEnabledModules, "bazel-force-enabled-modules", "", "additional modules to build with Bazel. Comma-delimited") flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file") flag.BoolVar(&cmdlineArgs.MultitreeBuild, "multitree-build", false, "this is a multitree build") @@ -517,6 +518,8 @@ func main() { var finalOutputFile string + writeSymlink := false + // Run Soong for a specific activity, like bp2build, queryview // or the actual Soong build for the build.ninja file. switch configuration.BuildMode { @@ -539,8 +542,13 @@ func main() { maybeQuit(err, "") } } + writeSymlink = true } else { finalOutputFile = runSoongOnlyBuild(ctx, extraNinjaDeps) + + if configuration.BuildMode == android.AnalysisNoBazel { + writeSymlink = true + } } writeMetrics(configuration, ctx.EventHandler, metricsDir) } @@ -551,6 +559,24 @@ func main() { // are ninja inputs to the main output file, then ninja would superfluously // rebuild this output file on the next build invocation. touch(shared.JoinPath(topDir, finalOutputFile)) + + // TODO(b/277029044): Remove this function once build..ninja lands + if writeSymlink { + writeBuildNinjaSymlink(configuration, finalOutputFile) + } +} + +// TODO(b/277029044): Remove this function once build..ninja lands +func writeBuildNinjaSymlink(config android.Config, source string) { + targetPath := shared.JoinPath(topDir, config.SoongOutDir(), "build.ninja") + sourcePath := shared.JoinPath(topDir, source) + + if targetPath == sourcePath { + return + } + + os.Remove(targetPath) + os.Symlink(sourcePath, targetPath) } func writeUsedEnvironmentFile(configuration android.Config) { diff --git a/ui/build/config.go b/ui/build/config.go index 8ec96800f3..89de01acd0 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -1229,6 +1229,13 @@ func (c *configImpl) TargetProduct() string { panic("TARGET_PRODUCT is not defined") } +func (c *configImpl) TargetProductOrErr() (string, error) { + if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { + return v, nil + } + return "", fmt.Errorf("TARGET_PRODUCT is not defined") +} + func (c *configImpl) TargetDevice() string { return c.targetDevice } @@ -1554,11 +1561,21 @@ func (c *configImpl) KatiPackageNinjaFile() string { } func (c *configImpl) SoongVarsFile() string { - return filepath.Join(c.SoongOutDir(), "soong.variables") + targetProduct, err := c.TargetProductOrErr() + if err != nil { + return filepath.Join(c.SoongOutDir(), "soong.variables") + } else { + return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".variables") + } } func (c *configImpl) SoongNinjaFile() string { - return filepath.Join(c.SoongOutDir(), "build.ninja") + targetProduct, err := c.TargetProductOrErr() + if err != nil { + return filepath.Join(c.SoongOutDir(), "build.ninja") + } else { + return filepath.Join(c.SoongOutDir(), "build."+targetProduct+".ninja") + } } func (c *configImpl) CombinedNinjaFile() string { diff --git a/ui/build/soong.go b/ui/build/soong.go index 18bf3b9121..6de8d6f989 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -235,7 +235,7 @@ func bootstrapEpochCleanup(ctx Context, config Config) { } else if !exists { // The tree is out of date for the current epoch, delete files used by bootstrap // and force the primary builder to rerun. - os.Remove(filepath.Join(config.SoongOutDir(), "build.ninja")) + os.Remove(config.SoongNinjaFile()) for _, globFile := range bootstrapGlobFileList(config) { os.Remove(globFile) } @@ -263,7 +263,9 @@ func bootstrapBlueprint(ctx Context, config Config) { // Clean up some files for incremental builds across incompatible changes. bootstrapEpochCleanup(ctx, config) - mainSoongBuildExtraArgs := []string{"-o", config.SoongNinjaFile()} + baseArgs := []string{"--soong_variables", config.SoongVarsFile()} + + mainSoongBuildExtraArgs := append(baseArgs, "-o", config.SoongNinjaFile()) if config.EmptyNinjaFile() { mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file") } @@ -306,49 +308,59 @@ func bootstrapBlueprint(ctx Context, config Config) { specificArgs: mainSoongBuildExtraArgs, }, { - name: bp2buildFilesTag, - description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()), - config: config, - output: config.Bp2BuildFilesMarkerFile(), - specificArgs: []string{"--bp2build_marker", config.Bp2BuildFilesMarkerFile()}, + name: bp2buildFilesTag, + description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()), + config: config, + output: config.Bp2BuildFilesMarkerFile(), + specificArgs: append(baseArgs, + "--bp2build_marker", config.Bp2BuildFilesMarkerFile(), + ), }, { - name: bp2buildWorkspaceTag, - description: "Creating Bazel symlink forest", - config: config, - output: config.Bp2BuildWorkspaceMarkerFile(), - specificArgs: []string{"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile()}, + name: bp2buildWorkspaceTag, + description: "Creating Bazel symlink forest", + config: config, + output: config.Bp2BuildWorkspaceMarkerFile(), + specificArgs: append(baseArgs, + "--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(), + ), }, { name: jsonModuleGraphTag, description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()), config: config, output: config.ModuleGraphFile(), - specificArgs: []string{ + specificArgs: append(baseArgs, "--module_graph_file", config.ModuleGraphFile(), "--module_actions_file", config.ModuleActionsFile(), - }, + ), }, { - name: queryviewTag, - description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir), - config: config, - output: config.QueryviewMarkerFile(), - specificArgs: []string{"--bazel_queryview_dir", queryviewDir}, + name: queryviewTag, + description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir), + config: config, + output: config.QueryviewMarkerFile(), + specificArgs: append(baseArgs, + "--bazel_queryview_dir", queryviewDir, + ), }, { - name: apiBp2buildTag, - description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir), - config: config, - output: config.ApiBp2buildMarkerFile(), - specificArgs: []string{"--bazel_api_bp2build_dir", apiBp2buildDir}, + name: apiBp2buildTag, + description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir), + config: config, + output: config.ApiBp2buildMarkerFile(), + specificArgs: append(baseArgs, + "--bazel_api_bp2build_dir", apiBp2buildDir, + ), }, { - name: soongDocsTag, - description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()), - config: config, - output: config.SoongDocsHtml(), - specificArgs: []string{"--soong_docs", config.SoongDocsHtml()}, + name: soongDocsTag, + description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()), + config: config, + output: config.SoongDocsHtml(), + specificArgs: append(baseArgs, + "--soong_docs", config.SoongDocsHtml(), + ), }, } From 160634ca3d9b9e23feccefea2ebfb64ed36f44d4 Mon Sep 17 00:00:00 2001 From: Jihoon Kang Date: Thu, 25 May 2023 05:28:29 +0000 Subject: [PATCH 0212/1460] Modify error emit condition for checking api files in java_api_library java_api_library currently emits error based when the java_api_contribution module has an empty api file or when the module does not have any api files. This may be problematic in partial manifests where allow missing dependency is set, thus add this condition for error emission. Test: go test ./java Bug: 284212875 Change-Id: I10b4baa5723e7a53d931823b3029ea20ac731911 --- java/java.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/java.go b/java/java.go index 33846bec6b..ed35a2ca85 100644 --- a/java/java.go +++ b/java/java.go @@ -1813,7 +1813,7 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { case javaApiContributionTag: provider := ctx.OtherModuleProvider(dep, JavaApiImportProvider).(JavaApiImportInfo) providerApiFile := provider.ApiFile - if providerApiFile == nil { + if providerApiFile == nil && !ctx.Config().AllowMissingDependencies() { ctx.ModuleErrorf("Error: %s has an empty api file.", dep.Name()) } srcFiles = append(srcFiles, android.PathForSource(ctx, providerApiFile.String())) @@ -1835,7 +1835,7 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { srcFiles = append(srcFiles, android.PathForModuleSrc(ctx, api)) } - if srcFiles == nil { + if srcFiles == nil && !ctx.Config().AllowMissingDependencies() { ctx.ModuleErrorf("Error: %s has an empty api file.", ctx.ModuleName()) } From 6324647bcce4a05568e9a5cb1237c2ca7f6781b3 Mon Sep 17 00:00:00 2001 From: Kousik Kumar Date: Thu, 25 May 2023 15:23:29 +0000 Subject: [PATCH 0213/1460] Remove env config fetcher code This code is no longer needed since we are now setting USE_RBE to true for 100% of users (except two users, who I'll reach out to over email). So fetching the configuration is no longer needed since sometimes this can result in RBE config flipping between on and off (depending on whether you have gcert or not). Tested: Ran the build with latest sync and ensured RBE was used for the build. Bug: b/283828386 Change-Id: I3cda78607b46a0e161b7841560dc587a44d91f27 --- ui/build/config.go | 88 ---------------------------------------------- 1 file changed, 88 deletions(-) diff --git a/ui/build/config.go b/ui/build/config.go index 8ec96800f3..45735ad66d 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -15,7 +15,6 @@ package build import ( - "context" "encoding/json" "fmt" "io/ioutil" @@ -39,9 +38,6 @@ import ( const ( envConfigDir = "vendor/google/tools/soong_config" jsonSuffix = "json" - - configFetcher = "vendor/google/tools/soong/expconfigfetcher" - envConfigFetchTimeout = 10 * time.Second ) var ( @@ -170,87 +166,6 @@ func checkTopDir(ctx Context) { } } -// fetchEnvConfig optionally fetches a configuration file that can then subsequently be -// loaded into Soong environment to control certain aspects of build behavior (e.g., enabling RBE). -// If a configuration file already exists on disk, the fetch is run in the background -// so as to NOT block the rest of the build execution. -func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error { - configName := envConfigName + "." + jsonSuffix - expConfigFetcher := &smpb.ExpConfigFetcher{Filename: &configName} - defer func() { - ctx.Metrics.ExpConfigFetcher(expConfigFetcher) - }() - if !config.GoogleProdCredsExist() { - status := smpb.ExpConfigFetcher_MISSING_GCERT - expConfigFetcher.Status = &status - return nil - } - - s, err := os.Stat(configFetcher) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if s.Mode()&0111 == 0 { - status := smpb.ExpConfigFetcher_ERROR - expConfigFetcher.Status = &status - return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode()) - } - - configExists := false - outConfigFilePath := filepath.Join(config.OutDir(), configName) - if _, err := os.Stat(outConfigFilePath); err == nil { - configExists = true - } - - tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout) - fetchStart := time.Now() - cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir(), - "-output_config_name", configName) - if err := cmd.Start(); err != nil { - status := smpb.ExpConfigFetcher_ERROR - expConfigFetcher.Status = &status - return err - } - - fetchCfg := func() error { - if err := cmd.Wait(); err != nil { - status := smpb.ExpConfigFetcher_ERROR - expConfigFetcher.Status = &status - return err - } - fetchEnd := time.Now() - expConfigFetcher.Micros = proto.Uint64(uint64(fetchEnd.Sub(fetchStart).Microseconds())) - expConfigFetcher.Filename = proto.String(outConfigFilePath) - - if _, err := os.Stat(outConfigFilePath); err != nil { - status := smpb.ExpConfigFetcher_NO_CONFIG - expConfigFetcher.Status = &status - return err - } - status := smpb.ExpConfigFetcher_CONFIG - expConfigFetcher.Status = &status - return nil - } - - // If a config file does not exist, wait for the config file to be fetched. Otherwise - // fetch the config file in the background and return immediately. - if !configExists { - defer cancel() - return fetchCfg() - } - - go func() { - defer cancel() - if err := fetchCfg(); err != nil { - ctx.Verbosef("Failed to fetch config file %v: %v\n", configName, err) - } - }() - return nil -} - func loadEnvConfig(ctx Context, config *configImpl, bc string) error { if bc == "" { return nil @@ -350,9 +265,6 @@ func NewConfig(ctx Context, args ...string) Config { bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") if bc != "" { - if err := fetchEnvConfig(ctx, ret, bc); err != nil { - ctx.Verbosef("Failed to fetch config file: %v\n", err) - } if err := loadEnvConfig(ctx, ret, bc); err != nil { ctx.Fatalln("Failed to parse env config files: %v", err) } From b1daccdc97ae08603b0e2f5eafe97bd11238ee58 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Thu, 25 May 2023 14:45:30 -0400 Subject: [PATCH 0214/1460] ensure that privapp_allowlist is installed before android_app AndroidMk assumes that the app is the last file installed, and it uses this assumption to populate the LOCAL_SOONG_INSTALLED_MODULE entry. This CL moves the privapp_allowlist installation to before the app installation to respect this assumption. Bug: 242509786 Test: go test Test: OUT_DIR=out.ref m nothing && cp aosp/2562351 && OUT_DIR=out.change m nothing && GOWORK=$PWD/build/bazel/mkcompare/go.work \ go run android/bazel/mkcompare/cmd -json \ <(sed -e "s/out\.ref/out/g" out.ref/soong/Android-aosp_cheetah.mk) \ <(sed -e "s/out\.change/out/g" out.change/soong/Android-aosp_cheetah.mk) && verify manually that the only diffs are related to the removal of the prebuilt_etc module. Change-Id: I95ec27070f575e79fb976de68493a219717ed89a --- android/test_asserts.go | 15 ++++++++ apex/apex.go | 7 ++-- apex/apex_test.go | 12 +++++++ java/app.go | 11 +++--- java/app_test.go | 79 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 8 deletions(-) diff --git a/android/test_asserts.go b/android/test_asserts.go index 4143f150dd..5cc7e4ac00 100644 --- a/android/test_asserts.go +++ b/android/test_asserts.go @@ -17,6 +17,7 @@ package android import ( "fmt" "reflect" + "regexp" "strings" "testing" ) @@ -137,6 +138,20 @@ func AssertStringContainsEquals(t *testing.T, message string, s string, substrin } } +// AssertStringMatches checks if the string matches the given regular expression. If it does not match, +// then an error is reported with the supplied message including a reason for why it failed. +func AssertStringMatches(t *testing.T, message, s, expectedRex string) { + t.Helper() + ok, err := regexp.MatchString(expectedRex, s) + if err != nil { + t.Fatalf("regexp failure trying to match %s against `%s` expression: %s", s, expectedRex, err) + return + } + if !ok { + t.Errorf("%s does not match regular expression %s", s, expectedRex) + } +} + // AssertStringListContains checks if the list of strings contains the expected string. If it does // not then it reports an error prefixed with the supplied message and including a reason for why it // failed. diff --git a/apex/apex.go b/apex/apex.go index 1c794639c2..f49492e84e 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1871,14 +1871,17 @@ func apexFilesForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) []ap }); ok { af.overriddenPackageName = app.OverriddenManifestPackageName() } - apexFiles := []apexFile{af} + + apexFiles := []apexFile{} if allowlist := aapp.PrivAppAllowlist(); allowlist.Valid() { dirInApex := filepath.Join("etc", "permissions") - privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"privapp", dirInApex, etc, aapp) + privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"_privapp", dirInApex, etc, aapp) apexFiles = append(apexFiles, privAppAllowlist) } + apexFiles = append(apexFiles, af) + return apexFiles } diff --git a/apex/apex_test.go b/apex/apex_test.go index 2dad22b06a..38e24e844c 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6305,6 +6305,18 @@ func TestApexWithApps(t *testing.T) { // ... and not directly inside the APEX ensureNotContains(t, copyCmds, "image.apex/lib64/"+jni+".so") } + + apexBundle := module.Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, ctx, apexBundle) + var builder strings.Builder + data.Custom(&builder, apexBundle.Name(), "TARGET_", "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := AppFooPriv.myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := AppFoo.myapex") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALLED_MODULE := \\S+AppFooPriv.apk") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALLED_MODULE := \\S+AppFoo.apk") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALL_PAIRS := \\S+AppFooPriv.apk") + ensureContains(t, androidMk, "LOCAL_SOONG_INSTALL_PAIRS := privapp_allowlist_com.android.AppFooPriv.xml:$(PRODUCT_OUT)/apex/myapex/etc/permissions/privapp_allowlist_com.android.AppFooPriv.xml") } func TestApexWithAppImportBuildId(t *testing.T) { diff --git a/java/app.go b/java/app.go index d0bde6f22d..561ce1d174 100755 --- a/java/app.go +++ b/java/app.go @@ -613,7 +613,6 @@ func processMainCert(m android.ModuleBase, certPropValue string, certificates [] } } - return mainCertificate, certificates } @@ -795,17 +794,17 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // Install the app package. shouldInstallAppPackage := (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && !a.appProperties.PreventInstall if shouldInstallAppPackage { + if a.privAppAllowlist.Valid() { + installPath := android.PathForModuleInstall(ctx, "etc", "permissions") + ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) + } + var extraInstalledPaths android.Paths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) } ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile, extraInstalledPaths...) - - if a.privAppAllowlist.Valid() { - installPath := android.PathForModuleInstall(ctx, "etc", "permissions") - ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) - } } a.buildAppDependencyInfo(ctx) diff --git a/java/app_test.go b/java/app_test.go index 4b78c48a5b..9293da48ed 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3589,3 +3589,82 @@ func TestPrivappAllowlist(t *testing.T) { app.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml") overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml") } + +func TestPrivappAllowlistAndroidMk(t *testing.T) { + result := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + android.PrepareForTestWithAndroidMk, + ).RunTestWithBp( + t, + ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", + privileged: true, + sdk_version: "current", + } + override_android_app { + name: "bar", + base: "foo", + package_name: "com.google.android.foo", + } + `, + ) + baseApp := result.ModuleForTests("foo", "android_common") + overrideApp := result.ModuleForTests("foo", "android_common_bar") + + baseAndroidApp := baseApp.Module().(*AndroidApp) + baseEntries := android.AndroidMkEntriesForTest(t, result.TestContext, baseAndroidApp)[0] + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALLED_MODULE; expected to find foo.apk", + baseEntries.EntryMap["LOCAL_SOONG_INSTALLED_MODULE"][0], + "\\S+foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include foo.apk", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include app", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+foo.apk:\\S+/target/product/test_device/system/priv-app/foo/foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "privapp_allowlist_com.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml", + ) + + overrideAndroidApp := overrideApp.Module().(*AndroidApp) + overrideEntries := android.AndroidMkEntriesForTest(t, result.TestContext, overrideAndroidApp)[0] + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALLED_MODULE; expected to find bar.apk", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALLED_MODULE"][0], + "\\S+bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include bar.apk", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include app", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+bar.apk:\\S+/target/product/test_device/system/priv-app/bar/bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml", + ) +} From 704146094a262f1f7e99ee8606469834ec142a34 Mon Sep 17 00:00:00 2001 From: Wei Li Date: Thu, 25 May 2023 16:29:20 -0700 Subject: [PATCH 0215/1460] Exclude diffs caught by sbom_test. Bug: 284279034 Test: build/soong/tests/sbom_test.sh Change-Id: I152c2e2c55b6547b2dddf7b1b2524681c9af952e --- tests/sbom_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/sbom_test.sh b/tests/sbom_test.sh index 2f154cde56..94fe51d2dd 100755 --- a/tests/sbom_test.sh +++ b/tests/sbom_test.sh @@ -90,10 +90,12 @@ diff_excludes[system]=\ -I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \ -I /system/lib64/android.security.compat-ndk.so \ -I /system/lib64/libkeymaster4_1support.so \ + -I /system/lib64/libkeymaster4support.so \ -I /system/lib64/libkeymint.so \ -I /system/lib64/libkeystore2_aaid.so \ -I /system/lib64/libkeystore2_apc_compat.so \ -I /system/lib64/libkeystore2_crypto.so \ + -I /system/lib64/libkeystore-attestation-application-id.so \ -I /system/lib64/libkm_compat_service.so \ -I /system/lib64/libkm_compat.so \ -I /system/lib64/vndk-29 \ From 85b7932892670b53f1a06fa01be109cf4d25e46a Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Fri, 26 May 2023 00:09:38 +0000 Subject: [PATCH 0216/1460] Add jihoonkang and mrziwang to OWNERS Test: N/A Change-Id: I39145a7115e8e3bda4fb4cb92561742089ad2414 --- OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/OWNERS b/OWNERS index 0234f27761..9221d3ee1c 100644 --- a/OWNERS +++ b/OWNERS @@ -10,10 +10,12 @@ dacek@google.com delmerico@google.com dwillemsen@google.com eakammer@google.com +jihoonkang@google.com jobredeaux@google.com joeo@google.com juu@google.com lamontjones@google.com +mrziwang@google.com spandandas@google.com tradical@google.com usta@google.com From f513b23ac432893a0fe0ac29acd0ef12106dd1fc Mon Sep 17 00:00:00 2001 From: Saagar Jha Date: Fri, 26 May 2023 00:17:55 +0000 Subject: [PATCH 0217/1460] Replace prodcertstatus with gcertstatus Prodaccess is apparently deprecated now. Test: Build with/without credentials and check that it works Change-Id: Iaadaebe84fe9531b2d7d5f0d265751462bb827d8 --- ui/build/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/build/config.go b/ui/build/config.go index 8ec96800f3..7a409652c9 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -1486,7 +1486,7 @@ func (c *configImpl) GoogleProdCredsExist() bool { if googleProdCredsExistCache { return googleProdCredsExistCache } - if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil { + if _, err := exec.Command("/usr/bin/gcertstatus").Output(); err != nil { return false } googleProdCredsExistCache = true From 022adb0b8cbe8ce17c1e612115068e327b0b4786 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Thu, 25 May 2023 14:45:30 -0400 Subject: [PATCH 0218/1460] ensure that privapp_allowlist is installed before android_app AndroidMk assumes that the app is the last file installed, and it uses this assumption to populate the LOCAL_SOONG_INSTALLED_MODULE entry. This CL moves the privapp_allowlist installation to before the app installation to respect this assumption. Bug: 242509786 Test: go test Test: OUT_DIR=out.ref m nothing && cp aosp/2562351 && OUT_DIR=out.change m nothing && GOWORK=$PWD/build/bazel/mkcompare/go.work \ go run android/bazel/mkcompare/cmd -json \ <(sed -e "s/out\.ref/out/g" out.ref/soong/Android-aosp_cheetah.mk) \ <(sed -e "s/out\.change/out/g" out.change/soong/Android-aosp_cheetah.mk) && verify manually that the only diffs are related to the removal of the prebuilt_etc module. (cherry picked from https://android-review.googlesource.com/q/commit:b1daccdc97ae08603b0e2f5eafe97bd11238ee58) Merged-In: I95ec27070f575e79fb976de68493a219717ed89a Change-Id: I95ec27070f575e79fb976de68493a219717ed89a --- android/test_asserts.go | 15 ++++++++ apex/apex.go | 7 ++-- apex/apex_test.go | 12 +++++++ java/app.go | 11 +++--- java/app_test.go | 79 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 8 deletions(-) diff --git a/android/test_asserts.go b/android/test_asserts.go index 4143f150dd..5cc7e4ac00 100644 --- a/android/test_asserts.go +++ b/android/test_asserts.go @@ -17,6 +17,7 @@ package android import ( "fmt" "reflect" + "regexp" "strings" "testing" ) @@ -137,6 +138,20 @@ func AssertStringContainsEquals(t *testing.T, message string, s string, substrin } } +// AssertStringMatches checks if the string matches the given regular expression. If it does not match, +// then an error is reported with the supplied message including a reason for why it failed. +func AssertStringMatches(t *testing.T, message, s, expectedRex string) { + t.Helper() + ok, err := regexp.MatchString(expectedRex, s) + if err != nil { + t.Fatalf("regexp failure trying to match %s against `%s` expression: %s", s, expectedRex, err) + return + } + if !ok { + t.Errorf("%s does not match regular expression %s", s, expectedRex) + } +} + // AssertStringListContains checks if the list of strings contains the expected string. If it does // not then it reports an error prefixed with the supplied message and including a reason for why it // failed. diff --git a/apex/apex.go b/apex/apex.go index 1773fcf23b..85be920eed 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1866,14 +1866,17 @@ func apexFilesForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) []ap }); ok { af.overriddenPackageName = app.OverriddenManifestPackageName() } - apexFiles := []apexFile{af} + + apexFiles := []apexFile{} if allowlist := aapp.PrivAppAllowlist(); allowlist.Valid() { dirInApex := filepath.Join("etc", "permissions") - privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"privapp", dirInApex, etc, aapp) + privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"_privapp", dirInApex, etc, aapp) apexFiles = append(apexFiles, privAppAllowlist) } + apexFiles = append(apexFiles, af) + return apexFiles } diff --git a/apex/apex_test.go b/apex/apex_test.go index 6e440ffe20..66ba0a8cfd 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6210,6 +6210,18 @@ func TestApexWithApps(t *testing.T) { // ... and not directly inside the APEX ensureNotContains(t, copyCmds, "image.apex/lib64/"+jni+".so") } + + apexBundle := module.Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, ctx, apexBundle) + var builder strings.Builder + data.Custom(&builder, apexBundle.Name(), "TARGET_", "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := AppFooPriv.myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := AppFoo.myapex") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALLED_MODULE := \\S+AppFooPriv.apk") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALLED_MODULE := \\S+AppFoo.apk") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALL_PAIRS := \\S+AppFooPriv.apk") + ensureContains(t, androidMk, "LOCAL_SOONG_INSTALL_PAIRS := privapp_allowlist_com.android.AppFooPriv.xml:$(PRODUCT_OUT)/apex/myapex/etc/permissions/privapp_allowlist_com.android.AppFooPriv.xml") } func TestApexWithAppImportBuildId(t *testing.T) { diff --git a/java/app.go b/java/app.go index e08e010484..cbdb502214 100755 --- a/java/app.go +++ b/java/app.go @@ -613,7 +613,6 @@ func processMainCert(m android.ModuleBase, certPropValue string, certificates [] } } - return mainCertificate, certificates } @@ -795,17 +794,17 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // Install the app package. shouldInstallAppPackage := (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && !a.appProperties.PreventInstall if shouldInstallAppPackage { + if a.privAppAllowlist.Valid() { + installPath := android.PathForModuleInstall(ctx, "etc", "permissions") + ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) + } + var extraInstalledPaths android.Paths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) } ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile, extraInstalledPaths...) - - if a.privAppAllowlist.Valid() { - installPath := android.PathForModuleInstall(ctx, "etc", "permissions") - ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) - } } a.buildAppDependencyInfo(ctx) diff --git a/java/app_test.go b/java/app_test.go index 6e5870f9e4..71cd38c227 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3589,3 +3589,82 @@ func TestPrivappAllowlist(t *testing.T) { app.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml") overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml") } + +func TestPrivappAllowlistAndroidMk(t *testing.T) { + result := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + android.PrepareForTestWithAndroidMk, + ).RunTestWithBp( + t, + ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", + privileged: true, + sdk_version: "current", + } + override_android_app { + name: "bar", + base: "foo", + package_name: "com.google.android.foo", + } + `, + ) + baseApp := result.ModuleForTests("foo", "android_common") + overrideApp := result.ModuleForTests("foo", "android_common_bar") + + baseAndroidApp := baseApp.Module().(*AndroidApp) + baseEntries := android.AndroidMkEntriesForTest(t, result.TestContext, baseAndroidApp)[0] + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALLED_MODULE; expected to find foo.apk", + baseEntries.EntryMap["LOCAL_SOONG_INSTALLED_MODULE"][0], + "\\S+foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include foo.apk", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include app", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+foo.apk:\\S+/target/product/test_device/system/priv-app/foo/foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "privapp_allowlist_com.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml", + ) + + overrideAndroidApp := overrideApp.Module().(*AndroidApp) + overrideEntries := android.AndroidMkEntriesForTest(t, result.TestContext, overrideAndroidApp)[0] + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALLED_MODULE; expected to find bar.apk", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALLED_MODULE"][0], + "\\S+bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include bar.apk", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include app", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+bar.apk:\\S+/target/product/test_device/system/priv-app/bar/bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml", + ) +} From 81b00a8db73491a4cac326853853be743000a2ac Mon Sep 17 00:00:00 2001 From: Inseob Kim Date: Mon, 15 May 2023 18:06:31 +0900 Subject: [PATCH 0219/1460] Use hashed subdir for soong_config modules This is to differentiate soong intermediate directories for soong config modules. This will help incremental build across different devices. Test result of building panther, building cheetah, and building panther again: Before this change - build time: 02:57 - # of tasks: 31044 After this change - build time: 01:48 - # of tasks: 1694 Build time includes build.ninja generating time (which is more than 1 minute), so the overriden artifacts become far fewer. And "NINJA_ARGS='-n -d explain' m" only gave 4 "command line changed" nodes. Bug: 279362051 Test: see above Change-Id: I4891cbe823ae21628465e5c6eb26a4837ccdd202 --- README.md | 6 + android/module.go | 89 ++++++++++++ android/paths.go | 6 +- android/soong_config_modules.go | 57 +++++++- android/soong_config_modules_test.go | 202 ++++++++++++++++++++++++++- 5 files changed, 355 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 70311cb449..2d8f0afdf2 100644 --- a/README.md +++ b/README.md @@ -565,6 +565,12 @@ modules (`cc_defaults`, `java_defaults`, etc.), which can then be referenced by all of the vendor's other modules using the normal namespace and visibility rules. +`soongConfigTraceMutator` enables modules affected by soong config variables to +write outputs into a hashed directory path. It does this by recording accesses +to soong config variables on each module, and then accumulating records of each +module's all dependencies. `m soong_config_trace` builds information about +hashes to `$OUT_DIR/soong/soong_config_trace.json`. + ## Build logic The build logic is written in Go using the diff --git a/android/module.go b/android/module.go index 604ba2401e..98084f359a 100644 --- a/android/module.go +++ b/android/module.go @@ -15,6 +15,9 @@ package android import ( + "crypto/md5" + "encoding/hex" + "encoding/json" "fmt" "net/url" "os" @@ -714,6 +717,31 @@ func SortedUniqueNamedPaths(l NamedPaths) NamedPaths { return l[:k+1] } +// soongConfigTrace holds all references to VendorVars. Uses []string for blueprint:"mutated" +type soongConfigTrace struct { + Bools []string `json:",omitempty"` + Strings []string `json:",omitempty"` + IsSets []string `json:",omitempty"` +} + +func (c *soongConfigTrace) isEmpty() bool { + return len(c.Bools) == 0 && len(c.Strings) == 0 && len(c.IsSets) == 0 +} + +// Returns hash of serialized trace records (empty string if there's no trace recorded) +func (c *soongConfigTrace) hash() string { + // Use MD5 for speed. We don't care collision or preimage attack + if c.isEmpty() { + return "" + } + j, err := json.Marshal(c) + if err != nil { + panic(fmt.Errorf("json marshal of %#v failed: %#v", *c, err)) + } + hash := md5.Sum(j) + return hex.EncodeToString(hash[:]) +} + type nameProperties struct { // The name of the module. Must be unique across all modules. Name *string @@ -958,6 +986,10 @@ type commonProperties struct { // Bazel conversion status BazelConversionStatus BazelConversionStatus `blueprint:"mutated"` + + // SoongConfigTrace records accesses to VendorVars (soong_config) + SoongConfigTrace soongConfigTrace `blueprint:"mutated"` + SoongConfigTraceHash string `blueprint:"mutated"` } // CommonAttributes represents the common Bazel attributes from which properties @@ -3160,6 +3192,10 @@ func (m *moduleContext) ModuleSubDir() string { return m.bp.ModuleSubDir() } +func (m *moduleContext) ModuleSoongConfigHash() string { + return m.module.base().commonProperties.SoongConfigTraceHash +} + func (b *baseModuleContext) Target() Target { return b.target } @@ -3744,6 +3780,8 @@ func (m *moduleContext) TargetRequiredModuleNames() []string { func init() { RegisterParallelSingletonType("buildtarget", BuildTargetSingleton) + RegisterParallelSingletonType("soongconfigtrace", soongConfigTraceSingletonFunc) + FinalDepsMutators(registerSoongConfigTraceMutator) } func BuildTargetSingleton() Singleton { @@ -3925,3 +3963,54 @@ func (d *installPathsDepSet) ToList() InstallPaths { } return d.depSet.ToList().(InstallPaths) } + +func registerSoongConfigTraceMutator(ctx RegisterMutatorsContext) { + ctx.BottomUp("soongconfigtrace", soongConfigTraceMutator).Parallel() +} + +// soongConfigTraceMutator accumulates recorded soong_config trace from children. Also it normalizes +// SoongConfigTrace to make it consistent. +func soongConfigTraceMutator(ctx BottomUpMutatorContext) { + trace := &ctx.Module().base().commonProperties.SoongConfigTrace + ctx.VisitDirectDeps(func(m Module) { + childTrace := &m.base().commonProperties.SoongConfigTrace + trace.Bools = append(trace.Bools, childTrace.Bools...) + trace.Strings = append(trace.Strings, childTrace.Strings...) + trace.IsSets = append(trace.IsSets, childTrace.IsSets...) + }) + trace.Bools = SortedUniqueStrings(trace.Bools) + trace.Strings = SortedUniqueStrings(trace.Strings) + trace.IsSets = SortedUniqueStrings(trace.IsSets) + + ctx.Module().base().commonProperties.SoongConfigTraceHash = trace.hash() +} + +// soongConfigTraceSingleton writes a map from each module's config hash value to trace data. +func soongConfigTraceSingletonFunc() Singleton { + return &soongConfigTraceSingleton{} +} + +type soongConfigTraceSingleton struct { +} + +func (s *soongConfigTraceSingleton) GenerateBuildActions(ctx SingletonContext) { + outFile := PathForOutput(ctx, "soong_config_trace.json") + + traces := make(map[string]*soongConfigTrace) + ctx.VisitAllModules(func(module Module) { + trace := &module.base().commonProperties.SoongConfigTrace + if !trace.isEmpty() { + hash := module.base().commonProperties.SoongConfigTraceHash + traces[hash] = trace + } + }) + + j, err := json.Marshal(traces) + if err != nil { + ctx.Errorf("json marshal to %q failed: %#v", outFile, err) + return + } + + WriteFileRule(ctx, outFile, string(j)) + ctx.Phony("soong_config_trace", outFile) +} diff --git a/android/paths.go b/android/paths.go index eaa6a8d0b4..0f3d972320 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1475,7 +1475,11 @@ type ModuleOutPathContext interface { } func pathForModuleOut(ctx ModuleOutPathContext) OutputPath { - return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir()) + soongConfigHash := "" + if i, ok := ctx.(interface{ ModuleSoongConfigHash() string }); ok { + soongConfigHash = i.ModuleSoongConfigHash() + } + return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir(), soongConfigHash) } // PathForModuleOut returns a Path representing the paths... under the module's diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go index 5fa60124ea..0246a08d85 100644 --- a/android/soong_config_modules.go +++ b/android/soong_config_modules.go @@ -421,6 +421,57 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s }).(map[string]blueprint.ModuleFactory) } +// tracingConfig is a wrapper to soongconfig.SoongConfig which records all accesses to SoongConfig. +type tracingConfig struct { + config soongconfig.SoongConfig + boolSet map[string]bool + stringSet map[string]string + isSetSet map[string]bool +} + +func (c *tracingConfig) Bool(name string) bool { + c.boolSet[name] = c.config.Bool(name) + return c.boolSet[name] +} + +func (c *tracingConfig) String(name string) string { + c.stringSet[name] = c.config.String(name) + return c.stringSet[name] +} + +func (c *tracingConfig) IsSet(name string) bool { + c.isSetSet[name] = c.config.IsSet(name) + return c.isSetSet[name] +} + +func (c *tracingConfig) getTrace() soongConfigTrace { + ret := soongConfigTrace{} + + for k, v := range c.boolSet { + ret.Bools = append(ret.Bools, fmt.Sprintf("%q:%t", k, v)) + } + for k, v := range c.stringSet { + ret.Strings = append(ret.Strings, fmt.Sprintf("%q:%q", k, v)) + } + for k, v := range c.isSetSet { + ret.IsSets = append(ret.IsSets, fmt.Sprintf("%q:%t", k, v)) + } + + return ret +} + +func newTracingConfig(config soongconfig.SoongConfig) *tracingConfig { + c := tracingConfig{ + config: config, + boolSet: make(map[string]bool), + stringSet: make(map[string]string), + isSetSet: make(map[string]bool), + } + return &c +} + +var _ soongconfig.SoongConfig = (*tracingConfig)(nil) + // configModuleFactory takes an existing soongConfigModuleFactory and a // ModuleType to create a new ModuleFactory that uses a custom loadhook. func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfig.ModuleType, bp2build bool) blueprint.ModuleFactory { @@ -485,8 +536,8 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi // conditional on Soong config variables by reading the product // config variables from Make. AddLoadHook(module, func(ctx LoadHookContext) { - config := ctx.Config().VendorConfig(moduleType.ConfigNamespace) - newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, config) + tracingConfig := newTracingConfig(ctx.Config().VendorConfig(moduleType.ConfigNamespace)) + newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, tracingConfig) if err != nil { ctx.ModuleErrorf("%s", err) return @@ -494,6 +545,8 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi for _, ps := range newProps { ctx.AppendProperties(ps) } + + module.(Module).base().commonProperties.SoongConfigTrace = tracingConfig.getTrace() }) } return module, props diff --git a/android/soong_config_modules_test.go b/android/soong_config_modules_test.go index cab3e2d6ba..79bdeb8297 100644 --- a/android/soong_config_modules_test.go +++ b/android/soong_config_modules_test.go @@ -16,6 +16,7 @@ package android import ( "fmt" + "path/filepath" "testing" ) @@ -34,7 +35,8 @@ func soongConfigTestDefaultsModuleFactory() Module { type soongConfigTestModule struct { ModuleBase DefaultableModuleBase - props soongConfigTestModuleProperties + props soongConfigTestModuleProperties + outputPath ModuleOutPath } type soongConfigTestModuleProperties struct { @@ -49,7 +51,9 @@ func soongConfigTestModuleFactory() Module { return m } -func (t soongConfigTestModule) GenerateAndroidBuildActions(ModuleContext) {} +func (t *soongConfigTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { + t.outputPath = PathForModuleOut(ctx, "test") +} var prepareForSoongConfigTestModule = FixtureRegisterWithContext(func(ctx RegistrationContext) { ctx.RegisterModuleType("test_defaults", soongConfigTestDefaultsModuleFactory) @@ -503,3 +507,197 @@ func TestSoongConfigModuleSingletonModule(t *testing.T) { }) } } + +func TestSoongConfigModuleTrace(t *testing.T) { + bp := ` + soong_config_module_type { + name: "acme_test", + module_type: "test", + config_namespace: "acme", + variables: ["board", "feature1", "FEATURE3", "unused_string_var"], + bool_variables: ["feature2", "unused_feature", "always_true"], + value_variables: ["size", "unused_size"], + properties: ["cflags", "srcs", "defaults"], + } + + soong_config_module_type { + name: "acme_test_defaults", + module_type: "test_defaults", + config_namespace: "acme", + variables: ["board", "feature1", "FEATURE3", "unused_string_var"], + bool_variables: ["feature2", "unused_feature", "always_true"], + value_variables: ["size", "unused_size"], + properties: ["cflags", "srcs", "defaults"], + } + + soong_config_string_variable { + name: "board", + values: ["soc_a", "soc_b", "soc_c"], + } + + soong_config_string_variable { + name: "unused_string_var", + values: ["a", "b"], + } + + soong_config_bool_variable { + name: "feature1", + } + + soong_config_bool_variable { + name: "FEATURE3", + } + + test_defaults { + name: "test_defaults", + cflags: ["DEFAULT"], + } + + test { + name: "normal", + defaults: ["test_defaults"], + } + + acme_test { + name: "board_1", + defaults: ["test_defaults"], + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + }, + }, + } + + acme_test { + name: "board_2", + defaults: ["test_defaults"], + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + }, + }, + } + + acme_test { + name: "size", + defaults: ["test_defaults"], + soong_config_variables: { + size: { + cflags: ["-DSIZE=%s"], + }, + }, + } + + acme_test { + name: "board_and_size", + defaults: ["test_defaults"], + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + }, + size: { + cflags: ["-DSIZE=%s"], + }, + }, + } + + acme_test_defaults { + name: "board_defaults", + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + }, + }, + } + + acme_test_defaults { + name: "size_defaults", + soong_config_variables: { + size: { + cflags: ["-DSIZE=%s"], + }, + }, + } + + test { + name: "board_and_size_with_defaults", + defaults: ["board_defaults", "size_defaults"], + } + ` + + fixtureForVendorVars := func(vars map[string]map[string]string) FixturePreparer { + return FixtureModifyProductVariables(func(variables FixtureProductVariables) { + variables.VendorVars = vars + }) + } + + preparer := fixtureForVendorVars(map[string]map[string]string{ + "acme": { + "board": "soc_a", + "size": "42", + "feature1": "true", + "feature2": "false", + // FEATURE3 unset + "unused_feature": "true", // unused + "unused_size": "1", // unused + "unused_string_var": "a", // unused + "always_true": "true", + }, + }) + + t.Run("soong config trace hash", func(t *testing.T) { + result := GroupFixturePreparers( + preparer, + PrepareForTestWithDefaults, + PrepareForTestWithSoongConfigModuleBuildComponents, + prepareForSoongConfigTestModule, + FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.FinalDepsMutators(registerSoongConfigTraceMutator) + }), + FixtureWithRootAndroidBp(bp), + ).RunTest(t) + + // Hashes of modules not using soong config should be empty + normal := result.ModuleForTests("normal", "").Module().(*soongConfigTestModule) + AssertDeepEquals(t, "normal hash", normal.base().commonProperties.SoongConfigTraceHash, "") + AssertDeepEquals(t, "normal hash out", normal.outputPath.RelativeToTop().String(), "out/soong/.intermediates/normal/test") + + board1 := result.ModuleForTests("board_1", "").Module().(*soongConfigTestModule) + board2 := result.ModuleForTests("board_2", "").Module().(*soongConfigTestModule) + size := result.ModuleForTests("size", "").Module().(*soongConfigTestModule) + + // Trace mutator sets soong config trace hash correctly + board1Hash := board1.base().commonProperties.SoongConfigTrace.hash() + board1Output := board1.outputPath.RelativeToTop().String() + AssertDeepEquals(t, "board hash calc", board1Hash, board1.base().commonProperties.SoongConfigTraceHash) + AssertDeepEquals(t, "board hash path", board1Output, filepath.Join("out/soong/.intermediates/board_1", board1Hash, "test")) + + sizeHash := size.base().commonProperties.SoongConfigTrace.hash() + sizeOutput := size.outputPath.RelativeToTop().String() + AssertDeepEquals(t, "size hash calc", sizeHash, size.base().commonProperties.SoongConfigTraceHash) + AssertDeepEquals(t, "size hash path", sizeOutput, filepath.Join("out/soong/.intermediates/size", sizeHash, "test")) + + // Trace should be identical for modules using the same set of variables + AssertDeepEquals(t, "board trace", board1.base().commonProperties.SoongConfigTrace, board2.base().commonProperties.SoongConfigTrace) + AssertDeepEquals(t, "board hash", board1.base().commonProperties.SoongConfigTraceHash, board2.base().commonProperties.SoongConfigTraceHash) + + // Trace hash should be different for different sets of soong variables + AssertBoolEquals(t, "board hash not equal to size hash", board1.base().commonProperties.SoongConfigTraceHash == size.commonProperties.SoongConfigTraceHash, false) + + boardSize := result.ModuleForTests("board_and_size", "").Module().(*soongConfigTestModule) + boardSizeDefaults := result.ModuleForTests("board_and_size_with_defaults", "").Module() + + // Trace should propagate + AssertDeepEquals(t, "board_size hash calc", boardSize.base().commonProperties.SoongConfigTrace.hash(), boardSize.base().commonProperties.SoongConfigTraceHash) + AssertDeepEquals(t, "board_size trace", boardSize.base().commonProperties.SoongConfigTrace, boardSizeDefaults.base().commonProperties.SoongConfigTrace) + AssertDeepEquals(t, "board_size hash", boardSize.base().commonProperties.SoongConfigTraceHash, boardSizeDefaults.base().commonProperties.SoongConfigTraceHash) + }) +} From ab5f1c56c833b40086e8ddcf43ec6781727f4138 Mon Sep 17 00:00:00 2001 From: Spandan Das Date: Fri, 19 May 2023 00:18:49 +0000 Subject: [PATCH 0220/1460] Allowlist xsdc This is a java_binary_host that is used to generate .cpp/.java files from .xsd file. Test: TH Bug: 211678537 Change-Id: I62ae9f1dc7c531fc59778a5a043bffc9960c12db --- android/allowlists/allowlists.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 6405e9fe90..0dd7dae696 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -766,6 +766,9 @@ var ( "libauto_value_plugin", "auto_value_plugin_resources", "auto_value_extension", + + // Used by xsd_config + "xsdc", } Bp2buildModuleTypeAlwaysConvertList = []string{ From ec7314336a2b35ea30ce5438b83949c28e3ac429 Mon Sep 17 00:00:00 2001 From: Dan Shi Date: Fri, 26 May 2023 04:21:44 +0000 Subject: [PATCH 0221/1460] Support test runnner option in auto generated test configs Bug: 284179405 Bug: 236980335 Test: unittest Change-Id: Ifae3d556ff79153ca6c3067347fc259b665fb2e1 --- cc/test.go | 5 +++++ java/java.go | 5 +++++ java/java_test.go | 24 ++++++++++++++++++++++++ tradefed/autogen.go | 37 +++++++++++++++++++++++++------------ 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/cc/test.go b/cc/test.go index 27de06b079..3f5f710078 100644 --- a/cc/test.go +++ b/cc/test.go @@ -70,6 +70,10 @@ type TestOptions struct { // Add MinApiLevelModuleController with ro.vndk.version property. If ro.vndk.version has an // integer value and the value is less than the min_vndk_version, skip this module. Min_vndk_version *int64 + + // Extra