Introduce "Follow repository references" for TPs

Adds
- The UI checkbox and corresponding help section
- Syntax (highlighting and autocomplete) for the .target source editor
- Doesn't serialize the default state to keep legibility

org.eclipse.pde.doc.user/guide/tools/target_shared/location_edit_site_wizard.htm

@@ -32,6 +32,8 @@ <h1>Software Site Location Wizard</h1>
 
 <p><strong>Include configure phase</strong> is an advanced option to control whether the configure phase of the download operation will be run.  The configure phase allows downloaded software to run additional operations.  If this causes problems for your software site, this option can be turned off.</p> 
 
+<p>The <strong>Follow repository references</strong> option will search repository references defined in your software sites. Disabling it in essence forces your software site to become self-contained.</p>
+
 <h3 class="related">Related references</h3>
 <a href="./edit_target_locations_tab.htm">Location Tab</a><br>
 <a href="./edit_target_wizard.htm">Edit Target Wizard</a><br>

ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/IUBundleContainer.java

@@ -111,6 +111,12 @@ public class IUBundleContainer extends AbstractBundleContainer {
 	 */
 	public static final int INCLUDE_CONFIGURE_PHASE = 1 << 3;
 
+
+	/**
+	 * Whether this container should follow repository references.
+	 */
+	public static final int FOLLOW_REPOSITORY_REFERENCES = 1 << 4;
+
 	/**
 	 * IU identifiers.
 	 */
@@ -152,10 +158,20 @@ public class IUBundleContainer extends AbstractBundleContainer {
 	/**
 	 * Constructs a installable unit bundle container for the specified units.
 	 *
-	 * @param ids IU identifiers
-	 * @param versions IU versions
-	 * @param repositories metadata repositories used to search for IU's or <code>null</code> for default set
-	 * @param resolutionFlags bitmask of flags to control IU resolution, possible flags are {@link IUBundleContainer#INCLUDE_ALL_ENVIRONMENTS}, {@link IUBundleContainer#INCLUDE_REQUIRED}, {@link IUBundleContainer#INCLUDE_SOURCE}, {@link IUBundleContainer#INCLUDE_CONFIGURE_PHASE}
+	 * @param ids
+	 *            IU identifiers
+	 * @param versions
+	 *            IU versions
+	 * @param repositories
+	 *            metadata repositories used to search for IU's or
+	 *            <code>null</code> for default set
+	 * @param resolutionFlags
+	 *            bitmask of flags to control IU resolution, possible flags are
+	 *            {@link IUBundleContainer#INCLUDE_ALL_ENVIRONMENTS},
+	 *            {@link IUBundleContainer#INCLUDE_REQUIRED},
+	 *            {@link IUBundleContainer#INCLUDE_SOURCE},
+	 *            {@link IUBundleContainer#INCLUDE_CONFIGURE_PHASE},
+	 *            {@link IUBundleContainer#FOLLOW_REPOSITORY_REFERENCES}
 	 */
 	IUBundleContainer(String[] ids, Version[] versions, URI[] repositories, int resolutionFlags) {
 		fIds = ids;
@@ -368,7 +384,8 @@ void synchronizerChanged(ITargetDefinition target) {
 	public synchronized IUBundleContainer update(Set<String> toUpdate, IProgressMonitor monitor) throws CoreException {
 		SubMonitor progress = SubMonitor.convert(monitor, 100);
 		URI[] updateRepos = fRepos == null ? null : fRepos.clone();
-		IQueryable<IInstallableUnit> source = P2TargetUtils.getQueryableMetadata(updateRepos, progress.split(30));
+		IQueryable<IInstallableUnit> source = P2TargetUtils.getQueryableMetadata(updateRepos, IsFollowRepositoryReferences(),
+				progress.split(30));
 		boolean updated = false;
 		String[] updateIDs = fIds.clone();
 		Version[] updateVersions = fVersions.clone();
@@ -550,7 +567,7 @@ public synchronized void removeInstallableUnit(IInstallableUnit unit) {
 	public boolean getIncludeAllRequired() {
 		// if this container has not been associated with a container, return its own value
 		if (fSynchronizer == null) {
-			return (fFlags & INCLUDE_REQUIRED) == INCLUDE_REQUIRED;
+			return (fFlags & INCLUDE_REQUIRED) != 0;
 		}
 		return fSynchronizer.getIncludeAllRequired();
 	}
@@ -566,7 +583,7 @@ public boolean getIncludeAllRequired() {
 	public boolean getIncludeAllEnvironments() {
 		// if this container has not been associated with a container, return its own value
 		if (fSynchronizer == null) {
-			return (fFlags & INCLUDE_ALL_ENVIRONMENTS) == INCLUDE_ALL_ENVIRONMENTS;
+			return (fFlags & INCLUDE_ALL_ENVIRONMENTS) != 0;
 		}
 		return fSynchronizer.getIncludeAllEnvironments();
 	}
@@ -580,7 +597,7 @@ public boolean getIncludeAllEnvironments() {
 	public boolean getIncludeSource() {
 		// if this container has not been associated with a container, return its own value
 		if (fSynchronizer == null) {
-			return (fFlags & INCLUDE_SOURCE) == INCLUDE_SOURCE;
+			return (fFlags & INCLUDE_SOURCE) != 0;
 		}
 		return fSynchronizer.getIncludeSource();
 	}
@@ -593,11 +610,25 @@ public boolean getIncludeSource() {
 	public boolean getIncludeConfigurePhase() {
 		// if this container has not been associated with a container, return its own value
 		if (fSynchronizer == null) {
-			return (fFlags & INCLUDE_CONFIGURE_PHASE) == INCLUDE_CONFIGURE_PHASE;
+			return (fFlags & INCLUDE_CONFIGURE_PHASE) != 0;
 		}
 		return fSynchronizer.getIncludeConfigurePhase();
 	}
 
+	/**
+	 * Returns whether or not repository references should be followed
+	 *
+	 * @return whether or not repository references should be followed
+	 */
+	public boolean IsFollowRepositoryReferences() {
+		// if this container has not been associated with a container, return
+		// its own value
+		if (fSynchronizer == null) {
+			return (fFlags & FOLLOW_REPOSITORY_REFERENCES) != 0;
+		}
+		return fSynchronizer.isFollowRepositoryReferences();
+	}
+
 	/**
 	 * Returns the installable units defined by this container
 	 *
@@ -663,6 +694,7 @@ protected void associateWithTarget(ITargetDefinition target) {
 		fSynchronizer.setIncludeAllEnvironments((fFlags & INCLUDE_ALL_ENVIRONMENTS) == INCLUDE_ALL_ENVIRONMENTS);
 		fSynchronizer.setIncludeSource((fFlags & INCLUDE_SOURCE) == INCLUDE_SOURCE);
 		fSynchronizer.setIncludeConfigurePhase((fFlags & INCLUDE_CONFIGURE_PHASE) == INCLUDE_CONFIGURE_PHASE);
+		fSynchronizer.setFollowRepositoryReferences((fFlags & FOLLOW_REPOSITORY_REFERENCES) == FOLLOW_REPOSITORY_REFERENCES);
 	}
 
 	@Override
@@ -684,6 +716,18 @@ public String serialize() {
 		containerElement.setAttribute(TargetDefinitionPersistenceHelper.ATTR_INCLUDE_ALL_PLATFORMS, Boolean.toString(getIncludeAllEnvironments()));
 		containerElement.setAttribute(TargetDefinitionPersistenceHelper.ATTR_INCLUDE_SOURCE, Boolean.toString(getIncludeSource()));
 		containerElement.setAttribute(TargetDefinitionPersistenceHelper.ATTR_INCLUDE_CONFIGURE_PHASE, Boolean.toString(getIncludeConfigurePhase()));
+		// Include references was only added quite late, defaults to true and
+		// most users will never edit it.
+		// As such, for stability, conciseness, and readability, we specifically
+		// don't serialize its default state.
+		boolean includeReferences = IsFollowRepositoryReferences();
+		if (includeReferences) {
+			containerElement.removeAttribute(TargetDefinitionPersistenceHelper.ATTR_FOLLOW_REPOSITORY_REFERENCES);
+		} else {
+			containerElement.setAttribute(TargetDefinitionPersistenceHelper.ATTR_FOLLOW_REPOSITORY_REFERENCES,
+					Boolean.toString(includeReferences));
+		}
+
 		URI[] repositories = getRepositories();
 		if (repositories != null) {
 			Arrays.sort(repositories);
@@ -739,7 +783,8 @@ private int[] getPredictableOrder(String[] ids, Version[] versions) {
 	}
 
 	IInstallableUnit[] getRootIUs(ITargetDefinition definition, IProgressMonitor monitor) throws CoreException {
-		IQueryable<IInstallableUnit> repos = P2TargetUtils.getQueryableMetadata(getRepositories(), monitor);
+		IQueryable<IInstallableUnit> repos = P2TargetUtils.getQueryableMetadata(getRepositories(),
+				IsFollowRepositoryReferences(), monitor);
 		MultiStatus status = new MultiStatus(PDECore.PLUGIN_ID, 0, Messages.IUBundleContainer_ProblemsLoadingRepositories, null);
 		List<IInstallableUnit> result = new ArrayList<>();
 		for (int j = 0; j < fIds.length; j++) {

ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/IULocationFactory.java

@@ -59,6 +59,7 @@ public ITargetLocation getTargetLocation(String type, String serializedXML) thro
 			String includeAllPlatforms = location.getAttribute(TargetDefinitionPersistenceHelper.ATTR_INCLUDE_ALL_PLATFORMS);
 			String includeSource = location.getAttribute(TargetDefinitionPersistenceHelper.ATTR_INCLUDE_SOURCE);
 			String includeConfigurePhase = location.getAttribute(TargetDefinitionPersistenceHelper.ATTR_INCLUDE_CONFIGURE_PHASE);
+			String followRepositoryReferences = location.getAttribute(TargetDefinitionPersistenceHelper.ATTR_FOLLOW_REPOSITORY_REFERENCES);
 
 			NodeList list = location.getChildNodes();
 			List<String> ids = new ArrayList<>();
@@ -101,6 +102,13 @@ public ITargetLocation getTargetLocation(String type, String serializedXML) thro
 			flags |= Boolean.parseBoolean(includeAllPlatforms) ? IUBundleContainer.INCLUDE_ALL_ENVIRONMENTS : 0;
 			flags |= Boolean.parseBoolean(includeSource) ? IUBundleContainer.INCLUDE_SOURCE : 0;
 			flags |= Boolean.parseBoolean(includeConfigurePhase) ? IUBundleContainer.INCLUDE_CONFIGURE_PHASE : 0;
+			// For backwards compatibility, followRepositoryReferences should be
+			// true when it's absent
+			if (followRepositoryReferences.isEmpty()) {
+				flags |= IUBundleContainer.FOLLOW_REPOSITORY_REFERENCES;
+			} else {
+				flags |= Boolean.parseBoolean(followRepositoryReferences) ? IUBundleContainer.FOLLOW_REPOSITORY_REFERENCES : 0;
+			}
 			return TargetPlatformService.getDefault().newIULocation(iuIDs, iuVer, uris,
 					flags);
 		}

ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/P2TargetUtils.java

@@ -216,8 +216,19 @@ public class P2TargetUtils {
 	private boolean fIncludeConfigurePhase = false;
 
 	/**
-	 * Deletes any profiles associated with target definitions that no longer exist
-	 * and returns a list of profile identifiers that were deleted.
+	 * Whether repository references should be resolved. If this option is
+	 * false, references will be skipped entirely. In essence, this forces
+	 * repositories to be self-contained.
+	 *
+	 * <p>
+	 * <code>true</code> by default
+	 * </p>
+	 */
+	private boolean fFollowRepositoryReferences = true;
+
+	/**
+	 * Deletes any profiles associated with target definitions that no longer
+	 * exist and returns a list of profile identifiers that were deleted.
 	 */
 	public static List<String> cleanOrphanedTargetDefinitionProfiles() throws CoreException {
 		List<String> list = new ArrayList<>();
@@ -703,6 +714,26 @@ public boolean getIncludeConfigurePhase() {
 		return fIncludeConfigurePhase;
 	}
 
+
+	/**
+	 * Set whether or not repository references should be followed
+	 *
+	 * @param value whether or not repository references should be followed
+	 */
+	public void setFollowRepositoryReferences(boolean value) {
+		fFollowRepositoryReferences = value;
+	}
+
+
+	/**
+	 * Return whether or not repository references should be followed
+	 *
+	 * @return whether or not repository references should be followed
+	 */
+	public boolean isFollowRepositoryReferences() {
+		return fFollowRepositoryReferences;
+	}
+
 	/**
 	 * Return whether or not the given target has a matching profile that is in sync
 	 * @param target the target to check
@@ -1004,11 +1035,12 @@ public static IMetadataRepositoryManager getRepoManager() throws CoreException {
 	 * Return a queryable on the metadata defined in the given repo locations
 	 *
 	 * @param repos the repos to lookup
+	 * @param followRepositoryReferences whether to follow repository references
 	 * @param monitor the progress monitor
 	 * @return the set of metadata repositories found
 	 * @throws CoreException if there is a problem getting the repositories
 	 */
-	static IQueryable<IInstallableUnit> getQueryableMetadata(URI[] repos, IProgressMonitor monitor) throws CoreException {
+	static IQueryable<IInstallableUnit> getQueryableMetadata(URI[] repos, boolean followRepositoryReferences, IProgressMonitor monitor) throws CoreException {
 		IMetadataRepositoryManager manager = getRepoManager();
 		if (repos == null) {
 			repos = manager.getKnownRepositories(IRepositoryManager.REPOSITORIES_ALL);
@@ -1017,15 +1049,16 @@ static IQueryable<IInstallableUnit> getQueryableMetadata(URI[] repos, IProgressM
 		int repoCount = repos.length;
 		SubMonitor subMonitor = SubMonitor.convert(monitor, repoCount * 2);
 
-		Set<IRepositoryReference> seen = new HashSet<>();
 		List<IMetadataRepository> result = new ArrayList<>(repoCount);
 		List<IMetadataRepository> additional = new ArrayList<>();
 		MultiStatus repoStatus = new MultiStatus(PDECore.PLUGIN_ID, 0, Messages.IUBundleContainer_ProblemsLoadingRepositories, null);
 		for (int i = 0; i < repoCount; ++i) {
 			try {
 				IMetadataRepository repository = manager.loadRepository(repos[i], subMonitor.split(1));
 				result.add(repository);
-				addReferences(repository, additional, seen, manager, subMonitor.split(1));
+				if (followRepositoryReferences) {
+					addReferences(repository, additional, new HashSet<>(), manager, subMonitor.split(1));
+				}
 			} catch (ProvisionException e) {
 				repoStatus.add(e.getStatus());
 			}
@@ -1046,8 +1079,7 @@ private static void addReferences(IMetadataRepository repository, List<IMetadata
 		Collection<IRepositoryReference> references = repository.getReferences();
 		SubMonitor subMonitor = SubMonitor.convert(monitor, references.size() * 2);
 		for (IRepositoryReference reference : references) {
-			if (reference.getType() == IRepository.TYPE_METADATA && reference.getOptions() == IRepository.ENABLED
-					&& seen.add(reference)) {
+			if (reference.getType() == IRepository.TYPE_METADATA && reference.isEnabled() && seen.add(reference)) {
 				try {
 					IMetadataRepository referencedRepository = manager.loadRepository(reference.getLocation(),
 							subMonitor.split(1));
@@ -1103,7 +1135,8 @@ public IQueryable<IInstallableUnit> getMetadata(IProgressMonitor monitor) {
 						QueryUtil.compoundQueryable(extraMetadataRepositories));
 			}
 		};
-		context.setProperty(ProvisioningContext.FOLLOW_REPOSITORY_REFERENCES, Boolean.toString(true));
+		context.setProperty(ProvisioningContext.FOLLOW_REPOSITORY_REFERENCES, Boolean.toString(isFollowRepositoryReferences()));
+		context.setProperty(ProvisioningContext.FOLLOW_ARTIFACT_REPOSITORY_REFERENCES, Boolean.toString(isFollowRepositoryReferences()));
 		context.setMetadataRepositories(getMetadataRepositories(target).toArray(URI[]::new));
 		context.setArtifactRepositories(getArtifactRepositories(target).toArray(URI[]::new));
 
@@ -1302,7 +1335,8 @@ private void resolveWithSlicer(ITargetDefinition target, IProfile profile, IProg
 			return;
 		}
 		URI[] uris = repositories.toArray(URI[]::new);
-		IQueryable<IInstallableUnit> allMetadata = getQueryableMetadata(uris, subMonitor.split(5));
+		IQueryable<IInstallableUnit> allMetadata = getQueryableMetadata(uris, isFollowRepositoryReferences(),
+				subMonitor.split(5));
 
 		// do an initial slice to add everything the user requested
 		IQueryResult<IInstallableUnit> queryResult = slice(units, allMetadata, target, subMonitor.split(5));
@@ -1329,7 +1363,8 @@ private void resolveWithSlicer(ITargetDefinition target, IProfile profile, IProg
 		ProvisioningContext context = new ProvisioningContext(getAgent());
 		context.setMetadataRepositories(uris);
 		context.setArtifactRepositories(getArtifactRepositories(target).toArray(URI[]::new));
-		context.setProperty(ProvisioningContext.FOLLOW_REPOSITORY_REFERENCES, Boolean.toString(true));
+		context.setProperty(ProvisioningContext.FOLLOW_REPOSITORY_REFERENCES, Boolean.toString(isFollowRepositoryReferences()));
+		context.setProperty(ProvisioningContext.FOLLOW_ARTIFACT_REPOSITORY_REFERENCES, Boolean.toString(isFollowRepositoryReferences()));
 		IProvisioningPlan plan = engine.createPlan(profile, context);
 		setPlanProperties(plan, target, TargetDefinitionPersistenceHelper.MODE_SLICER);
 

ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/TargetDefinitionPersistenceHelper.java

@@ -87,6 +87,7 @@ public class TargetDefinitionPersistenceHelper {
 	static final String ATTR_INCLUDE_ALL_PLATFORMS = "includeAllPlatforms"; //$NON-NLS-1$
 	static final String ATTR_INCLUDE_SOURCE = "includeSource"; //$NON-NLS-1$
 	static final String ATTR_INCLUDE_CONFIGURE_PHASE = "includeConfigurePhase"; //$NON-NLS-1$
+	static final String ATTR_FOLLOW_REPOSITORY_REFERENCES = "followRepositoryReferences"; //$NON-NLS-1$
 	static final String ATTR_VERSION = "version"; //$NON-NLS-1$
 	static final String ATTR_CONFIGURATION = "configuration"; //$NON-NLS-1$
 	static final String ATTR_SEQUENCE_NUMBER = "sequenceNumber"; //$NON-NLS-1$

ui/org.eclipse.pde.genericeditor.extension.tests/src/org/eclipse/pde/genericeditor/extension/tests/AttributeNameCompletionTests.java

@@ -30,8 +30,8 @@ public void testAttributeNameSuggestions() throws Exception {
 		// target
 		expectedProposalsByOffset.put(8, new String[] { "name", "sequenceNumber" });
 		// location
-		expectedProposalsByOffset.put(33, new String[] { "includeAllPlatforms", "includeConfigurePhase", "includeMode",
-				"includeSource", "type" });
+		expectedProposalsByOffset.put(33, new String[] { "followRepositoryReferences", "includeAllPlatforms",
+				"includeConfigurePhase", "includeMode", "includeSource", "type" });
 		// unit
 		expectedProposalsByOffset.put(41, new String[] { "id", "version" });
 		// repository
@@ -85,8 +85,7 @@ public void testNoAttributeNameRepeatSuggestions() throws Exception {
 				offset = Math.min(nextSpace, nextOpen);
 			}
 
-			ICompletionProposal[] completionProposals = contentAssist.computeCompletionProposals(textViewer,
-					offset);
+			ICompletionProposal[] completionProposals = contentAssist.computeCompletionProposals(textViewer, offset);
 			if (completionProposals.length != 0) {
 				Assert.fail("There should not be any proposals at index " + offset + ". Following proposals found: "
 						+ proposalListToString(completionProposals));

ui/org.eclipse.pde.genericeditor.extension.tests/testing-files/target-files/AttributeNamesFullTestCaseTarget.txt

@@ -1,6 +1,6 @@
 <target  name="" sequenceNumber=""  >
 <locations >
-<location  includeAllPlatforms="" includeConfigurePhase="" includeMode="" includeSource="" type=""  >
+<location  followRepositoryReferences="" includeAllPlatforms="" includeConfigurePhase="" includeMode="" includeSource="" type=""  >
 <unit  id="" version=""  />
 <repository  location=""  />
 </location>

ui/org.eclipse.pde.genericeditor.extension.tests/testing-files/target-files/ContainerContentsTestCaseTarget.txt

@@ -10,7 +10,7 @@
 		<location configuration="/test/path/to/configuration/location/" path="/test/path/to/eclipse/" type="Profile"/>
 		<location id="org.eclipse.test" path="${eclipse_home}" type="Feature" version="1.2.3"/>
 		<location path="/test/path/to/eclipse/" type="Profile"/>
-		<location includeAllPlatforms="false" includeConfigurePhase="false" includeMode="planner" includeSource="false" type="InstallableUnit">
+		<location followRepositoryReferences="false" includeAllPlatforms="false" includeConfigurePhase="false" includeMode="planner" includeSource="false" type="InstallableUnit">
 			<repository location="TESTURI"/>
 			<repository location="TESTURI2"/>
 			<unit id="unit1" version="1.0.0"/>

ui/org.eclipse.pde.genericeditor.extension.tests/testing-files/target-files/MultipleContainersSameRepoTestCaseTarget.txt

@@ -2,12 +2,12 @@
 <?pde version="3.8"?>
 <target>
 	<locations>
-		<location includeAllPlatforms="true" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
+		<location followRepositoryReferences="false" includeAllPlatforms="true" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
 			<repository location="TESTURI"/>
 			<unit id="unit1" version="1.0.0"/>
 			<unit id="unit2" version="2.0.0"/>
 		</location>
-		<location includeAllPlatforms="true" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
+		<location followRepositoryReferences="false" includeAllPlatforms="true" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
 			<repository location="TESTURI"/>
 		</location>
 	</locations>

ui/org.eclipse.pde.genericeditor.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/autocomplete/processors/AttributeNameCompletionProcessor.java

@@ -50,8 +50,9 @@ public class AttributeNameCompletionProcessor extends DelegateProcessor {
 	private final String[] target = new String[] { ITargetConstants.TARGET_NAME_ATTR, ITargetConstants.TARGET_SEQ_NO_ATTR };
 	private final String[] locations = new String[] {};
 	private final String[] location = new String[] { ITargetConstants.LOCATION_INCLUDE_PLATFORMS_ATTR,
-			ITargetConstants.LOCATION_INCLUDE_CONFIG_PHASE_ATTR, ITargetConstants.LOCATION_INCLUDE_MODE_ATTR,
-			ITargetConstants.LOCATION_INCLUDE_SOURCE_ATTR, ITargetConstants.LOCATION_TYPE_ATTR };
+			ITargetConstants.LOCATION_INCLUDE_CONFIG_PHASE_ATTR, ITargetConstants.LOCATION_FOLLOW_REPOSITORY_REFERENCES_ATTR,
+			ITargetConstants.LOCATION_INCLUDE_MODE_ATTR, ITargetConstants.LOCATION_INCLUDE_SOURCE_ATTR,
+			ITargetConstants.LOCATION_TYPE_ATTR };
 	private final String[] unit = new String[] { ITargetConstants.UNIT_ID_ATTR, ITargetConstants.UNIT_VERSION_ATTR };
 	private final String[] repository = new String[] { ITargetConstants.REPOSITORY_LOCATION_ATTR };
 	private final String[] targetJRE = new String[] { ITargetConstants.TARGET_JRE_PATH_ATTR };