From 913783b4797a74a77dc37168f0d182e6d4e60566 Mon Sep 17 00:00:00 2001 From: Vaclav Haisman Date: Thu, 25 May 2023 21:19:21 +0200 Subject: [PATCH 1/4] TAR: Implement extraction and archiving of hardlinks. --- .../plexus/archiver/AbstractUnArchiver.java | 30 +++++++-- .../plexus/archiver/tar/TarArchiver.java | 63 +++++++++++++++--- .../plexus/archiver/tar/TarUnArchiver.java | 3 +- .../archiver/zip/AbstractZipUnArchiver.java | 1 + .../archiver/AbstractUnArchiverTest.java | 2 +- .../plexus/archiver/HardlinkTest.java | 53 +++++++++++++++ .../plexus/archiver/tar/TarFileTest.java | 10 +-- src/test/resources/hardlinks/hardlinks.tar | Bin 0 -> 10240 bytes src/test/resources/symlinks/regen.sh | 4 +- 9 files changed, 144 insertions(+), 22 deletions(-) create mode 100644 src/test/java/org/codehaus/plexus/archiver/HardlinkTest.java create mode 100644 src/test/resources/hardlinks/hardlinks.tar diff --git a/src/main/java/org/codehaus/plexus/archiver/AbstractUnArchiver.java b/src/main/java/org/codehaus/plexus/archiver/AbstractUnArchiver.java index e8a6a2e2d..e499e36ce 100644 --- a/src/main/java/org/codehaus/plexus/archiver/AbstractUnArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/AbstractUnArchiver.java @@ -78,6 +78,8 @@ protected Logger getLogger() { */ private boolean ignorePermissions = false; + private boolean warnCannotHardlink = true; + public AbstractUnArchiver() { // no op } @@ -278,8 +280,9 @@ protected void extractFile( String entryName, final Date entryDate, final boolean isDirectory, + final boolean isSymlink, final Integer mode, - String symlinkDestination, + String linkDestination, final FileMapper[] fileMappers) throws IOException, ArchiverException { if (fileMappers != null) { @@ -312,11 +315,30 @@ protected void extractFile( dirF.mkdirs(); } - if (!StringUtils.isEmpty(symlinkDestination)) { - SymlinkUtils.createSymbolicLink(targetFileName, new File(symlinkDestination)); + boolean doCopy = true; + if (!StringUtils.isEmpty(linkDestination)) { + if (isSymlink) { + SymlinkUtils.createSymbolicLink(targetFileName, new File(linkDestination)); + doCopy = false; + } else { + try { + Files.createLink( + targetFileName.toPath(), + FileUtils.resolveFile(dir, linkDestination).toPath()); + doCopy = false; + } catch (final UnsupportedOperationException ex) { + if (warnCannotHardlink) { + getLogger().warn("Creating hardlinks is not supported"); + warnCannotHardlink = false; + } + // We will do a copy instead. + } + } } else if (isDirectory) { targetFileName.mkdirs(); - } else { + doCopy = false; + } + if (doCopy) { try (OutputStream out = Files.newOutputStream(targetFileName.toPath())) { IOUtil.copy(compressedInputStream, out); } diff --git a/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java b/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java index cae07804c..16a6a6c26 100644 --- a/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java @@ -23,6 +23,10 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributeView; +import java.util.HashMap; +import java.util.Map; import java.util.zip.GZIPOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -39,6 +43,7 @@ import org.codehaus.plexus.archiver.util.Streams; import org.codehaus.plexus.components.io.attributes.PlexusIoResourceAttributes; import org.codehaus.plexus.components.io.functions.SymlinkDestinationSupplier; +import org.codehaus.plexus.components.io.resources.PlexusIoFileResource; import org.codehaus.plexus.components.io.resources.PlexusIoResource; import org.codehaus.plexus.util.IOUtil; import org.codehaus.plexus.util.StringUtils; @@ -65,6 +70,8 @@ public class TarArchiver extends AbstractArchiver { private TarArchiveOutputStream tOut; + private final Map seenFiles = new HashMap<>(10); + /** * Set how to handle long files, those with a path>100 chars. * Optional, default=warn. @@ -177,7 +184,8 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v return; } - if (entry.getResource().isDirectory() && !vPath.endsWith("/")) { + final PlexusIoResource ioResource = entry.getResource(); + if (ioResource.isDirectory() && !vPath.endsWith("/")) { vPath += "/"; } @@ -194,7 +202,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v InputStream fIn = null; try { - TarArchiveEntry te; + TarArchiveEntry te = null; if (!longFileMode.isGnuMode() && pathLength >= org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN) { int maxPosixPathLen = org.apache.commons.compress.archivers.tar.TarConstants.NAMELEN @@ -233,18 +241,43 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v } } + boolean doCopy = true; if (entry.getType() == ArchiveEntry.SYMLINK) { - final SymlinkDestinationSupplier plexusIoSymlinkResource = - (SymlinkDestinationSupplier) entry.getResource(); + final SymlinkDestinationSupplier plexusIoSymlinkResource = (SymlinkDestinationSupplier) ioResource; te = new TarArchiveEntry(vPath, TarArchiveEntry.LF_SYMLINK); te.setLinkName(plexusIoSymlinkResource.getSymlinkDestination()); - } else { + doCopy = false; + } else if (options.getPreserveHardLinks() + && ioResource.isFile() + && ioResource instanceof PlexusIoFileResource) { + final PlexusIoFileResource fileResource = (PlexusIoFileResource) ioResource; + final Path file = fileResource.getFile().toPath(); + if (Files.exists(file)) { + final BasicFileAttributeView fileAttributeView = + Files.getFileAttributeView(file, BasicFileAttributeView.class); + if (fileAttributeView != null) { + final Object fileKey = + fileAttributeView.readAttributes().fileKey(); + if (fileKey != null) { + final String seenFile = this.seenFiles.get(fileKey); + if (seenFile != null) { + te = new TarArchiveEntry(vPath, TarArchiveEntry.LF_LINK); + te.setLinkName(seenFile); + doCopy = false; + } else { + this.seenFiles.put(fileKey, vPath); + } + } + } + } + } + if (te == null) { te = new TarArchiveEntry(vPath); } if (getLastModifiedTime() == null) { - long teLastModified = entry.getResource().getLastModified(); + long teLastModified = ioResource.getLastModified(); te.setModTime( teLastModified == PlexusIoResource.UNKNOWN_MODIFICATION_DATE ? System.currentTimeMillis() @@ -253,11 +286,11 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v te.setModTime(getLastModifiedTime().toMillis()); } - if (entry.getType() == ArchiveEntry.SYMLINK) { + if (!doCopy) { te.setSize(0); - } else if (!entry.getResource().isDirectory()) { - final long size = entry.getResource().getSize(); + } else if (!ioResource.isDirectory()) { + final long size = ioResource.getSize(); te.setSize(size == PlexusIoResource.UNKNOWN_RESOURCE_SIZE ? 0 : size); } te.setMode(entry.getMode()); @@ -289,7 +322,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v tOut.putArchiveEntry(te); try { - if (entry.getResource().isFile() && !(entry.getType() == ArchiveEntry.SYMLINK)) { + if (ioResource.isFile() && doCopy) { fIn = entry.getInputStream(); Streams.copyFullyDontCloseOutput(fIn, tOut, "xAR"); @@ -320,6 +353,8 @@ public class TarOptions { private boolean preserveLeadingSlashes = false; + private boolean preserveHardLinks = true; + /** * The username for the tar entry * This is not the same as the UID. @@ -405,6 +440,14 @@ public boolean getPreserveLeadingSlashes() { public void setPreserveLeadingSlashes(boolean preserveLeadingSlashes) { this.preserveLeadingSlashes = preserveLeadingSlashes; } + + public boolean getPreserveHardLinks() { + return preserveHardLinks; + } + + public void setPreserveHardLinks(boolean preserveHardLinks) { + this.preserveHardLinks = preserveHardLinks; + } } /** diff --git a/src/main/java/org/codehaus/plexus/archiver/tar/TarUnArchiver.java b/src/main/java/org/codehaus/plexus/archiver/tar/TarUnArchiver.java index 1b0b79509..b9fa61f61 100644 --- a/src/main/java/org/codehaus/plexus/archiver/tar/TarUnArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/tar/TarUnArchiver.java @@ -99,7 +99,7 @@ protected void execute(File sourceFile, File destDirectory, FileMapper[] fileMap while ((te = tis.getNextTarEntry()) != null) { TarResource fileInfo = new TarResource(tarFile, te); if (isSelected(te.getName(), fileInfo)) { - final String symlinkDestination = te.isSymbolicLink() ? te.getLinkName() : null; + final String symlinkDestination = te.isSymbolicLink() || te.isLink() ? te.getLinkName() : null; extractFile( sourceFile, destDirectory, @@ -107,6 +107,7 @@ protected void execute(File sourceFile, File destDirectory, FileMapper[] fileMap te.getName(), te.getModTime(), te.isDirectory(), + te.isSymbolicLink(), te.getMode() != 0 ? te.getMode() : null, symlinkDestination, fileMappers); diff --git a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipUnArchiver.java b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipUnArchiver.java index 4b2219de8..e27d823b6 100644 --- a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipUnArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipUnArchiver.java @@ -176,6 +176,7 @@ protected void execute(final String path, final File outputDirectory) throws Arc ze.getName(), new Date(ze.getTime()), ze.isDirectory(), + ze.isUnixSymlink(), ze.getUnixMode() != 0 ? ze.getUnixMode() : null, resolveSymlink(zipFile, ze), getFileMappers()); diff --git a/src/test/java/org/codehaus/plexus/archiver/AbstractUnArchiverTest.java b/src/test/java/org/codehaus/plexus/archiver/AbstractUnArchiverTest.java index 372d14fb3..a847c1186 100644 --- a/src/test/java/org/codehaus/plexus/archiver/AbstractUnArchiverTest.java +++ b/src/test/java/org/codehaus/plexus/archiver/AbstractUnArchiverTest.java @@ -74,7 +74,7 @@ public void shouldThrowExceptionBecauseRewrittenPathIsOutOfDirectory(@TempDir Fi Exception exception = assertThrows( ArchiverException.class, () -> abstractUnArchiver.extractFile( - null, targetFolder, null, "ENTRYNAME", null, false, null, null, fileMappers)); + null, targetFolder, null, "ENTRYNAME", null, false, false, null, null, fileMappers)); // then // ArchiverException is thrown providing the rewritten path assertEquals( diff --git a/src/test/java/org/codehaus/plexus/archiver/HardlinkTest.java b/src/test/java/org/codehaus/plexus/archiver/HardlinkTest.java new file mode 100644 index 000000000..4ed750494 --- /dev/null +++ b/src/test/java/org/codehaus/plexus/archiver/HardlinkTest.java @@ -0,0 +1,53 @@ +package org.codehaus.plexus.archiver; + +import java.io.File; +import java.nio.file.Files; + +import org.codehaus.plexus.archiver.tar.TarArchiver; +import org.codehaus.plexus.archiver.tar.TarLongFileMode; +import org.codehaus.plexus.archiver.tar.TarUnArchiver; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author Kristian Rosenvold + */ +public class HardlinkTest extends TestSupport { + + @Test + @DisabledOnOs(OS.WINDOWS) + public void testHardlinkTar() throws Exception { + // Extract test files + final File archiveFile = getTestFile("src/test/resources/hardlinks/hardlinks.tar"); + File output = getTestFile("target/output/untaredHardlinks"); + output.mkdirs(); + TarUnArchiver unarchiver = (TarUnArchiver) lookup(UnArchiver.class, "tar"); + unarchiver.setSourceFile(archiveFile); + unarchiver.setDestFile(output); + unarchiver.extract(); + // Check that we have hardlinks + assertTrue(Files.isSameFile( + output.toPath().resolve("fileR.txt"), output.toPath().resolve("hardlink"))); + + // Archive the extracted hardlinks to new archive + TarArchiver archiver = (TarArchiver) lookup(Archiver.class, "tar"); + archiver.setLongfile(TarLongFileMode.posix); + archiver.addDirectory(output); + final File testFile = getTestFile("target/output/untaredHardlinks2.tar"); + archiver.setDestFile(testFile); + archiver.createArchive(); + + // Check that our created archive actually contains hardlinks when extracted + unarchiver = (TarUnArchiver) lookup(UnArchiver.class, "tar"); + output = getTestFile("target/output/untaredHardlinks2"); + output.mkdirs(); + unarchiver.setSourceFile(testFile); + unarchiver.setDestFile(output); + unarchiver.extract(); + assertTrue(Files.isSameFile( + output.toPath().resolve("fileR.txt"), output.toPath().resolve("hardlink"))); + } +} diff --git a/src/test/java/org/codehaus/plexus/archiver/tar/TarFileTest.java b/src/test/java/org/codehaus/plexus/archiver/tar/TarFileTest.java index bebce92c0..09e4fb0dc 100644 --- a/src/test/java/org/codehaus/plexus/archiver/tar/TarFileTest.java +++ b/src/test/java/org/codehaus/plexus/archiver/tar/TarFileTest.java @@ -4,9 +4,9 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; -import java.util.Arrays; import java.util.Enumeration; +import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.codehaus.plexus.archiver.Archiver; import org.codehaus.plexus.archiver.TestSupport; @@ -18,7 +18,7 @@ import org.junit.jupiter.api.Test; import static org.codehaus.plexus.components.io.resources.ResourceFactory.createResource; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; /** * Test case for {@link TarFile}. @@ -92,15 +92,15 @@ private void testTarFile(Compressor compressor, String extension, TarFileCreator } final TarFile tarFile = tarFileCreator.newTarFile(file); - for (Enumeration en = tarFile.getEntries(); en.hasMoreElements(); ) { + for (Enumeration en = tarFile.getEntries(); en.hasMoreElements(); ) { final TarArchiveEntry te = (TarArchiveEntry) en.nextElement(); - if (te.isDirectory() || te.isSymbolicLink()) { + if (te.isDirectory() || te.isSymbolicLink() || te.isLink()) { continue; } final File teFile = new File("src", te.getName()); final InputStream teStream = tarFile.getInputStream(te); final InputStream fileStream = Files.newInputStream(teFile.toPath()); - assertTrue(Arrays.equals(IOUtil.toByteArray(teStream), IOUtil.toByteArray(fileStream))); + assertArrayEquals(IOUtil.toByteArray(teStream), IOUtil.toByteArray(fileStream)); teStream.close(); fileStream.close(); } diff --git a/src/test/resources/hardlinks/hardlinks.tar b/src/test/resources/hardlinks/hardlinks.tar new file mode 100644 index 0000000000000000000000000000000000000000..d43bcda3ca9fe862b5af5979f037df36502ccace GIT binary patch literal 10240 zcmeIy-3o#*6u|La_Y`{qIm>D95xsz Date: Sat, 27 May 2023 20:33:52 +0200 Subject: [PATCH 2/4] TAR: Implement extraction and archiving of hardlinks. [2] --- .../plexus/archiver/tar/TarArchiver.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java b/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java index 16a6a6c26..59e74b616 100644 --- a/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java @@ -184,8 +184,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v return; } - final PlexusIoResource ioResource = entry.getResource(); - if (ioResource.isDirectory() && !vPath.endsWith("/")) { + if (entry.getResource().isDirectory() && !vPath.endsWith("/")) { vPath += "/"; } @@ -243,15 +242,16 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v boolean doCopy = true; if (entry.getType() == ArchiveEntry.SYMLINK) { - final SymlinkDestinationSupplier plexusIoSymlinkResource = (SymlinkDestinationSupplier) ioResource; + final SymlinkDestinationSupplier plexusIoSymlinkResource = + (SymlinkDestinationSupplier) entry.getResource(); te = new TarArchiveEntry(vPath, TarArchiveEntry.LF_SYMLINK); te.setLinkName(plexusIoSymlinkResource.getSymlinkDestination()); doCopy = false; } else if (options.getPreserveHardLinks() - && ioResource.isFile() - && ioResource instanceof PlexusIoFileResource) { - final PlexusIoFileResource fileResource = (PlexusIoFileResource) ioResource; + && entry.getResource().isFile() + && entry.getResource() instanceof PlexusIoFileResource) { + final PlexusIoFileResource fileResource = (PlexusIoFileResource) entry.getResource(); final Path file = fileResource.getFile().toPath(); if (Files.exists(file)) { final BasicFileAttributeView fileAttributeView = @@ -277,7 +277,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v } if (getLastModifiedTime() == null) { - long teLastModified = ioResource.getLastModified(); + long teLastModified = entry.getResource().getLastModified(); te.setModTime( teLastModified == PlexusIoResource.UNKNOWN_MODIFICATION_DATE ? System.currentTimeMillis() @@ -289,8 +289,8 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v if (!doCopy) { te.setSize(0); - } else if (!ioResource.isDirectory()) { - final long size = ioResource.getSize(); + } else if (!entry.getResource().isDirectory()) { + final long size = entry.getResource().getSize(); te.setSize(size == PlexusIoResource.UNKNOWN_RESOURCE_SIZE ? 0 : size); } te.setMode(entry.getMode()); @@ -322,7 +322,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v tOut.putArchiveEntry(te); try { - if (ioResource.isFile() && doCopy) { + if (entry.getResource().isFile() && doCopy) { fIn = entry.getInputStream(); Streams.copyFullyDontCloseOutput(fIn, tOut, "xAR"); From dd5e14faca9a102e6e47d6a2304015917deaae32 Mon Sep 17 00:00:00 2001 From: Vaclav Haisman Date: Sat, 27 May 2023 20:41:46 +0200 Subject: [PATCH 3/4] TAR: Implement extraction and archiving of hardlinks. [3] doCopy -> isLink --- .../org/codehaus/plexus/archiver/tar/TarArchiver.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java b/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java index 59e74b616..d46a16e5e 100644 --- a/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java @@ -240,14 +240,14 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v } } - boolean doCopy = true; + boolean isLink = false; if (entry.getType() == ArchiveEntry.SYMLINK) { final SymlinkDestinationSupplier plexusIoSymlinkResource = (SymlinkDestinationSupplier) entry.getResource(); te = new TarArchiveEntry(vPath, TarArchiveEntry.LF_SYMLINK); te.setLinkName(plexusIoSymlinkResource.getSymlinkDestination()); - doCopy = false; + isLink = true; } else if (options.getPreserveHardLinks() && entry.getResource().isFile() && entry.getResource() instanceof PlexusIoFileResource) { @@ -264,7 +264,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v if (seenFile != null) { te = new TarArchiveEntry(vPath, TarArchiveEntry.LF_LINK); te.setLinkName(seenFile); - doCopy = false; + isLink = true; } else { this.seenFiles.put(fileKey, vPath); } @@ -286,7 +286,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v te.setModTime(getLastModifiedTime().toMillis()); } - if (!doCopy) { + if (isLink) { te.setSize(0); } else if (!entry.getResource().isDirectory()) { @@ -322,7 +322,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v tOut.putArchiveEntry(te); try { - if (entry.getResource().isFile() && doCopy) { + if (entry.getResource().isFile() && !isLink) { fIn = entry.getInputStream(); Streams.copyFullyDontCloseOutput(fIn, tOut, "xAR"); From cea72cc10d77565d0ee1c5124f2021039993d534 Mon Sep 17 00:00:00 2001 From: Vaclav Haisman Date: Sat, 27 May 2023 20:43:56 +0200 Subject: [PATCH 4/4] TAR: Implement extraction and archiving of hardlinks. [4] Use NOFOLLOW_LINKS to get file attributes for hardlink. --- .../java/org/codehaus/plexus/archiver/tar/TarArchiver.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java b/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java index d46a16e5e..c6e2ce0a9 100644 --- a/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/tar/TarArchiver.java @@ -23,6 +23,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributeView; import java.util.HashMap; @@ -255,7 +256,7 @@ protected void tarFile(ArchiveEntry entry, TarArchiveOutputStream tOut, String v final Path file = fileResource.getFile().toPath(); if (Files.exists(file)) { final BasicFileAttributeView fileAttributeView = - Files.getFileAttributeView(file, BasicFileAttributeView.class); + Files.getFileAttributeView(file, BasicFileAttributeView.class, LinkOption.NOFOLLOW_LINKS); if (fileAttributeView != null) { final Object fileKey = fileAttributeView.readAttributes().fileKey();