I have the following ZipCompressor Java class:
package lib.util.compressors.zip;
import java.util.Enumeration;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import lib.util.compressors.Compressor;
import lib.util.compressors.CompressorException;
import lib.util.compressors.Entry;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import java.util.zip.ZipFile;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipInputStream;
/**
* The ZipCompressor class supplies a simple way of writing zip files. The default zip level is set
* to maximum compression.
*
* @author REDACTED
*/
public class ZipCompressor extends Compressor {
private int zipLevel = 9;
/**
* @see lib.util.compressors.Compressor#compress
*/
public void compress(String fileName, String dirName) throws CompressorException {
File zipFile = new File(fileName);
try(FileOutputStream fos = new FileOutputStream(fileName);
ZipOutputStream zos = new ZipOutputStream(fos);
) {
// Open the file and set the compression level
zos.setLevel(zipLevel);
// Zip the directory
File dir = new File(dirName);
compress(zipFile, zos, dir, dir);
} catch (FileNotFoundException fnfe) {
throw new CompressorException(fnfe);
} catch (IOException ioe) {
throw new CompressorException(ioe);
}
}
/**
* @see lib.util.compressors.Compressor#uncompress
*/
public void uncompress(String fileName, String dirName) throws CompressorException {
try(// Open the zipfile
ZipFile zipFile = new ZipFile(fileName);
// Open streams
FileInputStream fis = new FileInputStream(zipFile.getName());
BufferedInputStream bis = new BufferedInputStream(fis);
ZipInputStream zis = new ZipInputStream(bis);) {
// Get the size of each entry
Map<String, Integer> zipEntrySizes = new HashMap<String, Integer> ();
Enumeration<? extends ZipEntry> e = zipFile.entries();
while (e.hasMoreElements()) {
ZipEntry zipEntry = (ZipEntry) e.nextElement();
zipEntrySizes.put(zipEntry.getName(), Integer.valueOf((int) zipEntry.getSize()));
}
// Start reading zipentries
ZipEntry zipEntry = null;
while ((zipEntry = zis.getNextEntry()) != null) {
// Zipentry is a file
if (!zipEntry.isDirectory()) {
// Get the size
int size = (int) zipEntry.getSize();
if (size == -1) {
size = ((Integer) zipEntrySizes.get(zipEntry.getName())).intValue();
}
// Get the content
byte[] buffer = new byte[size];
int bytesInBuffer = 0;
int bytesRead = 0;
while (((int) size - bytesInBuffer) > 0) {
bytesRead = zis.read(buffer, bytesInBuffer, size - bytesInBuffer);
if (bytesRead == -1) {
break;
}
bytesInBuffer += bytesRead;
}
String zipEntryName = zipEntry.getName();
// replace all "\" with "/"
zipEntryName = zipEntryName.replace('\\', '/');
// Get the full path name
File file = new File(dirName, zipEntryName);
// Create the parent directory
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
// Save file
try (FileOutputStream fos = new FileOutputStream(file.getPath());) {
fos.write(buffer, 0, bytesInBuffer);
}
if (zipEntry.getTime() >= 0L) {
// Set modification date to the date in the zipEntry
file.setLastModified(zipEntry.getTime());
}
}
// Zipentry is a directory
else {
String zipEntryName = zipEntry.getName();
// replace all "\" with "/"
zipEntryName = zipEntryName.replace('\\', '/');
// Create the directory
File dir = new File(dirName, zipEntryName);
dir.setLastModified(zipEntry.getTime());
if (!dir.exists()) {
dir.mkdirs();
}
}
}
} catch (IOException ioe) {
throw new CompressorException(ioe);
}
}
/**
* @see lib.util.compressors.Compressor#getEntries
*/
public List<Entry> getEntries(String fileName, boolean calculateCrc) throws CompressorException {
// List to return all entries
List<Entry> entries = new ArrayList<Entry>();
try( // Open the zipfile
ZipFile zipFile = new ZipFile(fileName);) {
// Get the size of each entry
Enumeration<? extends ZipEntry> e = zipFile.entries();
while (e.hasMoreElements()) {
ZipEntry zipEntry = (ZipEntry) e.nextElement();
Entry entry = new Entry();
entry.setName(zipEntry.getName());
if (calculateCrc) {
entry.setCrc(zipEntry.getCrc());
} else {
entry.setCrc(-1);
}
entry.setDirectory(zipEntry.isDirectory());
entry.setTime(zipEntry.getTime());
entry.setSize(zipEntry.getSize());
entries.add(entry);
}
// Sort entries by ascending name
sortEntries(entries);
// Return entries
return entries;
} catch (IOException ioe) {
throw new CompressorException(ioe);
}
}
/**
* Sets the zip level (1..9).
*
* @param zipLevel the desired zip level.
* @throws CompressorException
*/
public void setZipLevel(int zipLevel) throws CompressorException {
if ((zipLevel < 1) || (zipLevel > 9)) {
throw new CompressorException("Zip level " + zipLevel + " out of range (0 ... 9)");
}
this.zipLevel = zipLevel;
}
/**
* Add a new entry to the zip file.
*
* @param zos the output stream filter for writing files in the ZIP file format
* @param name the name of the entry.
* @param lastModified the modification date
* @param buffer an array of bytes
* @throws IOException
*/
private void addEntry(ZipOutputStream zos, String name, long lastModified, byte[] buffer) throws IOException {
ZipEntry zipEntry = new ZipEntry(name);
if (buffer != null) {
zipEntry.setSize(buffer.length);
}
zipEntry.setTime(lastModified);
zos.putNextEntry(zipEntry);
if (buffer != null) {
zos.write(buffer);
}
zos.closeEntry();
}
/**
* Zip the files of the given directory.
*
* @param zipFile the File which is used to store the compressed data
* @param zos the output stream filter for writing files in the ZIP file format
* @param dir the directory to zip
* @param relativeDir the name of each zip entry will be relative to this directory
* @throws FileNotFoundException
* @throws IOException
*/
private void compress(File zipFile, ZipOutputStream zos, File dir, File relativeDir) throws FileNotFoundException, IOException {
// Create an array of File objects
File[] fileList = dir.listFiles();
// Directory is not empty
if (fileList.length != 0) {
// Loop through File array
for (int i = 0; i < fileList.length; i++) {
// The zipfile itself may not be added
if (!zipFile.equals(fileList[i])) {
// Directory
if (fileList[i].isDirectory()) {
compress(zipFile, zos, fileList[i], relativeDir);
}
// File
else {
byte[] buffer = getFileContents(fileList[i]);
if (buffer != null) {
// Get the path names
String filePath = fileList[i].getPath();
String relativeDirPath = relativeDir.getPath();
// Convert the absolute path name to a relative path name
if (filePath.startsWith(relativeDirPath)) {
filePath = filePath.substring(relativeDirPath.length());
if (filePath.startsWith("/") || filePath.startsWith("\\")) {
if (filePath.length() == 1) {
filePath = "";
} else {
filePath = filePath.substring(1);
}
}
}
// Add the entry
addEntry(zos, filePath, fileList[i].lastModified(), buffer);
}
}
}
}
}
// Directory is empty
else {
// Get the path names
String filePath = dir.getPath();
String relativeDirPath = relativeDir.getPath();
// Convert the absolute path name to a relative path name
if (filePath.startsWith(relativeDirPath)) {
filePath = filePath.substring(relativeDirPath.length());
if (filePath.startsWith("/") || filePath.startsWith("\\")) {
if (filePath.length() == 1) {
filePath = "";
} else {
filePath = filePath.substring(1);
}
}
}
// Add the entry
if (!filePath.endsWith("\\") && !filePath.endsWith("/")) {
addEntry(zos, filePath + "/", dir.lastModified(), null);
}
else {
addEntry(zos, filePath, dir.lastModified(), null);
}
}
}
/**
* Read the contents of a file for zipping.
*
* @param file the File to read
* @return an array of bytes
* @throws FileNotFoundException
* @throws IOException
*/
private byte[] getFileContents(File file) throws FileNotFoundException, IOException {
try (FileInputStream fis = new FileInputStream(file);) {
long len = file.length();
byte[] buffer = new byte[(int) len];
fis.read(buffer);
return buffer;
}
}
}
The problem I have is that sometimes the compress() method produces an invalid archive with an incorrectly compressed first entry, that can't be opened by Windows or Java 11 and shows errors when opened with 7zip.
In Java, the error code I get is:
lib.util.compressors.CompressorException: java.util.zip.ZipException: invalid END header (bad central directory offset)
at lib.util.compressors.zip.ZipCompressor.uncompress(ZipCompressor.java:151)
at phases.core.deploy.UnCompressBuildResultPhase.execute(UnCompressBuildResultPhase.java:61)
at Proxy6f3235bf_85bf_485a_9577_a60fb3dad49c.execute(Unknown Source)
at phases.impl.DefaultPhaseExecutionImpl.execute(DefaultPhaseExecutionImpl.java:152)
at daemons.agent.deployer.DeployerThread.run(DeployerThread.java:144)
Caused by: java.util.zip.ZipException: invalid END header (bad central directory offset)
at java.base/java.util.zip.ZipFile$Source.zerror(ZipFile.java:1607)
at java.base/java.util.zip.ZipFile$Source.initCEN(ZipFile.java:1519)
at java.base/java.util.zip.ZipFile$Source.<init>(ZipFile.java:1308)
at java.base/java.util.zip.ZipFile$Source.get(ZipFile.java:1271)
at java.base/java.util.zip.ZipFile$CleanableResource.<init>(ZipFile.java:733)
at java.base/java.util.zip.ZipFile$CleanableResource.get(ZipFile.java:850)
at java.base/java.util.zip.ZipFile.<init>(ZipFile.java:248)
at java.base/java.util.zip.ZipFile.<init>(ZipFile.java:177)
at java.base/java.util.zip.ZipFile.<init>(ZipFile.java:148)
at lib.util.compressors.zip.ZipCompressor.uncompress(ZipCompressor.java:72)
... 4 more
The 7zip errors it shows when testing the archive are:
C:\Users\REDACTED\Downloads\b5_CONTBUILD_B-6-0-0_win.zip
Unavailable start of archive
Warning
The archive is open with offset
Unavailable data : ant\bin\ant
And the properties of the archive as 7zip gives them are:
Path: C:\Users\REDACTED\Downloads\b5_CONTBUILD_B-6-0-0_win.zip
Type: zip
Open WARNING:: Cannot open the file as expected archive type
Error Type: zip
Errors: Unavailable start of archive
Offset: -1024
Physical Size: 722 598 855
This is a transient error, in that it doesn't happen every time I try to create the same archive. However, when it happened the previous time, the error in the 7zip properties was slightly different:
Path: C:\Users\REDACTED\Downloads\b2355_CONTBUILD_win (1).zip
Type: zip
Open WARNING:: Cannot open the file as expected archive type
Error Type: zip
Errors: Unavailable start of archive
Offset: -256
Physical Size: 712 638 486
Note that both of these archives were generated using a Java 11.0.1.9 JVM.
Usually rerunning the script fixes it, but the entire script can take up to an hour to complete, of which the archive creation is only a small part.
I found some information on Java util zip creates "corrupt" zip files that says I should call zos.finish() and zos.flush() at the end of the try-with-resources, but if I'm not mistaken, zos.close() already calls those methods.
Edit: As I said above, I already found that question recommended as duplicate, I even linked it above, and to the best of my knowledge the code I currently use already does what that answer recommends, because the try-with-resources already automatically closes the ZipOutputStream using close(), and close() calls finish(). I didn't check the full stack call to verify, but I think .flush() is also called somewhere in that close stack.
After doing some additional testing, I have found that in fact the cause is NOT the Zip compression step. This code runs as part of a longer series of steps of which the compression step is one, and the step directly afterwards copies the created archive to another directory on a different drive. I have now verified that the archive before copying does work fine, but the archive AFTER copying doesn't. Hence the ZipCompressor does not seem related to this problem.