Java IO file slows down with large sequential write
I wrote a disk erasing program designed to reliably reload free disk space. Everything works fine at first, but over time, the speed dropped dramatically. I have one 1TB drive that starts at around 120MB / s and then slowly drops to 70. At first I thought it was a disk, so I tested it on my RAID0 drives which were getting 160MB / s almost half a minute before slowly dropping to about 110. It seems like filling the cache doesn't seem like it because it takes about a minute to slow down completely.
First, the problem is how can I write data to the disk potentially, or is this really a normal function for hard drives in other languages?
Second, can I see the potential benefit of moving to NIO related to speed? Using ISAAC firmware is multithreaded, so the bottleneck is really just the write speed.
Finally, maybe it could be in my speed calculations. But I left it very simple, so I don't understand how this could be.
EDIT: (Some info)
Both are conventional magnetic drives. 1TB drive - WD 7200 rpm. The raid0 setting is two WD 10,000 rpm. Running Windows 7 Ultimate.
java version "1.8.0_45". Java (TM) SE Runtime Environment (build 1.8.0_45-b15). Java HotSpot (TM) 64-bit Server VM (build 25.45-b02, mixed mode).
You can test this example: (cleaner.DriveCleaner)
package cleaner;
import java.awt.EventQueue;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.security.SecureRandom;
import java.text.NumberFormat;
import java.text.ParseException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import javax.swing.DefaultComboBoxModel;
import javax.swing.JProgressBar;
import javax.swing.text.NumberFormatter;
/**
* @author Colby
*/
public class DriveCleaner extends javax.swing.JFrame {
public DriveCleaner() {
initComponents();
refreshDrives();
}
protected static boolean running = false;
private Thread worker;
private void refreshDrives() {
File[] roots = File.listRoots();
drives.setModel(new DefaultComboBoxModel(roots));
}
@SuppressWarnings("unchecked")
// <editor-fold defaultstate="collapsed" desc="Generated Code">
private void initComponents() {
buttonGroup1 = new javax.swing.ButtonGroup();
jLabel1 = new javax.swing.JLabel();
jSeparator1 = new javax.swing.JSeparator();
drives = new javax.swing.JComboBox();
normSelect = new javax.swing.JRadioButton();
randSelect = new javax.swing.JRadioButton();
jLabel2 = new javax.swing.JLabel();
jLabel3 = new javax.swing.JLabel();
passes = new javax.swing.JComboBox();
jLabel4 = new javax.swing.JLabel();
runButton = new javax.swing.JButton();
jSeparator2 = new javax.swing.JSeparator();
jSeparator3 = new javax.swing.JSeparator();
progress = new javax.swing.JProgressBar();
jMenuBar1 = new javax.swing.JMenuBar();
jMenu1 = new javax.swing.JMenu();
jMenuItem1 = new javax.swing.JMenuItem();
setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);
setTitle("DriveCleaner V1.0");
jLabel1.setFont(new java.awt.Font("Consolas", 2, 17)); // NOI18N
jLabel1.setText("DriveCleaner");
buttonGroup1.add(normSelect);
normSelect.setText("Simple Clean");
normSelect.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
normSelectActionPerformed(evt);
}
});
buttonGroup1.add(randSelect);
randSelect.setSelected(true);
randSelect.setText("ISAAC 256 Clean");
randSelect.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
randSelectActionPerformed(evt);
}
});
jLabel2.setText("Drive:");
jLabel3.setText("Passes:");
passes.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "1", "2", "4", "8", "16", "32", "64", "128" }));
passes.setSelectedIndex(2);
jLabel4.setText("Method:");
runButton.setText("Clean");
runButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
runButtonActionPerformed(evt);
}
});
jSeparator2.setOrientation(javax.swing.SwingConstants.VERTICAL);
progress.setString("");
progress.setStringPainted(true);
jMenu1.setText("File");
jMenuItem1.setAccelerator(javax.swing.KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_F5, 0));
jMenuItem1.setText("Refresh Drives");
jMenuItem1.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
jMenuItem1ActionPerformed(evt);
}
});
jMenu1.add(jMenuItem1);
jMenuBar1.add(jMenu1);
setJMenuBar(jMenuBar1);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
getContentPane().setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jSeparator1, javax.swing.GroupLayout.Alignment.TRAILING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jSeparator3)
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jLabel1)
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(drives, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(jLabel2))
.addGap(18, 18, 18)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(passes, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(jLabel3))
.addGap(18, 18, 18)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addComponent(randSelect)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(normSelect))
.addComponent(jLabel4))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 31, Short.MAX_VALUE)
.addComponent(jSeparator2, javax.swing.GroupLayout.PREFERRED_SIZE, 12, javax.swing.GroupLayout.PREFERRED_SIZE)))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(runButton, javax.swing.GroupLayout.PREFERRED_SIZE, 75, javax.swing.GroupLayout.PREFERRED_SIZE))
.addComponent(progress, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addContainerGap())
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(jLabel1)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 10, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(jLabel2)
.addComponent(jLabel3)
.addComponent(jLabel4))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(drives, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(passes, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(randSelect)
.addComponent(normSelect)))
.addComponent(jSeparator2)
.addComponent(runButton, javax.swing.GroupLayout.DEFAULT_SIZE, 43, Short.MAX_VALUE))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(jSeparator3, javax.swing.GroupLayout.PREFERRED_SIZE, 10, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(progress, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
);
pack();
}// </editor-fold>
private void runButtonActionPerformed(java.awt.event.ActionEvent evt) {
if (running) {
runButton.setText("Halting");
runButton.setEnabled(false);
new Thread() {
@Override
public void run() {
try {
running = false;
worker.join();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
EventQueue.invokeLater(new Runnable() {
@Override
public void run() {
progress.setString("");
progress.setValue(0);
runButton.setEnabled(true);
runButton.setText("Clean");
}
});
}
}.start();
} else {
running = true;
runButton.setText("Stop");
worker = new Thread(new Runnable() {
@Override
public void run() {
try {
Wipe.wipe(progress, (File) drives.getSelectedItem(), Integer.parseInt((String) passes.getSelectedItem()), useRandomData);
} catch (Exception e) {
e.printStackTrace();
}
}
});
worker.start();
}
}
private void jMenuItem1ActionPerformed(java.awt.event.ActionEvent evt) {
refreshDrives();
}
private void randSelectActionPerformed(java.awt.event.ActionEvent evt) {
useRandomData = true;
}
private void normSelectActionPerformed(java.awt.event.ActionEvent evt) {
useRandomData = false;
}
protected static boolean useRandomData = true;
public static void main(String args[]) {
/* Set the Nimbus look and feel */
//<editor-fold defaultstate="collapsed" desc=" Look and feel setting code (optional) ">
/* If Nimbus (introduced in Java SE 6) is not available, stay with the default look and feel.
* For details see http://download.oracle.com/javase/tutorial/uiswing/lookandfeel/plaf.html
*/
try {
for (javax.swing.UIManager.LookAndFeelInfo info : javax.swing.UIManager.getInstalledLookAndFeels()) {
if ("Nimbus".equals(info.getName())) {
javax.swing.UIManager.setLookAndFeel(info.getClassName());
break;
}
}
} catch (ClassNotFoundException ex) {
java.util.logging.Logger.getLogger(DriveCleaner.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (InstantiationException ex) {
java.util.logging.Logger.getLogger(DriveCleaner.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (IllegalAccessException ex) {
java.util.logging.Logger.getLogger(DriveCleaner.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (javax.swing.UnsupportedLookAndFeelException ex) {
java.util.logging.Logger.getLogger(DriveCleaner.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
}
//</editor-fold>
/* Create and display the form */
java.awt.EventQueue.invokeLater(new Runnable() {
public void run() {
new DriveCleaner().setVisible(true);
}
});
}
// Variables declaration - do not modify
private javax.swing.ButtonGroup buttonGroup1;
private javax.swing.JComboBox drives;
private javax.swing.JLabel jLabel1;
private javax.swing.JLabel jLabel2;
private javax.swing.JLabel jLabel3;
private javax.swing.JLabel jLabel4;
private javax.swing.JMenu jMenu1;
private javax.swing.JMenuBar jMenuBar1;
private javax.swing.JMenuItem jMenuItem1;
private javax.swing.JSeparator jSeparator1;
private javax.swing.JSeparator jSeparator2;
private javax.swing.JSeparator jSeparator3;
private javax.swing.JRadioButton normSelect;
private javax.swing.JComboBox passes;
private javax.swing.JProgressBar progress;
private javax.swing.JRadioButton randSelect;
private javax.swing.JButton runButton;
// End of variables declaration
}
class ISAAC {
public ISAAC(int ai[]) {
cryptArray = new int[256];
keySetArray = new int[256];
System.arraycopy(ai, 0, keySetArray, 0, ai.length);
initializeKeySet();
}
public int getNextKey() {
if (keyArrayIdx-- == 0) {
generateNextKeySet();
keyArrayIdx = 255;
}
return keySetArray[keyArrayIdx];
}
public void generateNextKeySet() {
cryptVar2 += ++cryptVar3;
for (int i = 0; i < 256; i++) {
int j = cryptArray[i];
if ((i & 3) == 0) {
cryptVar1 ^= cryptVar1 << 13;
} else if ((i & 3) == 1) {
cryptVar1 ^= cryptVar1 >>> 6;
} else if ((i & 3) == 2) {
cryptVar1 ^= cryptVar1 << 2;
} else if ((i & 3) == 3) {
cryptVar1 ^= cryptVar1 >>> 16;
}
cryptVar1 += cryptArray[i + 128 & 0xff];
int k;
cryptArray[i] = k = cryptArray[(j & 0x3fc) >> 2] + cryptVar1 + cryptVar2;
keySetArray[i] = cryptVar2 = cryptArray[(k >> 8 & 0x3fc) >> 2] + j;
}
}
public void initializeKeySet() {
int i1;
int j1;
int k1;
int l1;
int i2;
int j2;
int k2;
int l = i1 = j1 = k1 = l1 = i2 = j2 = k2 = 0x9e3779b9;
for (int i = 0; i < 4; i++) {
l ^= i1 << 11;
k1 += l;
i1 += j1;
i1 ^= j1 >>> 2;
l1 += i1;
j1 += k1;
j1 ^= k1 << 8;
i2 += j1;
k1 += l1;
k1 ^= l1 >>> 16;
j2 += k1;
l1 += i2;
l1 ^= i2 << 10;
k2 += l1;
i2 += j2;
i2 ^= j2 >>> 4;
l += i2;
j2 += k2;
j2 ^= k2 << 8;
i1 += j2;
k2 += l;
k2 ^= l >>> 9;
j1 += k2;
l += i1;
}
for (int j = 0; j < 256; j += 8) {
l += keySetArray[j];
i1 += keySetArray[j + 1];
j1 += keySetArray[j + 2];
k1 += keySetArray[j + 3];
l1 += keySetArray[j + 4];
i2 += keySetArray[j + 5];
j2 += keySetArray[j + 6];
k2 += keySetArray[j + 7];
l ^= i1 << 11;
k1 += l;
i1 += j1;
i1 ^= j1 >>> 2;
l1 += i1;
j1 += k1;
j1 ^= k1 << 8;
i2 += j1;
k1 += l1;
k1 ^= l1 >>> 16;
j2 += k1;
l1 += i2;
l1 ^= i2 << 10;
k2 += l1;
i2 += j2;
i2 ^= j2 >>> 4;
l += i2;
j2 += k2;
j2 ^= k2 << 8;
i1 += j2;
k2 += l;
k2 ^= l >>> 9;
j1 += k2;
l += i1;
cryptArray[j] = l;
cryptArray[j + 1] = i1;
cryptArray[j + 2] = j1;
cryptArray[j + 3] = k1;
cryptArray[j + 4] = l1;
cryptArray[j + 5] = i2;
cryptArray[j + 6] = j2;
cryptArray[j + 7] = k2;
}
for (int k = 0; k < 256; k += 8) {
l += cryptArray[k];
i1 += cryptArray[k + 1];
j1 += cryptArray[k + 2];
k1 += cryptArray[k + 3];
l1 += cryptArray[k + 4];
i2 += cryptArray[k + 5];
j2 += cryptArray[k + 6];
k2 += cryptArray[k + 7];
l ^= i1 << 11;
k1 += l;
i1 += j1;
i1 ^= j1 >>> 2;
l1 += i1;
j1 += k1;
j1 ^= k1 << 8;
i2 += j1;
k1 += l1;
k1 ^= l1 >>> 16;
j2 += k1;
l1 += i2;
l1 ^= i2 << 10;
k2 += l1;
i2 += j2;
i2 ^= j2 >>> 4;
l += i2;
j2 += k2;
j2 ^= k2 << 8;
i1 += j2;
k2 += l;
k2 ^= l >>> 9;
j1 += k2;
l += i1;
cryptArray[k] = l;
cryptArray[k + 1] = i1;
cryptArray[k + 2] = j1;
cryptArray[k + 3] = k1;
cryptArray[k + 4] = l1;
cryptArray[k + 5] = i2;
cryptArray[k + 6] = j2;
cryptArray[k + 7] = k2;
}
generateNextKeySet();
keyArrayIdx = 256;
}
public int keyArrayIdx;
public int keySetArray[];
public int cryptArray[];
public int cryptVar1;
public int cryptVar2;
public int cryptVar3;
}
class Wipe {
private static BlockingQueue<byte[]> buffers, randata;
private static class SecureDataCreator implements Runnable {
@Override
public void run() {
try {
SecureRandom seeder = new SecureRandom();
ISAAC rand = new ISAAC(new int[]{seeder.nextInt(), seeder.nextInt(), seeder.nextInt(), seeder.nextInt()});
do {
byte[] next = buffers.take();
for (int i = 0; i < next.length; i++) {
next[i] = (byte) rand.getNextKey();
}
randata.add(next);
} while (true);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
public static void wipe(JProgressBar prog, File drive, int numPasses, boolean random) throws IOException, InterruptedException, ParseException {
NumberFormat format = NumberFormat.getPercentInstance();
format.setMinimumFractionDigits(2);
NumberFormatter formatter = new NumberFormatter(format);
prog.setValue(0);
prog.setString("Opening file handle");
File wipeFile = new File(drive, "wipefile.dat");
wipeFile.deleteOnExit();
try (RandomAccessFile raf = new RandomAccessFile(wipeFile, "rw")) {
try {
while (wipeFile.getFreeSpace() > raf.length()) {
try {
raf.setLength(drive.getFreeSpace());
} catch (IOException e) {
raf.setLength(0);
}
}
int dataSize = 1024 * 1024 * 32;
int numCores = Runtime.getRuntime().availableProcessors();
boolean needWorkers = buffers == null && random;
if (needWorkers) {
for (int i = 0; i < numCores; i++) {
Thread worker = new Thread(new SecureDataCreator());
worker.setPriority(Thread.MIN_PRIORITY);
worker.start();
}
buffers = new ArrayBlockingQueue<>(numCores + 1);
randata = new ArrayBlockingQueue<>(numCores + 1);
for (int i = 0; i < numCores + 1; i++) {
buffers.add(new byte[dataSize]);
}
}
long startTime = System.nanoTime();
byte[] data = random ? null : new byte[dataSize];
for (int pass = 0; DriveCleaner.running && (pass < numPasses); pass++) {
raf.seek(0);
do {
long writeLen = dataSize;
if (raf.getFilePointer() + writeLen > raf.length()) {
writeLen = raf.length() - raf.getFilePointer();
}
if (random) {
data = randata.take();
}
raf.write(data, 0, (int) writeLen);
if (random) {
buffers.add(data);
}
double total = numPasses * raf.length();
double done = (pass * (raf.length() - 1)) + raf.getFilePointer();
float percent = (float) (done / total);
double elapsed = (System.nanoTime() - startTime) / (1000000D * 1000D);
float bytesPerSec = (float) (done / elapsed) / (1024F * 1024F);
prog.setValue((int) percent);
prog.setString("Cleaning " + drive + ". Pass #" + pass + "/" + numPasses + ". "
+ formatter.valueToString(new Float(percent))
+ " @" + (int) bytesPerSec + "mBps");
} while (raf.getFilePointer() < raf.length() && DriveCleaner.running);
prog.setString("Complete.");
}
System.out.println("done");
} finally {
raf.setLength(0);
}
} finally {
wipeFile.delete();
}
}
}
source to share
I have a few suggestions for isolating the source of the problem, as there is no indication yet whether this is from the JVM, OS, or hardware:
- The random number generator can end up with entropy. As a test, use zero instead of calling random.
- Measure the JVM garbage collection time. It is possible that the creation of many temporary array objects is causing GC pauses.
- Try running your Java program on Linux (for example, from a bootable USB or CD) to see if the same problem occurs.
- Try using a different JVM implementation (like OpenJDK and Oracle JDK) to see if the same problem occurs.
source to share