Using multiple threads to download files using JSch

Question

I am trying to download multiple files from a remote server using multiple threads. However when I am using multiple threads I get

java.lang.IOException : Pipes closed.

The same code works fine when i am using exactly one thread.

Is it not possible to download multiple files simultaneously from the same remote server using JSch?

SftpTest.java

public class SftpTest {

private static List<SftpAccessor> accessorList = new ArrayList<SftpAccessor>();
private static List<Payload> files = new ArrayList<Payload>();
private static ExecutorService writerThreadPool = Executors.newFixedThreadPool(10);

public static void main(String args[]) throws JSchException, InterruptedException {
    SSH srcServer = new SSH();
    srcServer.setHostname("10.22.65.140");
    srcServer.setKey("D:\\jars\\dmwvmcol01.ec2user.pem");
    srcServer.setPort("22");
    srcServer.setUsername("ec2-user");

    SftpAccessor acc = new SftpAccessor(srcServer);
    accessorList.add(acc);

    files.addAll(acc.ls("/data/test/src", false, "*", 1));

    for (Payload file : files) {
        writerThreadPool.submit(new LocalWriterThread(file));
    }

    writerThreadPool.shutdown();
    writerThreadPool.awaitTermination(20, TimeUnit.MINUTES);
}}

LocalWriterThread.java

public class LocalWriterThread implements Callable<String> {

private Payload payload;

public LocalWriterThread(Payload payload) {
    this.payload = payload;
}

public String call() throws Exception {
    String dest = "D:\\output\\jobsystemnew";
    System.out.println(payload.getFilename());
    File file = new File(dest + "/" + payload.getFilename());
    file.createNewFile();
    FileOutputStream fos = new FileOutputStream(file);
    copy(payload.getInputStream(), fos);
    payload.setInputStream(null);
    return "";
}

private void copy(InputStream is, OutputStream os) throws IOException {

    BufferedInputStream bis = null;
    BufferedOutputStream bos = null;
    try {
        System.out.println("Started Copying file : " + payload.getFilename());

        bis = new BufferedInputStream(is);
        bos = new BufferedOutputStream(os);

        byte[] buffer = new byte[1024];

        while (bis.read(buffer) != -1) {
            bos.write(buffer);
        }

        bos.flush();
        System.out.println("Finished Copying file : " + payload.getFilename());
    }
    finally {
        System.out.println("Closing input stream for " + payload.getFilename());
        bis.close();
        bos.close();
        /*
         * try { if (is != null) { is.close(); } } catch (Exception e) { System.out.println(" " + e + " : " + e.getMessage() + " unable to close input stream : " + payload.getFilename());
         * e.printStackTrace(); }
         */
        /*
         * try { if (os != null) { os.close(); } } catch (Exception e) { System.out.println(" " + e + " : " + e.getMessage() + " unable to close output stream : " + payload.getFilename());
         * e.printStackTrace(); }
         */
    }

}}

SSH.java

public class SSH {
String hostname;
String port;
String username;
String key;

public String getHostname() {
    return hostname;
}
public void setHostname(String hostname) {
    this.hostname = hostname;
}
public String getPort() {
    return port;
}
public void setPort(String port) {
    this.port = port;
}
public String getUsername() {
    return username;
}
public void setUsername(String username) {
    this.username = username;
}
public String getKey() {
    return key;
}
public void setKey(String key) {
    this.key = key;
} }

SftpAccessor.java

public class SftpAccessor {

private JSch jsch = new JSch();
private ChannelSftp channelSftp = null;
private Session session = null;

String errorMsg = null;

public SftpAccessor(SSH ssh) throws JSchException {

    jsch.addIdentity(ssh.getKey());
    session = jsch.getSession(ssh.getUsername(), ssh.getHostname(), Integer.parseInt(ssh.getPort()));
    session.setHostKeyAlias(ssh.getKey());

    java.util.Properties config = new java.util.Properties();
    config.put("StrictHostKeyChecking", "no");
    session.setConfig(config);
    session.connect();
    channelSftp = (ChannelSftp) session.openChannel("sftp");
    channelSftp.connect();

}

public List<Payload> ls(String directory, boolean recursive, String filemask, int delayMinutes) {

    List<Payload> files = new ArrayList<Payload>();

    try {

        channelSftp.cd(directory);

        @SuppressWarnings("unchecked")
        Vector<ChannelSftp.LsEntry> entries = channelSftp.ls(filemask);

        for (ChannelSftp.LsEntry entry : entries) {
            try {
                if (entry.getAttrs().isDir()) {
                    if (recursive) {
                        List<Payload> filesToAdd = ls(directory + "/" + entry.getFilename(), recursive, filemask, delayMinutes);
                        files.addAll(filesToAdd);
                    }
                }
                else {

                    Date lastmodified = new Date(entry.getAttrs().getMTime() * 1000L);
                    Date currdate = new Date(new Date().getTime() - (delayMinutes * 60 * 1000L));

                    if (lastmodified.before(currdate)) {
                        String filename = entry.getFilename();
                        entry.getAttrs().getMTime();

                        Payload file = new Payload();
                        System.out.println("Getting input Stream for " + directory + "/" + filename);
                        file.setInputStream(channelSftp.get(directory + "/" + filename));
                        file.setFilename(filename);
                        file.setLastModified(lastmodified);
                        files.add(file);
                    }

                }
            }
            catch (Exception e) {
                e.printStackTrace();
            }
        }

    }
    catch (SftpException e) {

    }       

    return files;
}}

Payload.java

public class Payload {

private String filename;
private InputStream inputStream;
private Date lastModified;

public String getFilename() {
    return filename;
}

public void setFilename(String filename) {
    this.filename = filename;
}

public Date getLastModified() {
    return lastModified;
}

public void setLastModified(Date lastModified) {
    this.lastModified = lastModified;
}

public InputStream getInputStream() {
    return inputStream;
}

public void setInputStream(InputStream inputStream) {
    this.inputStream = inputStream;
}}

Answer 1

JSch is not thread-safe. Even if it were, there's hardly any performance advantage in using parallel downloads over a single SSH session. It would be as slow as serial downloads. Moreover, you may hit server-side limit of concurrent file handles.

You should open a separate session for each thread/download.