Logo Questions Linux Laravel Mysql Ubuntu Git Menu
 

Reading file from SFTP server using Node.js and SSH2

I have a pretty strange problem working with read streams in Node.js. I'm using SSH2 to create a sftp connection between me and a sftp server. I then try to create a read stream from the sftp stream. From the read stream's emitted 'data' event I append the data to an array. When the read stream's 'close' event occurs I make a call to Buffer.concat to create concat all the chunks of data I have retrieved into one buffer. This is the same technique described in other questions asked here at stack overflow. For example here. However, I'm not able to use the data I retrieve. It seems like the buffer is 32 bytes less in size then the file that I'm trying to retrieve (from counting the length of the data retrieved). Could this have something to do with my SFTP connection? Or how I create my read stream?

If it matters, the file is of type zip. When I'm trying to unzip the file (in node.js and manually) after reading it to buffer it doesn't work.

After investigating I have found out that:

  1. When I use readdir on the file, the size of the file is correct.
  2. Using FTP (JSFTP) against my development FTP server works just fine using the same technique above.

Any advise is appreciated!

Here is my code:

        var Client = require('ssh2').Client;
        var m_ssh2Credentials = {
           host: config.ftpHostName,
           port: config.ftpPort,
           username: config.ftpUser,
           password: config.ftpPassword,
           readyTimeout: 20000,
           algorithms: { cipher: ["3des-cbc", "aes256-cbc", "aes192-cbc","aes128-cbc"]}
        };
        ...
        var conn = new Client();
        var dataLength = 0;
        conn.on('ready', function() {
            conn.sftp(function(err, sftp) {
                if (err) {
                    writeToErrorLog("downloadFile(): Failed to open SFTP connection.");
                } else {
                    writeToLog("downloadFile(): Opened SFTP connection.");
                }

                var streamErr = "";
                var dataLength = 0;
                var stream = sftp.createReadStream(config.ftpPath + "/" + m_fileName)
                stream.on('data', function(d){
                    data.push(d);
                    dataLength += d.length;
                });
                .on('error', function(e){
                    streamErr = e;
                })
                .on('close', function(){
                    if(streamErr) {
                        writeToErrorLog("downloadFile(): Error retrieving the file: " + streamErr);
                    } else {
                        writeToLog("downloadFile(): No error using read stream.");
                        m_fileBuffer = Buffer.concat(data, dataLength);
                        writeToLog("Data length: " + dataLength);

                        writeToLog("downloadFile(): File saved to buffer.");
                    }
                    conn.end();
                });
            })
        })
        .on('error', function(err) {
            writeToErrorLog("downloadFile(): Error connecting: " + err);
        }).connect(m_ssh2Credentials);
like image 335
Adam Ramberg Avatar asked May 22 '16 13:05

Adam Ramberg


People also ask

Does SFTP use SSH2?

Subsystem sftp /usr/libexec/openssh/sftp-serverSftp uses SSH2 in data connections, so the file transport is as secure as possible.

What is SSH2 SFTP?

The SSH2 File Transfer Protocol (SFTP), supported by Cerberus FTP Server Professional and higher, is a network protocol that provides secure and reliable file access, file transfer, and file management functionality.


1 Answers

So after a lot of investigation I finally realized that there was something wrong with the last bits of data that was transmitted in the 'data' event. From my understanding it seems to be a bug in the implementation of the read stream. I was able to get around this problem by using the more simplistic functions (open, fstat, read) in the SSH2 library. This solution works for me. Wanted to share the solution if someone else comes across the same problem.

Working code:

sftp.open(config.ftpPath + "/" + m_fileName, "r", function(err, fd) {
sftp.fstat(fd, function(err, stats) {
    var bufferSize = stats.size,
        chunkSize = 16384,
        buffer = new Buffer(bufferSize),
        bytesRead = 0,
        errorOccured = false;

    while (bytesRead < bufferSize && !errorOccured) {
        if ((bytesRead + chunkSize) > bufferSize) {
            chunkSize = (bufferSize - bytesRead);
        }
        sftp.read(fd, buffer, bytesRead, chunkSize, bytesRead, callbackFunc);
        bytesRead += chunkSize;
    }

    var totalBytesRead = 0;
    function callbackFunc(err, bytesRead, buf, pos) {
        if(err) {
            writeToErrorLog("downloadFile(): Error retrieving the file.");
            errorOccured = true;
            sftp.close(fd);
        }
        totalBytesRead += bytesRead;
        data.push(buf);
        if(totalBytesRead === bufferSize) {
            m_fileBuffer = Buffer.concat(data);
            writeToLog("downloadFile(): File saved to buffer.");
            sftp.close(fd);
            m_eventEmitter.emit(' downloadFile_Completed ');
        }
    }
})
});   
like image 139
Adam Ramberg Avatar answered Sep 18 '22 19:09

Adam Ramberg