Coverage Report - org.mule.transport.sftp.SftpReceiverRequesterUtil
 
Classes in this File Line Coverage Branch Coverage Complexity
SftpReceiverRequesterUtil
0%
0/119
0%
0/78
0
 
 1  
 /*
 2  
  * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
 3  
  * The software in this package is published under the terms of the CPAL v1.0
 4  
  * license, a copy of which has been included with this distribution in the
 5  
  * LICENSE.txt file.
 6  
  */
 7  
 package org.mule.transport.sftp;
 8  
 
 9  
 import org.apache.commons.logging.Log;
 10  
 import org.apache.commons.logging.LogFactory;
 11  
 import org.mule.api.endpoint.ImmutableEndpoint;
 12  
 import org.mule.transport.sftp.notification.SftpNotifier;
 13  
 import org.mule.util.FileUtils;
 14  
 
 15  
 import java.io.File;
 16  
 import java.io.FilenameFilter;
 17  
 import java.io.IOException;
 18  
 import java.io.InputStream;
 19  
 import java.util.ArrayList;
 20  
 import java.util.List;
 21  
 
 22  
 /**
 23  
  * Contains reusable methods not directly related to usage of the jsch sftp library
 24  
  * (they can be found in the class SftpClient).
 25  
  * 
 26  
  * @author Magnus Larsson
 27  
  */
 28  
 public class SftpReceiverRequesterUtil
 29  
 {
 30  0
     private transient Log logger = LogFactory.getLog(getClass());
 31  
 
 32  
     private final SftpConnector connector;
 33  
     private final ImmutableEndpoint endpoint;
 34  
     private final FilenameFilter filenameFilter;
 35  
     private final SftpUtil sftpUtil;
 36  
 
 37  
     public SftpReceiverRequesterUtil(ImmutableEndpoint endpoint)
 38  0
     {
 39  0
         this.endpoint = endpoint;
 40  0
         this.connector = (SftpConnector) endpoint.getConnector();
 41  
 
 42  0
         sftpUtil = new SftpUtil(endpoint);
 43  
 
 44  0
         if (endpoint.getFilter() instanceof FilenameFilter)
 45  
         {
 46  0
             this.filenameFilter = (FilenameFilter) endpoint.getFilter();
 47  
         }
 48  
         else
 49  
         {
 50  0
             this.filenameFilter = null;
 51  
         }
 52  
 
 53  0
     }
 54  
 
 55  
     // Get files in directory configured on the endpoint
 56  
     public String[] getAvailableFiles(boolean onlyGetTheFirstOne) throws Exception
 57  
     {
 58  
         // This sftp client instance is only for checking available files. This
 59  
         // instance cannot be shared
 60  
         // with clients that retrieve files because of thread safety
 61  
 
 62  0
         if (logger.isDebugEnabled())
 63  
         {
 64  0
             logger.debug("Checking files at endpoint " + endpoint.getEndpointURI());
 65  
         }
 66  
 
 67  0
         SftpClient client = null;
 68  
 
 69  
         try
 70  
         {
 71  0
             client = connector.createSftpClient(endpoint);
 72  
 
 73  0
             long fileAge = connector.getFileAge();
 74  0
             boolean checkFileAge = connector.getCheckFileAge();
 75  
 
 76  
             // Override the value from the Endpoint?
 77  0
             if (endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE) != null)
 78  
             {
 79  0
                 checkFileAge = true;
 80  0
                 fileAge = Long.valueOf((String) endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE));
 81  
             }
 82  
 
 83  0
             logger.debug("fileAge : " + fileAge);
 84  
 
 85  
             // Get size check parameter
 86  0
             long sizeCheckDelayMs = sftpUtil.getSizeCheckWaitTime();
 87  
 
 88  0
             String[] files = client.listFiles();
 89  
 
 90  
             // Only return files that have completely been written and match
 91  
             // fileExtension
 92  0
             List<String> completedFiles = new ArrayList<String>(files.length);
 93  
 
 94  0
             for (String file : files)
 95  
             {
 96  
                 // Skip if no match.
 97  
                 // Note, Mule also uses this filter. We use the filter here because
 98  
                 // we don't want to
 99  
                 // run the below tests (checkFileAge, sizeCheckDelayMs etc) for files
 100  
                 // that Mule
 101  
                 // later should have ignored. Thus this is an "early" filter so that
 102  
                 // improves performance.
 103  0
                 if (filenameFilter != null && !filenameFilter.accept(null, file))
 104  
                 {
 105  0
                     continue;
 106  
                 }
 107  
 
 108  0
                 if (checkFileAge || sizeCheckDelayMs >= 0)
 109  
                 {
 110  
                     // See if the file is still growing (either by age or size),
 111  
                     // leave it alone if it is
 112  0
                     if (!hasChanged(file, client, fileAge, sizeCheckDelayMs))
 113  
                     {
 114  
                         // logger.debug("marking file [" + files[i] +
 115  
                         // "] as in transit.");
 116  
                         // client.rename(files[i], files[i] + ".transtit");
 117  
                         // completedFiles.add( files[i] + ".transtit" );
 118  0
                         completedFiles.add(file);
 119  0
                         if (onlyGetTheFirstOne)
 120  
                         {
 121  0
                             break;
 122  
                         }
 123  
                     }
 124  
                 }
 125  
                 else
 126  
                 {
 127  0
                     completedFiles.add(file);
 128  0
                     if (onlyGetTheFirstOne)
 129  
                     {
 130  0
                         break;
 131  
                     }
 132  
                 }
 133  
             }
 134  0
             return completedFiles.toArray(new String[completedFiles.size()]);
 135  
         }
 136  
         finally
 137  
         {
 138  0
             if (client != null)
 139  
             {
 140  0
                 connector.releaseClient(endpoint, client);
 141  
             }
 142  
         }
 143  
     }
 144  
 
 145  
     public InputStream retrieveFile(String fileName, SftpNotifier notifier) throws Exception
 146  
     {
 147  
         // Getting a new SFTP client dedicated to the SftpInputStream below
 148  0
         SftpClient client = connector.createSftpClient(endpoint, notifier);
 149  
 
 150  
         // Check usage of tmpSendingDir
 151  0
         String tmpSendingDir = sftpUtil.getTempDirInbound();
 152  0
         if (tmpSendingDir != null)
 153  
         {
 154  
             // Check usage of unique names of files during transfer
 155  0
             boolean addUniqueSuffix = sftpUtil.isUseTempFileTimestampSuffix();
 156  
 
 157  
             // TODO: is it possibly to move this to some kind of init method?
 158  0
             client.createSftpDirIfNotExists(endpoint, tmpSendingDir);
 159  0
             String tmpSendingFileName = tmpSendingDir + "/" + fileName;
 160  
 
 161  0
             if (addUniqueSuffix)
 162  
             {
 163  0
                 tmpSendingFileName = sftpUtil.createUniqueSuffix(tmpSendingFileName);
 164  
             }
 165  0
             String fullTmpSendingPath = endpoint.getEndpointURI().getPath() + "/" + tmpSendingFileName;
 166  
 
 167  0
             if (logger.isDebugEnabled())
 168  
             {
 169  0
                 logger.debug("Move " + fileName + " to " + fullTmpSendingPath);
 170  
             }
 171  0
             client.rename(fileName, fullTmpSendingPath);
 172  0
             fileName = tmpSendingFileName;
 173  0
             if (logger.isDebugEnabled())
 174  
             {
 175  0
                 logger.debug("Move done");
 176  
             }
 177  
         }
 178  
 
 179  
         // Archive functionality...
 180  0
         String archive = sftpUtil.getArchiveDir();
 181  
 
 182  
         // Retrieve the file stream
 183  0
         InputStream fileInputStream = client.retrieveFile(fileName);
 184  
 
 185  0
         if (!"".equals(archive))
 186  
         {
 187  0
             String archiveTmpReceivingDir = sftpUtil.getArchiveTempReceivingDir();
 188  0
             String archiveTmpSendingDir = sftpUtil.getArchiveTempSendingDir();
 189  
 
 190  0
             InputStream is = new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(),
 191  
                 endpoint);
 192  
 
 193  
             // TODO ML FIX. Refactor to util-class...
 194  0
             int idx = fileName.lastIndexOf('/');
 195  0
             String fileNamePart = fileName.substring(idx + 1);
 196  
 
 197  
             // don't use new File() directly, see MULE-1112
 198  0
             File archiveFile = FileUtils.newFile(archive, fileNamePart);
 199  
 
 200  
             // Should temp dirs be used when handling the archive file?
 201  0
             if ("".equals(archiveTmpReceivingDir) || "".equals(archiveTmpSendingDir))
 202  
             {
 203  0
                 return archiveFile(is, archiveFile);
 204  
             }
 205  
             else
 206  
             {
 207  0
                 return archiveFileUsingTempDirs(archive, archiveTmpReceivingDir, archiveTmpSendingDir, is,
 208  
                     fileNamePart, archiveFile);
 209  
             }
 210  
         }
 211  
 
 212  
         // This special InputStream closes the SftpClient when the stream is closed.
 213  
         // The stream will be materialized in a Message Dispatcher or Service
 214  
         // Component
 215  0
         return new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(), endpoint);
 216  
     }
 217  
 
 218  
     private InputStream archiveFileUsingTempDirs(String archive,
 219  
                                                  String archiveTmpReceivingDir,
 220  
                                                  String archiveTmpSendingDir,
 221  
                                                  InputStream is,
 222  
                                                  String fileNamePart,
 223  
                                                  File archiveFile) throws IOException
 224  
     {
 225  
 
 226  0
         File archiveTmpReceivingFolder = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir);
 227  0
         File archiveTmpReceivingFile = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir, fileNamePart);
 228  0
         if (!archiveTmpReceivingFolder.exists())
 229  
         {
 230  0
             if (logger.isInfoEnabled())
 231  
             {
 232  0
                 logger.info("Creates " + archiveTmpReceivingFolder.getAbsolutePath());
 233  
             }
 234  0
             if (!archiveTmpReceivingFolder.mkdirs())
 235  0
                 throw new IOException("Failed to create archive-tmp-receiving-folder: "
 236  
                                       + archiveTmpReceivingFolder);
 237  
         }
 238  
 
 239  0
         File archiveTmpSendingFolder = FileUtils.newFile(archive + '/' + archiveTmpSendingDir);
 240  0
         File archiveTmpSendingFile = FileUtils.newFile(archive + '/' + archiveTmpSendingDir, fileNamePart);
 241  0
         if (!archiveTmpSendingFolder.exists())
 242  
         {
 243  0
             if (logger.isInfoEnabled())
 244  
             {
 245  0
                 logger.info("Creates " + archiveTmpSendingFolder.getAbsolutePath());
 246  
             }
 247  0
             if (!archiveTmpSendingFolder.mkdirs())
 248  0
                 throw new IOException("Failed to create archive-tmp-sending-folder: "
 249  
                                       + archiveTmpSendingFolder);
 250  
         }
 251  
 
 252  0
         if (logger.isInfoEnabled())
 253  
         {
 254  0
             logger.info("Copy SftpInputStream to archiveTmpReceivingFile... "
 255  
                         + archiveTmpReceivingFile.getAbsolutePath());
 256  
         }
 257  0
         sftpUtil.copyStreamToFile(is, archiveTmpReceivingFile);
 258  
 
 259  
         // TODO. ML FIX. Should be performed before the sftp:delete - operation, i.e.
 260  
         // in the SftpInputStream in the operation above...
 261  0
         if (logger.isInfoEnabled())
 262  
         {
 263  0
             logger.info("Move archiveTmpReceivingFile (" + archiveTmpReceivingFile
 264  
                         + ") to archiveTmpSendingFile (" + archiveTmpSendingFile + ")...");
 265  
         }
 266  0
         FileUtils.moveFile(archiveTmpReceivingFile, archiveTmpSendingFile);
 267  
 
 268  0
         if (logger.isDebugEnabled())
 269  
         {
 270  0
             logger.debug("Return SftpFileArchiveInputStream for archiveTmpSendingFile ("
 271  
                          + archiveTmpSendingFile + ")...");
 272  
         }
 273  0
         return new SftpFileArchiveInputStream(archiveTmpSendingFile, archiveFile);
 274  
     }
 275  
 
 276  
     private InputStream archiveFile(InputStream is, File archiveFile) throws IOException
 277  
     {
 278  0
         File archiveFolder = FileUtils.newFile(archiveFile.getParentFile().getPath());
 279  0
         if (!archiveFolder.exists())
 280  
         {
 281  0
             if (logger.isInfoEnabled())
 282  
             {
 283  0
                 logger.info("Creates " + archiveFolder.getAbsolutePath());
 284  
             }
 285  0
             if (!archiveFolder.mkdirs())
 286  0
                 throw new IOException("Failed to create archive-folder: " + archiveFolder);
 287  
         }
 288  
 
 289  0
         if (logger.isInfoEnabled())
 290  
         {
 291  0
             logger.info("Copy SftpInputStream to archiveFile... " + archiveFile.getAbsolutePath());
 292  
         }
 293  0
         sftpUtil.copyStreamToFile(is, archiveFile);
 294  
 
 295  0
         if (logger.isDebugEnabled())
 296  
         {
 297  0
             logger.debug("*** Return SftpFileArchiveInputStream for archiveFile...");
 298  
         }
 299  0
         return new SftpFileArchiveInputStream(archiveFile);
 300  
     }
 301  
 
 302  
     /**
 303  
      * Checks if the file has been changed.
 304  
      * <p/>
 305  
      * Note! This assumes that the time on both servers are synchronized!
 306  
      * 
 307  
      * @param fileName The file to check
 308  
      * @param client instance of StftClient
 309  
      * @param fileAge How old the file should be to be considered "old" and not
 310  
      *            changed
 311  
      * @param sizeCheckDelayMs Wait time (in ms) between size-checks to determine if
 312  
      *            a file is ready to be processed.
 313  
      * @return true if the file has changed
 314  
      * @throws Exception Error
 315  
      */
 316  
     private boolean hasChanged(String fileName, SftpClient client, long fileAge, long sizeCheckDelayMs)
 317  
         throws Exception
 318  
     {
 319  
         // Perform fileAge test if configured
 320  
         // Note that for this to work it is required that the system clock on the
 321  
         // mule server
 322  
         // is synchronized with the system clock on the sftp server
 323  0
         if (fileAge > 0)
 324  
         {
 325  0
             long lastModifiedTime = client.getLastModifiedTime(fileName);
 326  
             // TODO Can we get the current time from the other server?
 327  0
             long now = System.currentTimeMillis();
 328  0
             long diff = now - lastModifiedTime;
 329  
             // If the diff is negative it's a sign that the time on the test server
 330  
             // and the ftps-server is not synchronized
 331  0
             if (diff < fileAge)
 332  
             {
 333  0
                 if (logger.isDebugEnabled())
 334  
                 {
 335  0
                     logger.debug("The file has not aged enough yet, will return nothing for: " + fileName
 336  
                                  + ". The file must be " + (fileAge - diff) + "ms older, was " + diff);
 337  
                 }
 338  0
                 return true;
 339  
             }
 340  0
             if (logger.isDebugEnabled())
 341  
             {
 342  0
                 logger.debug("The file " + fileName + " has aged enough. Was " + diff);
 343  
             }
 344  
         }
 345  
 
 346  
         // Perform a size check with a short configurable latencey between the
 347  
         // size-calls
 348  
         // Take consecutive file size snapshots to determine if file is still being
 349  
         // written
 350  0
         if (sizeCheckDelayMs > 0)
 351  
         {
 352  0
             logger.info("Perform size check with a delay of: " + sizeCheckDelayMs + " ms.");
 353  0
             long fileSize1 = client.getSize(fileName);
 354  0
             Thread.sleep(sizeCheckDelayMs);
 355  0
             long fileSize2 = client.getSize(fileName);
 356  
 
 357  0
             if (fileSize1 == fileSize2)
 358  
             {
 359  0
                 logger.info("File is stable (not growing), ready for retrieval: " + fileName);
 360  
             }
 361  
             else
 362  
             {
 363  0
                 logger.info("File is growing, deferring retrieval: " + fileName);
 364  0
                 return true;
 365  
             }
 366  
         }
 367  
 
 368  
         // None of file-change tests faile so we can retrieve this file
 369  0
         return false;
 370  
     }
 371  
 }