View Javadoc
1   /*
2    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
3    * The software in this package is published under the terms of the CPAL v1.0
4    * license, a copy of which has been included with this distribution in the
5    * LICENSE.txt file.
6    */
7   package org.mule.transport.sftp;
8   
9   import org.apache.commons.logging.Log;
10  import org.apache.commons.logging.LogFactory;
11  import org.mule.api.endpoint.ImmutableEndpoint;
12  import org.mule.transport.sftp.notification.SftpNotifier;
13  import org.mule.util.FileUtils;
14  
15  import java.io.File;
16  import java.io.FilenameFilter;
17  import java.io.IOException;
18  import java.io.InputStream;
19  import java.util.ArrayList;
20  import java.util.List;
21  
22  /**
23   * Contains reusable methods not directly related to usage of the jsch sftp library
24   * (they can be found in the class SftpClient).
25   * 
26   * @author Magnus Larsson
27   */
28  public class SftpReceiverRequesterUtil
29  {
30      private transient Log logger = LogFactory.getLog(getClass());
31  
32      private final SftpConnector connector;
33      private final ImmutableEndpoint endpoint;
34      private final FilenameFilter filenameFilter;
35      private final SftpUtil sftpUtil;
36  
37      public SftpReceiverRequesterUtil(ImmutableEndpoint endpoint)
38      {
39          this.endpoint = endpoint;
40          this.connector = (SftpConnector) endpoint.getConnector();
41  
42          sftpUtil = new SftpUtil(endpoint);
43  
44          if (endpoint.getFilter() instanceof FilenameFilter)
45          {
46              this.filenameFilter = (FilenameFilter) endpoint.getFilter();
47          }
48          else
49          {
50              this.filenameFilter = null;
51          }
52  
53      }
54  
55      // Get files in directory configured on the endpoint
56      public String[] getAvailableFiles(boolean onlyGetTheFirstOne) throws Exception
57      {
58          // This sftp client instance is only for checking available files. This
59          // instance cannot be shared
60          // with clients that retrieve files because of thread safety
61  
62          if (logger.isDebugEnabled())
63          {
64              logger.debug("Checking files at endpoint " + endpoint.getEndpointURI());
65          }
66  
67          SftpClient client = null;
68  
69          try
70          {
71              client = connector.createSftpClient(endpoint);
72  
73              long fileAge = connector.getFileAge();
74              boolean checkFileAge = connector.getCheckFileAge();
75  
76              // Override the value from the Endpoint?
77              if (endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE) != null)
78              {
79                  checkFileAge = true;
80                  fileAge = Long.valueOf((String) endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE));
81              }
82  
83              logger.debug("fileAge : " + fileAge);
84  
85              // Get size check parameter
86              long sizeCheckDelayMs = sftpUtil.getSizeCheckWaitTime();
87  
88              String[] files = client.listFiles();
89  
90              // Only return files that have completely been written and match
91              // fileExtension
92              List<String> completedFiles = new ArrayList<String>(files.length);
93  
94              for (String file : files)
95              {
96                  // Skip if no match.
97                  // Note, Mule also uses this filter. We use the filter here because
98                  // we don't want to
99                  // run the below tests (checkFileAge, sizeCheckDelayMs etc) for files
100                 // that Mule
101                 // later should have ignored. Thus this is an "early" filter so that
102                 // improves performance.
103                 if (filenameFilter != null && !filenameFilter.accept(null, file))
104                 {
105                     continue;
106                 }
107 
108                 if (checkFileAge || sizeCheckDelayMs >= 0)
109                 {
110                     // See if the file is still growing (either by age or size),
111                     // leave it alone if it is
112                     if (!hasChanged(file, client, fileAge, sizeCheckDelayMs))
113                     {
114                         // logger.debug("marking file [" + files[i] +
115                         // "] as in transit.");
116                         // client.rename(files[i], files[i] + ".transtit");
117                         // completedFiles.add( files[i] + ".transtit" );
118                         completedFiles.add(file);
119                         if (onlyGetTheFirstOne)
120                         {
121                             break;
122                         }
123                     }
124                 }
125                 else
126                 {
127                     completedFiles.add(file);
128                     if (onlyGetTheFirstOne)
129                     {
130                         break;
131                     }
132                 }
133             }
134             return completedFiles.toArray(new String[completedFiles.size()]);
135         }
136         finally
137         {
138             if (client != null)
139             {
140                 connector.releaseClient(endpoint, client);
141             }
142         }
143     }
144 
145     public InputStream retrieveFile(String fileName, SftpNotifier notifier) throws Exception
146     {
147         // Getting a new SFTP client dedicated to the SftpInputStream below
148         SftpClient client = connector.createSftpClient(endpoint, notifier);
149 
150         // Check usage of tmpSendingDir
151         String tmpSendingDir = sftpUtil.getTempDirInbound();
152         if (tmpSendingDir != null)
153         {
154             // Check usage of unique names of files during transfer
155             boolean addUniqueSuffix = sftpUtil.isUseTempFileTimestampSuffix();
156 
157             // TODO: is it possibly to move this to some kind of init method?
158             client.createSftpDirIfNotExists(endpoint, tmpSendingDir);
159             String tmpSendingFileName = tmpSendingDir + "/" + fileName;
160 
161             if (addUniqueSuffix)
162             {
163                 tmpSendingFileName = sftpUtil.createUniqueSuffix(tmpSendingFileName);
164             }
165             String fullTmpSendingPath = endpoint.getEndpointURI().getPath() + "/" + tmpSendingFileName;
166 
167             if (logger.isDebugEnabled())
168             {
169                 logger.debug("Move " + fileName + " to " + fullTmpSendingPath);
170             }
171             client.rename(fileName, fullTmpSendingPath);
172             fileName = tmpSendingFileName;
173             if (logger.isDebugEnabled())
174             {
175                 logger.debug("Move done");
176             }
177         }
178 
179         // Archive functionality...
180         String archive = sftpUtil.getArchiveDir();
181 
182         // Retrieve the file stream
183         InputStream fileInputStream = client.retrieveFile(fileName);
184 
185         if (!"".equals(archive))
186         {
187             String archiveTmpReceivingDir = sftpUtil.getArchiveTempReceivingDir();
188             String archiveTmpSendingDir = sftpUtil.getArchiveTempSendingDir();
189 
190             InputStream is = new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(),
191                 endpoint);
192 
193             // TODO ML FIX. Refactor to util-class...
194             int idx = fileName.lastIndexOf('/');
195             String fileNamePart = fileName.substring(idx + 1);
196 
197             // don't use new File() directly, see MULE-1112
198             File archiveFile = FileUtils.newFile(archive, fileNamePart);
199 
200             // Should temp dirs be used when handling the archive file?
201             if ("".equals(archiveTmpReceivingDir) || "".equals(archiveTmpSendingDir))
202             {
203                 return archiveFile(is, archiveFile);
204             }
205             else
206             {
207                 return archiveFileUsingTempDirs(archive, archiveTmpReceivingDir, archiveTmpSendingDir, is,
208                     fileNamePart, archiveFile);
209             }
210         }
211 
212         // This special InputStream closes the SftpClient when the stream is closed.
213         // The stream will be materialized in a Message Dispatcher or Service
214         // Component
215         return new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(), endpoint);
216     }
217 
218     private InputStream archiveFileUsingTempDirs(String archive,
219                                                  String archiveTmpReceivingDir,
220                                                  String archiveTmpSendingDir,
221                                                  InputStream is,
222                                                  String fileNamePart,
223                                                  File archiveFile) throws IOException
224     {
225 
226         File archiveTmpReceivingFolder = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir);
227         File archiveTmpReceivingFile = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir, fileNamePart);
228         if (!archiveTmpReceivingFolder.exists())
229         {
230             if (logger.isInfoEnabled())
231             {
232                 logger.info("Creates " + archiveTmpReceivingFolder.getAbsolutePath());
233             }
234             if (!archiveTmpReceivingFolder.mkdirs())
235                 throw new IOException("Failed to create archive-tmp-receiving-folder: "
236                                       + archiveTmpReceivingFolder);
237         }
238 
239         File archiveTmpSendingFolder = FileUtils.newFile(archive + '/' + archiveTmpSendingDir);
240         File archiveTmpSendingFile = FileUtils.newFile(archive + '/' + archiveTmpSendingDir, fileNamePart);
241         if (!archiveTmpSendingFolder.exists())
242         {
243             if (logger.isInfoEnabled())
244             {
245                 logger.info("Creates " + archiveTmpSendingFolder.getAbsolutePath());
246             }
247             if (!archiveTmpSendingFolder.mkdirs())
248                 throw new IOException("Failed to create archive-tmp-sending-folder: "
249                                       + archiveTmpSendingFolder);
250         }
251 
252         if (logger.isInfoEnabled())
253         {
254             logger.info("Copy SftpInputStream to archiveTmpReceivingFile... "
255                         + archiveTmpReceivingFile.getAbsolutePath());
256         }
257         sftpUtil.copyStreamToFile(is, archiveTmpReceivingFile);
258 
259         // TODO. ML FIX. Should be performed before the sftp:delete - operation, i.e.
260         // in the SftpInputStream in the operation above...
261         if (logger.isInfoEnabled())
262         {
263             logger.info("Move archiveTmpReceivingFile (" + archiveTmpReceivingFile
264                         + ") to archiveTmpSendingFile (" + archiveTmpSendingFile + ")...");
265         }
266         FileUtils.moveFile(archiveTmpReceivingFile, archiveTmpSendingFile);
267 
268         if (logger.isDebugEnabled())
269         {
270             logger.debug("Return SftpFileArchiveInputStream for archiveTmpSendingFile ("
271                          + archiveTmpSendingFile + ")...");
272         }
273         return new SftpFileArchiveInputStream(archiveTmpSendingFile, archiveFile);
274     }
275 
276     private InputStream archiveFile(InputStream is, File archiveFile) throws IOException
277     {
278         File archiveFolder = FileUtils.newFile(archiveFile.getParentFile().getPath());
279         if (!archiveFolder.exists())
280         {
281             if (logger.isInfoEnabled())
282             {
283                 logger.info("Creates " + archiveFolder.getAbsolutePath());
284             }
285             if (!archiveFolder.mkdirs())
286                 throw new IOException("Failed to create archive-folder: " + archiveFolder);
287         }
288 
289         if (logger.isInfoEnabled())
290         {
291             logger.info("Copy SftpInputStream to archiveFile... " + archiveFile.getAbsolutePath());
292         }
293         sftpUtil.copyStreamToFile(is, archiveFile);
294 
295         if (logger.isDebugEnabled())
296         {
297             logger.debug("*** Return SftpFileArchiveInputStream for archiveFile...");
298         }
299         return new SftpFileArchiveInputStream(archiveFile);
300     }
301 
302     /**
303      * Checks if the file has been changed.
304      * <p/>
305      * Note! This assumes that the time on both servers are synchronized!
306      * 
307      * @param fileName The file to check
308      * @param client instance of StftClient
309      * @param fileAge How old the file should be to be considered "old" and not
310      *            changed
311      * @param sizeCheckDelayMs Wait time (in ms) between size-checks to determine if
312      *            a file is ready to be processed.
313      * @return true if the file has changed
314      * @throws Exception Error
315      */
316     private boolean hasChanged(String fileName, SftpClient client, long fileAge, long sizeCheckDelayMs)
317         throws Exception
318     {
319         // Perform fileAge test if configured
320         // Note that for this to work it is required that the system clock on the
321         // mule server
322         // is synchronized with the system clock on the sftp server
323         if (fileAge > 0)
324         {
325             long lastModifiedTime = client.getLastModifiedTime(fileName);
326             // TODO Can we get the current time from the other server?
327             long now = System.currentTimeMillis();
328             long diff = now - lastModifiedTime;
329             // If the diff is negative it's a sign that the time on the test server
330             // and the ftps-server is not synchronized
331             if (diff < fileAge)
332             {
333                 if (logger.isDebugEnabled())
334                 {
335                     logger.debug("The file has not aged enough yet, will return nothing for: " + fileName
336                                  + ". The file must be " + (fileAge - diff) + "ms older, was " + diff);
337                 }
338                 return true;
339             }
340             if (logger.isDebugEnabled())
341             {
342                 logger.debug("The file " + fileName + " has aged enough. Was " + diff);
343             }
344         }
345 
346         // Perform a size check with a short configurable latencey between the
347         // size-calls
348         // Take consecutive file size snapshots to determine if file is still being
349         // written
350         if (sizeCheckDelayMs > 0)
351         {
352             logger.info("Perform size check with a delay of: " + sizeCheckDelayMs + " ms.");
353             long fileSize1 = client.getSize(fileName);
354             Thread.sleep(sizeCheckDelayMs);
355             long fileSize2 = client.getSize(fileName);
356 
357             if (fileSize1 == fileSize2)
358             {
359                 logger.info("File is stable (not growing), ready for retrieval: " + fileName);
360             }
361             else
362             {
363                 logger.info("File is growing, deferring retrieval: " + fileName);
364                 return true;
365             }
366         }
367 
368         // None of file-change tests faile so we can retrieve this file
369         return false;
370     }
371 }