View Javadoc

1   /*
2    * $Id: SftpReceiverRequesterUtil.java 21125 2011-01-26 21:21:10Z dzapata $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.transport.sftp;
12  
13  import org.apache.commons.logging.Log;
14  import org.apache.commons.logging.LogFactory;
15  import org.mule.api.endpoint.ImmutableEndpoint;
16  import org.mule.transport.sftp.notification.SftpNotifier;
17  import org.mule.util.FileUtils;
18  
19  import java.io.File;
20  import java.io.FilenameFilter;
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  /**
27   * Contains reusable methods not directly related to usage of the jsch sftp library
28   * (they can be found in the class SftpClient).
29   * 
30   * @author Magnus Larsson
31   */
32  public class SftpReceiverRequesterUtil
33  {
34      private transient Log logger = LogFactory.getLog(getClass());
35  
36      private final SftpConnector connector;
37      private final ImmutableEndpoint endpoint;
38      private final FilenameFilter filenameFilter;
39      private final SftpUtil sftpUtil;
40  
41      public SftpReceiverRequesterUtil(ImmutableEndpoint endpoint)
42      {
43          this.endpoint = endpoint;
44          this.connector = (SftpConnector) endpoint.getConnector();
45  
46          sftpUtil = new SftpUtil(endpoint);
47  
48          if (endpoint.getFilter() instanceof FilenameFilter)
49          {
50              this.filenameFilter = (FilenameFilter) endpoint.getFilter();
51          }
52          else
53          {
54              this.filenameFilter = null;
55          }
56  
57      }
58  
59      // Get files in directory configured on the endpoint
60      public String[] getAvailableFiles(boolean onlyGetTheFirstOne) throws Exception
61      {
62          // This sftp client instance is only for checking available files. This
63          // instance cannot be shared
64          // with clients that retrieve files because of thread safety
65  
66          if (logger.isDebugEnabled())
67          {
68              logger.debug("Checking files at endpoint " + endpoint.getEndpointURI());
69          }
70  
71          SftpClient client = null;
72  
73          try
74          {
75              client = connector.createSftpClient(endpoint);
76  
77              long fileAge = connector.getFileAge();
78              boolean checkFileAge = connector.getCheckFileAge();
79  
80              // Override the value from the Endpoint?
81              if (endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE) != null)
82              {
83                  checkFileAge = true;
84                  fileAge = Long.valueOf((String) endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE));
85              }
86  
87              logger.debug("fileAge : " + fileAge);
88  
89              // Get size check parameter
90              long sizeCheckDelayMs = sftpUtil.getSizeCheckWaitTime();
91  
92              String[] files = client.listFiles();
93  
94              // Only return files that have completely been written and match
95              // fileExtension
96              List<String> completedFiles = new ArrayList<String>(files.length);
97  
98              for (String file : files)
99              {
100                 // Skip if no match.
101                 // Note, Mule also uses this filter. We use the filter here because
102                 // we don't want to
103                 // run the below tests (checkFileAge, sizeCheckDelayMs etc) for files
104                 // that Mule
105                 // later should have ignored. Thus this is an "early" filter so that
106                 // improves performance.
107                 if (filenameFilter != null && !filenameFilter.accept(null, file))
108                 {
109                     continue;
110                 }
111 
112                 if (checkFileAge || sizeCheckDelayMs >= 0)
113                 {
114                     // See if the file is still growing (either by age or size),
115                     // leave it alone if it is
116                     if (!hasChanged(file, client, fileAge, sizeCheckDelayMs))
117                     {
118                         // logger.debug("marking file [" + files[i] +
119                         // "] as in transit.");
120                         // client.rename(files[i], files[i] + ".transtit");
121                         // completedFiles.add( files[i] + ".transtit" );
122                         completedFiles.add(file);
123                         if (onlyGetTheFirstOne)
124                         {
125                             break;
126                         }
127                     }
128                 }
129                 else
130                 {
131                     completedFiles.add(file);
132                     if (onlyGetTheFirstOne)
133                     {
134                         break;
135                     }
136                 }
137             }
138             return completedFiles.toArray(new String[completedFiles.size()]);
139         }
140         finally
141         {
142             if (client != null)
143             {
144                 connector.releaseClient(endpoint, client);
145             }
146         }
147     }
148 
149     public InputStream retrieveFile(String fileName, SftpNotifier notifier) throws Exception
150     {
151         // Getting a new SFTP client dedicated to the SftpInputStream below
152         SftpClient client = connector.createSftpClient(endpoint, notifier);
153 
154         // Check usage of tmpSendingDir
155         String tmpSendingDir = sftpUtil.getTempDirInbound();
156         if (tmpSendingDir != null)
157         {
158             // Check usage of unique names of files during transfer
159             boolean addUniqueSuffix = sftpUtil.isUseTempFileTimestampSuffix();
160 
161             // TODO: is it possibly to move this to some kind of init method?
162             client.createSftpDirIfNotExists(endpoint, tmpSendingDir);
163             String tmpSendingFileName = tmpSendingDir + "/" + fileName;
164 
165             if (addUniqueSuffix)
166             {
167                 tmpSendingFileName = sftpUtil.createUniqueSuffix(tmpSendingFileName);
168             }
169             String fullTmpSendingPath = endpoint.getEndpointURI().getPath() + "/" + tmpSendingFileName;
170 
171             if (logger.isDebugEnabled())
172             {
173                 logger.debug("Move " + fileName + " to " + fullTmpSendingPath);
174             }
175             client.rename(fileName, fullTmpSendingPath);
176             fileName = tmpSendingFileName;
177             if (logger.isDebugEnabled())
178             {
179                 logger.debug("Move done");
180             }
181         }
182 
183         // Archive functionality...
184         String archive = sftpUtil.getArchiveDir();
185 
186         // Retrieve the file stream
187         InputStream fileInputStream = client.retrieveFile(fileName);
188 
189         if (!"".equals(archive))
190         {
191             String archiveTmpReceivingDir = sftpUtil.getArchiveTempReceivingDir();
192             String archiveTmpSendingDir = sftpUtil.getArchiveTempSendingDir();
193 
194             InputStream is = new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(),
195                 endpoint);
196 
197             // TODO ML FIX. Refactor to util-class...
198             int idx = fileName.lastIndexOf('/');
199             String fileNamePart = fileName.substring(idx + 1);
200 
201             // don't use new File() directly, see MULE-1112
202             File archiveFile = FileUtils.newFile(archive, fileNamePart);
203 
204             // Should temp dirs be used when handling the archive file?
205             if ("".equals(archiveTmpReceivingDir) || "".equals(archiveTmpSendingDir))
206             {
207                 return archiveFile(is, archiveFile);
208             }
209             else
210             {
211                 return archiveFileUsingTempDirs(archive, archiveTmpReceivingDir, archiveTmpSendingDir, is,
212                     fileNamePart, archiveFile);
213             }
214         }
215 
216         // This special InputStream closes the SftpClient when the stream is closed.
217         // The stream will be materialized in a Message Dispatcher or Service
218         // Component
219         return new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(), endpoint);
220     }
221 
222     private InputStream archiveFileUsingTempDirs(String archive,
223                                                  String archiveTmpReceivingDir,
224                                                  String archiveTmpSendingDir,
225                                                  InputStream is,
226                                                  String fileNamePart,
227                                                  File archiveFile) throws IOException
228     {
229 
230         File archiveTmpReceivingFolder = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir);
231         File archiveTmpReceivingFile = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir, fileNamePart);
232         if (!archiveTmpReceivingFolder.exists())
233         {
234             if (logger.isInfoEnabled())
235             {
236                 logger.info("Creates " + archiveTmpReceivingFolder.getAbsolutePath());
237             }
238             if (!archiveTmpReceivingFolder.mkdirs())
239                 throw new IOException("Failed to create archive-tmp-receiving-folder: "
240                                       + archiveTmpReceivingFolder);
241         }
242 
243         File archiveTmpSendingFolder = FileUtils.newFile(archive + '/' + archiveTmpSendingDir);
244         File archiveTmpSendingFile = FileUtils.newFile(archive + '/' + archiveTmpSendingDir, fileNamePart);
245         if (!archiveTmpSendingFolder.exists())
246         {
247             if (logger.isInfoEnabled())
248             {
249                 logger.info("Creates " + archiveTmpSendingFolder.getAbsolutePath());
250             }
251             if (!archiveTmpSendingFolder.mkdirs())
252                 throw new IOException("Failed to create archive-tmp-sending-folder: "
253                                       + archiveTmpSendingFolder);
254         }
255 
256         if (logger.isInfoEnabled())
257         {
258             logger.info("Copy SftpInputStream to archiveTmpReceivingFile... "
259                         + archiveTmpReceivingFile.getAbsolutePath());
260         }
261         sftpUtil.copyStreamToFile(is, archiveTmpReceivingFile);
262 
263         // TODO. ML FIX. Should be performed before the sftp:delete - operation, i.e.
264         // in the SftpInputStream in the operation above...
265         if (logger.isInfoEnabled())
266         {
267             logger.info("Move archiveTmpReceivingFile (" + archiveTmpReceivingFile
268                         + ") to archiveTmpSendingFile (" + archiveTmpSendingFile + ")...");
269         }
270         FileUtils.moveFile(archiveTmpReceivingFile, archiveTmpSendingFile);
271 
272         if (logger.isDebugEnabled())
273         {
274             logger.debug("Return SftpFileArchiveInputStream for archiveTmpSendingFile ("
275                          + archiveTmpSendingFile + ")...");
276         }
277         return new SftpFileArchiveInputStream(archiveTmpSendingFile, archiveFile);
278     }
279 
280     private InputStream archiveFile(InputStream is, File archiveFile) throws IOException
281     {
282         File archiveFolder = FileUtils.newFile(archiveFile.getParentFile().getPath());
283         if (!archiveFolder.exists())
284         {
285             if (logger.isInfoEnabled())
286             {
287                 logger.info("Creates " + archiveFolder.getAbsolutePath());
288             }
289             if (!archiveFolder.mkdirs())
290                 throw new IOException("Failed to create archive-folder: " + archiveFolder);
291         }
292 
293         if (logger.isInfoEnabled())
294         {
295             logger.info("Copy SftpInputStream to archiveFile... " + archiveFile.getAbsolutePath());
296         }
297         sftpUtil.copyStreamToFile(is, archiveFile);
298 
299         if (logger.isDebugEnabled())
300         {
301             logger.debug("*** Return SftpFileArchiveInputStream for archiveFile...");
302         }
303         return new SftpFileArchiveInputStream(archiveFile);
304     }
305 
306     /**
307      * Checks if the file has been changed.
308      * <p/>
309      * Note! This assumes that the time on both servers are synchronized!
310      * 
311      * @param fileName The file to check
312      * @param client instance of StftClient
313      * @param fileAge How old the file should be to be considered "old" and not
314      *            changed
315      * @param sizeCheckDelayMs Wait time (in ms) between size-checks to determine if
316      *            a file is ready to be processed.
317      * @return true if the file has changed
318      * @throws Exception Error
319      */
320     private boolean hasChanged(String fileName, SftpClient client, long fileAge, long sizeCheckDelayMs)
321         throws Exception
322     {
323         // Perform fileAge test if configured
324         // Note that for this to work it is required that the system clock on the
325         // mule server
326         // is synchronized with the system clock on the sftp server
327         if (fileAge > 0)
328         {
329             long lastModifiedTime = client.getLastModifiedTime(fileName);
330             // TODO Can we get the current time from the other server?
331             long now = System.currentTimeMillis();
332             long diff = now - lastModifiedTime;
333             // If the diff is negative it's a sign that the time on the test server
334             // and the ftps-server is not synchronized
335             if (diff < fileAge)
336             {
337                 if (logger.isDebugEnabled())
338                 {
339                     logger.debug("The file has not aged enough yet, will return nothing for: " + fileName
340                                  + ". The file must be " + (fileAge - diff) + "ms older, was " + diff);
341                 }
342                 return true;
343             }
344             if (logger.isDebugEnabled())
345             {
346                 logger.debug("The file " + fileName + " has aged enough. Was " + diff);
347             }
348         }
349 
350         // Perform a size check with a short configurable latencey between the
351         // size-calls
352         // Take consecutive file size snapshots to determine if file is still being
353         // written
354         if (sizeCheckDelayMs > 0)
355         {
356             logger.info("Perform size check with a delay of: " + sizeCheckDelayMs + " ms.");
357             long fileSize1 = client.getSize(fileName);
358             Thread.sleep(sizeCheckDelayMs);
359             long fileSize2 = client.getSize(fileName);
360 
361             if (fileSize1 == fileSize2)
362             {
363                 logger.info("File is stable (not growing), ready for retrieval: " + fileName);
364             }
365             else
366             {
367                 logger.info("File is growing, deferring retrieval: " + fileName);
368                 return true;
369             }
370         }
371 
372         // None of file-change tests faile so we can retrieve this file
373         return false;
374     }
375 }