View Javadoc

1   /*
2    * $Id: SftpConnector.java 22029 2011-05-30 14:43:18Z dfeist $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.transport.sftp;
12  
13  import org.mule.api.MuleContext;
14  import org.mule.api.MuleException;
15  import org.mule.api.construct.FlowConstruct;
16  import org.mule.api.endpoint.EndpointURI;
17  import org.mule.api.endpoint.ImmutableEndpoint;
18  import org.mule.api.endpoint.InboundEndpoint;
19  import org.mule.api.lifecycle.InitialisationException;
20  import org.mule.api.transport.ConnectorException;
21  import org.mule.api.transport.MessageReceiver;
22  import org.mule.config.i18n.CoreMessages;
23  import org.mule.transport.AbstractConnector;
24  import org.mule.transport.file.ExpressionFilenameParser;
25  import org.mule.transport.file.FilenameParser;
26  import org.mule.transport.sftp.notification.SftpNotifier;
27  
28  import java.util.HashMap;
29  import java.util.Map;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.commons.pool.ObjectPool;
34  import org.apache.commons.pool.impl.GenericObjectPool;
35  
36  /**
37   * <code>SftpConnector</code> sends and receives file messages over sftp using jsch
38   * library Improves on SFTP with VFS Connector in the following ways: 1. Streams
39   * files instead of reading them into memory. The SftpMessageReceiver is a
40   * "non-materializing stream receiver" which does not read the file to memory. The
41   * SftpMessageDispatcher also never materializes the stream and delegates the jsch
42   * library for materialization. 3. Uses jsch library directly instead of using VFS as
43   * middle-man. 3. More explicit connection lifefecyle management. 4. Leverages sftp
44   * stat to determine if a file size changes (simpler and also less memory intensive)
45   */
46  public class SftpConnector extends AbstractConnector
47  {
48  
49      public static final String PROPERTY_POLLING_FREQUENCY = "pollingFrequency";
50      public static final String PROPERTY_DIRECTORY = "directory";
51      public static final String PROPERTY_OUTPUT_PATTERN = "outputPattern";
52      public static final String PROPERTY_FILENAME = "filename";
53      public static final String PROPERTY_ORIGINAL_FILENAME = "originalFilename";
54      public static final String PROPERTY_SELECT_EXPRESSION = "selectExpression";
55      public static final String PROPERTY_FILE_EXTENSION = "fileExtension";
56      public static final String PROPERTY_INCLUDE_SUBFOLDERS = "includeSubfolders";
57      public static final String PROPERTY_IDENTITY_FILE = "identityFile";
58      public static final String PROPERTY_PASS_PHRASE = "passphrase";
59      public static final String PROPERTY_FILE_AGE = "fileAge";
60      public static final String PROPERTY_TEMP_DIR = "tempDir";
61      public static final String PROPERTY_SIZE_CHECK_WAIT_TIME = "sizeCheckWaitTime";
62      public static final String PROPERTY_ARCHIVE_DIR = "archiveDir";
63      public static final String PROPERTY_ARCHIVE_TEMP_RECEIVING_DIR = "archiveTempReceivingDir";
64      public static final String PROPERTY_ARCHIVE_TEMP_SENDING_DIR = "archiveTempSendingDir";
65      public static final String PROPERTY_DUPLICATE_HANDLING = "duplicateHandling";
66      public static final String PROPERTY_USE_TEMP_FILE_TIMESTAMP_SUFFIX = "useTempFileTimestampSuffix";
67      public static final String PROPERTY_DUPLICATE_HANDLING_THROW_EXCEPTION = "throwException";
68      public static final String PROPERTY_DUPLICATE_HANDLING_OVERWRITE = "overwrite";
69      public static final String PROPERTY_DUPLICATE_HANDLING_ASS_SEQ_NO = "addSeqNo";
70      public static final String PROPERTY_MAX_CONNECTION_POOL_SIZE = "maxConnectionPoolSize";
71      public static final String PROPERTY_KEEP_FILE_ON_ERROR = "keepFileOnError";
72  
73      public static final int DEFAULT_POLLING_FREQUENCY = 1000;
74  
75      /**
76       * logger used by this class
77       */
78      protected final static Log logger = LogFactory.getLog(SftpConnector.class);
79  
80      private FilenameParser filenameParser = new ExpressionFilenameParser();
81  
82      private long pollingFrequency;
83      private boolean autoDelete = true;
84      private String outputPattern;
85  
86      private String identityFile;
87      private String passphrase;
88  
89      private boolean checkFileAge = false;
90      private long fileAge = 0;
91  
92      private String tempDirInbound = null;
93      private String tempDirOutbound = null;
94  
95      private Map<EndpointURI, GenericObjectPool> pools = new HashMap<EndpointURI, GenericObjectPool>();
96  
97      private String duplicateHandling = null;
98      private Boolean useTempFileTimestampSuffix = null;
99      private Long sizeCheckWaitTime = null;
100     private String archiveDir = "";
101     private String archiveTempReceivingDir = "";
102     private String archiveTempSendingDir = "";
103 
104     /**
105      * Should the file be kept if an error occurs when writing the file on the
106      * outbound endpoint?
107      */
108     private Boolean keepFileOnError;
109 
110     /**
111      * max pool size. 0 for no pool, -1 for no limit, otherwise the specified value
112      */
113     private int maxConnectionPoolSize;
114 
115     /**
116      * Value that can be set via the System property
117      * 'mule.sftp.transport.maxConnectionPoolSize'. If it's set the value is used
118      * instead of <i>maxConnectionPoolSize</i>
119      */
120     private static final Integer overrideMaxConnectionPoolSize;
121 
122     static
123     {
124         String propValue = System.getProperty("mule.sftp.transport.maxConnectionPoolSize");
125         if (propValue != null)
126         {
127             logger.info("Will override the maxConnectionPoolSize to " + propValue
128                         + " from the system property 'mule.sftp.transport.maxConnectionPoolSize'.");
129             overrideMaxConnectionPoolSize = Integer.parseInt(propValue);
130         }
131         else
132         {
133             overrideMaxConnectionPoolSize = null;
134         }
135     }
136 
137     public SftpConnector(MuleContext context)
138     {
139         super(context);
140         filenameParser = new ExpressionFilenameParser();
141     }
142 
143     public String getProtocol()
144     {
145         return "sftp";
146     }
147 
148     public MessageReceiver createReceiver(FlowConstruct flow, InboundEndpoint endpoint) throws Exception
149     {
150         long polling = pollingFrequency;
151 
152         // Override properties on the endpoint for the specific endpoint
153         String tempPolling = (String) endpoint.getProperty(PROPERTY_POLLING_FREQUENCY);
154         if (tempPolling != null)
155         {
156             polling = Long.parseLong(tempPolling);
157         }
158 
159         if (polling <= 0)
160         {
161             polling = DEFAULT_POLLING_FREQUENCY;
162         }
163         if (logger.isDebugEnabled())
164         {
165             logger.debug("Set polling frequency to: " + polling);
166         }
167 
168         return serviceDescriptor.createMessageReceiver(this, flow, endpoint, new Object[]{polling});
169     }
170 
171     public SftpClient createSftpClient(ImmutableEndpoint endpoint) throws Exception
172     {
173         return createSftpClient(endpoint, null);
174     }
175 
176     public SftpClient createSftpClient(ImmutableEndpoint endpoint, SftpNotifier notifier) throws Exception
177     {
178         SftpClient client = null;
179         boolean ok = false;
180 
181         try
182         {
183             if (useConnectionPool())
184             {
185                 ObjectPool pool = getClientPool(endpoint);
186                 client = (SftpClient) pool.borrowObject();
187             }
188             else
189             {
190                 client = SftpConnectionFactory.createClient(endpoint);
191             }
192 
193             // We have to set the working directory before returning
194             String dir = endpoint.getEndpointURI().getPath();
195             client.changeWorkingDirectory(dir);
196             if (logger.isDebugEnabled())
197             {
198                 logger.debug("Successfully changed working directory to: " + dir);
199             }
200 
201             // TODO ML: Is this always necessary?
202             client.setNotifier(notifier);
203 
204             ok = true;
205 
206         }
207         finally
208         {
209             // Release the client if it was created but something failed after that,
210             // otherwise we start to waste ssh-processes...
211             if (!ok && client != null)
212             {
213                 releaseClient(endpoint, client);
214             }
215         }
216 
217         return client;
218     }
219 
220     /**
221      * @return True if connection pooling is used, otherwise false
222      */
223     public boolean useConnectionPool()
224     {
225         return getMaxConnectionPoolSize() != 0;
226     }
227 
228     public void releaseClient(ImmutableEndpoint endpoint, SftpClient client) throws Exception
229     {
230         if (useConnectionPool())
231         {
232             if (getDispatcherFactory().isCreateDispatcherPerRequest())
233             {
234                 destroyClient(endpoint, client);
235             }
236             else
237             {
238                 if (client != null && client.isConnected())
239                 {
240                     ObjectPool pool = getClientPool(endpoint);
241                     if (logger.isDebugEnabled())
242                     {
243                         logger.debug("Releasing connection for endpoint " + endpoint.getEndpointURI());
244                     }
245                     pool.returnObject(client);
246                 }
247             }
248         }
249         else
250         {
251             client.disconnect();
252         }
253     }
254 
255     public void destroyClient(ImmutableEndpoint endpoint, SftpClient client) throws Exception
256     {
257         if (useConnectionPool())
258         {
259             if ((client != null) && (client.isConnected()))
260             {
261                 ObjectPool pool = getClientPool(endpoint);
262                 pool.invalidateObject(client);
263             }
264         }
265     }
266 
267     protected synchronized ObjectPool getClientPool(ImmutableEndpoint endpoint)
268     {
269         GenericObjectPool pool = pools.get(endpoint.getEndpointURI());
270 
271         if (pool == null)
272         {
273             if (logger.isDebugEnabled())
274             {
275                 logger.debug("Pool is null - creating one for endpoint " + endpoint.getEndpointURI()
276                              + " with max size " + getMaxConnectionPoolSize());
277             }
278             pool = new GenericObjectPool(new SftpConnectionFactory(endpoint), getMaxConnectionPoolSize());
279             pool.setTestOnBorrow(isValidateConnections());
280             pools.put(endpoint.getEndpointURI(), pool);
281         }
282         else
283         {
284             if (logger.isDebugEnabled())
285             {
286                 logger.debug("Using existing pool for endpoint " + endpoint.getEndpointURI() + ". Active: "
287                              + pool.getNumActive() + ", Idle:" + pool.getNumIdle());
288             }
289         }
290 
291         return pool;
292     }
293 
294     /*
295      * (non-Javadoc)
296      * @see org.mule.transport.AbstractConnector#doConnect()
297      */
298     protected void doConnect() throws Exception
299     {
300         // Do nothing!
301     }
302 
303     /*
304      * (non-Javadoc)
305      * @see org.mule.transport.AbstractConnector#doDisconnect()
306      */
307     protected void doDisconnect() throws Exception
308     {
309         // Do nothing!
310     }
311 
312     /*
313      * (non-Javadoc)
314      * @see org.mule.transport.AbstractConnector#doDispose()
315      */
316     protected void doDispose()
317     {
318         // Do nothing!
319     }
320 
321     /*
322      * (non-Javadoc)
323      * @see org.mule.transport.AbstractConnector#doInitialise()
324      */
325     protected void doInitialise() throws InitialisationException
326     {
327         if (filenameParser != null)
328         {
329             filenameParser.setMuleContext(muleContext);
330         }
331     }
332 
333     /*
334      * (non-Javadoc)
335      * @see org.mule.transport.AbstractConnector#doStart()
336      */
337     protected void doStart() throws MuleException
338     {
339         // Do nothing!
340     }
341 
342     /*
343      * (non-Javadoc)
344      * @see org.mule.transport.AbstractConnector#doStop()
345      */
346     protected void doStop() throws MuleException
347     {
348         if (logger.isDebugEnabled())
349         {
350             logger.debug("Stopping all pools");
351         }
352         try
353         {
354             for (ObjectPool pool : pools.values())
355             {
356                 pool.close();
357             }
358         }
359         catch (Exception e)
360         {
361             throw new ConnectorException(CoreMessages.failedToStop("SFTP Connector"), this, e);
362         }
363         finally
364         {
365             pools.clear();
366         }
367     }
368 
369     public long getPollingFrequency()
370     {
371         return pollingFrequency;
372     }
373 
374     public void setPollingFrequency(long pollingFrequency)
375     {
376         this.pollingFrequency = pollingFrequency;
377     }
378 
379     public FilenameParser getFilenameParser()
380     {
381         return filenameParser;
382     }
383 
384     public void setFilenameParser(FilenameParser filenameParser)
385     {
386         this.filenameParser = filenameParser;
387         if (filenameParser != null)
388         {
389             filenameParser.setMuleContext(muleContext);
390         }
391     }
392 
393     public String getOutputPattern()
394     {
395         return outputPattern;
396     }
397 
398     public void setOutputPattern(String outputPattern)
399     {
400         this.outputPattern = outputPattern;
401     }
402 
403     public boolean isAutoDelete()
404     {
405         return autoDelete;
406     }
407 
408     public void setAutoDelete(boolean autoDelete)
409     {
410         this.autoDelete = autoDelete;
411     }
412 
413     public String getIdentityFile()
414     {
415         return identityFile;
416     }
417 
418     public void setIdentityFile(String identityFile)
419     {
420         this.identityFile = identityFile;
421     }
422 
423     public String getPassphrase()
424     {
425         return passphrase;
426     }
427 
428     public void setPassphrase(String passphrase)
429     {
430         this.passphrase = passphrase;
431     }
432 
433     /**
434      * Returns the file age.
435      * 
436      * @return Returns the fileAge in milliseconds.
437      */
438     public long getFileAge()
439     {
440         return fileAge;
441     }
442 
443     /**
444      * Sets the file age.
445      * 
446      * @param fileAge the fileAge in milliseconds to set.
447      */
448     public void setFileAge(long fileAge)
449     {
450         this.fileAge = fileAge;
451         this.checkFileAge = true;
452     }
453 
454     public boolean getCheckFileAge()
455     {
456         return checkFileAge;
457     }
458 
459     public String getTempDirInbound()
460     {
461         return tempDirInbound;
462     }
463 
464     public void setTempDirInbound(String pTempDirInbound)
465     {
466         tempDirInbound = pTempDirInbound;
467     }
468 
469     public String getTempDirOutbound()
470     {
471         return tempDirOutbound;
472     }
473 
474     public void setTempDirOutbound(String pTempDirOutbound)
475     {
476         tempDirOutbound = pTempDirOutbound;
477     }
478 
479     // Need this method to be public for SftpNotifier
480     @Override
481     public boolean isEnableMessageEvents()
482     {
483         return super.isEnableMessageEvents();
484     }
485 
486     public void setDuplicateHandling(String duplicateHandling)
487     {
488         this.duplicateHandling = duplicateHandling;
489     }
490 
491     public String getDuplicateHandling()
492     {
493         return duplicateHandling;
494     }
495 
496     public void setUseTempFileTimestampSuffix(Boolean useTempFileTimestampSuffix)
497     {
498         this.useTempFileTimestampSuffix = useTempFileTimestampSuffix;
499     }
500 
501     public Boolean isUseTempFileTimestampSuffix()
502     {
503         return useTempFileTimestampSuffix;
504     }
505 
506     public void setSizeCheckWaitTime(Long sizeCheckWaitTime)
507     {
508         this.sizeCheckWaitTime = sizeCheckWaitTime;
509     }
510 
511     public Long getSizeCheckWaitTime()
512     {
513         return sizeCheckWaitTime;
514     }
515 
516     public void setArchiveDir(String archiveDir)
517     {
518         this.archiveDir = archiveDir;
519     }
520 
521     public String getArchiveDir()
522     {
523         return archiveDir;
524     }
525 
526     public void setArchiveTempReceivingDir(String archiveTempReceivingDir)
527     {
528         this.archiveTempReceivingDir = archiveTempReceivingDir;
529     }
530 
531     public String getArchiveTempReceivingDir()
532     {
533         return archiveTempReceivingDir;
534     }
535 
536     public void setArchiveTempSendingDir(String archiveTempSendingDir)
537     {
538         this.archiveTempSendingDir = archiveTempSendingDir;
539     }
540 
541     public String getArchiveTempSendingDir()
542     {
543         return archiveTempSendingDir;
544     }
545 
546     /**
547      * @see SftpConnector#maxConnectionPoolSize
548      */
549     public void setMaxConnectionPoolSize(int maxConnectionPoolSize)
550     {
551         this.maxConnectionPoolSize = maxConnectionPoolSize;
552     }
553 
554     /**
555      * @return the max connection pool size. If the system parameter
556      *         mule.sftp.transport.maxConnectionPoolSize is set, that value will be
557      *         used instead.
558      */
559     public int getMaxConnectionPoolSize()
560     {
561         if (overrideMaxConnectionPoolSize != null)
562         {
563             return overrideMaxConnectionPoolSize;
564         }
565         return maxConnectionPoolSize;
566     }
567 
568     public Boolean isKeepFileOnError()
569     {
570         return keepFileOnError;
571     }
572 
573     public void setKeepFileOnError(Boolean pKeepFileOnError)
574     {
575         keepFileOnError = pKeepFileOnError;
576     }
577 }