View Javadoc
1   /*
2    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
3    * The software in this package is published under the terms of the CPAL v1.0
4    * license, a copy of which has been included with this distribution in the
5    * LICENSE.txt file.
6    */
7   package org.mule.transport.sftp;
8   
9   import org.mule.api.MuleContext;
10  import org.mule.api.MuleException;
11  import org.mule.api.construct.FlowConstruct;
12  import org.mule.api.endpoint.EndpointURI;
13  import org.mule.api.endpoint.ImmutableEndpoint;
14  import org.mule.api.endpoint.InboundEndpoint;
15  import org.mule.api.lifecycle.InitialisationException;
16  import org.mule.api.transport.ConnectorException;
17  import org.mule.api.transport.MessageReceiver;
18  import org.mule.config.i18n.CoreMessages;
19  import org.mule.transport.AbstractConnector;
20  import org.mule.transport.file.ExpressionFilenameParser;
21  import org.mule.transport.file.FilenameParser;
22  import org.mule.transport.sftp.notification.SftpNotifier;
23  
24  import java.util.HashMap;
25  import java.util.Map;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.commons.pool.ObjectPool;
30  import org.apache.commons.pool.impl.GenericObjectPool;
31  
32  /**
33   * <code>SftpConnector</code> sends and receives file messages over sftp using jsch
34   * library Improves on SFTP with VFS Connector in the following ways: 1. Streams
35   * files instead of reading them into memory. The SftpMessageReceiver is a
36   * "non-materializing stream receiver" which does not read the file to memory. The
37   * SftpMessageDispatcher also never materializes the stream and delegates the jsch
38   * library for materialization. 3. Uses jsch library directly instead of using VFS as
39   * middle-man. 3. More explicit connection lifefecyle management. 4. Leverages sftp
40   * stat to determine if a file size changes (simpler and also less memory intensive)
41   */
42  public class SftpConnector extends AbstractConnector
43  {
44  
45      public static final String PROPERTY_POLLING_FREQUENCY = "pollingFrequency";
46      public static final String PROPERTY_DIRECTORY = "directory";
47      public static final String PROPERTY_OUTPUT_PATTERN = "outputPattern";
48      public static final String PROPERTY_FILENAME = "filename";
49      public static final String PROPERTY_ORIGINAL_FILENAME = "originalFilename";
50      public static final String PROPERTY_SELECT_EXPRESSION = "selectExpression";
51      public static final String PROPERTY_FILE_EXTENSION = "fileExtension";
52      public static final String PROPERTY_INCLUDE_SUBFOLDERS = "includeSubfolders";
53      public static final String PROPERTY_IDENTITY_FILE = "identityFile";
54      public static final String PROPERTY_PASS_PHRASE = "passphrase";
55      public static final String PROPERTY_FILE_AGE = "fileAge";
56      public static final String PROPERTY_TEMP_DIR = "tempDir";
57      public static final String PROPERTY_SIZE_CHECK_WAIT_TIME = "sizeCheckWaitTime";
58      public static final String PROPERTY_ARCHIVE_DIR = "archiveDir";
59      public static final String PROPERTY_ARCHIVE_TEMP_RECEIVING_DIR = "archiveTempReceivingDir";
60      public static final String PROPERTY_ARCHIVE_TEMP_SENDING_DIR = "archiveTempSendingDir";
61      public static final String PROPERTY_DUPLICATE_HANDLING = "duplicateHandling";
62      public static final String PROPERTY_USE_TEMP_FILE_TIMESTAMP_SUFFIX = "useTempFileTimestampSuffix";
63      public static final String PROPERTY_DUPLICATE_HANDLING_THROW_EXCEPTION = "throwException";
64      public static final String PROPERTY_DUPLICATE_HANDLING_OVERWRITE = "overwrite";
65      public static final String PROPERTY_DUPLICATE_HANDLING_ASS_SEQ_NO = "addSeqNo";
66      public static final String PROPERTY_MAX_CONNECTION_POOL_SIZE = "maxConnectionPoolSize";
67      public static final String PROPERTY_KEEP_FILE_ON_ERROR = "keepFileOnError";
68  
69      public static final int DEFAULT_POLLING_FREQUENCY = 1000;
70  
71      /**
72       * logger used by this class
73       */
74      protected final static Log logger = LogFactory.getLog(SftpConnector.class);
75  
76      private FilenameParser filenameParser = new ExpressionFilenameParser();
77  
78      private long pollingFrequency;
79      private boolean autoDelete = true;
80      private String outputPattern;
81  
82      private String identityFile;
83      private String passphrase;
84  
85      private boolean checkFileAge = false;
86      private long fileAge = 0;
87  
88      private String tempDirInbound = null;
89      private String tempDirOutbound = null;
90  
91      private Map<EndpointURI, GenericObjectPool> pools = new HashMap<EndpointURI, GenericObjectPool>();
92  
93      private String duplicateHandling = null;
94      private Boolean useTempFileTimestampSuffix = null;
95      private Long sizeCheckWaitTime = null;
96      private String archiveDir = "";
97      private String archiveTempReceivingDir = "";
98      private String archiveTempSendingDir = "";
99  
100     /**
101      * Should the file be kept if an error occurs when writing the file on the
102      * outbound endpoint?
103      */
104     private Boolean keepFileOnError;
105 
106     /**
107      * max pool size. 0 for no pool, -1 for no limit, otherwise the specified value
108      */
109     private int maxConnectionPoolSize;
110 
111     /**
112      * Value that can be set via the System property
113      * 'mule.sftp.transport.maxConnectionPoolSize'. If it's set the value is used
114      * instead of <i>maxConnectionPoolSize</i>
115      */
116     private static final Integer overrideMaxConnectionPoolSize;
117 
118     static
119     {
120         String propValue = System.getProperty("mule.sftp.transport.maxConnectionPoolSize");
121         if (propValue != null)
122         {
123             logger.info("Will override the maxConnectionPoolSize to " + propValue
124                         + " from the system property 'mule.sftp.transport.maxConnectionPoolSize'.");
125             overrideMaxConnectionPoolSize = Integer.parseInt(propValue);
126         }
127         else
128         {
129             overrideMaxConnectionPoolSize = null;
130         }
131     }
132 
133     public SftpConnector(MuleContext context)
134     {
135         super(context);
136         filenameParser = new ExpressionFilenameParser();
137     }
138 
139     public String getProtocol()
140     {
141         return "sftp";
142     }
143 
144     @Override
145     public MessageReceiver createReceiver(FlowConstruct flow, InboundEndpoint endpoint) throws Exception
146     {
147         long polling = pollingFrequency;
148 
149         // Override properties on the endpoint for the specific endpoint
150         String tempPolling = (String) endpoint.getProperty(PROPERTY_POLLING_FREQUENCY);
151         if (tempPolling != null)
152         {
153             polling = Long.parseLong(tempPolling);
154         }
155 
156         if (polling <= 0)
157         {
158             polling = DEFAULT_POLLING_FREQUENCY;
159         }
160         if (logger.isDebugEnabled())
161         {
162             logger.debug("Set polling frequency to: " + polling);
163         }
164 
165         return serviceDescriptor.createMessageReceiver(this, flow, endpoint, new Object[]{polling});
166     }
167 
168     public SftpClient createSftpClient(ImmutableEndpoint endpoint) throws Exception
169     {
170         return createSftpClient(endpoint, null);
171     }
172 
173     public SftpClient createSftpClient(ImmutableEndpoint endpoint, SftpNotifier notifier) throws Exception
174     {
175         SftpClient client = null;
176         boolean ok = false;
177 
178         try
179         {
180             if (useConnectionPool())
181             {
182                 ObjectPool pool = getClientPool(endpoint);
183                 client = (SftpClient) pool.borrowObject();
184             }
185             else
186             {
187                 client = SftpConnectionFactory.createClient(endpoint);
188             }
189 
190             // We have to set the working directory before returning
191             String dir = endpoint.getEndpointURI().getPath();
192             client.changeWorkingDirectory(dir);
193             if (logger.isDebugEnabled())
194             {
195                 logger.debug("Successfully changed working directory to: " + dir);
196             }
197 
198             // TODO ML: Is this always necessary?
199             client.setNotifier(notifier);
200 
201             ok = true;
202 
203         }
204         finally
205         {
206             // Release the client if it was created but something failed after that,
207             // otherwise we start to waste ssh-processes...
208             if (!ok && client != null)
209             {
210                 releaseClient(endpoint, client);
211             }
212         }
213 
214         return client;
215     }
216 
217     /**
218      * @return True if connection pooling is used, otherwise false
219      */
220     public boolean useConnectionPool()
221     {
222         return getMaxConnectionPoolSize() != 0;
223     }
224 
225     public void releaseClient(ImmutableEndpoint endpoint, SftpClient client) throws Exception
226     {
227         if (useConnectionPool())
228         {
229             if (getDispatcherFactory().isCreateDispatcherPerRequest())
230             {
231                 destroyClient(endpoint, client);
232             }
233             else
234             {
235                 if (client != null && client.isConnected())
236                 {
237                     ObjectPool pool = getClientPool(endpoint);
238                     if (logger.isDebugEnabled())
239                     {
240                         logger.debug("Releasing connection for endpoint " + endpoint.getEndpointURI());
241                     }
242                     pool.returnObject(client);
243                 }
244             }
245         }
246         else
247         {
248             client.disconnect();
249         }
250     }
251 
252     public void destroyClient(ImmutableEndpoint endpoint, SftpClient client) throws Exception
253     {
254         if (useConnectionPool())
255         {
256             if ((client != null) && (client.isConnected()))
257             {
258                 ObjectPool pool = getClientPool(endpoint);
259                 pool.invalidateObject(client);
260             }
261         }
262     }
263 
264     protected synchronized ObjectPool getClientPool(ImmutableEndpoint endpoint)
265     {
266         GenericObjectPool pool = pools.get(endpoint.getEndpointURI());
267 
268         if (pool == null)
269         {
270             if (logger.isDebugEnabled())
271             {
272                 logger.debug("Pool is null - creating one for endpoint " + endpoint.getEndpointURI()
273                              + " with max size " + getMaxConnectionPoolSize());
274             }
275             pool = new GenericObjectPool(new SftpConnectionFactory(endpoint), getMaxConnectionPoolSize());
276             pool.setTestOnBorrow(isValidateConnections());
277             pools.put(endpoint.getEndpointURI(), pool);
278         }
279         else
280         {
281             if (logger.isDebugEnabled())
282             {
283                 logger.debug("Using existing pool for endpoint " + endpoint.getEndpointURI() + ". Active: "
284                              + pool.getNumActive() + ", Idle:" + pool.getNumIdle());
285             }
286         }
287 
288         return pool;
289     }
290 
291     /*
292      * (non-Javadoc)
293      * @see org.mule.transport.AbstractConnector#doConnect()
294      */
295     protected void doConnect() throws Exception
296     {
297         // Do nothing!
298     }
299 
300     /*
301      * (non-Javadoc)
302      * @see org.mule.transport.AbstractConnector#doDisconnect()
303      */
304     protected void doDisconnect() throws Exception
305     {
306         // Do nothing!
307     }
308 
309     /*
310      * (non-Javadoc)
311      * @see org.mule.transport.AbstractConnector#doDispose()
312      */
313     protected void doDispose()
314     {
315         // Do nothing!
316     }
317 
318     /*
319      * (non-Javadoc)
320      * @see org.mule.transport.AbstractConnector#doInitialise()
321      */
322     protected void doInitialise() throws InitialisationException
323     {
324         if (filenameParser != null)
325         {
326             filenameParser.setMuleContext(muleContext);
327         }
328     }
329 
330     /*
331      * (non-Javadoc)
332      * @see org.mule.transport.AbstractConnector#doStart()
333      */
334     protected void doStart() throws MuleException
335     {
336         // Do nothing!
337     }
338 
339     /*
340      * (non-Javadoc)
341      * @see org.mule.transport.AbstractConnector#doStop()
342      */
343     protected void doStop() throws MuleException
344     {
345         if (logger.isDebugEnabled())
346         {
347             logger.debug("Stopping all pools");
348         }
349         try
350         {
351             for (ObjectPool pool : pools.values())
352             {
353                 pool.close();
354             }
355         }
356         catch (Exception e)
357         {
358             throw new ConnectorException(CoreMessages.failedToStop("SFTP Connector"), this, e);
359         }
360         finally
361         {
362             pools.clear();
363         }
364     }
365 
366     public long getPollingFrequency()
367     {
368         return pollingFrequency;
369     }
370 
371     public void setPollingFrequency(long pollingFrequency)
372     {
373         this.pollingFrequency = pollingFrequency;
374     }
375 
376     public FilenameParser getFilenameParser()
377     {
378         return filenameParser;
379     }
380 
381     public void setFilenameParser(FilenameParser filenameParser)
382     {
383         this.filenameParser = filenameParser;
384         if (filenameParser != null)
385         {
386             filenameParser.setMuleContext(muleContext);
387         }
388     }
389 
390     public String getOutputPattern()
391     {
392         return outputPattern;
393     }
394 
395     public void setOutputPattern(String outputPattern)
396     {
397         this.outputPattern = outputPattern;
398     }
399 
400     public boolean isAutoDelete()
401     {
402         return autoDelete;
403     }
404 
405     public void setAutoDelete(boolean autoDelete)
406     {
407         this.autoDelete = autoDelete;
408     }
409 
410     public String getIdentityFile()
411     {
412         return identityFile;
413     }
414 
415     public void setIdentityFile(String identityFile)
416     {
417         this.identityFile = identityFile;
418     }
419 
420     public String getPassphrase()
421     {
422         return passphrase;
423     }
424 
425     public void setPassphrase(String passphrase)
426     {
427         this.passphrase = passphrase;
428     }
429 
430     /**
431      * Returns the file age.
432      * 
433      * @return Returns the fileAge in milliseconds.
434      */
435     public long getFileAge()
436     {
437         return fileAge;
438     }
439 
440     /**
441      * Sets the file age.
442      * 
443      * @param fileAge the fileAge in milliseconds to set.
444      */
445     public void setFileAge(long fileAge)
446     {
447         this.fileAge = fileAge;
448         this.checkFileAge = true;
449     }
450 
451     public boolean getCheckFileAge()
452     {
453         return checkFileAge;
454     }
455 
456     public String getTempDirInbound()
457     {
458         return tempDirInbound;
459     }
460 
461     public void setTempDirInbound(String pTempDirInbound)
462     {
463         tempDirInbound = pTempDirInbound;
464     }
465 
466     public String getTempDirOutbound()
467     {
468         return tempDirOutbound;
469     }
470 
471     public void setTempDirOutbound(String pTempDirOutbound)
472     {
473         tempDirOutbound = pTempDirOutbound;
474     }
475 
476     // Need this method to be public for SftpNotifier
477     @Override
478     public boolean isEnableMessageEvents()
479     {
480         return super.isEnableMessageEvents();
481     }
482 
483     public void setDuplicateHandling(String duplicateHandling)
484     {
485         this.duplicateHandling = duplicateHandling;
486     }
487 
488     public String getDuplicateHandling()
489     {
490         return duplicateHandling;
491     }
492 
493     public void setUseTempFileTimestampSuffix(Boolean useTempFileTimestampSuffix)
494     {
495         this.useTempFileTimestampSuffix = useTempFileTimestampSuffix;
496     }
497 
498     public Boolean isUseTempFileTimestampSuffix()
499     {
500         return useTempFileTimestampSuffix;
501     }
502 
503     public void setSizeCheckWaitTime(Long sizeCheckWaitTime)
504     {
505         this.sizeCheckWaitTime = sizeCheckWaitTime;
506     }
507 
508     public Long getSizeCheckWaitTime()
509     {
510         return sizeCheckWaitTime;
511     }
512 
513     public void setArchiveDir(String archiveDir)
514     {
515         this.archiveDir = archiveDir;
516     }
517 
518     public String getArchiveDir()
519     {
520         return archiveDir;
521     }
522 
523     public void setArchiveTempReceivingDir(String archiveTempReceivingDir)
524     {
525         this.archiveTempReceivingDir = archiveTempReceivingDir;
526     }
527 
528     public String getArchiveTempReceivingDir()
529     {
530         return archiveTempReceivingDir;
531     }
532 
533     public void setArchiveTempSendingDir(String archiveTempSendingDir)
534     {
535         this.archiveTempSendingDir = archiveTempSendingDir;
536     }
537 
538     public String getArchiveTempSendingDir()
539     {
540         return archiveTempSendingDir;
541     }
542 
543     /**
544      * @see SftpConnector#maxConnectionPoolSize
545      */
546     public void setMaxConnectionPoolSize(int maxConnectionPoolSize)
547     {
548         this.maxConnectionPoolSize = maxConnectionPoolSize;
549     }
550 
551     /**
552      * @return the max connection pool size. If the system parameter
553      *         mule.sftp.transport.maxConnectionPoolSize is set, that value will be
554      *         used instead.
555      */
556     public int getMaxConnectionPoolSize()
557     {
558         if (overrideMaxConnectionPoolSize != null)
559         {
560             return overrideMaxConnectionPoolSize;
561         }
562         return maxConnectionPoolSize;
563     }
564 
565     public Boolean isKeepFileOnError()
566     {
567         return keepFileOnError;
568     }
569 
570     public void setKeepFileOnError(Boolean pKeepFileOnError)
571     {
572         keepFileOnError = pKeepFileOnError;
573     }
574 }