View Javadoc

1   /*
2    * $Id: FtpConnector.java 22713 2011-08-22 07:46:36Z dirk.olmes $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.transport.ftp;
12  
13  import org.mule.api.MuleContext;
14  import org.mule.api.MuleEvent;
15  import org.mule.api.MuleException;
16  import org.mule.api.MuleMessage;
17  import org.mule.api.MuleRuntimeException;
18  import org.mule.api.config.ThreadingProfile;
19  import org.mule.api.construct.FlowConstruct;
20  import org.mule.api.endpoint.EndpointURI;
21  import org.mule.api.endpoint.ImmutableEndpoint;
22  import org.mule.api.endpoint.InboundEndpoint;
23  import org.mule.api.endpoint.OutboundEndpoint;
24  import org.mule.api.lifecycle.InitialisationException;
25  import org.mule.api.transport.ConnectorException;
26  import org.mule.api.transport.DispatchException;
27  import org.mule.api.transport.MessageReceiver;
28  import org.mule.config.i18n.CoreMessages;
29  import org.mule.config.i18n.MessageFactory;
30  import org.mule.model.streaming.CallbackOutputStream;
31  import org.mule.transport.AbstractConnector;
32  import org.mule.transport.ConnectException;
33  import org.mule.transport.file.ExpressionFilenameParser;
34  import org.mule.transport.file.FilenameParser;
35  import org.mule.util.ClassUtils;
36  import org.mule.util.StringUtils;
37  
38  import java.io.IOException;
39  import java.io.OutputStream;
40  import java.text.MessageFormat;
41  import java.util.ArrayList;
42  import java.util.HashMap;
43  import java.util.List;
44  import java.util.Map;
45  
46  import org.apache.commons.net.ftp.FTPClient;
47  import org.apache.commons.net.ftp.FTPFile;
48  import org.apache.commons.pool.ObjectPool;
49  import org.apache.commons.pool.impl.GenericObjectPool;
50  
51  public class FtpConnector extends AbstractConnector
52  {
53  
54      public static final String FTP = "ftp";
55  
56      // endpoint properties
57      public static final int DEFAULT_POLLING_FREQUENCY = 1000;
58      public static final String PROPERTY_OUTPUT_PATTERN = "outputPattern"; // outbound only
59      public static final String PROPERTY_PASSIVE_MODE = "passive";
60      public static final String PROPERTY_BINARY_TRANSFER = "binary";
61  
62      // message properties
63      public static final String PROPERTY_FILENAME = "filename";
64  
65  
66      /**
67       *  TODO it makes sense to have a type-safe adapter for FTP specifically, but without
68       *  Java 5's covariant return types the benefits are diminished. Keeping it simple for now.
69       */
70      public static final String DEFAULT_FTP_CONNECTION_FACTORY_CLASS = "org.mule.transport.ftp.FtpConnectionFactory";
71  
72      /**
73       * Time in milliseconds to poll. On each poll the poll() method is called
74       */
75      private long pollingFrequency;
76  
77      private String outputPattern;
78  
79      private FilenameParser filenameParser = new ExpressionFilenameParser();
80  
81      private boolean passive = true;
82  
83      private boolean binary = true;
84  
85      /** Streaming is off by default until MULE-3192 gets fixed */
86      private boolean streaming = false;
87  
88      private Map<String, ObjectPool> pools;
89  
90      private String connectionFactoryClass = DEFAULT_FTP_CONNECTION_FACTORY_CLASS;
91  
92      public FtpConnector(MuleContext context)
93      {
94          super(context);
95      }
96  
97      public String getProtocol()
98      {
99          return FTP;
100     }
101 
102     @Override
103     public MessageReceiver createReceiver(FlowConstruct flowConstruct, InboundEndpoint endpoint) throws Exception
104     {
105         List<?> args = getReceiverArguments(endpoint.getProperties());
106         return serviceDescriptor.createMessageReceiver(this, flowConstruct, endpoint, args.toArray());
107     }
108 
109     protected List<?> getReceiverArguments(Map endpointProperties)
110     {
111         List<Object> args = new ArrayList<Object>();
112 
113         long polling = getPollingFrequency();
114         if (endpointProperties != null)
115         {
116             // Override properties on the endpoint for the specific endpoint
117             String tempPolling = (String) endpointProperties.get(PROPERTY_POLLING_FREQUENCY);
118             if (tempPolling != null)
119             {
120                 polling = Long.parseLong(tempPolling);
121             }
122         }
123         if (polling <= 0)
124         {
125             polling = DEFAULT_POLLING_FREQUENCY;
126         }
127         logger.debug("set polling frequency to " + polling);
128         args.add(polling);
129 
130         return args;
131     }
132 
133     /**
134      * @return Returns the pollingFrequency.
135      */
136     public long getPollingFrequency()
137     {
138         return pollingFrequency;
139     }
140 
141     /**
142      * @param pollingFrequency The pollingFrequency to set.
143      */
144     public void setPollingFrequency(long pollingFrequency)
145     {
146         this.pollingFrequency = pollingFrequency;
147     }
148 
149     /**
150      * Getter for property 'connectionFactoryClass'.
151      *
152      * @return Value for property 'connectionFactoryClass'.
153      */
154     public String getConnectionFactoryClass()
155     {
156         return connectionFactoryClass;
157     }
158 
159     /**
160      * Setter for property 'connectionFactoryClass'. Should be an instance of
161      * {@link FtpConnectionFactory}.
162      *
163      * @param connectionFactoryClass Value to set for property 'connectionFactoryClass'.
164      */
165     public void setConnectionFactoryClass(final String connectionFactoryClass)
166     {
167         this.connectionFactoryClass = connectionFactoryClass;
168     }
169 
170     public FTPClient getFtp(EndpointURI uri) throws Exception
171     {
172         if (logger.isDebugEnabled())
173         {
174             logger.debug(">>> retrieving client for " + uri);
175         }
176         return (FTPClient) getFtpPool(uri).borrowObject();
177     }
178 
179     public void releaseFtp(EndpointURI uri, FTPClient client) throws Exception
180     {
181         if (logger.isDebugEnabled())
182         {
183             logger.debug("<<< releasing client for " + uri);
184         }
185         if (dispatcherFactory.isCreateDispatcherPerRequest())
186         {
187             destroyFtp(uri, client);
188         }
189         else
190         {
191             getFtpPool(uri).returnObject(client);
192         }
193     }
194 
195     public void destroyFtp(EndpointURI uri, FTPClient client) throws Exception
196     {
197         if (logger.isDebugEnabled())
198         {
199             logger.debug("<<< destroying client for " + uri);
200         }
201         try
202         {
203             getFtpPool(uri).invalidateObject(client);
204         }
205         catch (Exception e)
206         {
207             // no way to test if pool is closed except try to access it
208             logger.debug(e.getMessage());
209         }
210     }
211 
212     protected synchronized ObjectPool getFtpPool(EndpointURI uri)
213     {
214         if (logger.isDebugEnabled())
215         {
216             logger.debug("=== get pool for " + uri);
217         }
218         String key = uri.getUser() + ":" + uri.getPassword() + "@" + uri.getHost() + ":" + uri.getPort();
219         ObjectPool pool = pools.get(key);
220         if (pool == null)
221         {
222             try
223             {
224                 FtpConnectionFactory connectionFactory =
225                         (FtpConnectionFactory) ClassUtils.instanciateClass(getConnectionFactoryClass(),
226                                                                             new Object[] {uri}, getClass());
227                 GenericObjectPool genericPool = createPool(connectionFactory);
228                 pools.put(key, genericPool);
229                 pool = genericPool;
230             }
231             catch (Exception ex)
232             {
233                 throw new MuleRuntimeException(
234                         MessageFactory.createStaticMessage("Hmm, couldn't instanciate FTP connection factory."), ex);
235             }
236         }
237         return pool;
238     }
239 
240     protected GenericObjectPool createPool(FtpConnectionFactory connectionFactory)
241     {
242         GenericObjectPool genericPool = new GenericObjectPool(connectionFactory);
243         byte poolExhaustedAction = ThreadingProfile.DEFAULT_POOL_EXHAUST_ACTION;
244 
245         ThreadingProfile receiverThreadingProfile = this.getReceiverThreadingProfile();
246         if (receiverThreadingProfile != null)
247         {
248             int threadingProfilePoolExhaustedAction = receiverThreadingProfile.getPoolExhaustedAction();
249             if (threadingProfilePoolExhaustedAction == ThreadingProfile.WHEN_EXHAUSTED_WAIT)
250             {
251                 poolExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
252             }
253             else if (threadingProfilePoolExhaustedAction == ThreadingProfile.WHEN_EXHAUSTED_ABORT)
254             {
255                 poolExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_FAIL;
256             }
257             else if (threadingProfilePoolExhaustedAction == ThreadingProfile.WHEN_EXHAUSTED_RUN)
258             {
259                 poolExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_GROW;
260             }
261         }
262 
263         genericPool.setWhenExhaustedAction(poolExhaustedAction);
264         genericPool.setTestOnBorrow(isValidateConnections());
265         return genericPool;
266     }
267 
268     @Override
269     protected void doInitialise() throws InitialisationException
270     {
271         if (filenameParser != null)
272         {
273             filenameParser.setMuleContext(muleContext);
274         }
275 
276         try
277         {
278             Class<?> objectFactoryClass = ClassUtils.loadClass(this.connectionFactoryClass, getClass());
279             if (!FtpConnectionFactory.class.isAssignableFrom(objectFactoryClass))
280             {
281                 throw new InitialisationException(MessageFactory.createStaticMessage(
282                         "FTP connectionFactoryClass is not an instance of org.mule.transport.ftp.FtpConnectionFactory"),
283                         this);
284             }
285         }
286         catch (ClassNotFoundException e)
287         {
288             throw new InitialisationException(e, this);
289         }
290 
291         pools = new HashMap<String, ObjectPool>();
292     }
293 
294     @Override
295     protected void doDispose()
296     {
297         // template method
298     }
299 
300     @Override
301     protected void doConnect() throws Exception
302     {
303         // template method
304     }
305 
306     @Override
307     protected void doDisconnect() throws Exception
308     {
309         // template method
310     }
311 
312     @Override
313     protected void doStart() throws MuleException
314     {
315         // template method
316     }
317 
318     @Override
319     protected void doStop() throws MuleException
320     {
321         if (logger.isDebugEnabled())
322         {
323             logger.debug("Stopping all pools");
324         }
325         try
326         {
327             for (ObjectPool pool : pools.values())
328             {
329                 pool.close();
330             }
331         }
332         catch (Exception e)
333         {
334             throw new ConnectorException(CoreMessages.failedToStop("FTP Connector"), this, e);
335         }
336         finally
337         {
338             pools.clear();
339         }
340     }
341 
342     /**
343      * @return Returns the outputPattern.
344      */
345     public String getOutputPattern()
346     {
347         return outputPattern;
348     }
349 
350     /**
351      * @param outputPattern The outputPattern to set.
352      */
353     public void setOutputPattern(String outputPattern)
354     {
355         this.outputPattern = outputPattern;
356     }
357 
358     /**
359      * @return Returns the filenameParser.
360      */
361     public FilenameParser getFilenameParser()
362     {
363         return filenameParser;
364     }
365 
366     /**
367      * @param filenameParser The filenameParser to set.
368      */
369     public void setFilenameParser(FilenameParser filenameParser)
370     {
371         this.filenameParser = filenameParser;
372         if (filenameParser != null)
373         {
374             filenameParser.setMuleContext(muleContext);
375         }
376     }
377 
378     /**
379      * Getter for FTP passive mode.
380      *
381      * @return true if using FTP passive mode
382      */
383     public boolean isPassive()
384     {
385         return passive;
386     }
387 
388     /**
389      * Setter for FTP passive mode.
390      *
391      * @param passive passive mode flag
392      */
393     public void setPassive(final boolean passive)
394     {
395         this.passive = passive;
396     }
397 
398     /**
399      * Passive mode is OFF by default. The value is taken from the connector
400      * settings. In case there are any overriding properties set on the endpoint,
401      * those will be used.
402      *
403      * @see #setPassive(boolean)
404      */
405     public void enterActiveOrPassiveMode(FTPClient client, ImmutableEndpoint endpoint)
406     {
407         // well, no endpoint URI here, as we have to use the most common denominator
408         // in API :(
409         final String passiveString = (String)endpoint.getProperty(FtpConnector.PROPERTY_PASSIVE_MODE);
410         if (passiveString == null)
411         {
412             // try the connector properties then
413             if (isPassive())
414             {
415                 if (logger.isTraceEnabled())
416                 {
417                     logger.trace("Entering FTP passive mode");
418                 }
419                 client.enterLocalPassiveMode();
420             }
421             else
422             {
423                 if (logger.isTraceEnabled())
424                 {
425                     logger.trace("Entering FTP active mode");
426                 }
427                 client.enterLocalActiveMode();
428             }
429         }
430         else
431         {
432             // override with endpoint's definition
433             final boolean passiveMode = Boolean.valueOf(passiveString).booleanValue();
434             if (passiveMode)
435             {
436                 if (logger.isTraceEnabled())
437                 {
438                     logger.trace("Entering FTP passive mode (endpoint override)");
439                 }
440                 client.enterLocalPassiveMode();
441             }
442             else
443             {
444                 if (logger.isTraceEnabled())
445                 {
446                     logger.trace("Entering FTP active mode (endpoint override)");
447                 }
448                 client.enterLocalActiveMode();
449             }
450         }
451     }
452 
453     /**
454      * Getter for FTP transfer type.
455      *
456      * @return true if using FTP binary type
457      */
458     public boolean isBinary()
459     {
460         return binary;
461     }
462 
463     /**
464      * Setter for FTP transfer type.
465      *
466      * @param binary binary type flag
467      */
468     public void setBinary(final boolean binary)
469     {
470         this.binary = binary;
471     }
472 
473     /**
474      * Transfer type is BINARY by default. The value is taken from the connector
475      * settings. In case there are any overriding properties set on the endpoint,
476      * those will be used. <p/> The alternative type is ASCII. <p/>
477      *
478      * @see #setBinary(boolean)
479      */
480     public void setupFileType(FTPClient client, ImmutableEndpoint endpoint) throws Exception
481     {
482         int type;
483 
484         // well, no endpoint URI here, as we have to use the most common denominator
485         // in API :(
486         final String binaryTransferString = (String)endpoint.getProperty(FtpConnector.PROPERTY_BINARY_TRANSFER);
487         if (binaryTransferString == null)
488         {
489             // try the connector properties then
490             if (isBinary())
491             {
492                 if (logger.isTraceEnabled())
493                 {
494                     logger.trace("Using FTP BINARY type");
495                 }
496                 type = org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE;
497             }
498             else
499             {
500                 if (logger.isTraceEnabled())
501                 {
502                     logger.trace("Using FTP ASCII type");
503                 }
504                 type = org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE;
505             }
506         }
507         else
508         {
509             // override with endpoint's definition
510             final boolean binaryTransfer = Boolean.valueOf(binaryTransferString).booleanValue();
511             if (binaryTransfer)
512             {
513                 if (logger.isTraceEnabled())
514                 {
515                     logger.trace("Using FTP BINARY type (endpoint override)");
516                 }
517                 type = org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE;
518             }
519             else
520             {
521                 if (logger.isTraceEnabled())
522                 {
523                     logger.trace("Using FTP ASCII type (endpoint override)");
524                 }
525                 type = org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE;
526             }
527         }
528 
529         client.setFileType(type);
530     }
531 
532     /**
533      * Well get the output stream (if any) for this type of transport. Typically this
534      * will be called only when Streaming is being used on an outbound endpoint
535      *
536      * @param endpoint the endpoint that releates to this Dispatcher
537      * @param event the current event being processed
538      * @return the output stream to use for this request or null if the transport
539      *         does not support streaming
540      */
541     @Override
542     public OutputStream getOutputStream(OutboundEndpoint endpoint, MuleEvent event) throws MuleException
543     {
544         try
545         {
546             final EndpointURI uri = endpoint.getEndpointURI();
547             String filename = getFilename(endpoint, event.getMessage());
548 
549             final FTPClient client;
550             try
551             {
552                 client = this.createFtpClient(endpoint);
553             }
554             catch (Exception e)
555             {
556                 throw new ConnectException(e, this);
557             }
558 
559             try
560             {
561                 OutputStream out = client.storeFileStream(filename);
562                 if (out == null)
563                 {
564                     throw new IOException("FTP operation failed: " + client.getReplyString());
565                 }
566 
567                 return new CallbackOutputStream(out,
568                         new CallbackOutputStream.Callback()
569                         {
570                             public void onClose() throws Exception
571                             {
572                                 try
573                                 {
574                                     if (!client.completePendingCommand())
575                                     {
576                                         client.logout();
577                                         client.disconnect();
578                                         throw new IOException("FTP Stream failed to complete pending request");
579                                     }
580                                 }
581                                 finally
582                                 {
583                                     releaseFtp(uri, client);
584                                 }
585                             }
586                         });
587             }
588             catch (Exception e)
589             {
590                 logger.debug("Error getting output stream: ", e);
591                 releaseFtp(uri, client);
592                 throw e;
593             }
594         }
595         catch (ConnectException ce)
596         {
597             // Don't wrap a ConnectException, otherwise the retry policy will not go into effect.
598             throw ce;
599         }
600         catch (Exception e)
601         {
602             throw new DispatchException(CoreMessages.streamingFailedNoStream(), event, endpoint, e);
603         }
604     }
605 
606     private String getFilename(ImmutableEndpoint endpoint, MuleMessage message) throws IOException
607     {
608         String filename = message.getOutboundProperty(FtpConnector.PROPERTY_FILENAME);
609         String outPattern = (String) endpoint.getProperty(FtpConnector.PROPERTY_OUTPUT_PATTERN);
610         if (outPattern == null)
611         {
612             outPattern = message.getOutboundProperty(FtpConnector.PROPERTY_OUTPUT_PATTERN, getOutputPattern());
613         }
614         if (outPattern != null || filename == null)
615         {
616             filename = generateFilename(message, outPattern);
617         }
618         if (filename == null)
619         {
620             throw new IOException("Filename is null");
621         }
622         return filename;
623     }
624 
625     private String generateFilename(MuleMessage message, String pattern)
626     {
627         if (pattern == null)
628         {
629             pattern = getOutputPattern();
630         }
631         return getFilenameParser().getFilename(message, pattern);
632     }
633 
634     /**
635      * Creates a new FTPClient that logs in and changes the working directory using the data
636      * provided in <code>endpoint</code>.
637      */
638     protected FTPClient createFtpClient(ImmutableEndpoint endpoint) throws Exception
639     {
640         EndpointURI uri = endpoint.getEndpointURI();
641         FTPClient client = this.getFtp(uri);
642 
643         this.enterActiveOrPassiveMode(client, endpoint);
644         this.setupFileType(client, endpoint);
645 
646         String path = uri.getPath();
647 
648         // only change directory if one was configured
649         if (StringUtils.isNotBlank(path))
650         {
651             // MULE-2400: if the path begins with '~' we must strip the first '/' to make things
652             // work with FTPClient
653             if ((path.length() >= 2) && (path.charAt(1) == '~'))
654             {
655                 path = path.substring(1);
656             }
657 
658             if (!client.changeWorkingDirectory(path))
659             {
660                 throw new IOException(MessageFormat.format("Failed to change working directory to {0}. Ftp error: {1}",
661                                                            path, client.getReplyCode()));
662             }
663         }
664         return client;
665     }
666 
667     /**
668      * Override this method to do extra checking on the file.
669      */
670     protected boolean validateFile(FTPFile file)
671     {
672         return true;
673     }
674 
675     public boolean isStreaming()
676     {
677         return streaming;
678     }
679 
680     public void setStreaming(boolean streaming)
681     {
682         this.streaming = streaming;
683     }
684 
685 }