View Javadoc
1   /*
2    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
3    * The software in this package is published under the terms of the CPAL v1.0
4    * license, a copy of which has been included with this distribution in the
5    * LICENSE.txt file.
6    */
7   package org.mule.transport.ftp;
8   
9   import org.mule.api.MuleContext;
10  import org.mule.api.MuleEvent;
11  import org.mule.api.MuleException;
12  import org.mule.api.MuleMessage;
13  import org.mule.api.MuleRuntimeException;
14  import org.mule.api.config.ThreadingProfile;
15  import org.mule.api.construct.FlowConstruct;
16  import org.mule.api.endpoint.EndpointURI;
17  import org.mule.api.endpoint.ImmutableEndpoint;
18  import org.mule.api.endpoint.InboundEndpoint;
19  import org.mule.api.endpoint.OutboundEndpoint;
20  import org.mule.api.lifecycle.InitialisationException;
21  import org.mule.api.transport.ConnectorException;
22  import org.mule.api.transport.DispatchException;
23  import org.mule.api.transport.MessageReceiver;
24  import org.mule.config.i18n.CoreMessages;
25  import org.mule.config.i18n.MessageFactory;
26  import org.mule.model.streaming.CallbackOutputStream;
27  import org.mule.transport.AbstractConnector;
28  import org.mule.transport.ConnectException;
29  import org.mule.transport.file.ExpressionFilenameParser;
30  import org.mule.transport.file.FilenameParser;
31  import org.mule.util.ClassUtils;
32  import org.mule.util.StringUtils;
33  
34  import java.io.IOException;
35  import java.io.OutputStream;
36  import java.text.MessageFormat;
37  import java.util.ArrayList;
38  import java.util.HashMap;
39  import java.util.List;
40  import java.util.Map;
41  
42  import org.apache.commons.net.ftp.FTPClient;
43  import org.apache.commons.net.ftp.FTPFile;
44  import org.apache.commons.pool.ObjectPool;
45  import org.apache.commons.pool.impl.GenericObjectPool;
46  
47  public class FtpConnector extends AbstractConnector
48  {
49  
50      public static final String FTP = "ftp";
51  
52      // endpoint properties
53      public static final int DEFAULT_POLLING_FREQUENCY = 1000;
54      public static final String PROPERTY_OUTPUT_PATTERN = "outputPattern"; // outbound only
55      public static final String PROPERTY_PASSIVE_MODE = "passive";
56      public static final String PROPERTY_BINARY_TRANSFER = "binary";
57  
58      // message properties
59      public static final String PROPERTY_FILENAME = "filename";
60  
61  
62      /**
63       *  TODO it makes sense to have a type-safe adapter for FTP specifically, but without
64       *  Java 5's covariant return types the benefits are diminished. Keeping it simple for now.
65       */
66      public static final String DEFAULT_FTP_CONNECTION_FACTORY_CLASS = "org.mule.transport.ftp.FtpConnectionFactory";
67  
68      /**
69       * Time in milliseconds to poll. On each poll the poll() method is called
70       */
71      private long pollingFrequency;
72  
73      private String outputPattern;
74  
75      private FilenameParser filenameParser = new ExpressionFilenameParser();
76  
77      private boolean passive = true;
78  
79      private boolean binary = true;
80  
81      /** Streaming is off by default until MULE-3192 gets fixed */
82      private boolean streaming = false;
83  
84      private Map<String, ObjectPool> pools;
85  
86      private String connectionFactoryClass = DEFAULT_FTP_CONNECTION_FACTORY_CLASS;
87  
88      public FtpConnector(MuleContext context)
89      {
90          super(context);
91      }
92  
93      public String getProtocol()
94      {
95          return FTP;
96      }
97  
98      @Override
99      public MessageReceiver createReceiver(FlowConstruct flowConstruct, InboundEndpoint endpoint) throws Exception
100     {
101         List<?> args = getReceiverArguments(endpoint.getProperties());
102         return serviceDescriptor.createMessageReceiver(this, flowConstruct, endpoint, args.toArray());
103     }
104 
105     protected List<?> getReceiverArguments(Map endpointProperties)
106     {
107         List<Object> args = new ArrayList<Object>();
108 
109         long polling = getPollingFrequency();
110         if (endpointProperties != null)
111         {
112             // Override properties on the endpoint for the specific endpoint
113             String tempPolling = (String) endpointProperties.get(PROPERTY_POLLING_FREQUENCY);
114             if (tempPolling != null)
115             {
116                 polling = Long.parseLong(tempPolling);
117             }
118         }
119         if (polling <= 0)
120         {
121             polling = DEFAULT_POLLING_FREQUENCY;
122         }
123         logger.debug("set polling frequency to " + polling);
124         args.add(polling);
125 
126         return args;
127     }
128 
129     /**
130      * @return Returns the pollingFrequency.
131      */
132     public long getPollingFrequency()
133     {
134         return pollingFrequency;
135     }
136 
137     /**
138      * @param pollingFrequency The pollingFrequency to set.
139      */
140     public void setPollingFrequency(long pollingFrequency)
141     {
142         this.pollingFrequency = pollingFrequency;
143     }
144 
145     /**
146      * Getter for property 'connectionFactoryClass'.
147      *
148      * @return Value for property 'connectionFactoryClass'.
149      */
150     public String getConnectionFactoryClass()
151     {
152         return connectionFactoryClass;
153     }
154 
155     /**
156      * Setter for property 'connectionFactoryClass'. Should be an instance of
157      * {@link FtpConnectionFactory}.
158      *
159      * @param connectionFactoryClass Value to set for property 'connectionFactoryClass'.
160      */
161     public void setConnectionFactoryClass(final String connectionFactoryClass)
162     {
163         this.connectionFactoryClass = connectionFactoryClass;
164     }
165 
166     public FTPClient getFtp(EndpointURI uri) throws Exception
167     {
168         if (logger.isDebugEnabled())
169         {
170             logger.debug(">>> retrieving client for " + uri);
171         }
172         return (FTPClient) getFtpPool(uri).borrowObject();
173     }
174 
175     public void releaseFtp(EndpointURI uri, FTPClient client) throws Exception
176     {
177         if (logger.isDebugEnabled())
178         {
179             logger.debug("<<< releasing client for " + uri);
180         }
181         if (dispatcherFactory.isCreateDispatcherPerRequest())
182         {
183             destroyFtp(uri, client);
184         }
185         else
186         {
187             getFtpPool(uri).returnObject(client);
188         }
189     }
190 
191     public void destroyFtp(EndpointURI uri, FTPClient client) throws Exception
192     {
193         if (logger.isDebugEnabled())
194         {
195             logger.debug("<<< destroying client for " + uri);
196         }
197         try
198         {
199             getFtpPool(uri).invalidateObject(client);
200         }
201         catch (Exception e)
202         {
203             // no way to test if pool is closed except try to access it
204             logger.debug(e.getMessage());
205         }
206     }
207 
208     protected synchronized ObjectPool getFtpPool(EndpointURI uri)
209     {
210         if (logger.isDebugEnabled())
211         {
212             logger.debug("=== get pool for " + uri);
213         }
214         String key = uri.getUser() + ":" + uri.getPassword() + "@" + uri.getHost() + ":" + uri.getPort();
215         ObjectPool pool = pools.get(key);
216         if (pool == null)
217         {
218             try
219             {
220                 FtpConnectionFactory connectionFactory =
221                         (FtpConnectionFactory) ClassUtils.instanciateClass(getConnectionFactoryClass(),
222                                                                             new Object[] {uri}, getClass());
223                 GenericObjectPool genericPool = createPool(connectionFactory);
224                 pools.put(key, genericPool);
225                 pool = genericPool;
226             }
227             catch (Exception ex)
228             {
229                 throw new MuleRuntimeException(
230                         MessageFactory.createStaticMessage("Hmm, couldn't instanciate FTP connection factory."), ex);
231             }
232         }
233         return pool;
234     }
235 
236     protected GenericObjectPool createPool(FtpConnectionFactory connectionFactory)
237     {
238         GenericObjectPool genericPool = new GenericObjectPool(connectionFactory);
239         byte poolExhaustedAction = ThreadingProfile.DEFAULT_POOL_EXHAUST_ACTION;
240 
241         ThreadingProfile receiverThreadingProfile = this.getReceiverThreadingProfile();
242         if (receiverThreadingProfile != null)
243         {
244             int threadingProfilePoolExhaustedAction = receiverThreadingProfile.getPoolExhaustedAction();
245             if (threadingProfilePoolExhaustedAction == ThreadingProfile.WHEN_EXHAUSTED_WAIT)
246             {
247                 poolExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
248             }
249             else if (threadingProfilePoolExhaustedAction == ThreadingProfile.WHEN_EXHAUSTED_ABORT)
250             {
251                 poolExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_FAIL;
252             }
253             else if (threadingProfilePoolExhaustedAction == ThreadingProfile.WHEN_EXHAUSTED_RUN)
254             {
255                 poolExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_GROW;
256             }
257         }
258 
259         genericPool.setWhenExhaustedAction(poolExhaustedAction);
260         genericPool.setTestOnBorrow(isValidateConnections());
261         return genericPool;
262     }
263 
264     @Override
265     protected void doInitialise() throws InitialisationException
266     {
267         if (filenameParser != null)
268         {
269             filenameParser.setMuleContext(muleContext);
270         }
271 
272         try
273         {
274             Class<?> objectFactoryClass = ClassUtils.loadClass(this.connectionFactoryClass, getClass());
275             if (!FtpConnectionFactory.class.isAssignableFrom(objectFactoryClass))
276             {
277                 throw new InitialisationException(MessageFactory.createStaticMessage(
278                         "FTP connectionFactoryClass is not an instance of org.mule.transport.ftp.FtpConnectionFactory"),
279                         this);
280             }
281         }
282         catch (ClassNotFoundException e)
283         {
284             throw new InitialisationException(e, this);
285         }
286 
287         pools = new HashMap<String, ObjectPool>();
288     }
289 
290     @Override
291     protected void doDispose()
292     {
293         // template method
294     }
295 
296     @Override
297     protected void doConnect() throws Exception
298     {
299         // template method
300     }
301 
302     @Override
303     protected void doDisconnect() throws Exception
304     {
305         // template method
306     }
307 
308     @Override
309     protected void doStart() throws MuleException
310     {
311         // template method
312     }
313 
314     @Override
315     protected void doStop() throws MuleException
316     {
317         if (logger.isDebugEnabled())
318         {
319             logger.debug("Stopping all pools");
320         }
321         try
322         {
323             for (ObjectPool pool : pools.values())
324             {
325                 pool.close();
326             }
327         }
328         catch (Exception e)
329         {
330             throw new ConnectorException(CoreMessages.failedToStop("FTP Connector"), this, e);
331         }
332         finally
333         {
334             pools.clear();
335         }
336     }
337 
338     /**
339      * @return Returns the outputPattern.
340      */
341     public String getOutputPattern()
342     {
343         return outputPattern;
344     }
345 
346     /**
347      * @param outputPattern The outputPattern to set.
348      */
349     public void setOutputPattern(String outputPattern)
350     {
351         this.outputPattern = outputPattern;
352     }
353 
354     /**
355      * @return Returns the filenameParser.
356      */
357     public FilenameParser getFilenameParser()
358     {
359         return filenameParser;
360     }
361 
362     /**
363      * @param filenameParser The filenameParser to set.
364      */
365     public void setFilenameParser(FilenameParser filenameParser)
366     {
367         this.filenameParser = filenameParser;
368         if (filenameParser != null)
369         {
370             filenameParser.setMuleContext(muleContext);
371         }
372     }
373 
374     /**
375      * Getter for FTP passive mode.
376      *
377      * @return true if using FTP passive mode
378      */
379     public boolean isPassive()
380     {
381         return passive;
382     }
383 
384     /**
385      * Setter for FTP passive mode.
386      *
387      * @param passive passive mode flag
388      */
389     public void setPassive(final boolean passive)
390     {
391         this.passive = passive;
392     }
393 
394     /**
395      * Passive mode is OFF by default. The value is taken from the connector
396      * settings. In case there are any overriding properties set on the endpoint,
397      * those will be used.
398      *
399      * @see #setPassive(boolean)
400      */
401     public void enterActiveOrPassiveMode(FTPClient client, ImmutableEndpoint endpoint)
402     {
403         // well, no endpoint URI here, as we have to use the most common denominator
404         // in API :(
405         final String passiveString = (String)endpoint.getProperty(FtpConnector.PROPERTY_PASSIVE_MODE);
406         if (passiveString == null)
407         {
408             // try the connector properties then
409             if (isPassive())
410             {
411                 if (logger.isTraceEnabled())
412                 {
413                     logger.trace("Entering FTP passive mode");
414                 }
415                 client.enterLocalPassiveMode();
416             }
417             else
418             {
419                 if (logger.isTraceEnabled())
420                 {
421                     logger.trace("Entering FTP active mode");
422                 }
423                 client.enterLocalActiveMode();
424             }
425         }
426         else
427         {
428             // override with endpoint's definition
429             final boolean passiveMode = Boolean.valueOf(passiveString).booleanValue();
430             if (passiveMode)
431             {
432                 if (logger.isTraceEnabled())
433                 {
434                     logger.trace("Entering FTP passive mode (endpoint override)");
435                 }
436                 client.enterLocalPassiveMode();
437             }
438             else
439             {
440                 if (logger.isTraceEnabled())
441                 {
442                     logger.trace("Entering FTP active mode (endpoint override)");
443                 }
444                 client.enterLocalActiveMode();
445             }
446         }
447     }
448 
449     /**
450      * Getter for FTP transfer type.
451      *
452      * @return true if using FTP binary type
453      */
454     public boolean isBinary()
455     {
456         return binary;
457     }
458 
459     /**
460      * Setter for FTP transfer type.
461      *
462      * @param binary binary type flag
463      */
464     public void setBinary(final boolean binary)
465     {
466         this.binary = binary;
467     }
468 
469     /**
470      * Transfer type is BINARY by default. The value is taken from the connector
471      * settings. In case there are any overriding properties set on the endpoint,
472      * those will be used. <p/> The alternative type is ASCII. <p/>
473      *
474      * @see #setBinary(boolean)
475      */
476     public void setupFileType(FTPClient client, ImmutableEndpoint endpoint) throws Exception
477     {
478         int type;
479 
480         // well, no endpoint URI here, as we have to use the most common denominator
481         // in API :(
482         final String binaryTransferString = (String)endpoint.getProperty(FtpConnector.PROPERTY_BINARY_TRANSFER);
483         if (binaryTransferString == null)
484         {
485             // try the connector properties then
486             if (isBinary())
487             {
488                 if (logger.isTraceEnabled())
489                 {
490                     logger.trace("Using FTP BINARY type");
491                 }
492                 type = org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE;
493             }
494             else
495             {
496                 if (logger.isTraceEnabled())
497                 {
498                     logger.trace("Using FTP ASCII type");
499                 }
500                 type = org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE;
501             }
502         }
503         else
504         {
505             // override with endpoint's definition
506             final boolean binaryTransfer = Boolean.valueOf(binaryTransferString).booleanValue();
507             if (binaryTransfer)
508             {
509                 if (logger.isTraceEnabled())
510                 {
511                     logger.trace("Using FTP BINARY type (endpoint override)");
512                 }
513                 type = org.apache.commons.net.ftp.FTP.BINARY_FILE_TYPE;
514             }
515             else
516             {
517                 if (logger.isTraceEnabled())
518                 {
519                     logger.trace("Using FTP ASCII type (endpoint override)");
520                 }
521                 type = org.apache.commons.net.ftp.FTP.ASCII_FILE_TYPE;
522             }
523         }
524 
525         client.setFileType(type);
526     }
527 
528     /**
529      * Well get the output stream (if any) for this type of transport. Typically this
530      * will be called only when Streaming is being used on an outbound endpoint
531      *
532      * @param endpoint the endpoint that releates to this Dispatcher
533      * @param event the current event being processed
534      * @return the output stream to use for this request or null if the transport
535      *         does not support streaming
536      */
537     @Override
538     public OutputStream getOutputStream(OutboundEndpoint endpoint, MuleEvent event) throws MuleException
539     {
540         try
541         {
542             final EndpointURI uri = endpoint.getEndpointURI();
543             String filename = getFilename(endpoint, event.getMessage());
544 
545             final FTPClient client;
546             try
547             {
548                 client = this.createFtpClient(endpoint);
549             }
550             catch (Exception e)
551             {
552                 throw new ConnectException(e, this);
553             }
554 
555             try
556             {
557                 OutputStream out = client.storeFileStream(filename);
558                 if (out == null)
559                 {
560                     throw new IOException("FTP operation failed: " + client.getReplyString());
561                 }
562 
563                 return new CallbackOutputStream(out,
564                         new CallbackOutputStream.Callback()
565                         {
566                             public void onClose() throws Exception
567                             {
568                                 try
569                                 {
570                                     if (!client.completePendingCommand())
571                                     {
572                                         client.logout();
573                                         client.disconnect();
574                                         throw new IOException("FTP Stream failed to complete pending request");
575                                     }
576                                 }
577                                 finally
578                                 {
579                                     releaseFtp(uri, client);
580                                 }
581                             }
582                         });
583             }
584             catch (Exception e)
585             {
586                 logger.debug("Error getting output stream: ", e);
587                 releaseFtp(uri, client);
588                 throw e;
589             }
590         }
591         catch (ConnectException ce)
592         {
593             // Don't wrap a ConnectException, otherwise the retry policy will not go into effect.
594             throw ce;
595         }
596         catch (Exception e)
597         {
598             throw new DispatchException(CoreMessages.streamingFailedNoStream(), event, endpoint, e);
599         }
600     }
601 
602     private String getFilename(ImmutableEndpoint endpoint, MuleMessage message) throws IOException
603     {
604         String filename = message.getOutboundProperty(FtpConnector.PROPERTY_FILENAME);
605         String outPattern = (String) endpoint.getProperty(FtpConnector.PROPERTY_OUTPUT_PATTERN);
606         if (outPattern == null)
607         {
608             outPattern = message.getOutboundProperty(FtpConnector.PROPERTY_OUTPUT_PATTERN, getOutputPattern());
609         }
610         if (outPattern != null || filename == null)
611         {
612             filename = generateFilename(message, outPattern);
613         }
614         if (filename == null)
615         {
616             throw new IOException("Filename is null");
617         }
618         return filename;
619     }
620 
621     private String generateFilename(MuleMessage message, String pattern)
622     {
623         if (pattern == null)
624         {
625             pattern = getOutputPattern();
626         }
627         return getFilenameParser().getFilename(message, pattern);
628     }
629 
630     /**
631      * Creates a new FTPClient that logs in and changes the working directory using the data
632      * provided in <code>endpoint</code>.
633      */
634     protected FTPClient createFtpClient(ImmutableEndpoint endpoint) throws Exception
635     {
636         EndpointURI uri = endpoint.getEndpointURI();
637         FTPClient client = this.getFtp(uri);
638 
639         this.enterActiveOrPassiveMode(client, endpoint);
640         this.setupFileType(client, endpoint);
641 
642         String path = uri.getPath();
643 
644         // only change directory if one was configured
645         if (StringUtils.isNotBlank(path))
646         {
647             // MULE-2400: if the path begins with '~' we must strip the first '/' to make things
648             // work with FTPClient
649             if ((path.length() >= 2) && (path.charAt(1) == '~'))
650             {
651                 path = path.substring(1);
652             }
653 
654             if (!client.changeWorkingDirectory(path))
655             {
656                 throw new IOException(MessageFormat.format("Failed to change working directory to {0}. Ftp error: {1}",
657                                                            path, client.getReplyCode()));
658             }
659         }
660         return client;
661     }
662 
663     /**
664      * Override this method to do extra checking on the file.
665      */
666     protected boolean validateFile(FTPFile file)
667     {
668         return true;
669     }
670 
671     public boolean isStreaming()
672     {
673         return streaming;
674     }
675 
676     public void setStreaming(boolean streaming)
677     {
678         this.streaming = streaming;
679     }
680 
681 }