View Javadoc

1   /*
2    * $Id: FileMessageReceiver.java 23227 2011-10-20 15:03:14Z pablo.kraan $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.transport.file;
12  
13  import org.mule.DefaultMuleMessage;
14  import org.mule.api.DefaultMuleException;
15  import org.mule.api.MessagingException;
16  import org.mule.api.MuleEvent;
17  import org.mule.api.MuleException;
18  import org.mule.api.MuleMessage;
19  import org.mule.api.config.MuleProperties;
20  import org.mule.api.construct.FlowConstruct;
21  import org.mule.api.endpoint.InboundEndpoint;
22  import org.mule.api.exception.RollbackSourceCallback;
23  import org.mule.api.lifecycle.CreateException;
24  import org.mule.api.transport.Connector;
25  import org.mule.api.transport.PropertyScope;
26  import org.mule.transport.AbstractPollingMessageReceiver;
27  import org.mule.transport.ConnectException;
28  import org.mule.transport.file.i18n.FileMessages;
29  import org.mule.util.FileUtils;
30  
31  import java.io.File;
32  import java.io.FileFilter;
33  import java.io.FileNotFoundException;
34  import java.io.FilenameFilter;
35  import java.io.IOException;
36  import java.io.RandomAccessFile;
37  import java.nio.channels.FileChannel;
38  import java.nio.channels.FileLock;
39  import java.util.ArrayList;
40  import java.util.Collections;
41  import java.util.Comparator;
42  import java.util.List;
43  
44  import org.apache.commons.collections.comparators.ReverseComparator;
45  
46  /**
47   * <code>FileMessageReceiver</code> is a polling listener that reads files from a
48   * directory.
49   */
50  
51  public class FileMessageReceiver extends AbstractPollingMessageReceiver
52  {
53      public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator";
54      public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder";
55  
56      private static final List<File> NO_FILES = new ArrayList<File>();
57  
58      private String readDir = null;
59      private String moveDir = null;
60      private String workDir = null;
61      private File readDirectory = null;
62      private File moveDirectory = null;
63      private String moveToPattern = null;
64      private String workFileNamePattern = null;
65      private FilenameFilter filenameFilter = null;
66      private FileFilter fileFilter = null;
67      private boolean forceSync;
68  
69      public FileMessageReceiver(Connector connector,
70                                 FlowConstruct flowConstruct,
71                                 InboundEndpoint endpoint,
72                                 String readDir,
73                                 String moveDir,
74                                 String moveToPattern,
75                                 long frequency) throws CreateException
76      {
77          super(connector, flowConstruct, endpoint);
78          this.setFrequency(frequency);
79  
80          this.readDir = readDir;
81          this.moveDir = moveDir;
82          this.moveToPattern = moveToPattern;
83          this.workDir = ((FileConnector) connector).getWorkDirectory();
84          this.workFileNamePattern = ((FileConnector) connector).getWorkFileNamePattern();
85  
86          if (endpoint.getFilter() instanceof FilenameFilter)
87          {
88              filenameFilter = (FilenameFilter) endpoint.getFilter();
89          }
90          else if (endpoint.getFilter() instanceof FileFilter)
91          {
92              fileFilter = (FileFilter) endpoint.getFilter();
93          }
94          else if (endpoint.getFilter() != null)
95          {
96              throw new CreateException(FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
97          }
98          
99          checkMustForceSync();
100     }
101 
102     /**
103      * If we will be autodeleting File objects, events must be processed synchronously to avoid a race
104      */
105     protected void checkMustForceSync() throws CreateException
106     {
107         boolean connectorIsAutoDelete = false;
108         boolean isStreaming = false;
109         if (connector instanceof FileConnector)
110         {
111             connectorIsAutoDelete = ((FileConnector) connector).isAutoDelete();
112             isStreaming = ((FileConnector) connector).isStreaming();
113         }
114 
115         boolean messageFactoryConsumes = (createMuleMessageFactory() instanceof FileContentsMuleMessageFactory);
116         
117         forceSync = connectorIsAutoDelete && !messageFactoryConsumes && !isStreaming;
118     }
119 
120     @Override
121     protected void doConnect() throws Exception
122     {
123         if (readDir != null)
124         {
125             readDirectory = FileUtils.openDirectory(readDir);
126             if (!(readDirectory.canRead()))
127             {
128                 throw new ConnectException(FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
129             }
130             else
131             {
132                 logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
133             }
134         }
135 
136         if (moveDir != null)
137         {
138             moveDirectory = FileUtils.openDirectory((moveDir));
139             if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
140             {
141                 throw new ConnectException(FileMessages.moveToDirectoryNotWritable(), this);
142             }
143         }
144     }
145 
146     @Override
147     protected void doDisconnect() throws Exception
148     {
149         // template method
150     }
151 
152     @Override
153     protected void doDispose()
154     {
155         // nothing to do
156     }
157 
158     @Override
159     public void poll()
160     {
161         try
162         {
163             List<File> files = this.listFiles();
164             if (logger.isDebugEnabled())
165             {
166                 logger.debug("Files: " + files.toString());
167             }
168             Comparator<File> comparator = getComparator();
169             if (comparator != null)
170             {
171                 Collections.sort(files, comparator);
172             }
173             for (File file : files)
174             {
175                 if (getLifecycleState().isStopping())
176                 {
177                     break;
178                 }
179                 // don't process directories
180                 if (file.isFile())
181                 {
182                     processFile(file);
183                 }
184             }
185         }
186         catch (MessagingException e)
187         {
188             MuleEvent event = e.getEvent();
189             event.getFlowConstruct().getExceptionListener().handleException(e, event);
190         }
191         catch (Exception e)
192         {
193             getConnector().getMuleContext().getExceptionListener().handleException(e);
194         }
195     }
196 
197     @Override
198     protected boolean pollOnPrimaryInstanceOnly()
199     {
200         return true;
201     }
202 
203     public void processFile(File file) throws MuleException
204     {
205         //TODO RM*: This can be put in a Filter. Also we can add an AndFileFilter/OrFileFilter to allow users to
206         //combine file filters (since we can only pass a single filter to File.listFiles, we would need to wrap
207         //the current And/Or filters to extend {@link FilenameFilter}
208         boolean checkFileAge = ((FileConnector) connector).getCheckFileAge();
209         if (checkFileAge)
210         {
211             long fileAge = ((FileConnector) connector).getFileAge();
212             long lastMod = file.lastModified();
213             long now = System.currentTimeMillis();
214             long thisFileAge = now - lastMod;
215             if (thisFileAge < fileAge)
216             {
217                 if (logger.isDebugEnabled())
218                 {
219                     logger.debug("The file has not aged enough yet, will return nothing for: " + file);
220                 }
221                 return;
222             }
223         }
224 
225         // Perform some quick checks to make sure file can be processed
226         if (!(file.canRead() && file.exists() && file.isFile()))
227         {
228             throw new DefaultMuleException(FileMessages.fileDoesNotExist(file.getName()));
229         }
230 
231         // don't process a file that is locked by another process (probably still being written)
232         if (!attemptFileLock(file))
233         {
234             return;
235         }
236         else if(logger.isInfoEnabled())
237         {
238             logger.info("Lock obtained on file: " + file.getAbsolutePath());
239         }
240 
241         // This isn't nice but is needed as MuleMessage is required to resolve
242         // destination file name
243         DefaultMuleMessage fileParserMessasge = new DefaultMuleMessage(null, connector.getMuleContext());
244         fileParserMessasge.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, file.getName());
245 
246         // The file may get moved/renamed here so store the original file info.
247         final String originalSourceFile = file.getAbsolutePath();
248         final String originalSourceFileName = file.getName();        
249         final File sourceFile;
250         if (workDir != null)
251         {
252             String workFileName = file.getName();
253             
254             workFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMessasge, workFileNamePattern);
255             // don't use new File() directly, see MULE-1112
256             File workFile = FileUtils.newFile(workDir, workFileName);
257             
258             move(file, workFile);
259             // Now the Work File is the Source file
260             sourceFile = workFile;
261         }
262         else
263         {
264             sourceFile = file;
265         }
266         // Do not use the original file handle beyond this point since it may have moved.
267         file = null;
268         
269         // set up destination file
270         File destinationFile = null;
271         if (moveDir != null)
272         {
273             String destinationFileName = originalSourceFileName;
274             if (moveToPattern != null)
275             {
276                 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMessasge,
277                     moveToPattern);
278             }
279             // don't use new File() directly, see MULE-1112
280             destinationFile = FileUtils.newFile(moveDir, destinationFileName);
281         }
282 
283         MuleMessage message = null;
284         String encoding = endpoint.getEncoding();
285         try
286         {
287             if (((FileConnector) connector).isStreaming())
288             {
289                 ReceiverFileInputStream payload = new ReceiverFileInputStream(sourceFile,
290                     ((FileConnector) connector).isAutoDelete(), destinationFile);
291                 message = createMuleMessage(payload, encoding);
292             }
293             else
294             {
295                 message = createMuleMessage(sourceFile, encoding);
296             }
297         }
298         catch (FileNotFoundException e)
299         {
300             // we can ignore since we did manage to acquire a lock, but just in case
301             logger.error("File being read disappeared!", e);
302             return;
303         }
304 
305         message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, originalSourceFileName);
306         if (forceSync)
307         {
308             message.setProperty(MuleProperties.MULE_FORCE_SYNC_PROPERTY, Boolean.TRUE, PropertyScope.INBOUND);
309         }
310         
311         try
312         {
313             if (!((FileConnector) connector).isStreaming())
314             {
315                 moveAndDelete(sourceFile, destinationFile, originalSourceFileName, message);
316             }
317             else
318             {
319                 // If we are streaming no need to move/delete now, that will be done when
320                 // stream is closed
321                 message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, sourceFile.getName());
322                 this.routeMessage(message);
323             }
324         }
325         catch (Exception e)
326         {
327             RollbackSourceCallback rollbackMethod = null;
328 
329             if (((FileConnector) connector).isStreaming())
330             {
331                 if (message.getPayload() instanceof ReceiverFileInputStream)
332                 {
333                     final ReceiverFileInputStream receiverFileInputStream = (ReceiverFileInputStream) message.getPayload();
334                     rollbackMethod = new RollbackSourceCallback()
335                     {
336                         @Override
337                         public void rollback()
338                         {
339                             receiverFileInputStream.setStreamProcessingError(true);
340                         }
341                     };
342                 }
343             }
344             else if (!sourceFile.getAbsolutePath().equals(originalSourceFile))
345             {
346                 rollbackMethod = new RollbackSourceCallback()
347                 {
348                     @Override
349                     public void rollback()
350                     {
351                         try
352                         {
353                             rollbackFileMove(sourceFile, originalSourceFile);
354                         }
355                         catch (IOException iox)
356                         {
357                             logger.warn(iox);
358                         }
359                     }
360                 };
361             }
362 
363             if (e instanceof MessagingException)
364             {            
365                 MuleEvent event = ((MessagingException) e).getEvent();
366                 event.getFlowConstruct().getExceptionListener().handleException(e, event, rollbackMethod);                
367             }
368             else
369             {
370                 connector.getMuleContext().getExceptionListener().handleException(e, rollbackMethod);
371             }
372         }
373     }
374 
375     private void moveAndDelete(final File sourceFile, File destinationFile,
376         String sourceFileOriginalName, MuleMessage message) throws MuleException
377     {
378         // If we are moving the file to a read directory, move it there now and
379         // hand over a reference to the
380         // File in its moved location
381         if (destinationFile != null)
382         {
383             // move sourceFile to new destination
384             try
385             {
386                 FileUtils.moveFile(sourceFile, destinationFile);
387             }
388             catch (IOException e)
389             {
390                 // move didn't work - bail out (will attempt rollback)
391                 throw new DefaultMuleException(FileMessages.failedToMoveFile(
392                     sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath()));
393             }
394 
395             // create new Message for destinationFile
396             message = createMuleMessage(destinationFile, endpoint.getEncoding());
397             message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
398             message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
399         }
400 
401         // finally deliver the file message
402         this.routeMessage(message);
403 
404         // at this point msgAdapter either points to the old sourceFile
405         // or the new destinationFile.
406         if (((FileConnector) connector).isAutoDelete())
407         {
408             // no moveTo directory
409             if (destinationFile == null)
410             {
411                 // delete source
412                 if (!sourceFile.delete())
413                 {
414                     throw new DefaultMuleException(FileMessages.failedToDeleteFile(sourceFile));
415                 }
416             }
417             else
418             {
419                 // nothing to do here since moveFile() should have deleted
420                 // the source file for us
421             }
422         }
423     }
424 
425     /**
426      * Try to acquire a lock on a file and release it immediately. Usually used as a
427      * quick check to see if another process is still holding onto the file, e.g. a
428      * large file (more than 100MB) is still being written to.
429      * 
430      * @param sourceFile file to check
431      * @return <code>true</code> if the file can be locked
432      */
433     protected boolean attemptFileLock(final File sourceFile) throws MuleException
434     {
435         // check if the file can be processed, be sure that it's not still being
436         // written
437         // if the file can't be locked don't process it yet, since creating
438         // a new FileInputStream() will throw an exception
439         FileLock lock = null;
440         FileChannel channel = null;
441         boolean fileCanBeLocked = false;
442         try
443         {
444             channel = new RandomAccessFile(sourceFile, "rw").getChannel();
445 
446             // Try acquiring the lock without blocking. This method returns
447             // null or throws an exception if the file is already locked.
448             lock = channel.tryLock();
449         }
450         catch (FileNotFoundException fnfe)
451         {
452             throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFile.getName()));
453         }
454         catch (IOException e)
455         {
456             // Unable to create a lock. This exception should only be thrown when
457             // the file is already locked. No sense in repeating the message over
458             // and over.
459         }
460         finally
461         {
462             if (lock != null)
463             {
464                 // if lock is null the file is locked by another process
465                 fileCanBeLocked = true;
466                 try
467                 {
468                     // Release the lock
469                     lock.release();
470                 }
471                 catch (IOException e)
472                 {
473                     // ignore
474                 }
475             }
476 
477             if (channel != null)
478             {
479                 try
480                 {
481                     // Close the file
482                     channel.close();
483                 }
484                 catch (IOException e)
485                 {
486                     // ignore
487                 }
488             }
489         }
490 
491         return fileCanBeLocked;
492     }
493 
494     /**
495      * Get a list of files to be processed.
496      * 
497      * @return an array of files to be processed.
498      * @throws org.mule.api.MuleException which will wrap any other exceptions or
499      *             errors.
500      */
501     List<File> listFiles() throws MuleException
502     {
503         try
504         {
505             List<File> files = new ArrayList<File>();
506             this.basicListFiles(readDirectory, files);
507             return (files.isEmpty() ? NO_FILES : files);
508         }
509         catch (Exception e)
510         {
511             throw new DefaultMuleException(FileMessages.errorWhileListingFiles(), e);
512         }
513     }
514 
515     protected void basicListFiles(File currentDirectory, List<File> discoveredFiles)
516     {
517         File[] files;
518         if (fileFilter != null)
519         {
520             files = currentDirectory.listFiles(fileFilter);
521         }
522         else
523         {
524             files = currentDirectory.listFiles(filenameFilter);
525         }
526         
527         // the listFiles calls above may actually return null (check the JDK code).
528         if (files == null)
529         {
530             return;
531         }
532 
533         for (File file : files)
534         {
535             if (!file.isDirectory())
536             {
537                 discoveredFiles.add(file);
538             }
539             else
540             {
541                 if (((FileConnector) this.getConnector()).isRecursive())
542                 {
543                     this.basicListFiles(file, discoveredFiles);
544                 }
545             }
546         }
547     }
548 
549     /**
550      * Exception tolerant roll back method
551      * 
552      * @throws Throwable
553      */
554     protected void rollbackFileMove(File sourceFile, String destinationFilePath) throws IOException
555     {
556         try
557         {
558             FileUtils.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
559         }
560         catch (IOException t)
561         {
562             logger.debug("rollback of file move failed: " + t.getMessage());
563             throw t;
564         }
565     }
566 
567     protected Comparator<File> getComparator() throws Exception
568     {
569         Object comparatorClassName = getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY);
570         if (comparatorClassName != null)
571         {
572             Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY);
573             boolean reverse = false;
574             if (reverseProperty != null)
575             {
576                 reverse = Boolean.valueOf((String) reverseProperty);
577             }
578             
579             Class<?> clazz = Class.forName(comparatorClassName.toString());
580             Comparator<?> comparator = (Comparator<?>)clazz.newInstance();
581             return reverse ? new ReverseComparator(comparator) : comparator;
582         }
583         return null;
584     }
585     
586     private void move(final File sourceFile,File destinationFile) throws DefaultMuleException
587     {
588         if (destinationFile != null)
589         {
590             // move sourceFile to new destination. Do not use FileUtils here as it ultimately
591             // falls back to copying the file which will cause problems on large files again -
592             // which is what we're trying to avoid in the first place
593             boolean fileWasMoved = sourceFile.renameTo(destinationFile);
594 
595             // move didn't work - bail out
596             if (!fileWasMoved)
597             {
598                 throw new DefaultMuleException(FileMessages.failedToMoveFile(sourceFile.getAbsolutePath(),
599                     destinationFile.getAbsolutePath()));
600             }
601         }
602     }
603     
604 }