View Javadoc
1   /*
2    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
3    * The software in this package is published under the terms of the CPAL v1.0
4    * license, a copy of which has been included with this distribution in the
5    * LICENSE.txt file.
6    */
7   package org.mule.transport.file;
8   
9   import org.mule.DefaultMuleMessage;
10  import org.mule.api.DefaultMuleException;
11  import org.mule.api.MessagingException;
12  import org.mule.api.MuleException;
13  import org.mule.api.MuleMessage;
14  import org.mule.api.config.MuleProperties;
15  import org.mule.api.construct.FlowConstruct;
16  import org.mule.api.endpoint.InboundEndpoint;
17  import org.mule.api.lifecycle.CreateException;
18  import org.mule.api.lifecycle.InitialisationException;
19  import org.mule.api.store.ObjectAlreadyExistsException;
20  import org.mule.api.store.ObjectStore;
21  import org.mule.api.store.ObjectStoreException;
22  import org.mule.api.transport.Connector;
23  import org.mule.api.transport.PropertyScope;
24  import org.mule.config.i18n.Message;
25  import org.mule.transport.AbstractPollingMessageReceiver;
26  import org.mule.transport.ConnectException;
27  import org.mule.transport.file.i18n.FileMessages;
28  import org.mule.util.FileUtils;
29  import org.mule.util.store.InMemoryObjectStore;
30  
31  import java.io.File;
32  import java.io.FileFilter;
33  import java.io.FileNotFoundException;
34  import java.io.FilenameFilter;
35  import java.io.IOException;
36  import java.io.RandomAccessFile;
37  import java.nio.channels.FileChannel;
38  import java.nio.channels.FileLock;
39  import java.util.ArrayList;
40  import java.util.Collections;
41  import java.util.Comparator;
42  import java.util.List;
43  
44  import org.apache.commons.collections.comparators.ReverseComparator;
45  
46  /**
47   * <code>FileMessageReceiver</code> is a polling listener that reads files from a
48   * directory.
49   */
50  
51  public class FileMessageReceiver extends AbstractPollingMessageReceiver
52  {
53      public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator";
54      public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder";
55  
56      private static final List<File> NO_FILES = new ArrayList<File>();
57  
58      private String readDir = null;
59      private String moveDir = null;
60      private String workDir = null;
61      private File readDirectory = null;
62      private File moveDirectory = null;
63      private String moveToPattern = null;
64      private String workFileNamePattern = null;
65      private FilenameFilter filenameFilter = null;
66      private FileFilter fileFilter = null;
67      private boolean forceSync;
68      private ObjectStore<String> filesBeingProcessingObjectStore;
69  
70      public FileMessageReceiver(Connector connector,
71                                 FlowConstruct flowConstruct,
72                                 InboundEndpoint endpoint,
73                                 String readDir,
74                                 String moveDir,
75                                 String moveToPattern,
76                                 long frequency) throws CreateException
77      {
78          super(connector, flowConstruct, endpoint);
79          this.setFrequency(frequency);
80  
81          this.readDir = readDir;
82          this.moveDir = moveDir;
83          this.moveToPattern = moveToPattern;
84          this.workDir = ((FileConnector) connector).getWorkDirectory();
85          this.workFileNamePattern = ((FileConnector) connector).getWorkFileNamePattern();
86  
87          if (endpoint.getFilter() instanceof FilenameFilter)
88          {
89              filenameFilter = (FilenameFilter) endpoint.getFilter();
90          }
91          else if (endpoint.getFilter() instanceof FileFilter)
92          {
93              fileFilter = (FileFilter) endpoint.getFilter();
94          }
95          else if (endpoint.getFilter() != null)
96          {
97              throw new CreateException(FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
98          }
99  
100         checkMustForceSync(endpoint);
101     }
102 
103     /**
104      * If we will be autodeleting File objects, events must be processed synchronously to avoid a race
105      */
106     private void checkMustForceSync(InboundEndpoint ep) throws CreateException
107     {
108         boolean connectorIsAutoDelete = false;
109         boolean isStreaming = false;
110         if (connector instanceof FileConnector)
111         {
112             connectorIsAutoDelete = ((FileConnector) connector).isAutoDelete();
113             isStreaming = ((FileConnector) connector).isStreaming();
114         }
115 
116         boolean messageFactoryConsumes = (createMuleMessageFactory() instanceof FileContentsMuleMessageFactory);
117 
118         forceSync = connectorIsAutoDelete && !messageFactoryConsumes && !isStreaming;
119     }
120 
121     @Override
122     protected void doConnect() throws Exception
123     {
124         if (readDir != null)
125         {
126             readDirectory = FileUtils.openDirectory(readDir);
127             if (!(readDirectory.canRead()))
128             {
129                 throw new ConnectException(FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
130             }
131             else
132             {
133                 logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
134             }
135         }
136 
137         if (moveDir != null)
138         {
139             moveDirectory = FileUtils.openDirectory((moveDir));
140             if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
141             {
142                 throw new ConnectException(FileMessages.moveToDirectoryNotWritable(), this);
143             }
144         }
145     }
146 
147     @Override
148     protected void doInitialise() throws InitialisationException
149     {
150         InMemoryObjectStore objectStore = new InMemoryObjectStore<String>();
151         objectStore.setMaxEntries(1000);
152         objectStore.setExpirationInterval(20000);
153         objectStore.setEntryTTL(60000);
154         filesBeingProcessingObjectStore = objectStore;
155     }
156 
157     @Override
158     protected void doDisconnect() throws Exception
159     {
160         // template method
161     }
162 
163     @Override
164     protected void doDispose()
165     {
166         // nothing to do
167     }
168 
169     @Override
170     public void poll()
171     {
172         try
173         {
174             List<File> files = this.listFiles();
175             if (logger.isDebugEnabled())
176             {
177                 logger.debug("Files: " + files.toString());
178             }
179             Comparator<File> comparator = getComparator();
180             if (comparator != null)
181             {
182                 Collections.sort(files, comparator);
183             }
184             for (File file : files)
185             {
186                 if (getLifecycleState().isStopping())
187                 {
188                     break;
189                 }
190                 // don't process directories
191                 if (file.isFile())
192                 {
193                     try
194                     {
195                         filesBeingProcessingObjectStore.store(file.getAbsolutePath(),file.getAbsolutePath());
196                         processFile(file);
197                     }
198                     catch (ObjectAlreadyExistsException e)
199                     {
200                         logger.debug("file " + file.getAbsolutePath() + " it's being processed. Skipping it.");
201                     }
202                 }
203             }
204         }
205         catch (Exception e)
206         {
207             getConnector().getMuleContext().getExceptionListener().handleException(e);
208         }
209     }
210 
211     public void processFile(File file) throws MuleException
212     {
213         FileConnector fileConnector = (FileConnector) connector;
214 
215         //TODO RM*: This can be put in a Filter. Also we can add an AndFileFilter/OrFileFilter to allow users to
216         //combine file filters (since we can only pass a single filter to File.listFiles, we would need to wrap
217         //the current And/Or filters to extend {@link FilenameFilter}
218         boolean checkFileAge = fileConnector.getCheckFileAge();
219         if (checkFileAge)
220         {
221             long fileAge = ((FileConnector) connector).getFileAge();
222             long lastMod = file.lastModified();
223             long now = System.currentTimeMillis();
224             long thisFileAge = now - lastMod;
225             if (thisFileAge < fileAge)
226             {
227                 if (logger.isDebugEnabled())
228                 {
229                     logger.debug("The file has not aged enough yet, will return nothing for: " + file);
230                 }
231                 return;
232             }
233         }
234 
235         String sourceFileOriginalName = file.getName();
236         final String sourceFileOriginalAbsolutePath = file.getAbsolutePath();
237 
238         // Perform some quick checks to make sure file can be processed
239         if (!(file.canRead() && file.exists() && file.isFile()))
240         {
241             throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName));
242         }
243 
244         // when polling with more than one mule instance attemptFileLock may create
245         // zero-sized files depending on the timing, so we skip attemptFileLock in
246         // combination with workDir to avoid this. Some sensible fileAge value should
247         // be used to avoid processing files still being written
248         if (workDir == null)
249         {
250             // don't process a file that is locked by another process (probably still
251             // being written)
252             if (!attemptFileLock(file))
253             {
254                 return;
255             }
256             else if (logger.isInfoEnabled())
257             {
258                 logger.info("Lock obtained on file: " + file.getAbsolutePath());
259             }
260         }
261 
262         // This isn't nice but is needed as MuleMessage is required to resolve
263         // destination file name
264         DefaultMuleMessage fileParserMessasge = new DefaultMuleMessage(null, connector.getMuleContext());
265         fileParserMessasge.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
266 
267         File workFile = null;
268         final File sourceFile;
269         if (workDir != null)
270         {
271             String workFileName = sourceFileOriginalName;
272 
273             workFileName = fileConnector.getFilenameParser().getFilename(fileParserMessasge,
274                     workFileNamePattern);
275             // don't use new File() directly, see MULE-1112
276             workFile = FileUtils.newFile(workDir, workFileName);
277 
278             fileConnector.move(file, workFile);
279             // Now the Work File is the Source file
280             sourceFile = workFile;
281         }
282         else
283         {
284             sourceFile = file;
285         }
286 
287         // set up destination file
288         File destinationFile = null;
289         if (moveDir != null)
290         {
291             String destinationFileName = sourceFileOriginalName;
292             if (moveToPattern != null)
293             {
294                 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMessasge,
295                     moveToPattern);
296             }
297             // don't use new File() directly, see MULE-1112
298             destinationFile = FileUtils.newFile(moveDir, destinationFileName);
299         }
300 
301         MuleMessage message = null;
302         String encoding = endpoint.getEncoding();
303         try
304         {
305             if (fileConnector.isStreaming())
306             {
307                 ReceiverFileInputStream payload = new ReceiverFileInputStream(sourceFile, fileConnector.isAutoDelete(), destinationFile, new InputStreamCloseListener()
308                 {
309                     public void fileClose(File file)
310                     {
311                         try
312                         {
313                             if (logger.isDebugEnabled())
314                             {
315                                 logger.debug(String.format("Removing processing flag for $ ", file.getAbsolutePath()));
316                             }
317                             filesBeingProcessingObjectStore.remove(sourceFileOriginalAbsolutePath);
318                         }
319                         catch (ObjectStoreException e)
320                         {
321                             logger.warn("Failure trying to remove file " + sourceFileOriginalAbsolutePath + " from list of files under processing");
322                         }
323                     }
324                 });
325                 message = createMuleMessage(payload, encoding);
326             }
327             else
328             {
329                 message = createMuleMessage(sourceFile, encoding);
330             }
331         }
332         catch (FileNotFoundException e)
333         {
334             // we can ignore since we did manage to acquire a lock, but just in case
335             logger.error("File being read disappeared!", e);
336             return;
337         }
338 
339         if (workDir != null)
340         {
341             message.setProperty(FileConnector.PROPERTY_SOURCE_DIRECTORY, file.getParent(), PropertyScope.INBOUND);
342             message.setProperty(FileConnector.PROPERTY_SOURCE_FILENAME, file.getName(), PropertyScope.INBOUND);
343         }
344 
345         message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
346         if (forceSync)
347         {
348             message.setProperty(MuleProperties.MULE_FORCE_SYNC_PROPERTY, Boolean.TRUE, PropertyScope.INBOUND);
349         }
350         if (!fileConnector.isStreaming())
351         {
352             moveAndDelete(sourceFile, destinationFile, sourceFileOriginalName, sourceFileOriginalAbsolutePath,message);
353         }
354         else
355         {
356             // If we are streaming no need to move/delete now, that will be done when
357             // stream is closed
358             message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, sourceFile.getName());
359             this.routeMessage(message);
360         }
361     }
362 
363     private void moveAndDelete(final File sourceFile, File destinationFile,
364         String sourceFileOriginalName, String sourceFileOriginalAbsolutePath, MuleMessage message)
365     {
366         boolean fileWasMoved = false;
367 
368         try
369         {
370             // If we are moving the file to a read directory, move it there now and
371             // hand over a reference to the
372             // File in its moved location
373             if (destinationFile != null)
374             {
375                 // move sourceFile to new destination
376                 try
377                 {
378                     FileUtils.moveFile(sourceFile, destinationFile);
379                 }
380                 catch (IOException e)
381                 {
382                     // move didn't work - bail out (will attempt rollback)
383                     throw new DefaultMuleException(FileMessages.failedToMoveFile(
384                         sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath()));
385                 }
386 
387                 // create new Message for destinationFile
388                 message = createMuleMessage(destinationFile, endpoint.getEncoding());
389                 message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
390                 message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
391             }
392 
393             // finally deliver the file message
394             this.routeMessage(message);
395 
396             // at this point msgAdapter either points to the old sourceFile
397             // or the new destinationFile.
398             if (((FileConnector) connector).isAutoDelete())
399             {
400                 // no moveTo directory
401                 if (destinationFile == null)
402                 {
403                     // delete source
404                     if (!sourceFile.delete())
405                     {
406                         throw new DefaultMuleException(FileMessages.failedToDeleteFile(sourceFile));
407                     }
408                 }
409                 else
410                 {
411                     // nothing to do here since moveFile() should have deleted
412                     // the source file for us
413                 }
414             }
415         }
416         catch (Exception e)
417         {
418             boolean fileWasRolledBack = false;
419 
420             // only attempt rollback if file move was successful
421             if (fileWasMoved)
422             {
423                 try
424                 {
425                     rollbackFileMove(destinationFile, sourceFile.getAbsolutePath());
426                     fileWasRolledBack = true;
427                 }
428                 catch (IOException ioException)
429                 {
430                     // eat it
431                 }
432             }
433 
434             // wrap exception & handle it
435             Message msg = FileMessages.exceptionWhileProcessing(sourceFile.getName(),
436                 (fileWasRolledBack ? "successful" : "unsuccessful"));
437             getConnector().getMuleContext().getExceptionListener().handleException(new MessagingException(msg, message, e));
438         }
439         finally
440         {
441             try
442             {
443                 filesBeingProcessingObjectStore.remove(sourceFileOriginalAbsolutePath);
444                 if (logger.isDebugEnabled())
445                 {
446                     logger.debug(String.format("Removing processing flag for $ ", sourceFileOriginalAbsolutePath));
447                 }
448             }
449             catch (ObjectStoreException e)
450             {
451                 logger.warn("Failure trying to remove file " + sourceFileOriginalAbsolutePath + " from list of files under processing");
452             }
453         }
454 }
455 
456     /**
457      * Try to acquire a lock on a file and release it immediately. Usually used as a
458      * quick check to see if another process is still holding onto the file, e.g. a
459      * large file (more than 100MB) is still being written to.
460      *
461      * @param sourceFile file to check
462      * @return <code>true</code> if the file can be locked
463      */
464     protected boolean attemptFileLock(final File sourceFile) throws MuleException
465     {
466         // check if the file can be processed, be sure that it's not still being
467         // written
468         // if the file can't be locked don't process it yet, since creating
469         // a new FileInputStream() will throw an exception
470         FileLock lock = null;
471         FileChannel channel = null;
472         boolean fileCanBeLocked = false;
473         try
474         {
475             channel = new RandomAccessFile(sourceFile, "rw").getChannel();
476 
477             // Try acquiring the lock without blocking. This method returns
478             // null or throws an exception if the file is already locked.
479             lock = channel.tryLock();
480         }
481         catch (FileNotFoundException fnfe)
482         {
483             throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFile.getName()));
484         }
485         catch (IOException e)
486         {
487             // Unable to create a lock. This exception should only be thrown when
488             // the file is already locked. No sense in repeating the message over
489             // and over.
490         }
491         finally
492         {
493             if (lock != null)
494             {
495                 // if lock is null the file is locked by another process
496                 fileCanBeLocked = true;
497                 try
498                 {
499                     // Release the lock
500                     lock.release();
501                 }
502                 catch (IOException e)
503                 {
504                     // ignore
505                 }
506             }
507 
508             if (channel != null)
509             {
510                 try
511                 {
512                     // Close the file
513                     channel.close();
514                 }
515                 catch (IOException e)
516                 {
517                     // ignore
518                 }
519             }
520         }
521 
522         return fileCanBeLocked;
523     }
524 
525     /**
526      * Get a list of files to be processed.
527      *
528      * @return an array of files to be processed.
529      * @throws org.mule.api.MuleException which will wrap any other exceptions or
530      *             errors.
531      */
532     List<File> listFiles() throws MuleException
533     {
534         try
535         {
536             List<File> files = new ArrayList<File>();
537             this.basicListFiles(readDirectory, files);
538             return (files.isEmpty() ? NO_FILES : files);
539         }
540         catch (Exception e)
541         {
542             throw new DefaultMuleException(FileMessages.errorWhileListingFiles(), e);
543         }
544     }
545 
546     protected void basicListFiles(File currentDirectory, List<File> discoveredFiles)
547     {
548         File[] files;
549         if (fileFilter != null)
550         {
551             files = currentDirectory.listFiles(fileFilter);
552         }
553         else
554         {
555             files = currentDirectory.listFiles(filenameFilter);
556         }
557 
558         // the listFiles calls above may actually return null (check the JDK code).
559         if (files == null)
560         {
561             return;
562         }
563 
564         for (File file : files)
565         {
566             if (!file.isDirectory())
567             {
568                 discoveredFiles.add(file);
569             }
570             else
571             {
572                 if (((FileConnector) this.getConnector()).isRecursive())
573                 {
574                     this.basicListFiles(file, discoveredFiles);
575                 }
576             }
577         }
578     }
579 
580     /**
581      * Exception tolerant roll back method
582      *
583      * @throws Throwable
584      */
585     protected void rollbackFileMove(File sourceFile, String destinationFilePath) throws IOException
586     {
587         try
588         {
589             FileUtils.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
590         }
591         catch (IOException t)
592         {
593             logger.debug("rollback of file move failed: " + t.getMessage());
594             throw t;
595         }
596     }
597 
598     @SuppressWarnings("unchecked")
599     protected Comparator<File> getComparator() throws Exception
600     {
601         Object comparatorClassName = getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY);
602         if (comparatorClassName != null)
603         {
604             Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY);
605             boolean reverse = false;
606             if (reverseProperty != null)
607             {
608                 reverse = Boolean.valueOf((String) reverseProperty);
609             }
610 
611             Class<?> clazz = Class.forName(comparatorClassName.toString());
612             Comparator<?> comparator = (Comparator<?>)clazz.newInstance();
613             return reverse ? new ReverseComparator(comparator) : comparator;
614         }
615         return null;
616     }
617 }