View Javadoc

1   /*
2    * $Id: FileMessageReceiver.java 20477 2010-12-06 23:38:52Z mike.schilling $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.transport.file;
12  
13  import org.mule.DefaultMuleMessage;
14  import org.mule.api.DefaultMuleException;
15  import org.mule.api.MessagingException;
16  import org.mule.api.MuleException;
17  import org.mule.api.MuleMessage;
18  import org.mule.api.config.MuleProperties;
19  import org.mule.api.construct.FlowConstruct;
20  import org.mule.api.endpoint.InboundEndpoint;
21  import org.mule.api.lifecycle.CreateException;
22  import org.mule.api.transport.Connector;
23  import org.mule.api.transport.PropertyScope;
24  import org.mule.config.i18n.Message;
25  import org.mule.transport.AbstractPollingMessageReceiver;
26  import org.mule.transport.ConnectException;
27  import org.mule.transport.file.i18n.FileMessages;
28  import org.mule.util.FileUtils;
29  
30  import java.io.File;
31  import java.io.FileFilter;
32  import java.io.FileNotFoundException;
33  import java.io.FilenameFilter;
34  import java.io.IOException;
35  import java.io.RandomAccessFile;
36  import java.nio.channels.FileChannel;
37  import java.nio.channels.FileLock;
38  import java.util.ArrayList;
39  import java.util.Collections;
40  import java.util.Comparator;
41  import java.util.List;
42  
43  import org.apache.commons.collections.comparators.ReverseComparator;
44  
45  /**
46   * <code>FileMessageReceiver</code> is a polling listener that reads files from a
47   * directory.
48   */
49  
50  public class FileMessageReceiver extends AbstractPollingMessageReceiver
51  {
52      public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator";
53      public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder";
54  
55      private static final List<File> NO_FILES = new ArrayList<File>();
56  
57      private String readDir = null;
58      private String moveDir = null;
59      private String workDir = null;
60      private File readDirectory = null;
61      private File moveDirectory = null;
62      private String moveToPattern = null;
63      private String workFileNamePattern = null;
64      private FilenameFilter filenameFilter = null;
65      private FileFilter fileFilter = null;
66      private boolean forceSync;
67  
68      public FileMessageReceiver(Connector connector,
69                                 FlowConstruct flowConstruct,
70                                 InboundEndpoint endpoint,
71                                 String readDir,
72                                 String moveDir,
73                                 String moveToPattern,
74                                 long frequency) throws CreateException
75      {
76          super(connector, flowConstruct, endpoint);
77          this.setFrequency(frequency);
78  
79          this.readDir = readDir;
80          this.moveDir = moveDir;
81          this.moveToPattern = moveToPattern;
82          this.workDir = ((FileConnector) connector).getWorkDirectory();
83          this.workFileNamePattern = ((FileConnector) connector).getWorkFileNamePattern();
84  
85          if (endpoint.getFilter() instanceof FilenameFilter)
86          {
87              filenameFilter = (FilenameFilter) endpoint.getFilter();
88          }
89          else if (endpoint.getFilter() instanceof FileFilter)
90          {
91              fileFilter = (FileFilter) endpoint.getFilter();
92          }
93          else if (endpoint.getFilter() != null)
94          {
95              throw new CreateException(FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
96          }
97          
98          checkMustForceSync(endpoint);
99      }
100 
101     /**
102      * If we will be autodeleting File objects, events must be processed synchronously to avoid a race
103      */
104     private void checkMustForceSync(InboundEndpoint endpoint) throws CreateException
105     {
106         boolean connectorIsAutoDelete = false;
107         boolean isStreaming = false;
108         if (connector instanceof FileConnector)
109         {
110             connectorIsAutoDelete = ((FileConnector) connector).isAutoDelete();
111             isStreaming = ((FileConnector) connector).isStreaming();
112         }
113 
114         boolean messageFactoryConsumes = (createMuleMessageFactory() instanceof FileContentsMuleMessageFactory);
115         
116         forceSync = connectorIsAutoDelete && !messageFactoryConsumes && !isStreaming;
117     }
118 
119     @Override
120     protected void doConnect() throws Exception
121     {
122         if (readDir != null)
123         {
124             readDirectory = FileUtils.openDirectory(readDir);
125             if (!(readDirectory.canRead()))
126             {
127                 throw new ConnectException(FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
128             }
129             else
130             {
131                 logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
132             }
133         }
134 
135         if (moveDir != null)
136         {
137             moveDirectory = FileUtils.openDirectory((moveDir));
138             if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
139             {
140                 throw new ConnectException(FileMessages.moveToDirectoryNotWritable(), this);
141             }
142         }
143     }
144 
145     @Override
146     protected void doDisconnect() throws Exception
147     {
148         // template method
149     }
150 
151     @Override
152     protected void doDispose()
153     {
154         // nothing to do
155     }
156 
157     @Override
158     public void poll()
159     {
160         try
161         {
162             List<File> files = this.listFiles();
163             if (logger.isDebugEnabled())
164             {
165                 logger.debug("Files: " + files.toString());
166             }
167             Comparator<File> comparator = getComparator();
168             if (comparator != null)
169             {
170                 Collections.sort(files, comparator);
171             }
172             for (File file : files)
173             {
174                 // don't process directories
175                 if (file.isFile())
176                 {
177                     processFile(file);
178                 }
179             }
180         }
181         catch (Exception e)
182         {
183             getConnector().getMuleContext().getExceptionListener().handleException(e);
184         }
185     }
186 
187     public synchronized void processFile(File sourceFile) throws MuleException
188     {
189         FileConnector fileConnector = (FileConnector) connector;
190         
191         //TODO RM*: This can be put in a Filter. Also we can add an AndFileFilter/OrFileFilter to allow users to
192         //combine file filters (since we can only pass a single filter to File.listFiles, we would need to wrap
193         //the current And/Or filters to extend {@link FilenameFilter}
194         boolean checkFileAge = fileConnector.getCheckFileAge();
195         if (checkFileAge)
196         {
197             long fileAge = ((FileConnector) connector).getFileAge();
198             long lastMod = sourceFile.lastModified();
199             long now = System.currentTimeMillis();
200             long thisFileAge = now - lastMod;
201             if (thisFileAge < fileAge)
202             {
203                 if (logger.isDebugEnabled())
204                 {
205                     logger.debug("The file has not aged enough yet, will return nothing for: " + sourceFile);
206                 }
207                 return;
208             }
209         }
210 
211         // don't process a file that is locked by another process (probably still being written)
212         if (!attemptFileLock(sourceFile))
213         {
214             return;
215         }
216         else if(logger.isInfoEnabled())
217         {
218             logger.info("Lock obtained on file: " + sourceFile.getAbsolutePath());
219         }
220 
221         String sourceFileOriginalName = sourceFile.getName();
222 
223         // Perform some quick checks to make sure file can be processed
224         if (!(sourceFile.canRead() && sourceFile.exists() && sourceFile.isFile()))
225         {
226             throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName));
227         }
228 
229         // This isn't nice but is needed as MuleMessage is required to resolve
230         // destination file name
231         DefaultMuleMessage fileParserMessasge = new DefaultMuleMessage(null, connector.getMuleContext());
232         fileParserMessasge.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
233         
234         File workFile = null;
235         if (workDir != null)
236         {
237             String workFileName = sourceFileOriginalName;
238             
239             workFileName = fileConnector.getFilenameParser().getFilename(fileParserMessasge,
240                     workFileNamePattern);
241             // don't use new File() directly, see MULE-1112
242             workFile = FileUtils.newFile(workDir, workFileName);
243             
244             move(sourceFile, workFile);
245             // Now the Work File is the Source file
246             sourceFile = workFile;
247         }
248         
249         // set up destination file
250         File destinationFile = null;
251         if (moveDir != null)
252         {
253             String destinationFileName = sourceFileOriginalName;
254             if (moveToPattern != null)
255             {
256                 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMessasge,
257                     moveToPattern);
258             }
259             // don't use new File() directly, see MULE-1112
260             destinationFile = FileUtils.newFile(moveDir, destinationFileName);
261         }
262 
263         MuleMessage message = null;
264         String encoding = endpoint.getEncoding();
265         try
266         {
267             if (fileConnector.isStreaming())
268             {
269                 ReceiverFileInputStream payload = new ReceiverFileInputStream(sourceFile,
270                     fileConnector.isAutoDelete(), destinationFile);
271                 message = createMuleMessage(payload, encoding);
272             }
273             else
274             {
275                 message = createMuleMessage(sourceFile, encoding);
276             }
277         }
278         catch (FileNotFoundException e)
279         {
280             // we can ignore since we did manage to acquire a lock, but just in case
281             logger.error("File being read disappeared!", e);
282             return;
283         }
284 
285         message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
286         if (forceSync)
287         {
288             message.setProperty(MuleProperties.MULE_FORCE_SYNC_PROPERTY, Boolean.TRUE, PropertyScope.INBOUND);
289         }
290         if (!fileConnector.isStreaming())
291         {
292             moveAndDelete(sourceFile, destinationFile, sourceFileOriginalName, message);
293         }
294         else
295         {
296             // If we are streaming no need to move/delete now, that will be done when
297             // stream is closed
298             message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, sourceFile.getName());
299             this.routeMessage(message);
300         }
301     }
302 
303     private void moveAndDelete(final File sourceFile, File destinationFile,
304         String sourceFileOriginalName, MuleMessage message)
305     {
306         boolean fileWasMoved = false;
307 
308         try
309         {
310             // If we are moving the file to a read directory, move it there now and
311             // hand over a reference to the
312             // File in its moved location
313             if (destinationFile != null)
314             {
315                 // move sourceFile to new destination
316                 try
317                 {
318                     FileUtils.moveFile(sourceFile, destinationFile);
319                 }
320                 catch (IOException e)
321                 {
322                     // move didn't work - bail out (will attempt rollback)
323                     throw new DefaultMuleException(FileMessages.failedToMoveFile(
324                         sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath()));
325                 }
326 
327                 // create new Message for destinationFile
328                 message = createMuleMessage(destinationFile, endpoint.getEncoding());
329                 message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
330                 message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
331             }
332 
333             // finally deliver the file message
334             this.routeMessage(message);
335 
336             // at this point msgAdapter either points to the old sourceFile
337             // or the new destinationFile.
338             if (((FileConnector) connector).isAutoDelete())
339             {
340                 // no moveTo directory
341                 if (destinationFile == null)
342                 {
343                     // delete source
344                     if (!sourceFile.delete())
345                     {
346                         throw new DefaultMuleException(FileMessages.failedToDeleteFile(sourceFile));
347                     }
348                 }
349                 else
350                 {
351                     // nothing to do here since moveFile() should have deleted
352                     // the source file for us
353                 }
354             }
355         }
356         catch (Exception e)
357         {
358             boolean fileWasRolledBack = false;
359 
360             // only attempt rollback if file move was successful
361             if (fileWasMoved)
362             {
363                 try
364                 {
365                     rollbackFileMove(destinationFile, sourceFile.getAbsolutePath());
366                     fileWasRolledBack = true;
367                 }
368                 catch (IOException ioException)
369                 {
370                     // eat it
371                 }
372             }
373 
374             // wrap exception & handle it
375             Message msg = FileMessages.exceptionWhileProcessing(sourceFile.getName(),
376                 (fileWasRolledBack ? "successful" : "unsuccessful"));
377             getConnector().getMuleContext().getExceptionListener().handleException(new MessagingException(msg, message, e));
378         }
379     }
380 
381     /**
382      * Try to acquire a lock on a file and release it immediately. Usually used as a
383      * quick check to see if another process is still holding onto the file, e.g. a
384      * large file (more than 100MB) is still being written to.
385      * 
386      * @param sourceFile file to check
387      * @return <code>true</code> if the file can be locked
388      */
389     protected boolean attemptFileLock(final File sourceFile)
390     {
391         // check if the file can be processed, be sure that it's not still being
392         // written
393         // if the file can't be locked don't process it yet, since creating
394         // a new FileInputStream() will throw an exception
395         FileLock lock = null;
396         FileChannel channel = null;
397         boolean fileCanBeLocked = false;
398         try
399         {
400             channel = new RandomAccessFile(sourceFile, "rw").getChannel();
401 
402             // Try acquiring the lock without blocking. This method returns
403             // null or throws an exception if the file is already locked.
404             lock = channel.tryLock();
405         }
406         catch (FileNotFoundException fnfe)
407         {
408             logger.warn("Unable to open " + sourceFile.getAbsolutePath(), fnfe);
409         }
410         catch (IOException e)
411         {
412             // Unable to create a lock. This exception should only be thrown when
413             // the file is already locked. No sense in repeating the message over
414             // and over.
415         }
416         finally
417         {
418             if (lock != null)
419             {
420                 // if lock is null the file is locked by another process
421                 fileCanBeLocked = true;
422                 try
423                 {
424                     // Release the lock
425                     lock.release();
426                 }
427                 catch (IOException e)
428                 {
429                     // ignore
430                 }
431             }
432 
433             if (channel != null)
434             {
435                 try
436                 {
437                     // Close the file
438                     channel.close();
439                 }
440                 catch (IOException e)
441                 {
442                     // ignore
443                 }
444             }
445         }
446 
447         return fileCanBeLocked;
448     }
449 
450     /**
451      * Get a list of files to be processed.
452      * 
453      * @return an array of files to be processed.
454      * @throws org.mule.api.MuleException which will wrap any other exceptions or
455      *             errors.
456      */
457     List<File> listFiles() throws MuleException
458     {
459         try
460         {
461             List<File> files = new ArrayList<File>();
462             this.basicListFiles(readDirectory, files);
463             return (files.isEmpty() ? NO_FILES : files);
464         }
465         catch (Exception e)
466         {
467             throw new DefaultMuleException(FileMessages.errorWhileListingFiles(), e);
468         }
469     }
470 
471     protected void basicListFiles(File currentDirectory, List<File> discoveredFiles)
472     {
473         File[] files;
474         if (fileFilter != null)
475         {
476             files = currentDirectory.listFiles(fileFilter);
477         }
478         else
479         {
480             files = currentDirectory.listFiles(filenameFilter);
481         }
482         
483         // the listFiles calls above may actually return null (check the JDK code).
484         if (files == null)
485         {
486             return;
487         }
488 
489         for (File file : files)
490         {
491             if (!file.isDirectory())
492             {
493                 discoveredFiles.add(file);
494             }
495             else
496             {
497                 if (((FileConnector) this.getConnector()).isRecursive())
498                 {
499                     this.basicListFiles(file, discoveredFiles);
500                 }
501             }
502         }
503     }
504 
505     /**
506      * Exception tolerant roll back method
507      * 
508      * @throws Throwable
509      */
510     protected void rollbackFileMove(File sourceFile, String destinationFilePath) throws IOException
511     {
512         try
513         {
514             FileUtils.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
515         }
516         catch (IOException t)
517         {
518             logger.debug("rollback of file move failed: " + t.getMessage());
519             throw t;
520         }
521     }
522 
523     protected Comparator<File> getComparator() throws Exception
524     {
525         Object comparatorClassName = getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY);
526         if (comparatorClassName != null)
527         {
528             Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY);
529             boolean reverse = false;
530             if (reverseProperty != null)
531             {
532                 reverse = Boolean.valueOf((String) reverseProperty);
533             }
534             
535             Class<?> clazz = Class.forName(comparatorClassName.toString());
536             Comparator<?> comparator = (Comparator<?>)clazz.newInstance();
537             return reverse ? new ReverseComparator(comparator) : comparator;
538         }
539         return null;
540     }
541     
542     private void move(final File sourceFile,File destinationFile) throws DefaultMuleException
543     {
544         if (destinationFile != null)
545         {
546             // move sourceFile to new destination. Do not use FileUtils here as it ultimately
547             // falls back to copying the file which will cause problems on large files again -
548             // which is what we're trying to avoid in the first place
549             boolean fileWasMoved = sourceFile.renameTo(destinationFile);
550 
551             // move didn't work - bail out
552             if (!fileWasMoved)
553             {
554                 throw new DefaultMuleException(FileMessages.failedToMoveFile(sourceFile.getAbsolutePath(),
555                     destinationFile.getAbsolutePath()));
556             }
557         }
558     }
559     
560 }