View Javadoc

1   /*
2    * $Id: FileMessageReceiver.java 20109 2010-11-07 05:20:30Z mike.schilling $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.transport.file;
12  
13  import org.mule.DefaultMuleMessage;
14  import org.mule.api.DefaultMuleException;
15  import org.mule.api.MessagingException;
16  import org.mule.api.MuleException;
17  import org.mule.api.MuleMessage;
18  import org.mule.api.construct.FlowConstruct;
19  import org.mule.api.endpoint.InboundEndpoint;
20  import org.mule.api.lifecycle.CreateException;
21  import org.mule.api.transport.Connector;
22  import org.mule.config.i18n.Message;
23  import org.mule.transport.AbstractPollingMessageReceiver;
24  import org.mule.transport.ConnectException;
25  import org.mule.transport.file.i18n.FileMessages;
26  import org.mule.util.FileUtils;
27  
28  import java.io.File;
29  import java.io.FileFilter;
30  import java.io.FileNotFoundException;
31  import java.io.FilenameFilter;
32  import java.io.IOException;
33  import java.io.RandomAccessFile;
34  import java.nio.channels.FileChannel;
35  import java.nio.channels.FileLock;
36  import java.util.ArrayList;
37  import java.util.Collections;
38  import java.util.Comparator;
39  import java.util.List;
40  
41  import org.apache.commons.collections.comparators.ReverseComparator;
42  
43  /**
44   * <code>FileMessageReceiver</code> is a polling listener that reads files from a
45   * directory.
46   */
47  
48  public class FileMessageReceiver extends AbstractPollingMessageReceiver
49  {
50      public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator";
51      public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder";
52  
53      private static final List<File> NO_FILES = new ArrayList<File>();
54  
55      private String readDir = null;
56      private String moveDir = null;
57      private String workDir = null;
58      private File readDirectory = null;
59      private File moveDirectory = null;
60      private String moveToPattern = null;
61      private String workFileNamePattern = null;
62      private FilenameFilter filenameFilter = null;
63      private FileFilter fileFilter = null;
64  
65      public FileMessageReceiver(Connector connector,
66                                 FlowConstruct flowConstruct,
67                                 InboundEndpoint endpoint,
68                                 String readDir,
69                                 String moveDir,
70                                 String moveToPattern,
71                                 long frequency) throws CreateException
72      {
73          super(connector, flowConstruct, endpoint);
74          this.setFrequency(frequency);
75  
76          this.readDir = readDir;
77          this.moveDir = moveDir;
78          this.moveToPattern = moveToPattern;
79          this.workDir = ((FileConnector) connector).getWorkDirectory();
80          this.workFileNamePattern = ((FileConnector) connector).getWorkFileNamePattern();
81  
82          if (endpoint.getFilter() instanceof FilenameFilter)
83          {
84              filenameFilter = (FilenameFilter) endpoint.getFilter();
85          }
86          else if (endpoint.getFilter() instanceof FileFilter)
87          {
88              fileFilter = (FileFilter) endpoint.getFilter();
89          }
90          else if (endpoint.getFilter() != null)
91          {
92              throw new CreateException(FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
93          }
94          
95          logWarningWhenConnectorAutoDeletesAndEndpointIsOneWay();
96      }
97  
98      private void logWarningWhenConnectorAutoDeletesAndEndpointIsOneWay() throws CreateException
99      {
100         boolean connectorIsAutoDelete = false;
101         if (connector instanceof FileConnector)
102         {
103             connectorIsAutoDelete = ((FileConnector) connector).isAutoDelete();
104         }
105 
106         boolean messageFactoryConsumes = (createMuleMessageFactory() instanceof FileContentsMuleMessageFactory);
107         
108         if (connectorIsAutoDelete && !messageFactoryConsumes)
109         {
110             logger.warn(FileMessages.connectorAutodeletesWithoutConsuming(connector).getMessage());
111         }
112     }
113 
114     @Override
115     protected void doConnect() throws Exception
116     {
117         if (readDir != null)
118         {
119             readDirectory = FileUtils.openDirectory(readDir);
120             if (!(readDirectory.canRead()))
121             {
122                 throw new ConnectException(FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
123             }
124             else
125             {
126                 logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
127             }
128         }
129 
130         if (moveDir != null)
131         {
132             moveDirectory = FileUtils.openDirectory((moveDir));
133             if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
134             {
135                 throw new ConnectException(FileMessages.moveToDirectoryNotWritable(), this);
136             }
137         }
138     }
139 
140     @Override
141     protected void doDisconnect() throws Exception
142     {
143         // template method
144     }
145 
146     @Override
147     protected void doDispose()
148     {
149         // nothing to do
150     }
151 
152     @Override
153     public void poll()
154     {
155         try
156         {
157             List<File> files = this.listFiles();
158             if (logger.isDebugEnabled())
159             {
160                 logger.debug("Files: " + files.toString());
161             }
162             Comparator<File> comparator = getComparator();
163             if (comparator != null)
164             {
165                 Collections.sort(files, comparator);
166             }
167             for (File file : files)
168             {
169                 // don't process directories
170                 if (file.isFile())
171                 {
172                     processFile(file);
173                 }
174             }
175         }
176         catch (Exception e)
177         {
178             getConnector().getMuleContext().getExceptionListener().handleException(e);
179         }
180     }
181 
182     public synchronized void processFile(File sourceFile) throws MuleException
183     {
184         FileConnector fileConnector = (FileConnector) connector;
185         
186         //TODO RM*: This can be put in a Filter. Also we can add an AndFileFilter/OrFileFilter to allow users to
187         //combine file filters (since we can only pass a single filter to File.listFiles, we would need to wrap
188         //the current And/Or filters to extend {@link FilenameFilter}
189         boolean checkFileAge = fileConnector.getCheckFileAge();
190         if (checkFileAge)
191         {
192             long fileAge = ((FileConnector) connector).getFileAge();
193             long lastMod = sourceFile.lastModified();
194             long now = System.currentTimeMillis();
195             long thisFileAge = now - lastMod;
196             if (thisFileAge < fileAge)
197             {
198                 if (logger.isDebugEnabled())
199                 {
200                     logger.debug("The file has not aged enough yet, will return nothing for: " + sourceFile);
201                 }
202                 return;
203             }
204         }
205 
206         // don't process a file that is locked by another process (probably still being written)
207         if (!attemptFileLock(sourceFile))
208         {
209             return;
210         }
211         else if(logger.isInfoEnabled())
212         {
213             logger.info("Lock obtained on file: " + sourceFile.getAbsolutePath());
214         }
215 
216         String sourceFileOriginalName = sourceFile.getName();
217 
218         // Perform some quick checks to make sure file can be processed
219         if (!(sourceFile.canRead() && sourceFile.exists() && sourceFile.isFile()))
220         {
221             throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName));
222         }
223 
224         // This isn't nice but is needed as MuleMessage is required to resolve
225         // destination file name
226         DefaultMuleMessage fileParserMessasge = new DefaultMuleMessage(null, connector.getMuleContext());
227         fileParserMessasge.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
228         
229         File workFile = null;
230         if (workDir != null && (moveDir == null || (moveDir != null && fileConnector.isStreaming())))
231         {
232             String workFileName = sourceFileOriginalName;
233             
234             workFileName = fileConnector.getFilenameParser().getFilename(fileParserMessasge,
235                     workFileNamePattern);
236             // don't use new File() directly, see MULE-1112
237             workFile = FileUtils.newFile(workDir, workFileName);
238             
239             move(sourceFile, workFile);
240             // Now the Work File is the Source file
241             sourceFile = workFile;
242         }
243         
244         // set up destination file
245         File destinationFile = null;
246         if (moveDir != null)
247         {
248             String destinationFileName = sourceFileOriginalName;
249             if (moveToPattern != null)
250             {
251                 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMessasge,
252                     moveToPattern);
253             }
254             // don't use new File() directly, see MULE-1112
255             destinationFile = FileUtils.newFile(moveDir, destinationFileName);
256         }
257 
258         MuleMessage message = null;
259         String encoding = endpoint.getEncoding();
260         try
261         {
262             if (fileConnector.isStreaming())
263             {
264                 ReceiverFileInputStream payload = new ReceiverFileInputStream(sourceFile,
265                     fileConnector.isAutoDelete(), destinationFile);
266                 message = createMuleMessage(payload, encoding);
267             }
268             else
269             {
270                 message = createMuleMessage(sourceFile, encoding);
271             }
272         }
273         catch (FileNotFoundException e)
274         {
275             // we can ignore since we did manage to acquire a lock, but just in case
276             logger.error("File being read disappeared!", e);
277             return;
278         }
279 
280         message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
281 
282         if (!fileConnector.isStreaming())
283         {
284             moveAndDelete(sourceFile, destinationFile, sourceFileOriginalName, message);
285         }
286         else
287         {
288             // If we are streaming no need to move/delete now, that will be done when
289             // stream is closed
290             message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, sourceFile.getName());
291             this.routeMessage(message);
292         }
293     }
294 
295     private void moveAndDelete(final File sourceFile, File destinationFile,
296         String sourceFileOriginalName, MuleMessage message)
297     {
298         boolean fileWasMoved = false;
299 
300         try
301         {
302             // If we are moving the file to a read directory, move it there now and
303             // hand over a reference to the
304             // File in its moved location
305             if (destinationFile != null)
306             {
307                 // move sourceFile to new destination
308                 try
309                 {
310                     FileUtils.moveFile(sourceFile, destinationFile);
311                 }
312                 catch (IOException e)
313                 {
314                     // move didn't work - bail out (will attempt rollback)
315                     throw new DefaultMuleException(FileMessages.failedToMoveFile(
316                         sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath()));
317                 }
318 
319                 // create new Message for destinationFile
320                 message = createMuleMessage(destinationFile, endpoint.getEncoding());
321                 message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
322                 message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
323             }
324 
325             // finally deliver the file message
326             this.routeMessage(message);
327 
328             // at this point msgAdapter either points to the old sourceFile
329             // or the new destinationFile.
330             if (((FileConnector) connector).isAutoDelete())
331             {
332                 // no moveTo directory
333                 if (destinationFile == null)
334                 {
335                     // delete source
336                     if (!sourceFile.delete())
337                     {
338                         throw new DefaultMuleException(FileMessages.failedToDeleteFile(sourceFile));
339                     }
340                 }
341                 else
342                 {
343                     // nothing to do here since moveFile() should have deleted
344                     // the source file for us
345                 }
346             }
347         }
348         catch (Exception e)
349         {
350             boolean fileWasRolledBack = false;
351 
352             // only attempt rollback if file move was successful
353             if (fileWasMoved)
354             {
355                 try
356                 {
357                     rollbackFileMove(destinationFile, sourceFile.getAbsolutePath());
358                     fileWasRolledBack = true;
359                 }
360                 catch (IOException ioException)
361                 {
362                     // eat it
363                 }
364             }
365 
366             // wrap exception & handle it
367             Message msg = FileMessages.exceptionWhileProcessing(sourceFile.getName(),
368                 (fileWasRolledBack ? "successful" : "unsuccessful"));
369             getConnector().getMuleContext().getExceptionListener().handleException(new MessagingException(msg, message, e));
370         }
371     }
372 
373     /**
374      * Try to acquire a lock on a file and release it immediately. Usually used as a
375      * quick check to see if another process is still holding onto the file, e.g. a
376      * large file (more than 100MB) is still being written to.
377      * 
378      * @param sourceFile file to check
379      * @return <code>true</code> if the file can be locked
380      */
381     protected boolean attemptFileLock(final File sourceFile)
382     {
383         // check if the file can be processed, be sure that it's not still being
384         // written
385         // if the file can't be locked don't process it yet, since creating
386         // a new FileInputStream() will throw an exception
387         FileLock lock = null;
388         FileChannel channel = null;
389         boolean fileCanBeLocked = false;
390         try
391         {
392             channel = new RandomAccessFile(sourceFile, "rw").getChannel();
393 
394             // Try acquiring the lock without blocking. This method returns
395             // null or throws an exception if the file is already locked.
396             lock = channel.tryLock();
397         }
398         catch (FileNotFoundException fnfe)
399         {
400             logger.warn("Unable to open " + sourceFile.getAbsolutePath(), fnfe);
401         }
402         catch (IOException e)
403         {
404             // Unable to create a lock. This exception should only be thrown when
405             // the file is already locked. No sense in repeating the message over
406             // and over.
407         }
408         finally
409         {
410             if (lock != null)
411             {
412                 // if lock is null the file is locked by another process
413                 fileCanBeLocked = true;
414                 try
415                 {
416                     // Release the lock
417                     lock.release();
418                 }
419                 catch (IOException e)
420                 {
421                     // ignore
422                 }
423             }
424 
425             if (channel != null)
426             {
427                 try
428                 {
429                     // Close the file
430                     channel.close();
431                 }
432                 catch (IOException e)
433                 {
434                     // ignore
435                 }
436             }
437         }
438 
439         return fileCanBeLocked;
440     }
441 
442     /**
443      * Get a list of files to be processed.
444      * 
445      * @return an array of files to be processed.
446      * @throws org.mule.api.MuleException which will wrap any other exceptions or
447      *             errors.
448      */
449     List<File> listFiles() throws MuleException
450     {
451         try
452         {
453             List<File> files = new ArrayList<File>();
454             this.basicListFiles(readDirectory, files);
455             return (files.isEmpty() ? NO_FILES : files);
456         }
457         catch (Exception e)
458         {
459             throw new DefaultMuleException(FileMessages.errorWhileListingFiles(), e);
460         }
461     }
462 
463     protected void basicListFiles(File currentDirectory, List<File> discoveredFiles)
464     {
465         File[] files;
466         if (fileFilter != null)
467         {
468             files = currentDirectory.listFiles(fileFilter);
469         }
470         else
471         {
472             files = currentDirectory.listFiles(filenameFilter);
473         }
474         
475         // the listFiles calls above may actually return null (check the JDK code).
476         if (files == null)
477         {
478             return;
479         }
480 
481         for (File file : files)
482         {
483             if (!file.isDirectory())
484             {
485                 discoveredFiles.add(file);
486             }
487             else
488             {
489                 if (((FileConnector) this.getConnector()).isRecursive())
490                 {
491                     this.basicListFiles(file, discoveredFiles);
492                 }
493             }
494         }
495     }
496 
497     /**
498      * Exception tolerant roll back method
499      * 
500      * @throws Throwable
501      */
502     protected void rollbackFileMove(File sourceFile, String destinationFilePath) throws IOException
503     {
504         try
505         {
506             FileUtils.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
507         }
508         catch (IOException t)
509         {
510             logger.debug("rollback of file move failed: " + t.getMessage());
511             throw t;
512         }
513     }
514 
515     protected Comparator<File> getComparator() throws Exception
516     {
517         Object comparatorClassName = getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY);
518         if (comparatorClassName != null)
519         {
520             Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY);
521             boolean reverse = false;
522             if (reverseProperty != null)
523             {
524                 reverse = Boolean.valueOf((String) reverseProperty);
525             }
526             
527             Class<?> clazz = Class.forName(comparatorClassName.toString());
528             Comparator<?> comparator = (Comparator<?>)clazz.newInstance();
529             return reverse ? new ReverseComparator(comparator) : comparator;
530         }
531         return null;
532     }
533     
534     private void move(final File sourceFile,File destinationFile) throws DefaultMuleException
535     {
536         if (destinationFile != null)
537         {
538             // move sourceFile to new destination. Do not use FileUtils here as it ultimately
539             // falls back to copying the file which will cause problems on large files again -
540             // which is what we're trying to avoid in the first place
541             boolean fileWasMoved = sourceFile.renameTo(destinationFile);
542 
543             // move didn't work - bail out
544             if (!fileWasMoved)
545             {
546                 throw new DefaultMuleException(FileMessages.failedToMoveFile(sourceFile.getAbsolutePath(),
547                     destinationFile.getAbsolutePath()));
548             }
549         }
550     }
551     
552 }