View Javadoc

1   /*
2    * $Id: FileMessageReceiver.java 9917 2007-11-28 00:05:51Z aperepel $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSource, Inc.  All rights reserved.  http://www.mulesource.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.providers.file;
12  
13  import org.mule.MuleException;
14  import org.mule.impl.MuleMessage;
15  import org.mule.providers.AbstractPollingMessageReceiver;
16  import org.mule.providers.ConnectException;
17  import org.mule.providers.file.i18n.FileMessages;
18  import org.mule.umo.UMOComponent;
19  import org.mule.umo.UMOException;
20  import org.mule.umo.endpoint.UMOEndpoint;
21  import org.mule.umo.lifecycle.InitialisationException;
22  import org.mule.umo.provider.UMOConnector;
23  import org.mule.umo.provider.UMOMessageAdapter;
24  import org.mule.umo.routing.RoutingException;
25  import org.mule.util.FileUtils;
26  
27  import java.io.File;
28  import java.io.FileFilter;
29  import java.io.FileInputStream;
30  import java.io.FileNotFoundException;
31  import java.io.FileOutputStream;
32  import java.io.FilenameFilter;
33  import java.io.IOException;
34  import java.io.RandomAccessFile;
35  import java.nio.channels.FileChannel;
36  import java.nio.channels.FileLock;
37  import java.util.Comparator;
38  
39  import edu.emory.mathcs.backport.java.util.Arrays;
40  import org.apache.commons.collections.comparators.ReverseComparator;
41  import org.apache.commons.io.IOUtils;
42  
43  /**
44   * <code>FileMessageReceiver</code> is a polling listener that reads files from a
45   * directory.
46   */
47  
48  public class FileMessageReceiver extends AbstractPollingMessageReceiver
49  {
50  
51      public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator";
52      public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder";
53  
54      private String readDir = null;
55      private String moveDir = null;
56      private File readDirectory = null;
57      private File moveDirectory = null;
58      private String moveToPattern = null;
59      private FilenameFilter filenameFilter = null;
60      private FileFilter fileFilter = null;
61  
62      public FileMessageReceiver(UMOConnector connector,
63                                 UMOComponent component,
64                                 UMOEndpoint endpoint,
65                                 String readDir,
66                                 String moveDir,
67                                 String moveToPattern,
68                                 long frequency) throws InitialisationException
69      {
70          super(connector, component, endpoint);
71          this.setFrequency(frequency);
72  
73          this.readDir = readDir;
74          this.moveDir = moveDir;
75          this.moveToPattern = moveToPattern;
76  
77          if (endpoint.getFilter() instanceof FilenameFilter)
78          {
79              filenameFilter = (FilenameFilter) endpoint.getFilter();
80          }
81          else if (endpoint.getFilter() instanceof FileFilter)
82          {
83              fileFilter = (FileFilter) endpoint.getFilter();
84          }
85          else if (endpoint.getFilter() != null)
86          {
87              throw new InitialisationException(
88                      FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
89          }
90      }
91  
92      protected void doConnect() throws Exception
93      {
94          if (readDir != null)
95          {
96              readDirectory = FileUtils.openDirectory(readDir);
97              if (!(readDirectory.canRead()))
98              {
99                  throw new ConnectException(
100                         FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
101             }
102             else
103             {
104                 logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
105             }
106         }
107 
108         if (moveDir != null)
109         {
110             moveDirectory = FileUtils.openDirectory((moveDir));
111             if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
112             {
113                 throw new ConnectException(
114                         FileMessages.moveToDirectoryNotWritable(), this);
115             }
116         }
117     }
118 
119     protected void doDisconnect() throws Exception
120     {
121         // template method
122     }
123 
124     protected void doDispose()
125     {
126         // nothing to do               
127     }
128 
129     public void poll()
130     {
131         try
132         {
133             File[] files = this.listFiles();
134             Comparator comparator = getComparator();
135             if (comparator != null)
136             {
137                 Arrays.sort(files, comparator);
138             }
139             for (int i = 0; i < files.length; i++)
140             {
141                 // don't process directories
142                 if (files[i].isFile())
143                 {
144                     this.processFile(files[i]);
145                 }
146             }
147         }
148         catch (Exception e)
149         {
150             this.handleException(e);
151         }
152     }
153 
154     public synchronized void processFile(final File sourceFile) throws UMOException
155     {
156         //TODO RM*: This can be put in a Filter. Also we can add an AndFileFilter/OrFileFilter to allow users to
157         //combine file filters (since we can only pass a single filter to File.listFiles, we would need to wrap
158         //the current And/Or filters to extend {@link FilenameFilter}
159         boolean checkFileAge = ((FileConnector) connector).getCheckFileAge();
160         if (checkFileAge)
161         {
162             long fileAge = ((FileConnector) connector).getFileAge();
163             long lastMod = sourceFile.lastModified();
164             long now = System.currentTimeMillis();
165             long thisFileAge = now - lastMod;
166             if (thisFileAge < fileAge)
167             {
168                 if (logger.isDebugEnabled())
169                 {
170                     logger.debug("The file has not aged enough yet, will return nothing for: " + sourceFile);
171                 }
172                 return;
173             }
174         }
175 
176         // don't process a file that is locked by another process (probably still being written)
177         if (!attemptFileLock(sourceFile))
178         {
179             return;
180         }
181 
182         File destinationFile = null;
183         String sourceFileOriginalName = sourceFile.getName();
184         //Create a message adapter here to pass to the fileName parser
185         UMOMessageAdapter msgAdapter;
186         FileInputStream fileIn = null;
187         if (endpoint.isStreaming())
188         {
189             try
190             {
191                 fileIn = new FileInputStream(sourceFile);
192                 msgAdapter = connector.getStreamMessageAdapter(fileIn, null);
193             }
194             catch (FileNotFoundException e)
195             {
196                 // we can ignore since we did manage to acquire a lock, but just in case
197                 logger.error("File being read disappeared!", e);
198                 return;
199             }
200         }
201         else
202         {
203             msgAdapter = connector.getMessageAdapter(sourceFile);
204         }
205         msgAdapter.setProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
206 
207         // set up destination file
208         if (moveDir != null)
209         {
210             String destinationFileName = sourceFileOriginalName;
211 
212             if (moveToPattern != null)
213             {
214                 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(msgAdapter,
215                                                                                                   moveToPattern);
216             }
217 
218             // don't use new File() directly, see MULE-1112
219             destinationFile = FileUtils.newFile(moveDir, destinationFileName);
220         }
221 
222         boolean fileWasMoved = false;
223 
224         try
225         {
226             // Perform some quick checks to make sure file can be processed
227             if (!(sourceFile.canRead() && sourceFile.exists() && sourceFile.isFile()))
228             {
229                 throw new MuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName));
230             }
231 
232             //If we are moving the file to a read directory, move it there now and hand over a reference to the
233             //File in its moved location
234             if (destinationFile != null)
235             {
236                 if (fileIn != null)
237                 {
238                     fileIn.close();
239                 }
240                 
241                 // move sourceFile to new destination
242                 fileWasMoved = this.moveFile(sourceFile, destinationFile);
243 
244                 // move didn't work - bail out (will attempt rollback)
245                 if (!fileWasMoved)
246                 {
247                     throw new MuleException(
248                             FileMessages.failedToMoveFile(
249                                     sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath()));
250                 }
251 
252                 // create new MessageAdapter for destinationFile
253                 if (endpoint.isStreaming())
254                 {
255                     msgAdapter = connector.getStreamMessageAdapter(new FileInputStream(destinationFile), null);
256                 }
257                 else
258                 {
259                     msgAdapter = connector.getMessageAdapter(destinationFile);
260                 }
261                 msgAdapter.setProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
262                 msgAdapter.setProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
263             }
264 
265             // finally deliver the file message
266             this.routeMessage(new MuleMessage(msgAdapter), endpoint.isSynchronous());
267 
268             // at this point msgAdapter either points to the old sourceFile
269             // or the new destinationFile.
270             if (((FileConnector) connector).isAutoDelete())
271             {
272                 // no moveTo directory
273                 if (destinationFile == null)
274                 {
275                     // delete source
276                     if (!sourceFile.delete())
277                     {
278                         throw new MuleException(
279                                 FileMessages.failedToDeleteFile(sourceFile.getAbsolutePath()));
280                     }
281                 }
282                 else
283                 {
284                     // nothing to do here since moveFile() should have deleted
285                     // the source file for us
286                 }
287             }
288         }
289         catch (Exception e)
290         {
291             boolean fileWasRolledBack = false;
292 
293             // only attempt rollback if file move was successful
294             if (fileWasMoved)
295             {
296                 fileWasRolledBack = this.rollbackFileMove(destinationFile, sourceFile.getAbsolutePath());
297             }
298 
299             // wrap exception & handle it
300             Exception ex = new RoutingException(
301                     FileMessages.exceptionWhileProcessing(sourceFile.getName(),
302                                                           (fileWasRolledBack ? "successful" : "unsuccessful")), new MuleMessage(msgAdapter), endpoint,
303                                                                                                                 e);
304             this.handleException(ex);
305         }
306     }
307 
308     /**
309      * Try to acquire a lock on a file and release it immediately. Usually used as a quick check to
310      * see if another process is still holding onto the file, e.g. a large file (more than 100MB) is
311      * still being written to.
312      *
313      * @param sourceFile file to check
314      * @return <code>true</code> if the file can be locked
315      */
316     protected boolean attemptFileLock(final File sourceFile)
317     {
318         // check if the file can be processed, be sure that it's not still being written
319         // if the file can't be locked don't process it yet, since creating
320         // a new FileInputStream() will throw an exception
321         FileLock lock = null;
322         FileChannel channel = null;
323         boolean fileCanBeLocked = false;
324         try
325         {
326             channel = new RandomAccessFile(sourceFile, "rw").getChannel();
327 
328             // Try acquiring the lock without blocking. This method returns
329             // null or throws an exception if the file is already locked.
330             lock = channel.tryLock();
331         }
332         catch (FileNotFoundException fnfe)
333         {
334             logger.warn("Unable to open " + sourceFile.getAbsolutePath(), fnfe);
335         }
336         catch (IOException e)
337         {
338             // Unable to create a lock. This exception should only be thrown when
339             // the file is already locked. No sense in repeating the message over
340             // and over.
341         }
342         finally
343         {
344             if (lock != null)
345             {
346                 // if lock is null the file is locked by another process
347                 fileCanBeLocked = true;
348                 try
349                 {
350                     // Release the lock
351                     lock.release();
352                 }
353                 catch (IOException e)
354                 {
355                     // ignore
356                 }
357             }
358 
359             if (channel != null)
360             {
361                 try
362                 {
363                     // Close the file
364                     channel.close();
365                 }
366                 catch (IOException e)
367                 {
368                     // ignore
369                 }
370             }
371         }
372 
373         return fileCanBeLocked;
374     }
375 
376     /**
377      * Try to move a file by renaming with backup attempt by copying/deleting via NIO
378      */
379     protected boolean moveFile(File sourceFile, File destinationFile)
380     {
381         // try fast file-system-level move/rename first
382         boolean success = sourceFile.renameTo(destinationFile);
383 
384         if (!success)
385         {
386             // try again using NIO copy
387             FileInputStream fis = null;
388             FileOutputStream fos = null;
389             try
390             {
391                 fis = new FileInputStream(sourceFile);
392                 fos = new FileOutputStream(destinationFile);
393                 FileChannel srcChannel = fis.getChannel();
394                 FileChannel dstChannel = fos.getChannel();
395                 dstChannel.transferFrom(srcChannel, 0, srcChannel.size());
396                 srcChannel.close();
397                 dstChannel.close();
398                 success = sourceFile.delete();
399             }
400             catch (IOException ioex)
401             {
402                 // grr!
403                 success = false;
404             }
405             finally
406             {
407                 IOUtils.closeQuietly(fis);
408                 IOUtils.closeQuietly(fos);
409             }
410         }
411 
412         return success;
413     }
414 
415     /**
416      * Exception tolerant roll back method
417      */
418     protected boolean rollbackFileMove(File sourceFile, String destinationFilePath)
419     {
420         boolean result = false;
421         try
422         {
423             result = this.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
424         }
425         catch (Throwable t)
426         {
427             logger.debug("rollback of file move failed: " + t.getMessage());
428         }
429         return result;
430     }
431 
432     /**
433      * Get a list of files to be processed.
434      *
435      * @return an array of files to be processed.
436      * @throws org.mule.MuleException which will wrap any other exceptions or errors.
437      */
438     File[] listFiles() throws MuleException
439     {
440         try
441         {
442             File[] todoFiles;
443             if (fileFilter != null)
444             {
445                 todoFiles = readDirectory.listFiles(fileFilter);
446             }
447             else
448             {
449                 todoFiles = readDirectory.listFiles(filenameFilter);
450             }
451             // logger.trace("Reading directory " + readDirectory.getAbsolutePath() +
452             // " -> " + todoFiles.length + " file(s)");
453             return (todoFiles == null ? new File[0] : todoFiles);
454         }
455         catch (Exception e)
456         {
457             throw new MuleException(FileMessages.errorWhileListingFiles(), e);
458         }
459     }
460 
461     protected Comparator getComparator() throws Exception
462     {
463 
464         Object o = this.getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY);
465         Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY);
466         boolean reverse = false;
467         if (o != null)
468         {
469             if (reverseProperty != null)
470             {
471                 reverse = Boolean.valueOf((String) reverseProperty).booleanValue();
472             }
473             Class clazz = Class.forName(o.toString());
474             o = clazz.newInstance();
475             return reverse ? new ReverseComparator((Comparator) o) : (Comparator) o;
476         }
477         return null;
478     }
479 
480 }