View Javadoc

1   /*
2    * $Id: FileMessageReceiver.java 10961 2008-02-22 19:01:02Z dfeist $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSource, Inc.  All rights reserved.  http://www.mulesource.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.transport.file;
12  
13  import org.mule.DefaultMuleMessage;
14  import org.mule.api.DefaultMuleException;
15  import org.mule.api.MuleException;
16  import org.mule.api.endpoint.InboundEndpoint;
17  import org.mule.api.lifecycle.CreateException;
18  import org.mule.api.routing.RoutingException;
19  import org.mule.api.service.Service;
20  import org.mule.api.transport.Connector;
21  import org.mule.api.transport.MessageAdapter;
22  import org.mule.transport.AbstractPollingMessageReceiver;
23  import org.mule.transport.ConnectException;
24  import org.mule.transport.DefaultMessageAdapter;
25  import org.mule.transport.file.i18n.FileMessages;
26  import org.mule.util.FileUtils;
27  
28  import java.io.File;
29  import java.io.FileFilter;
30  import java.io.FileNotFoundException;
31  import java.io.FilenameFilter;
32  import java.io.IOException;
33  import java.io.RandomAccessFile;
34  import java.nio.channels.FileChannel;
35  import java.nio.channels.FileLock;
36  import java.util.Comparator;
37  
38  import edu.emory.mathcs.backport.java.util.Arrays;
39  
40  import org.apache.commons.collections.comparators.ReverseComparator;
41  
42  /**
43   * <code>FileMessageReceiver</code> is a polling listener that reads files from a
44   * directory.
45   */
46  
47  public class FileMessageReceiver extends AbstractPollingMessageReceiver
48  {
49      public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator";
50      public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder";
51  
52      private String readDir = null;
53      private String moveDir = null;
54      private File readDirectory = null;
55      private File moveDirectory = null;
56      private String moveToPattern = null;
57      private FilenameFilter filenameFilter = null;
58      private FileFilter fileFilter = null;
59  
60      public FileMessageReceiver(Connector connector,
61                                 Service service,
62                                 InboundEndpoint endpoint,
63                                 String readDir,
64                                 String moveDir,
65                                 String moveToPattern,
66                                 long frequency) throws CreateException
67      {
68          super(connector, service, endpoint);
69          this.setFrequency(frequency);
70  
71          this.readDir = readDir;
72          this.moveDir = moveDir;
73          this.moveToPattern = moveToPattern;
74  
75          if (endpoint.getFilter() instanceof FilenameFilter)
76          {
77              filenameFilter = (FilenameFilter) endpoint.getFilter();
78          }
79          else if (endpoint.getFilter() instanceof FileFilter)
80          {
81              fileFilter = (FileFilter) endpoint.getFilter();
82          }
83          else if (endpoint.getFilter() != null)
84          {
85              throw new CreateException(FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
86          }
87      }
88  
89      protected void doConnect() throws Exception
90      {
91          if (readDir != null)
92          {
93              readDirectory = FileUtils.openDirectory(readDir);
94              if (!(readDirectory.canRead()))
95              {
96                  throw new ConnectException(FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
97              }
98              else
99              {
100                 logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
101             }
102         }
103 
104         if (moveDir != null)
105         {
106             moveDirectory = FileUtils.openDirectory((moveDir));
107             if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
108             {
109                 throw new ConnectException(FileMessages.moveToDirectoryNotWritable(), this);
110             }
111         }
112     }
113 
114     protected void doDisconnect() throws Exception
115     {
116         // template method
117     }
118 
119     protected void doDispose()
120     {
121         // nothing to do
122     }
123 
124     public void poll()
125     {
126         try
127         {
128             File[] files = this.listFiles();
129             if (logger.isDebugEnabled())
130             {
131                 logger.debug("Files: " + files);
132             }
133             Comparator comparator = getComparator();
134             if (comparator != null)
135             {
136                 Arrays.sort(files, comparator);
137             }
138             for (int i = 0; i < files.length; i++)
139             {
140                 // don't process directories
141                 if (files[i].isFile())
142                 {
143                     this.processFile(files[i]);
144                 }
145             }
146         }
147         catch (Exception e)
148         {
149             this.handleException(e);
150         }
151     }
152 
153     public synchronized void processFile(final File sourceFile) throws MuleException
154     {
155         //TODO RM*: This can be put in a Filter. Also we can add an AndFileFilter/OrFileFilter to allow users to
156         //combine file filters (since we can only pass a single filter to File.listFiles, we would need to wrap
157         //the current And/Or filters to extend {@link FilenameFilter}
158         boolean checkFileAge = ((FileConnector) connector).getCheckFileAge();
159         if (checkFileAge)
160         {
161             long fileAge = ((FileConnector) connector).getFileAge();
162             long lastMod = sourceFile.lastModified();
163             long now = System.currentTimeMillis();
164             long thisFileAge = now - lastMod;
165             if (thisFileAge < fileAge)
166             {
167                 if (logger.isDebugEnabled())
168                 {
169                     logger.debug("The file has not aged enough yet, will return nothing for: " + sourceFile);
170                 }
171                 return;
172             }
173         }
174 
175         // don't process a file that is locked by another process (probably still being written)
176         if (!attemptFileLock(sourceFile))
177         {
178             return;
179         }
180 
181         FileConnector fc = ((FileConnector) connector);
182         String sourceFileOriginalName = sourceFile.getName();
183 
184         // Perform some quick checks to make sure file can be processed
185         if (!(sourceFile.canRead() && sourceFile.exists() && sourceFile.isFile()))
186         {
187             throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName));
188         }
189 
190         // This isn't nice but is needed as MessageAdaptor is required to resolve
191         // destination file name, and StreamingReceiverFileInputStream is
192         // required to create MessageAdaptor
193         DefaultMessageAdapter fileParserMsgAdaptor = new DefaultMessageAdapter(null);
194         fileParserMsgAdaptor.setProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
195 
196         // set up destination file
197         File destinationFile = null;
198         if (moveDir != null)
199         {
200             String destinationFileName = sourceFileOriginalName;
201             if (moveToPattern != null)
202             {
203                 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMsgAdaptor,
204                     moveToPattern);
205             }
206             // don't use new File() directly, see MULE-1112
207             destinationFile = FileUtils.newFile(moveDir, destinationFileName);
208         }
209 
210         MessageAdapter msgAdapter = null;
211         try
212         {
213             if (fc.isStreaming())
214             {
215                 msgAdapter = connector.getMessageAdapter(new ReceiverFileInputStream(sourceFile, fc.isAutoDelete(),
216                     destinationFile));
217             }
218             else
219             {
220                 msgAdapter = connector.getMessageAdapter(sourceFile);
221             }
222         }
223         catch (FileNotFoundException e)
224         {
225             // we can ignore since we did manage to acquire a lock, but just in case
226             logger.error("File being read disappeared!", e);
227             return;
228         }
229         msgAdapter.setProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
230 
231         if (!fc.isStreaming())
232         {
233             moveAndDelete(sourceFile, destinationFile, sourceFileOriginalName, msgAdapter);
234         }
235         else
236         {
237             // If we are streaming no need to move/delete now, that will be done when
238             // stream is closed
239             this.routeMessage(new DefaultMuleMessage(msgAdapter), endpoint.isSynchronous());
240         }
241     }
242 
243     private void moveAndDelete(final File sourceFile,
244                                File destinationFile,
245                                String sourceFileOriginalName,
246                                MessageAdapter msgAdapter)
247     {
248 
249         boolean fileWasMoved = false;
250 
251         try
252         {
253             // If we are moving the file to a read directory, move it there now and
254             // hand over a reference to the
255             // File in its moved location
256             if (destinationFile != null)
257             {
258                 // move sourceFile to new destination
259                 fileWasMoved = FileUtils.moveFile(sourceFile, destinationFile);
260 
261                 // move didn't work - bail out (will attempt rollback)
262                 if (!fileWasMoved)
263                 {
264                     throw new DefaultMuleException(FileMessages.failedToMoveFile(sourceFile.getAbsolutePath(),
265                         destinationFile.getAbsolutePath()));
266                 }
267 
268                 // create new MessageAdapter for destinationFile
269                 msgAdapter = connector.getMessageAdapter(destinationFile);
270 
271                 msgAdapter.setProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
272                 msgAdapter.setProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
273             }
274 
275             // finally deliver the file message
276             this.routeMessage(new DefaultMuleMessage(msgAdapter), endpoint.isSynchronous());
277 
278             // at this point msgAdapter either points to the old sourceFile
279             // or the new destinationFile.
280             if (((FileConnector) connector).isAutoDelete())
281             {
282                 // no moveTo directory
283                 if (destinationFile == null)
284                 {
285                     // delete source
286                     if (!sourceFile.delete())
287                     {
288                         throw new DefaultMuleException(FileMessages.failedToDeleteFile(sourceFile.getAbsolutePath()));
289                     }
290                 }
291                 else
292                 {
293                     // nothing to do here since moveFile() should have deleted
294                     // the source file for us
295                 }
296             }
297         }
298         catch (Exception e)
299         {
300             boolean fileWasRolledBack = false;
301 
302             // only attempt rollback if file move was successful
303             if (fileWasMoved)
304             {
305                 fileWasRolledBack = rollbackFileMove(destinationFile, sourceFile.getAbsolutePath());
306             }
307 
308             // wrap exception & handle it
309             Exception ex = new RoutingException(FileMessages.exceptionWhileProcessing(sourceFile.getName(),
310                 (fileWasRolledBack ? "successful" : "unsuccessful")), new DefaultMuleMessage(msgAdapter), endpoint, e);
311             this.handleException(ex);
312         }
313     }
314 
315     /**
316      * Try to acquire a lock on a file and release it immediately. Usually used as a
317      * quick check to see if another process is still holding onto the file, e.g. a
318      * large file (more than 100MB) is still being written to.
319      * 
320      * @param sourceFile file to check
321      * @return <code>true</code> if the file can be locked
322      */
323     protected boolean attemptFileLock(final File sourceFile)
324     {
325         // check if the file can be processed, be sure that it's not still being
326         // written
327         // if the file can't be locked don't process it yet, since creating
328         // a new FileInputStream() will throw an exception
329         FileLock lock = null;
330         FileChannel channel = null;
331         boolean fileCanBeLocked = false;
332         try
333         {
334             channel = new RandomAccessFile(sourceFile, "rw").getChannel();
335 
336             // Try acquiring the lock without blocking. This method returns
337             // null or throws an exception if the file is already locked.
338             lock = channel.tryLock();
339         }
340         catch (FileNotFoundException fnfe)
341         {
342             logger.warn("Unable to open " + sourceFile.getAbsolutePath(), fnfe);
343         }
344         catch (IOException e)
345         {
346             // Unable to create a lock. This exception should only be thrown when
347             // the file is already locked. No sense in repeating the message over
348             // and over.
349         }
350         finally
351         {
352             if (lock != null)
353             {
354                 // if lock is null the file is locked by another process
355                 fileCanBeLocked = true;
356                 try
357                 {
358                     // Release the lock
359                     lock.release();
360                 }
361                 catch (IOException e)
362                 {
363                     // ignore
364                 }
365             }
366 
367             if (channel != null)
368             {
369                 try
370                 {
371                     // Close the file
372                     channel.close();
373                 }
374                 catch (IOException e)
375                 {
376                     // ignore
377                 }
378             }
379         }
380 
381         return fileCanBeLocked;
382     }
383 
384     /**
385      * Get a list of files to be processed.
386      * 
387      * @return an array of files to be processed.
388      * @throws org.mule.api.MuleException which will wrap any other exceptions or
389      *             errors.
390      */
391     File[] listFiles() throws MuleException
392     {
393         try
394         {
395             File[] todoFiles = null;
396             if (fileFilter != null)
397             {
398                 todoFiles = readDirectory.listFiles(fileFilter);
399             }
400             else
401             {
402                 todoFiles = readDirectory.listFiles(filenameFilter);
403             }
404             // logger.trace("Reading directory " + readDirectory.getAbsolutePath() +
405             // " -> " + TODOFiles.length + " file(s)");
406             return (todoFiles == null ? new File[0] : todoFiles);
407         }
408         catch (Exception e)
409         {
410             throw new DefaultMuleException(FileMessages.errorWhileListingFiles(), e);
411         }
412     }
413 
414     /** Exception tolerant roll back method */
415     protected boolean rollbackFileMove(File sourceFile, String destinationFilePath)
416     {
417         boolean result = false;
418         try
419         {
420             result = FileUtils.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
421         }
422         catch (Throwable t)
423         {
424             logger.debug("rollback of file move failed: " + t.getMessage());
425         }
426         return result;
427     }
428 
429     protected Comparator getComparator() throws Exception
430     {
431 
432         Object o = getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY);
433         Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY);
434         boolean reverse = false;
435         if (o != null)
436         {
437             if (reverseProperty != null)
438             {
439                 reverse = Boolean.valueOf((String) reverseProperty).booleanValue();
440             }
441             Class clazz = Class.forName(o.toString());
442             o = clazz.newInstance();
443             return reverse ? new ReverseComparator((Comparator) o) : (Comparator) o;
444         }
445         return null;
446     }
447 }