View Javadoc

1   /*
2    * $Id: FileMessageReceiver.java 7976 2007-08-21 14:26:13Z dirk.olmes $
3    * --------------------------------------------------------------------------------------
4    * Copyright (c) MuleSource, Inc.  All rights reserved.  http://www.mulesource.com
5    *
6    * The software in this package is published under the terms of the CPAL v1.0
7    * license, a copy of which has been included with this distribution in the
8    * LICENSE.txt file.
9    */
10  
11  package org.mule.providers.file;
12  
13  import org.mule.MuleException;
14  import org.mule.impl.MuleMessage;
15  import org.mule.providers.AbstractPollingMessageReceiver;
16  import org.mule.providers.ConnectException;
17  import org.mule.providers.file.i18n.FileMessages;
18  import org.mule.umo.UMOComponent;
19  import org.mule.umo.UMOException;
20  import org.mule.umo.endpoint.UMOEndpoint;
21  import org.mule.umo.lifecycle.InitialisationException;
22  import org.mule.umo.provider.UMOConnector;
23  import org.mule.umo.provider.UMOMessageAdapter;
24  import org.mule.umo.routing.RoutingException;
25  import org.mule.util.FileUtils;
26  
27  import java.io.File;
28  import java.io.FileFilter;
29  import java.io.FileInputStream;
30  import java.io.FileNotFoundException;
31  import java.io.FileOutputStream;
32  import java.io.FilenameFilter;
33  import java.io.IOException;
34  import java.io.RandomAccessFile;
35  import java.nio.channels.FileChannel;
36  import java.nio.channels.FileLock;
37  
38  import org.apache.commons.io.IOUtils;
39  
40  /**
41   * <code>FileMessageReceiver</code> is a polling listener that reads files from a
42   * directory.
43   */
44  
45  public class FileMessageReceiver extends AbstractPollingMessageReceiver
46  {
47      private String readDir = null;
48      private String moveDir = null;
49      private File readDirectory = null;
50      private File moveDirectory = null;
51      private String moveToPattern = null;
52      private FilenameFilter filenameFilter = null;
53      private FileFilter fileFilter = null;
54  
55      public FileMessageReceiver(UMOConnector connector,
56                                 UMOComponent component,
57                                 UMOEndpoint endpoint,
58                                 String readDir,
59                                 String moveDir,
60                                 String moveToPattern,
61                                 long frequency) throws InitialisationException
62      {
63          super(connector, component, endpoint);
64          this.setFrequency(frequency);
65  
66          this.readDir = readDir;
67          this.moveDir = moveDir;
68          this.moveToPattern = moveToPattern;
69  
70          if (endpoint.getFilter() instanceof FilenameFilter)
71          {
72              filenameFilter = (FilenameFilter)endpoint.getFilter();
73          }
74          else if (endpoint.getFilter() instanceof FileFilter)
75          {
76              fileFilter = (FileFilter)endpoint.getFilter();
77          }
78          else if(endpoint.getFilter()!=null)
79          {
80              throw new InitialisationException(
81                  FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
82          }
83      }
84  
85      protected void doConnect() throws Exception
86      {
87          if (readDir != null)
88          {
89              readDirectory = FileUtils.openDirectory(readDir);
90              if (!(readDirectory.canRead()))
91              {
92                  throw new ConnectException(
93                      FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
94              }
95              else
96              {
97                  logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
98              }
99          }
100 
101         if (moveDir != null)
102         {
103             moveDirectory = FileUtils.openDirectory((moveDir));
104             if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
105             {
106                 throw new ConnectException(
107                     FileMessages.moveToDirectoryNotWritable(), this);
108             }
109         }
110     }
111 
112     protected void doDisconnect() throws Exception
113     {
114         // template method
115     }
116 
117     protected void doDispose()
118     {
119         // nothing to do               
120     }
121 
122     public void poll()
123     {
124         try
125         {
126             File[] files = this.listFiles();
127             for (int i = 0; i < files.length; i++)
128             {
129                 this.processFile(files[i]);
130             }
131         }
132         catch (Exception e)
133         {
134             this.handleException(e);
135         }
136     }
137 
138     public synchronized void processFile(final File sourceFile) throws UMOException
139     {
140         //TODO RM*: This can be put in a Filter. Also we can add an AndFileFilter/OrFileFilter to allow users to
141         //combine file filters (since we can only pass a single filter to File.listFiles, we would need to wrap
142         //the current And/Or filters to extend {@link FilenameFilter}
143         boolean checkFileAge = ((FileConnector) connector).getCheckFileAge();
144         if (checkFileAge)
145         {
146             long fileAge = ((FileConnector) connector).getFileAge();
147             long lastMod = sourceFile.lastModified();
148             long now = System.currentTimeMillis();
149             long thisFileAge = now - lastMod;
150             if (thisFileAge < fileAge)
151             {
152                 if (logger.isDebugEnabled()) {
153                     logger.debug("The file has not aged enough yet, will return nothing for: " + sourceFile);
154                 }
155                 return;
156             }
157         }
158 
159         // don't process a file that is locked by another process (probably still being written)
160         if (!attemptFileLock(sourceFile))
161         {
162             return;
163         }
164 
165         File destinationFile = null;
166         String sourceFileOriginalName = sourceFile.getName();
167         //Create a message adapter here to pass to the fileName parser
168         UMOMessageAdapter msgAdapter;
169         if(endpoint.isStreaming())
170         {
171             try
172             {
173                 msgAdapter = connector.getStreamMessageAdapter(new FileInputStream(sourceFile), null);
174             }
175             catch (FileNotFoundException e)
176             {
177                 //we can ignore since we did manage to acquire a lock, but just in case
178                 logger.error("File being read disappeared!", e);
179                 return;
180             }
181         }
182         else
183         {
184             msgAdapter = connector.getMessageAdapter(sourceFile);
185         }
186         msgAdapter.setProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
187 
188         // set up destination file
189         if (moveDir != null)
190         {
191             String destinationFileName = sourceFileOriginalName;
192 
193             if (moveToPattern != null)
194             {
195                 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(msgAdapter,
196                     moveToPattern);
197             }
198 
199             // don't use new File() directly, see MULE-1112
200             destinationFile = FileUtils.newFile(moveDir, destinationFileName);
201         }
202 
203         boolean fileWasMoved = false;
204 
205         try
206         {
207             // Perform some quick checks to make sure file can be processed
208             if (!(sourceFile.canRead() && sourceFile.exists() && sourceFile.isFile()))
209             {
210                 throw new MuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName));
211             }
212 
213             //If we are moving the file to a read directory, move it there now and hand over a reference to the
214             //File in its moved location
215             if (destinationFile != null)
216             {
217                 // move sourceFile to new destination
218                 fileWasMoved = this.moveFile(sourceFile, destinationFile);
219 
220                 // move didn't work - bail out (will attempt rollback)
221                 if (!fileWasMoved)
222                 {
223                     throw new MuleException(
224                         FileMessages.failedToMoveFile(
225                             sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath()));
226                 }
227 
228                 // create new MessageAdapter for destinationFile
229                 if(endpoint.isStreaming())
230                 {
231                     msgAdapter = connector.getStreamMessageAdapter(new FileInputStream(destinationFile), null);
232                 }
233                 else
234                 {
235                     msgAdapter = connector.getMessageAdapter(destinationFile);
236                 }
237                 msgAdapter.setProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
238                 msgAdapter.setProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
239             }
240 
241             // finally deliver the file message
242             this.routeMessage(new MuleMessage(msgAdapter), endpoint.isSynchronous());
243             
244             // at this point msgAdapter either points to the old sourceFile
245             // or the new destinationFile.
246             if (((FileConnector) connector).isAutoDelete())
247             {
248                 // no moveTo directory
249                 if (destinationFile == null)
250                 {
251                     // delete source
252                     if (!sourceFile.delete())
253                     {
254                         throw new MuleException(
255                             FileMessages.failedToDeleteFile(sourceFile.getAbsolutePath()));
256                     }
257                 }
258                 else
259                 {
260                     // nothing to do here since moveFile() should have deleted
261                     // the source file for us
262                 }
263             }
264         }
265         catch (Exception e)
266         {
267             boolean fileWasRolledBack = false;
268 
269             // only attempt rollback if file move was successful
270             if (fileWasMoved)
271             {
272                 fileWasRolledBack = this.rollbackFileMove(destinationFile, sourceFile.getAbsolutePath());
273             }
274 
275             // wrap exception & handle it
276             Exception ex = new RoutingException(
277                 FileMessages.exceptionWhileProcessing(sourceFile.getName(),
278                 (fileWasRolledBack ? "successful" : "unsuccessful")), new MuleMessage(msgAdapter), endpoint,
279                 e);
280             this.handleException(ex);
281         }
282     }
283 
284     /**
285      * Try to acquire a lock on a file and release it immediately. Usually used as a quick check to
286      * see if another process is still holding onto the file, e.g. a large file (more than 100MB) is
287      * still being written to.
288      * @param sourceFile file to check
289      * @return <code>true</code> if the file can be locked
290      */
291     protected boolean attemptFileLock(final File sourceFile)
292     {
293         // check if the file can be processed, be sure that it's not still being written
294         // if the file can't be locked don't process it yet, since creating
295         // a new FileInputStream() will throw an exception
296         FileLock lock = null;
297         FileChannel channel = null;
298         boolean fileCanBeLocked = false;
299         try
300         {
301             channel = new RandomAccessFile(sourceFile, "rw").getChannel();
302 
303             // Try acquiring the lock without blocking. This method returns
304             // null or throws an exception if the file is already locked.
305             lock = channel.tryLock();
306         }
307         catch (FileNotFoundException fnfe)
308         {
309            logger.warn("Unable to open " + sourceFile.getAbsolutePath(), fnfe);
310         }
311         catch (IOException e)
312         {
313             // Unable to create a lock. This exception should only be thrown when
314             // the file is already locked. No sense in repeating the message over
315             // and over.
316         }
317         finally {
318             if (lock != null)
319             {
320                 // if lock is null the file is locked by another process
321                 fileCanBeLocked = true;
322                 try
323                 {
324                     // Release the lock
325                     lock.release();
326                 }
327                 catch (IOException e)
328                 {
329                     // ignore
330                 }
331             }
332 
333             if (channel != null)
334             {
335                 try
336                 {
337                     // Close the file
338                     channel.close();
339                 }
340                 catch (IOException e)
341                 {
342                     // ignore
343                 }
344             }
345         }
346 
347         return fileCanBeLocked;
348     }
349 
350     /**
351      * Try to move a file by renaming with backup attempt by copying/deleting via NIO
352      */
353     protected boolean moveFile(File sourceFile, File destinationFile)
354     {
355         // try fast file-system-level move/rename first
356         boolean success = sourceFile.renameTo(destinationFile);
357 
358         if (!success)
359         {
360             // try again using NIO copy
361             FileInputStream fis = null;
362             FileOutputStream fos = null;
363             try
364             {
365                 fis = new FileInputStream(sourceFile);
366                 fos = new FileOutputStream(destinationFile);
367                 FileChannel srcChannel = fis.getChannel();
368                 FileChannel dstChannel = fos.getChannel();
369                 dstChannel.transferFrom(srcChannel, 0, srcChannel.size());
370                 srcChannel.close();
371                 dstChannel.close();
372                 success = sourceFile.delete();
373             }
374             catch (IOException ioex)
375             {
376                 // grr!
377                 success = false;
378             }
379             finally
380             {
381                 IOUtils.closeQuietly(fis);
382                 IOUtils.closeQuietly(fos);
383             }
384         }
385 
386         return success;
387     }
388 
389     /**
390      * Exception tolerant roll back method
391      */
392     protected boolean rollbackFileMove(File sourceFile, String destinationFilePath)
393     {
394         boolean result = false;
395         try
396         {
397             result = this.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
398         }
399         catch (Throwable t)
400         {
401             logger.debug("rollback of file move failed: " + t.getMessage());
402         }
403         return result;
404     }
405 
406     /**
407      * Get a list of files to be processed.
408      * 
409      * @return an array of files to be processed.
410      * @throws org.mule.MuleException which will wrap any other exceptions or errors.
411      */
412     File[] listFiles() throws MuleException
413     {
414         try
415         {
416             File[] todoFiles;
417             if(fileFilter!=null)
418             {
419                 todoFiles = readDirectory.listFiles(fileFilter);
420             }
421             else
422             {
423                 todoFiles = readDirectory.listFiles(filenameFilter);
424             }
425             // logger.trace("Reading directory " + readDirectory.getAbsolutePath() +
426             // " -> " + todoFiles.length + " file(s)");
427             return (todoFiles == null ? new File[0] : todoFiles);
428         }
429         catch (Exception e)
430         {
431             throw new MuleException(FileMessages.errorWhileListingFiles(), e);
432         }
433     }
434 
435 }