Coverage Report - org.mule.transport.file.FileMessageReceiver
 
Classes in this File Line Coverage Branch Coverage Complexity
FileMessageReceiver
0%
0/190
0%
0/98
0
 
 1  
 /*
 2  
  * $Id: FileMessageReceiver.java 20477 2010-12-06 23:38:52Z mike.schilling $
 3  
  * --------------------------------------------------------------------------------------
 4  
  * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
 5  
  *
 6  
  * The software in this package is published under the terms of the CPAL v1.0
 7  
  * license, a copy of which has been included with this distribution in the
 8  
  * LICENSE.txt file.
 9  
  */
 10  
 
 11  
 package org.mule.transport.file;
 12  
 
 13  
 import org.mule.DefaultMuleMessage;
 14  
 import org.mule.api.DefaultMuleException;
 15  
 import org.mule.api.MessagingException;
 16  
 import org.mule.api.MuleException;
 17  
 import org.mule.api.MuleMessage;
 18  
 import org.mule.api.config.MuleProperties;
 19  
 import org.mule.api.construct.FlowConstruct;
 20  
 import org.mule.api.endpoint.InboundEndpoint;
 21  
 import org.mule.api.lifecycle.CreateException;
 22  
 import org.mule.api.transport.Connector;
 23  
 import org.mule.api.transport.PropertyScope;
 24  
 import org.mule.config.i18n.Message;
 25  
 import org.mule.transport.AbstractPollingMessageReceiver;
 26  
 import org.mule.transport.ConnectException;
 27  
 import org.mule.transport.file.i18n.FileMessages;
 28  
 import org.mule.util.FileUtils;
 29  
 
 30  
 import java.io.File;
 31  
 import java.io.FileFilter;
 32  
 import java.io.FileNotFoundException;
 33  
 import java.io.FilenameFilter;
 34  
 import java.io.IOException;
 35  
 import java.io.RandomAccessFile;
 36  
 import java.nio.channels.FileChannel;
 37  
 import java.nio.channels.FileLock;
 38  
 import java.util.ArrayList;
 39  
 import java.util.Collections;
 40  
 import java.util.Comparator;
 41  
 import java.util.List;
 42  
 
 43  
 import org.apache.commons.collections.comparators.ReverseComparator;
 44  
 
 45  
 /**
 46  
  * <code>FileMessageReceiver</code> is a polling listener that reads files from a
 47  
  * directory.
 48  
  */
 49  
 
 50  
 public class FileMessageReceiver extends AbstractPollingMessageReceiver
 51  
 {
 52  
     public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator";
 53  
     public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder";
 54  
 
 55  0
     private static final List<File> NO_FILES = new ArrayList<File>();
 56  
 
 57  0
     private String readDir = null;
 58  0
     private String moveDir = null;
 59  0
     private String workDir = null;
 60  0
     private File readDirectory = null;
 61  0
     private File moveDirectory = null;
 62  0
     private String moveToPattern = null;
 63  0
     private String workFileNamePattern = null;
 64  0
     private FilenameFilter filenameFilter = null;
 65  0
     private FileFilter fileFilter = null;
 66  
     private boolean forceSync;
 67  
 
 68  
     public FileMessageReceiver(Connector connector,
 69  
                                FlowConstruct flowConstruct,
 70  
                                InboundEndpoint endpoint,
 71  
                                String readDir,
 72  
                                String moveDir,
 73  
                                String moveToPattern,
 74  
                                long frequency) throws CreateException
 75  
     {
 76  0
         super(connector, flowConstruct, endpoint);
 77  0
         this.setFrequency(frequency);
 78  
 
 79  0
         this.readDir = readDir;
 80  0
         this.moveDir = moveDir;
 81  0
         this.moveToPattern = moveToPattern;
 82  0
         this.workDir = ((FileConnector) connector).getWorkDirectory();
 83  0
         this.workFileNamePattern = ((FileConnector) connector).getWorkFileNamePattern();
 84  
 
 85  0
         if (endpoint.getFilter() instanceof FilenameFilter)
 86  
         {
 87  0
             filenameFilter = (FilenameFilter) endpoint.getFilter();
 88  
         }
 89  0
         else if (endpoint.getFilter() instanceof FileFilter)
 90  
         {
 91  0
             fileFilter = (FileFilter) endpoint.getFilter();
 92  
         }
 93  0
         else if (endpoint.getFilter() != null)
 94  
         {
 95  0
             throw new CreateException(FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
 96  
         }
 97  
         
 98  0
         checkMustForceSync(endpoint);
 99  0
     }
 100  
 
 101  
     /**
 102  
      * If we will be autodeleting File objects, events must be processed synchronously to avoid a race
 103  
      */
 104  
     private void checkMustForceSync(InboundEndpoint endpoint) throws CreateException
 105  
     {
 106  0
         boolean connectorIsAutoDelete = false;
 107  0
         boolean isStreaming = false;
 108  0
         if (connector instanceof FileConnector)
 109  
         {
 110  0
             connectorIsAutoDelete = ((FileConnector) connector).isAutoDelete();
 111  0
             isStreaming = ((FileConnector) connector).isStreaming();
 112  
         }
 113  
 
 114  0
         boolean messageFactoryConsumes = (createMuleMessageFactory() instanceof FileContentsMuleMessageFactory);
 115  
         
 116  0
         forceSync = connectorIsAutoDelete && !messageFactoryConsumes && !isStreaming;
 117  0
     }
 118  
 
 119  
     @Override
 120  
     protected void doConnect() throws Exception
 121  
     {
 122  0
         if (readDir != null)
 123  
         {
 124  0
             readDirectory = FileUtils.openDirectory(readDir);
 125  0
             if (!(readDirectory.canRead()))
 126  
             {
 127  0
                 throw new ConnectException(FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
 128  
             }
 129  
             else
 130  
             {
 131  0
                 logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
 132  
             }
 133  
         }
 134  
 
 135  0
         if (moveDir != null)
 136  
         {
 137  0
             moveDirectory = FileUtils.openDirectory((moveDir));
 138  0
             if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
 139  
             {
 140  0
                 throw new ConnectException(FileMessages.moveToDirectoryNotWritable(), this);
 141  
             }
 142  
         }
 143  0
     }
 144  
 
 145  
     @Override
 146  
     protected void doDisconnect() throws Exception
 147  
     {
 148  
         // template method
 149  0
     }
 150  
 
 151  
     @Override
 152  
     protected void doDispose()
 153  
     {
 154  
         // nothing to do
 155  0
     }
 156  
 
 157  
     @Override
 158  
     public void poll()
 159  
     {
 160  
         try
 161  
         {
 162  0
             List<File> files = this.listFiles();
 163  0
             if (logger.isDebugEnabled())
 164  
             {
 165  0
                 logger.debug("Files: " + files.toString());
 166  
             }
 167  0
             Comparator<File> comparator = getComparator();
 168  0
             if (comparator != null)
 169  
             {
 170  0
                 Collections.sort(files, comparator);
 171  
             }
 172  0
             for (File file : files)
 173  
             {
 174  
                 // don't process directories
 175  0
                 if (file.isFile())
 176  
                 {
 177  0
                     processFile(file);
 178  
                 }
 179  
             }
 180  
         }
 181  0
         catch (Exception e)
 182  
         {
 183  0
             getConnector().getMuleContext().getExceptionListener().handleException(e);
 184  0
         }
 185  0
     }
 186  
 
 187  
     public synchronized void processFile(File sourceFile) throws MuleException
 188  
     {
 189  0
         FileConnector fileConnector = (FileConnector) connector;
 190  
         
 191  
         //TODO RM*: This can be put in a Filter. Also we can add an AndFileFilter/OrFileFilter to allow users to
 192  
         //combine file filters (since we can only pass a single filter to File.listFiles, we would need to wrap
 193  
         //the current And/Or filters to extend {@link FilenameFilter}
 194  0
         boolean checkFileAge = fileConnector.getCheckFileAge();
 195  0
         if (checkFileAge)
 196  
         {
 197  0
             long fileAge = ((FileConnector) connector).getFileAge();
 198  0
             long lastMod = sourceFile.lastModified();
 199  0
             long now = System.currentTimeMillis();
 200  0
             long thisFileAge = now - lastMod;
 201  0
             if (thisFileAge < fileAge)
 202  
             {
 203  0
                 if (logger.isDebugEnabled())
 204  
                 {
 205  0
                     logger.debug("The file has not aged enough yet, will return nothing for: " + sourceFile);
 206  
                 }
 207  0
                 return;
 208  
             }
 209  
         }
 210  
 
 211  
         // don't process a file that is locked by another process (probably still being written)
 212  0
         if (!attemptFileLock(sourceFile))
 213  
         {
 214  0
             return;
 215  
         }
 216  0
         else if(logger.isInfoEnabled())
 217  
         {
 218  0
             logger.info("Lock obtained on file: " + sourceFile.getAbsolutePath());
 219  
         }
 220  
 
 221  0
         String sourceFileOriginalName = sourceFile.getName();
 222  
 
 223  
         // Perform some quick checks to make sure file can be processed
 224  0
         if (!(sourceFile.canRead() && sourceFile.exists() && sourceFile.isFile()))
 225  
         {
 226  0
             throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName));
 227  
         }
 228  
 
 229  
         // This isn't nice but is needed as MuleMessage is required to resolve
 230  
         // destination file name
 231  0
         DefaultMuleMessage fileParserMessasge = new DefaultMuleMessage(null, connector.getMuleContext());
 232  0
         fileParserMessasge.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
 233  
         
 234  0
         File workFile = null;
 235  0
         if (workDir != null)
 236  
         {
 237  0
             String workFileName = sourceFileOriginalName;
 238  
             
 239  0
             workFileName = fileConnector.getFilenameParser().getFilename(fileParserMessasge,
 240  
                     workFileNamePattern);
 241  
             // don't use new File() directly, see MULE-1112
 242  0
             workFile = FileUtils.newFile(workDir, workFileName);
 243  
             
 244  0
             move(sourceFile, workFile);
 245  
             // Now the Work File is the Source file
 246  0
             sourceFile = workFile;
 247  
         }
 248  
         
 249  
         // set up destination file
 250  0
         File destinationFile = null;
 251  0
         if (moveDir != null)
 252  
         {
 253  0
             String destinationFileName = sourceFileOriginalName;
 254  0
             if (moveToPattern != null)
 255  
             {
 256  0
                 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMessasge,
 257  
                     moveToPattern);
 258  
             }
 259  
             // don't use new File() directly, see MULE-1112
 260  0
             destinationFile = FileUtils.newFile(moveDir, destinationFileName);
 261  
         }
 262  
 
 263  0
         MuleMessage message = null;
 264  0
         String encoding = endpoint.getEncoding();
 265  
         try
 266  
         {
 267  0
             if (fileConnector.isStreaming())
 268  
             {
 269  0
                 ReceiverFileInputStream payload = new ReceiverFileInputStream(sourceFile,
 270  
                     fileConnector.isAutoDelete(), destinationFile);
 271  0
                 message = createMuleMessage(payload, encoding);
 272  0
             }
 273  
             else
 274  
             {
 275  0
                 message = createMuleMessage(sourceFile, encoding);
 276  
             }
 277  
         }
 278  0
         catch (FileNotFoundException e)
 279  
         {
 280  
             // we can ignore since we did manage to acquire a lock, but just in case
 281  0
             logger.error("File being read disappeared!", e);
 282  0
             return;
 283  0
         }
 284  
 
 285  0
         message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
 286  0
         if (forceSync)
 287  
         {
 288  0
             message.setProperty(MuleProperties.MULE_FORCE_SYNC_PROPERTY, Boolean.TRUE, PropertyScope.INBOUND);
 289  
         }
 290  0
         if (!fileConnector.isStreaming())
 291  
         {
 292  0
             moveAndDelete(sourceFile, destinationFile, sourceFileOriginalName, message);
 293  
         }
 294  
         else
 295  
         {
 296  
             // If we are streaming no need to move/delete now, that will be done when
 297  
             // stream is closed
 298  0
             message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, sourceFile.getName());
 299  0
             this.routeMessage(message);
 300  
         }
 301  0
     }
 302  
 
 303  
     private void moveAndDelete(final File sourceFile, File destinationFile,
 304  
         String sourceFileOriginalName, MuleMessage message)
 305  
     {
 306  0
         boolean fileWasMoved = false;
 307  
 
 308  
         try
 309  
         {
 310  
             // If we are moving the file to a read directory, move it there now and
 311  
             // hand over a reference to the
 312  
             // File in its moved location
 313  0
             if (destinationFile != null)
 314  
             {
 315  
                 // move sourceFile to new destination
 316  
                 try
 317  
                 {
 318  0
                     FileUtils.moveFile(sourceFile, destinationFile);
 319  
                 }
 320  0
                 catch (IOException e)
 321  
                 {
 322  
                     // move didn't work - bail out (will attempt rollback)
 323  0
                     throw new DefaultMuleException(FileMessages.failedToMoveFile(
 324  
                         sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath()));
 325  0
                 }
 326  
 
 327  
                 // create new Message for destinationFile
 328  0
                 message = createMuleMessage(destinationFile, endpoint.getEncoding());
 329  0
                 message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
 330  0
                 message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
 331  
             }
 332  
 
 333  
             // finally deliver the file message
 334  0
             this.routeMessage(message);
 335  
 
 336  
             // at this point msgAdapter either points to the old sourceFile
 337  
             // or the new destinationFile.
 338  0
             if (((FileConnector) connector).isAutoDelete())
 339  
             {
 340  
                 // no moveTo directory
 341  0
                 if (destinationFile == null)
 342  
                 {
 343  
                     // delete source
 344  0
                     if (!sourceFile.delete())
 345  
                     {
 346  0
                         throw new DefaultMuleException(FileMessages.failedToDeleteFile(sourceFile));
 347  
                     }
 348  
                 }
 349  
                 else
 350  
                 {
 351  
                     // nothing to do here since moveFile() should have deleted
 352  
                     // the source file for us
 353  
                 }
 354  
             }
 355  
         }
 356  0
         catch (Exception e)
 357  
         {
 358  0
             boolean fileWasRolledBack = false;
 359  
 
 360  
             // only attempt rollback if file move was successful
 361  0
             if (fileWasMoved)
 362  
             {
 363  
                 try
 364  
                 {
 365  0
                     rollbackFileMove(destinationFile, sourceFile.getAbsolutePath());
 366  0
                     fileWasRolledBack = true;
 367  
                 }
 368  0
                 catch (IOException ioException)
 369  
                 {
 370  
                     // eat it
 371  0
                 }
 372  
             }
 373  
 
 374  
             // wrap exception & handle it
 375  0
             Message msg = FileMessages.exceptionWhileProcessing(sourceFile.getName(),
 376  
                 (fileWasRolledBack ? "successful" : "unsuccessful"));
 377  0
             getConnector().getMuleContext().getExceptionListener().handleException(new MessagingException(msg, message, e));
 378  0
         }
 379  0
     }
 380  
 
 381  
     /**
 382  
      * Try to acquire a lock on a file and release it immediately. Usually used as a
 383  
      * quick check to see if another process is still holding onto the file, e.g. a
 384  
      * large file (more than 100MB) is still being written to.
 385  
      * 
 386  
      * @param sourceFile file to check
 387  
      * @return <code>true</code> if the file can be locked
 388  
      */
 389  
     protected boolean attemptFileLock(final File sourceFile)
 390  
     {
 391  
         // check if the file can be processed, be sure that it's not still being
 392  
         // written
 393  
         // if the file can't be locked don't process it yet, since creating
 394  
         // a new FileInputStream() will throw an exception
 395  0
         FileLock lock = null;
 396  0
         FileChannel channel = null;
 397  0
         boolean fileCanBeLocked = false;
 398  
         try
 399  
         {
 400  0
             channel = new RandomAccessFile(sourceFile, "rw").getChannel();
 401  
 
 402  
             // Try acquiring the lock without blocking. This method returns
 403  
             // null or throws an exception if the file is already locked.
 404  0
             lock = channel.tryLock();
 405  
         }
 406  0
         catch (FileNotFoundException fnfe)
 407  
         {
 408  0
             logger.warn("Unable to open " + sourceFile.getAbsolutePath(), fnfe);
 409  
         }
 410  0
         catch (IOException e)
 411  
         {
 412  
             // Unable to create a lock. This exception should only be thrown when
 413  
             // the file is already locked. No sense in repeating the message over
 414  
             // and over.
 415  
         }
 416  
         finally
 417  
         {
 418  0
             if (lock != null)
 419  
             {
 420  
                 // if lock is null the file is locked by another process
 421  0
                 fileCanBeLocked = true;
 422  
                 try
 423  
                 {
 424  
                     // Release the lock
 425  0
                     lock.release();
 426  
                 }
 427  0
                 catch (IOException e)
 428  
                 {
 429  
                     // ignore
 430  0
                 }
 431  
             }
 432  
 
 433  0
             if (channel != null)
 434  
             {
 435  
                 try
 436  
                 {
 437  
                     // Close the file
 438  0
                     channel.close();
 439  
                 }
 440  0
                 catch (IOException e)
 441  
                 {
 442  
                     // ignore
 443  0
                 }
 444  
             }
 445  
         }
 446  
 
 447  0
         return fileCanBeLocked;
 448  
     }
 449  
 
 450  
     /**
 451  
      * Get a list of files to be processed.
 452  
      * 
 453  
      * @return an array of files to be processed.
 454  
      * @throws org.mule.api.MuleException which will wrap any other exceptions or
 455  
      *             errors.
 456  
      */
 457  
     List<File> listFiles() throws MuleException
 458  
     {
 459  
         try
 460  
         {
 461  0
             List<File> files = new ArrayList<File>();
 462  0
             this.basicListFiles(readDirectory, files);
 463  0
             return (files.isEmpty() ? NO_FILES : files);
 464  
         }
 465  0
         catch (Exception e)
 466  
         {
 467  0
             throw new DefaultMuleException(FileMessages.errorWhileListingFiles(), e);
 468  
         }
 469  
     }
 470  
 
 471  
     protected void basicListFiles(File currentDirectory, List<File> discoveredFiles)
 472  
     {
 473  
         File[] files;
 474  0
         if (fileFilter != null)
 475  
         {
 476  0
             files = currentDirectory.listFiles(fileFilter);
 477  
         }
 478  
         else
 479  
         {
 480  0
             files = currentDirectory.listFiles(filenameFilter);
 481  
         }
 482  
         
 483  
         // the listFiles calls above may actually return null (check the JDK code).
 484  0
         if (files == null)
 485  
         {
 486  0
             return;
 487  
         }
 488  
 
 489  0
         for (File file : files)
 490  
         {
 491  0
             if (!file.isDirectory())
 492  
             {
 493  0
                 discoveredFiles.add(file);
 494  
             }
 495  
             else
 496  
             {
 497  0
                 if (((FileConnector) this.getConnector()).isRecursive())
 498  
                 {
 499  0
                     this.basicListFiles(file, discoveredFiles);
 500  
                 }
 501  
             }
 502  
         }
 503  0
     }
 504  
 
 505  
     /**
 506  
      * Exception tolerant roll back method
 507  
      * 
 508  
      * @throws Throwable
 509  
      */
 510  
     protected void rollbackFileMove(File sourceFile, String destinationFilePath) throws IOException
 511  
     {
 512  
         try
 513  
         {
 514  0
             FileUtils.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
 515  
         }
 516  0
         catch (IOException t)
 517  
         {
 518  0
             logger.debug("rollback of file move failed: " + t.getMessage());
 519  0
             throw t;
 520  0
         }
 521  0
     }
 522  
 
 523  
     protected Comparator<File> getComparator() throws Exception
 524  
     {
 525  0
         Object comparatorClassName = getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY);
 526  0
         if (comparatorClassName != null)
 527  
         {
 528  0
             Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY);
 529  0
             boolean reverse = false;
 530  0
             if (reverseProperty != null)
 531  
             {
 532  0
                 reverse = Boolean.valueOf((String) reverseProperty);
 533  
             }
 534  
             
 535  0
             Class<?> clazz = Class.forName(comparatorClassName.toString());
 536  0
             Comparator<?> comparator = (Comparator<?>)clazz.newInstance();
 537  0
             return reverse ? new ReverseComparator(comparator) : comparator;
 538  
         }
 539  0
         return null;
 540  
     }
 541  
     
 542  
     private void move(final File sourceFile,File destinationFile) throws DefaultMuleException
 543  
     {
 544  0
         if (destinationFile != null)
 545  
         {
 546  
             // move sourceFile to new destination. Do not use FileUtils here as it ultimately
 547  
             // falls back to copying the file which will cause problems on large files again -
 548  
             // which is what we're trying to avoid in the first place
 549  0
             boolean fileWasMoved = sourceFile.renameTo(destinationFile);
 550  
 
 551  
             // move didn't work - bail out
 552  0
             if (!fileWasMoved)
 553  
             {
 554  0
                 throw new DefaultMuleException(FileMessages.failedToMoveFile(sourceFile.getAbsolutePath(),
 555  
                     destinationFile.getAbsolutePath()));
 556  
             }
 557  
         }
 558  0
     }
 559  
     
 560  
 }