1
2
3
4
5
6
7 package org.mule.transport.file;
8
9 import org.mule.DefaultMuleMessage;
10 import org.mule.api.DefaultMuleException;
11 import org.mule.api.MessagingException;
12 import org.mule.api.MuleException;
13 import org.mule.api.MuleMessage;
14 import org.mule.api.config.MuleProperties;
15 import org.mule.api.construct.FlowConstruct;
16 import org.mule.api.endpoint.InboundEndpoint;
17 import org.mule.api.lifecycle.CreateException;
18 import org.mule.api.lifecycle.InitialisationException;
19 import org.mule.api.store.ObjectAlreadyExistsException;
20 import org.mule.api.store.ObjectStore;
21 import org.mule.api.store.ObjectStoreException;
22 import org.mule.api.transport.Connector;
23 import org.mule.api.transport.PropertyScope;
24 import org.mule.config.i18n.Message;
25 import org.mule.transport.AbstractPollingMessageReceiver;
26 import org.mule.transport.ConnectException;
27 import org.mule.transport.file.i18n.FileMessages;
28 import org.mule.util.FileUtils;
29 import org.mule.util.store.InMemoryObjectStore;
30
31 import java.io.File;
32 import java.io.FileFilter;
33 import java.io.FileNotFoundException;
34 import java.io.FilenameFilter;
35 import java.io.IOException;
36 import java.io.RandomAccessFile;
37 import java.nio.channels.FileChannel;
38 import java.nio.channels.FileLock;
39 import java.util.ArrayList;
40 import java.util.Collections;
41 import java.util.Comparator;
42 import java.util.List;
43
44 import org.apache.commons.collections.comparators.ReverseComparator;
45
46
47
48
49
50
51 public class FileMessageReceiver extends AbstractPollingMessageReceiver
52 {
53 public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator";
54 public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder";
55
56 private static final List<File> NO_FILES = new ArrayList<File>();
57
58 private String readDir = null;
59 private String moveDir = null;
60 private String workDir = null;
61 private File readDirectory = null;
62 private File moveDirectory = null;
63 private String moveToPattern = null;
64 private String workFileNamePattern = null;
65 private FilenameFilter filenameFilter = null;
66 private FileFilter fileFilter = null;
67 private boolean forceSync;
68 private ObjectStore<String> filesBeingProcessingObjectStore;
69
70 public FileMessageReceiver(Connector connector,
71 FlowConstruct flowConstruct,
72 InboundEndpoint endpoint,
73 String readDir,
74 String moveDir,
75 String moveToPattern,
76 long frequency) throws CreateException
77 {
78 super(connector, flowConstruct, endpoint);
79 this.setFrequency(frequency);
80
81 this.readDir = readDir;
82 this.moveDir = moveDir;
83 this.moveToPattern = moveToPattern;
84 this.workDir = ((FileConnector) connector).getWorkDirectory();
85 this.workFileNamePattern = ((FileConnector) connector).getWorkFileNamePattern();
86
87 if (endpoint.getFilter() instanceof FilenameFilter)
88 {
89 filenameFilter = (FilenameFilter) endpoint.getFilter();
90 }
91 else if (endpoint.getFilter() instanceof FileFilter)
92 {
93 fileFilter = (FileFilter) endpoint.getFilter();
94 }
95 else if (endpoint.getFilter() != null)
96 {
97 throw new CreateException(FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this);
98 }
99
100 checkMustForceSync(endpoint);
101 }
102
103
104
105
106 private void checkMustForceSync(InboundEndpoint ep) throws CreateException
107 {
108 boolean connectorIsAutoDelete = false;
109 boolean isStreaming = false;
110 if (connector instanceof FileConnector)
111 {
112 connectorIsAutoDelete = ((FileConnector) connector).isAutoDelete();
113 isStreaming = ((FileConnector) connector).isStreaming();
114 }
115
116 boolean messageFactoryConsumes = (createMuleMessageFactory() instanceof FileContentsMuleMessageFactory);
117
118 forceSync = connectorIsAutoDelete && !messageFactoryConsumes && !isStreaming;
119 }
120
121 @Override
122 protected void doConnect() throws Exception
123 {
124 if (readDir != null)
125 {
126 readDirectory = FileUtils.openDirectory(readDir);
127 if (!(readDirectory.canRead()))
128 {
129 throw new ConnectException(FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this);
130 }
131 else
132 {
133 logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath());
134 }
135 }
136
137 if (moveDir != null)
138 {
139 moveDirectory = FileUtils.openDirectory((moveDir));
140 if (!(moveDirectory.canRead()) || !moveDirectory.canWrite())
141 {
142 throw new ConnectException(FileMessages.moveToDirectoryNotWritable(), this);
143 }
144 }
145 }
146
147 @Override
148 protected void doInitialise() throws InitialisationException
149 {
150 InMemoryObjectStore objectStore = new InMemoryObjectStore<String>();
151 objectStore.setMaxEntries(1000);
152 objectStore.setExpirationInterval(20000);
153 objectStore.setEntryTTL(60000);
154 filesBeingProcessingObjectStore = objectStore;
155 }
156
157 @Override
158 protected void doDisconnect() throws Exception
159 {
160
161 }
162
163 @Override
164 protected void doDispose()
165 {
166
167 }
168
169 @Override
170 public void poll()
171 {
172 try
173 {
174 List<File> files = this.listFiles();
175 if (logger.isDebugEnabled())
176 {
177 logger.debug("Files: " + files.toString());
178 }
179 Comparator<File> comparator = getComparator();
180 if (comparator != null)
181 {
182 Collections.sort(files, comparator);
183 }
184 for (File file : files)
185 {
186 if (getLifecycleState().isStopping())
187 {
188 break;
189 }
190
191 if (file.isFile())
192 {
193 try
194 {
195 filesBeingProcessingObjectStore.store(file.getAbsolutePath(),file.getAbsolutePath());
196 processFile(file);
197 }
198 catch (ObjectAlreadyExistsException e)
199 {
200 logger.debug("file " + file.getAbsolutePath() + " it's being processed. Skipping it.");
201 }
202 }
203 }
204 }
205 catch (Exception e)
206 {
207 getConnector().getMuleContext().getExceptionListener().handleException(e);
208 }
209 }
210
211 public void processFile(File file) throws MuleException
212 {
213 FileConnector fileConnector = (FileConnector) connector;
214
215
216
217
218 boolean checkFileAge = fileConnector.getCheckFileAge();
219 if (checkFileAge)
220 {
221 long fileAge = ((FileConnector) connector).getFileAge();
222 long lastMod = file.lastModified();
223 long now = System.currentTimeMillis();
224 long thisFileAge = now - lastMod;
225 if (thisFileAge < fileAge)
226 {
227 if (logger.isDebugEnabled())
228 {
229 logger.debug("The file has not aged enough yet, will return nothing for: " + file);
230 }
231 return;
232 }
233 }
234
235 String sourceFileOriginalName = file.getName();
236 final String sourceFileOriginalAbsolutePath = file.getAbsolutePath();
237
238
239 if (!(file.canRead() && file.exists() && file.isFile()))
240 {
241 throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName));
242 }
243
244
245
246
247
248 if (workDir == null)
249 {
250
251
252 if (!attemptFileLock(file))
253 {
254 return;
255 }
256 else if (logger.isInfoEnabled())
257 {
258 logger.info("Lock obtained on file: " + file.getAbsolutePath());
259 }
260 }
261
262
263
264 DefaultMuleMessage fileParserMessasge = new DefaultMuleMessage(null, connector.getMuleContext());
265 fileParserMessasge.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
266
267 File workFile = null;
268 final File sourceFile;
269 if (workDir != null)
270 {
271 String workFileName = sourceFileOriginalName;
272
273 workFileName = fileConnector.getFilenameParser().getFilename(fileParserMessasge,
274 workFileNamePattern);
275
276 workFile = FileUtils.newFile(workDir, workFileName);
277
278 fileConnector.move(file, workFile);
279
280 sourceFile = workFile;
281 }
282 else
283 {
284 sourceFile = file;
285 }
286
287
288 File destinationFile = null;
289 if (moveDir != null)
290 {
291 String destinationFileName = sourceFileOriginalName;
292 if (moveToPattern != null)
293 {
294 destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMessasge,
295 moveToPattern);
296 }
297
298 destinationFile = FileUtils.newFile(moveDir, destinationFileName);
299 }
300
301 MuleMessage message = null;
302 String encoding = endpoint.getEncoding();
303 try
304 {
305 if (fileConnector.isStreaming())
306 {
307 ReceiverFileInputStream payload = new ReceiverFileInputStream(sourceFile, fileConnector.isAutoDelete(), destinationFile, new InputStreamCloseListener()
308 {
309 public void fileClose(File file)
310 {
311 try
312 {
313 if (logger.isDebugEnabled())
314 {
315 logger.debug(String.format("Removing processing flag for $ ", file.getAbsolutePath()));
316 }
317 filesBeingProcessingObjectStore.remove(sourceFileOriginalAbsolutePath);
318 }
319 catch (ObjectStoreException e)
320 {
321 logger.warn("Failure trying to remove file " + sourceFileOriginalAbsolutePath + " from list of files under processing");
322 }
323 }
324 });
325 message = createMuleMessage(payload, encoding);
326 }
327 else
328 {
329 message = createMuleMessage(sourceFile, encoding);
330 }
331 }
332 catch (FileNotFoundException e)
333 {
334
335 logger.error("File being read disappeared!", e);
336 return;
337 }
338
339 if (workDir != null)
340 {
341 message.setProperty(FileConnector.PROPERTY_SOURCE_DIRECTORY, file.getParent(), PropertyScope.INBOUND);
342 message.setProperty(FileConnector.PROPERTY_SOURCE_FILENAME, file.getName(), PropertyScope.INBOUND);
343 }
344
345 message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
346 if (forceSync)
347 {
348 message.setProperty(MuleProperties.MULE_FORCE_SYNC_PROPERTY, Boolean.TRUE, PropertyScope.INBOUND);
349 }
350 if (!fileConnector.isStreaming())
351 {
352 moveAndDelete(sourceFile, destinationFile, sourceFileOriginalName, sourceFileOriginalAbsolutePath,message);
353 }
354 else
355 {
356
357
358 message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, sourceFile.getName());
359 this.routeMessage(message);
360 }
361 }
362
363 private void moveAndDelete(final File sourceFile, File destinationFile,
364 String sourceFileOriginalName, String sourceFileOriginalAbsolutePath, MuleMessage message)
365 {
366 boolean fileWasMoved = false;
367
368 try
369 {
370
371
372
373 if (destinationFile != null)
374 {
375
376 try
377 {
378 FileUtils.moveFile(sourceFile, destinationFile);
379 }
380 catch (IOException e)
381 {
382
383 throw new DefaultMuleException(FileMessages.failedToMoveFile(
384 sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath()));
385 }
386
387
388 message = createMuleMessage(destinationFile, endpoint.getEncoding());
389 message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName());
390 message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName);
391 }
392
393
394 this.routeMessage(message);
395
396
397
398 if (((FileConnector) connector).isAutoDelete())
399 {
400
401 if (destinationFile == null)
402 {
403
404 if (!sourceFile.delete())
405 {
406 throw new DefaultMuleException(FileMessages.failedToDeleteFile(sourceFile));
407 }
408 }
409 else
410 {
411
412
413 }
414 }
415 }
416 catch (Exception e)
417 {
418 boolean fileWasRolledBack = false;
419
420
421 if (fileWasMoved)
422 {
423 try
424 {
425 rollbackFileMove(destinationFile, sourceFile.getAbsolutePath());
426 fileWasRolledBack = true;
427 }
428 catch (IOException ioException)
429 {
430
431 }
432 }
433
434
435 Message msg = FileMessages.exceptionWhileProcessing(sourceFile.getName(),
436 (fileWasRolledBack ? "successful" : "unsuccessful"));
437 getConnector().getMuleContext().getExceptionListener().handleException(new MessagingException(msg, message, e));
438 }
439 finally
440 {
441 try
442 {
443 filesBeingProcessingObjectStore.remove(sourceFileOriginalAbsolutePath);
444 if (logger.isDebugEnabled())
445 {
446 logger.debug(String.format("Removing processing flag for $ ", sourceFileOriginalAbsolutePath));
447 }
448 }
449 catch (ObjectStoreException e)
450 {
451 logger.warn("Failure trying to remove file " + sourceFileOriginalAbsolutePath + " from list of files under processing");
452 }
453 }
454 }
455
456
457
458
459
460
461
462
463
464 protected boolean attemptFileLock(final File sourceFile) throws MuleException
465 {
466
467
468
469
470 FileLock lock = null;
471 FileChannel channel = null;
472 boolean fileCanBeLocked = false;
473 try
474 {
475 channel = new RandomAccessFile(sourceFile, "rw").getChannel();
476
477
478
479 lock = channel.tryLock();
480 }
481 catch (FileNotFoundException fnfe)
482 {
483 throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFile.getName()));
484 }
485 catch (IOException e)
486 {
487
488
489
490 }
491 finally
492 {
493 if (lock != null)
494 {
495
496 fileCanBeLocked = true;
497 try
498 {
499
500 lock.release();
501 }
502 catch (IOException e)
503 {
504
505 }
506 }
507
508 if (channel != null)
509 {
510 try
511 {
512
513 channel.close();
514 }
515 catch (IOException e)
516 {
517
518 }
519 }
520 }
521
522 return fileCanBeLocked;
523 }
524
525
526
527
528
529
530
531
532 List<File> listFiles() throws MuleException
533 {
534 try
535 {
536 List<File> files = new ArrayList<File>();
537 this.basicListFiles(readDirectory, files);
538 return (files.isEmpty() ? NO_FILES : files);
539 }
540 catch (Exception e)
541 {
542 throw new DefaultMuleException(FileMessages.errorWhileListingFiles(), e);
543 }
544 }
545
546 protected void basicListFiles(File currentDirectory, List<File> discoveredFiles)
547 {
548 File[] files;
549 if (fileFilter != null)
550 {
551 files = currentDirectory.listFiles(fileFilter);
552 }
553 else
554 {
555 files = currentDirectory.listFiles(filenameFilter);
556 }
557
558
559 if (files == null)
560 {
561 return;
562 }
563
564 for (File file : files)
565 {
566 if (!file.isDirectory())
567 {
568 discoveredFiles.add(file);
569 }
570 else
571 {
572 if (((FileConnector) this.getConnector()).isRecursive())
573 {
574 this.basicListFiles(file, discoveredFiles);
575 }
576 }
577 }
578 }
579
580
581
582
583
584
585 protected void rollbackFileMove(File sourceFile, String destinationFilePath) throws IOException
586 {
587 try
588 {
589 FileUtils.moveFile(sourceFile, FileUtils.newFile(destinationFilePath));
590 }
591 catch (IOException t)
592 {
593 logger.debug("rollback of file move failed: " + t.getMessage());
594 throw t;
595 }
596 }
597
598 @SuppressWarnings("unchecked")
599 protected Comparator<File> getComparator() throws Exception
600 {
601 Object comparatorClassName = getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY);
602 if (comparatorClassName != null)
603 {
604 Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY);
605 boolean reverse = false;
606 if (reverseProperty != null)
607 {
608 reverse = Boolean.valueOf((String) reverseProperty);
609 }
610
611 Class<?> clazz = Class.forName(comparatorClassName.toString());
612 Comparator<?> comparator = (Comparator<?>)clazz.newInstance();
613 return reverse ? new ReverseComparator(comparator) : comparator;
614 }
615 return null;
616 }
617 }