1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
package org.mule.transport.file; |
8 | |
|
9 | |
import org.mule.DefaultMuleMessage; |
10 | |
import org.mule.api.DefaultMuleException; |
11 | |
import org.mule.api.MessagingException; |
12 | |
import org.mule.api.MuleException; |
13 | |
import org.mule.api.MuleMessage; |
14 | |
import org.mule.api.config.MuleProperties; |
15 | |
import org.mule.api.construct.FlowConstruct; |
16 | |
import org.mule.api.endpoint.InboundEndpoint; |
17 | |
import org.mule.api.lifecycle.CreateException; |
18 | |
import org.mule.api.lifecycle.InitialisationException; |
19 | |
import org.mule.api.store.ObjectAlreadyExistsException; |
20 | |
import org.mule.api.store.ObjectStore; |
21 | |
import org.mule.api.store.ObjectStoreException; |
22 | |
import org.mule.api.transport.Connector; |
23 | |
import org.mule.api.transport.PropertyScope; |
24 | |
import org.mule.config.i18n.Message; |
25 | |
import org.mule.transport.AbstractPollingMessageReceiver; |
26 | |
import org.mule.transport.ConnectException; |
27 | |
import org.mule.transport.file.i18n.FileMessages; |
28 | |
import org.mule.util.FileUtils; |
29 | |
import org.mule.util.store.InMemoryObjectStore; |
30 | |
|
31 | |
import java.io.File; |
32 | |
import java.io.FileFilter; |
33 | |
import java.io.FileNotFoundException; |
34 | |
import java.io.FilenameFilter; |
35 | |
import java.io.IOException; |
36 | |
import java.io.RandomAccessFile; |
37 | |
import java.nio.channels.FileChannel; |
38 | |
import java.nio.channels.FileLock; |
39 | |
import java.util.ArrayList; |
40 | |
import java.util.Collections; |
41 | |
import java.util.Comparator; |
42 | |
import java.util.List; |
43 | |
|
44 | |
import org.apache.commons.collections.comparators.ReverseComparator; |
45 | |
|
46 | |
|
47 | |
|
48 | |
|
49 | |
|
50 | |
|
51 | 0 | public class FileMessageReceiver extends AbstractPollingMessageReceiver |
52 | |
{ |
53 | |
public static final String COMPARATOR_CLASS_NAME_PROPERTY = "comparator"; |
54 | |
public static final String COMPARATOR_REVERSE_ORDER_PROPERTY = "reverseOrder"; |
55 | |
|
56 | 0 | private static final List<File> NO_FILES = new ArrayList<File>(); |
57 | |
|
58 | 0 | private String readDir = null; |
59 | 0 | private String moveDir = null; |
60 | 0 | private String workDir = null; |
61 | 0 | private File readDirectory = null; |
62 | 0 | private File moveDirectory = null; |
63 | 0 | private String moveToPattern = null; |
64 | 0 | private String workFileNamePattern = null; |
65 | 0 | private FilenameFilter filenameFilter = null; |
66 | 0 | private FileFilter fileFilter = null; |
67 | |
private boolean forceSync; |
68 | |
private ObjectStore<String> filesBeingProcessingObjectStore; |
69 | |
|
70 | |
public FileMessageReceiver(Connector connector, |
71 | |
FlowConstruct flowConstruct, |
72 | |
InboundEndpoint endpoint, |
73 | |
String readDir, |
74 | |
String moveDir, |
75 | |
String moveToPattern, |
76 | |
long frequency) throws CreateException |
77 | |
{ |
78 | 0 | super(connector, flowConstruct, endpoint); |
79 | 0 | this.setFrequency(frequency); |
80 | |
|
81 | 0 | this.readDir = readDir; |
82 | 0 | this.moveDir = moveDir; |
83 | 0 | this.moveToPattern = moveToPattern; |
84 | 0 | this.workDir = ((FileConnector) connector).getWorkDirectory(); |
85 | 0 | this.workFileNamePattern = ((FileConnector) connector).getWorkFileNamePattern(); |
86 | |
|
87 | 0 | if (endpoint.getFilter() instanceof FilenameFilter) |
88 | |
{ |
89 | 0 | filenameFilter = (FilenameFilter) endpoint.getFilter(); |
90 | |
} |
91 | 0 | else if (endpoint.getFilter() instanceof FileFilter) |
92 | |
{ |
93 | 0 | fileFilter = (FileFilter) endpoint.getFilter(); |
94 | |
} |
95 | 0 | else if (endpoint.getFilter() != null) |
96 | |
{ |
97 | 0 | throw new CreateException(FileMessages.invalidFileFilter(endpoint.getEndpointURI()), this); |
98 | |
} |
99 | |
|
100 | 0 | checkMustForceSync(endpoint); |
101 | 0 | } |
102 | |
|
103 | |
|
104 | |
|
105 | |
|
106 | |
private void checkMustForceSync(InboundEndpoint ep) throws CreateException |
107 | |
{ |
108 | 0 | boolean connectorIsAutoDelete = false; |
109 | 0 | boolean isStreaming = false; |
110 | 0 | if (connector instanceof FileConnector) |
111 | |
{ |
112 | 0 | connectorIsAutoDelete = ((FileConnector) connector).isAutoDelete(); |
113 | 0 | isStreaming = ((FileConnector) connector).isStreaming(); |
114 | |
} |
115 | |
|
116 | 0 | boolean messageFactoryConsumes = (createMuleMessageFactory() instanceof FileContentsMuleMessageFactory); |
117 | |
|
118 | 0 | forceSync = connectorIsAutoDelete && !messageFactoryConsumes && !isStreaming; |
119 | 0 | } |
120 | |
|
121 | |
@Override |
122 | |
protected void doConnect() throws Exception |
123 | |
{ |
124 | 0 | if (readDir != null) |
125 | |
{ |
126 | 0 | readDirectory = FileUtils.openDirectory(readDir); |
127 | 0 | if (!(readDirectory.canRead())) |
128 | |
{ |
129 | 0 | throw new ConnectException(FileMessages.fileDoesNotExist(readDirectory.getAbsolutePath()), this); |
130 | |
} |
131 | |
else |
132 | |
{ |
133 | 0 | logger.debug("Listening on endpointUri: " + readDirectory.getAbsolutePath()); |
134 | |
} |
135 | |
} |
136 | |
|
137 | 0 | if (moveDir != null) |
138 | |
{ |
139 | 0 | moveDirectory = FileUtils.openDirectory((moveDir)); |
140 | 0 | if (!(moveDirectory.canRead()) || !moveDirectory.canWrite()) |
141 | |
{ |
142 | 0 | throw new ConnectException(FileMessages.moveToDirectoryNotWritable(), this); |
143 | |
} |
144 | |
} |
145 | 0 | } |
146 | |
|
147 | |
@Override |
148 | |
protected void doInitialise() throws InitialisationException |
149 | |
{ |
150 | 0 | InMemoryObjectStore objectStore = new InMemoryObjectStore<String>(); |
151 | 0 | objectStore.setMaxEntries(1000); |
152 | 0 | objectStore.setExpirationInterval(20000); |
153 | 0 | objectStore.setEntryTTL(60000); |
154 | 0 | filesBeingProcessingObjectStore = objectStore; |
155 | 0 | } |
156 | |
|
157 | |
@Override |
158 | |
protected void doDisconnect() throws Exception |
159 | |
{ |
160 | |
|
161 | 0 | } |
162 | |
|
163 | |
@Override |
164 | |
protected void doDispose() |
165 | |
{ |
166 | |
|
167 | 0 | } |
168 | |
|
169 | |
@Override |
170 | |
public void poll() |
171 | |
{ |
172 | |
try |
173 | |
{ |
174 | 0 | List<File> files = this.listFiles(); |
175 | 0 | if (logger.isDebugEnabled()) |
176 | |
{ |
177 | 0 | logger.debug("Files: " + files.toString()); |
178 | |
} |
179 | 0 | Comparator<File> comparator = getComparator(); |
180 | 0 | if (comparator != null) |
181 | |
{ |
182 | 0 | Collections.sort(files, comparator); |
183 | |
} |
184 | 0 | for (File file : files) |
185 | |
{ |
186 | 0 | if (getLifecycleState().isStopping()) |
187 | |
{ |
188 | 0 | break; |
189 | |
} |
190 | |
|
191 | 0 | if (file.isFile()) |
192 | |
{ |
193 | |
try |
194 | |
{ |
195 | 0 | filesBeingProcessingObjectStore.store(file.getAbsolutePath(),file.getAbsolutePath()); |
196 | 0 | processFile(file); |
197 | |
} |
198 | 0 | catch (ObjectAlreadyExistsException e) |
199 | |
{ |
200 | 0 | logger.debug("file " + file.getAbsolutePath() + " it's being processed. Skipping it."); |
201 | 0 | } |
202 | |
} |
203 | |
} |
204 | |
} |
205 | 0 | catch (Exception e) |
206 | |
{ |
207 | 0 | getConnector().getMuleContext().getExceptionListener().handleException(e); |
208 | 0 | } |
209 | 0 | } |
210 | |
|
211 | |
public void processFile(File file) throws MuleException |
212 | |
{ |
213 | 0 | FileConnector fileConnector = (FileConnector) connector; |
214 | |
|
215 | |
|
216 | |
|
217 | |
|
218 | 0 | boolean checkFileAge = fileConnector.getCheckFileAge(); |
219 | 0 | if (checkFileAge) |
220 | |
{ |
221 | 0 | long fileAge = ((FileConnector) connector).getFileAge(); |
222 | 0 | long lastMod = file.lastModified(); |
223 | 0 | long now = System.currentTimeMillis(); |
224 | 0 | long thisFileAge = now - lastMod; |
225 | 0 | if (thisFileAge < fileAge) |
226 | |
{ |
227 | 0 | if (logger.isDebugEnabled()) |
228 | |
{ |
229 | 0 | logger.debug("The file has not aged enough yet, will return nothing for: " + file); |
230 | |
} |
231 | 0 | return; |
232 | |
} |
233 | |
} |
234 | |
|
235 | 0 | String sourceFileOriginalName = file.getName(); |
236 | 0 | final String sourceFileOriginalAbsolutePath = file.getAbsolutePath(); |
237 | |
|
238 | |
|
239 | 0 | if (!(file.canRead() && file.exists() && file.isFile())) |
240 | |
{ |
241 | 0 | throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFileOriginalName)); |
242 | |
} |
243 | |
|
244 | |
|
245 | |
|
246 | |
|
247 | |
|
248 | 0 | if (workDir == null) |
249 | |
{ |
250 | |
|
251 | |
|
252 | 0 | if (!attemptFileLock(file)) |
253 | |
{ |
254 | 0 | return; |
255 | |
} |
256 | 0 | else if (logger.isInfoEnabled()) |
257 | |
{ |
258 | 0 | logger.info("Lock obtained on file: " + file.getAbsolutePath()); |
259 | |
} |
260 | |
} |
261 | |
|
262 | |
|
263 | |
|
264 | 0 | DefaultMuleMessage fileParserMessasge = new DefaultMuleMessage(null, connector.getMuleContext()); |
265 | 0 | fileParserMessasge.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName); |
266 | |
|
267 | 0 | File workFile = null; |
268 | |
final File sourceFile; |
269 | 0 | if (workDir != null) |
270 | |
{ |
271 | 0 | String workFileName = sourceFileOriginalName; |
272 | |
|
273 | 0 | workFileName = fileConnector.getFilenameParser().getFilename(fileParserMessasge, |
274 | |
workFileNamePattern); |
275 | |
|
276 | 0 | workFile = FileUtils.newFile(workDir, workFileName); |
277 | |
|
278 | 0 | fileConnector.move(file, workFile); |
279 | |
|
280 | 0 | sourceFile = workFile; |
281 | 0 | } |
282 | |
else |
283 | |
{ |
284 | 0 | sourceFile = file; |
285 | |
} |
286 | |
|
287 | |
|
288 | 0 | File destinationFile = null; |
289 | 0 | if (moveDir != null) |
290 | |
{ |
291 | 0 | String destinationFileName = sourceFileOriginalName; |
292 | 0 | if (moveToPattern != null) |
293 | |
{ |
294 | 0 | destinationFileName = ((FileConnector) connector).getFilenameParser().getFilename(fileParserMessasge, |
295 | |
moveToPattern); |
296 | |
} |
297 | |
|
298 | 0 | destinationFile = FileUtils.newFile(moveDir, destinationFileName); |
299 | |
} |
300 | |
|
301 | 0 | MuleMessage message = null; |
302 | 0 | String encoding = endpoint.getEncoding(); |
303 | |
try |
304 | |
{ |
305 | 0 | if (fileConnector.isStreaming()) |
306 | |
{ |
307 | 0 | ReceiverFileInputStream payload = new ReceiverFileInputStream(sourceFile, fileConnector.isAutoDelete(), destinationFile, new InputStreamCloseListener() |
308 | 0 | { |
309 | |
public void fileClose(File file) |
310 | |
{ |
311 | |
try |
312 | |
{ |
313 | 0 | if (logger.isDebugEnabled()) |
314 | |
{ |
315 | 0 | logger.debug(String.format("Removing processing flag for $ ", file.getAbsolutePath())); |
316 | |
} |
317 | 0 | filesBeingProcessingObjectStore.remove(sourceFileOriginalAbsolutePath); |
318 | |
} |
319 | 0 | catch (ObjectStoreException e) |
320 | |
{ |
321 | 0 | logger.warn("Failure trying to remove file " + sourceFileOriginalAbsolutePath + " from list of files under processing"); |
322 | 0 | } |
323 | 0 | } |
324 | |
}); |
325 | 0 | message = createMuleMessage(payload, encoding); |
326 | 0 | } |
327 | |
else |
328 | |
{ |
329 | 0 | message = createMuleMessage(sourceFile, encoding); |
330 | |
} |
331 | |
} |
332 | 0 | catch (FileNotFoundException e) |
333 | |
{ |
334 | |
|
335 | 0 | logger.error("File being read disappeared!", e); |
336 | 0 | return; |
337 | 0 | } |
338 | |
|
339 | 0 | if (workDir != null) |
340 | |
{ |
341 | 0 | message.setProperty(FileConnector.PROPERTY_SOURCE_DIRECTORY, file.getParent(), PropertyScope.INBOUND); |
342 | 0 | message.setProperty(FileConnector.PROPERTY_SOURCE_FILENAME, file.getName(), PropertyScope.INBOUND); |
343 | |
} |
344 | |
|
345 | 0 | message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName); |
346 | 0 | if (forceSync) |
347 | |
{ |
348 | 0 | message.setProperty(MuleProperties.MULE_FORCE_SYNC_PROPERTY, Boolean.TRUE, PropertyScope.INBOUND); |
349 | |
} |
350 | 0 | if (!fileConnector.isStreaming()) |
351 | |
{ |
352 | 0 | moveAndDelete(sourceFile, destinationFile, sourceFileOriginalName, sourceFileOriginalAbsolutePath,message); |
353 | |
} |
354 | |
else |
355 | |
{ |
356 | |
|
357 | |
|
358 | 0 | message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, sourceFile.getName()); |
359 | 0 | this.routeMessage(message); |
360 | |
} |
361 | 0 | } |
362 | |
|
363 | |
private void moveAndDelete(final File sourceFile, File destinationFile, |
364 | |
String sourceFileOriginalName, String sourceFileOriginalAbsolutePath, MuleMessage message) |
365 | |
{ |
366 | 0 | boolean fileWasMoved = false; |
367 | |
|
368 | |
try |
369 | |
{ |
370 | |
|
371 | |
|
372 | |
|
373 | 0 | if (destinationFile != null) |
374 | |
{ |
375 | |
|
376 | |
try |
377 | |
{ |
378 | 0 | FileUtils.moveFile(sourceFile, destinationFile); |
379 | |
} |
380 | 0 | catch (IOException e) |
381 | |
{ |
382 | |
|
383 | 0 | throw new DefaultMuleException(FileMessages.failedToMoveFile( |
384 | |
sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath())); |
385 | 0 | } |
386 | |
|
387 | |
|
388 | 0 | message = createMuleMessage(destinationFile, endpoint.getEncoding()); |
389 | 0 | message.setOutboundProperty(FileConnector.PROPERTY_FILENAME, destinationFile.getName()); |
390 | 0 | message.setOutboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, sourceFileOriginalName); |
391 | |
} |
392 | |
|
393 | |
|
394 | 0 | this.routeMessage(message); |
395 | |
|
396 | |
|
397 | |
|
398 | 0 | if (((FileConnector) connector).isAutoDelete()) |
399 | |
{ |
400 | |
|
401 | 0 | if (destinationFile == null) |
402 | |
{ |
403 | |
|
404 | 0 | if (!sourceFile.delete()) |
405 | |
{ |
406 | 0 | throw new DefaultMuleException(FileMessages.failedToDeleteFile(sourceFile)); |
407 | |
} |
408 | |
} |
409 | |
else |
410 | |
{ |
411 | |
|
412 | |
|
413 | |
} |
414 | |
} |
415 | |
} |
416 | 0 | catch (Exception e) |
417 | |
{ |
418 | 0 | boolean fileWasRolledBack = false; |
419 | |
|
420 | |
|
421 | 0 | if (fileWasMoved) |
422 | |
{ |
423 | |
try |
424 | |
{ |
425 | 0 | rollbackFileMove(destinationFile, sourceFile.getAbsolutePath()); |
426 | 0 | fileWasRolledBack = true; |
427 | |
} |
428 | 0 | catch (IOException ioException) |
429 | |
{ |
430 | |
|
431 | 0 | } |
432 | |
} |
433 | |
|
434 | |
|
435 | 0 | Message msg = FileMessages.exceptionWhileProcessing(sourceFile.getName(), |
436 | |
(fileWasRolledBack ? "successful" : "unsuccessful")); |
437 | 0 | getConnector().getMuleContext().getExceptionListener().handleException(new MessagingException(msg, message, e)); |
438 | |
} |
439 | |
finally |
440 | |
{ |
441 | 0 | try |
442 | |
{ |
443 | 0 | filesBeingProcessingObjectStore.remove(sourceFileOriginalAbsolutePath); |
444 | 0 | if (logger.isDebugEnabled()) |
445 | |
{ |
446 | 0 | logger.debug(String.format("Removing processing flag for $ ", sourceFileOriginalAbsolutePath)); |
447 | |
} |
448 | |
} |
449 | 0 | catch (ObjectStoreException e) |
450 | |
{ |
451 | 0 | logger.warn("Failure trying to remove file " + sourceFileOriginalAbsolutePath + " from list of files under processing"); |
452 | 0 | } |
453 | 0 | } |
454 | 0 | } |
455 | |
|
456 | |
|
457 | |
|
458 | |
|
459 | |
|
460 | |
|
461 | |
|
462 | |
|
463 | |
|
464 | |
protected boolean attemptFileLock(final File sourceFile) throws MuleException |
465 | |
{ |
466 | |
|
467 | |
|
468 | |
|
469 | |
|
470 | 0 | FileLock lock = null; |
471 | 0 | FileChannel channel = null; |
472 | 0 | boolean fileCanBeLocked = false; |
473 | |
try |
474 | |
{ |
475 | 0 | channel = new RandomAccessFile(sourceFile, "rw").getChannel(); |
476 | |
|
477 | |
|
478 | |
|
479 | 0 | lock = channel.tryLock(); |
480 | |
} |
481 | 0 | catch (FileNotFoundException fnfe) |
482 | |
{ |
483 | 0 | throw new DefaultMuleException(FileMessages.fileDoesNotExist(sourceFile.getName())); |
484 | |
} |
485 | 0 | catch (IOException e) |
486 | |
{ |
487 | |
|
488 | |
|
489 | |
|
490 | |
} |
491 | |
finally |
492 | |
{ |
493 | 0 | if (lock != null) |
494 | |
{ |
495 | |
|
496 | 0 | fileCanBeLocked = true; |
497 | |
try |
498 | |
{ |
499 | |
|
500 | 0 | lock.release(); |
501 | |
} |
502 | 0 | catch (IOException e) |
503 | |
{ |
504 | |
|
505 | 0 | } |
506 | |
} |
507 | |
|
508 | 0 | if (channel != null) |
509 | |
{ |
510 | |
try |
511 | |
{ |
512 | |
|
513 | 0 | channel.close(); |
514 | |
} |
515 | 0 | catch (IOException e) |
516 | |
{ |
517 | |
|
518 | 0 | } |
519 | |
} |
520 | |
} |
521 | |
|
522 | 0 | return fileCanBeLocked; |
523 | |
} |
524 | |
|
525 | |
|
526 | |
|
527 | |
|
528 | |
|
529 | |
|
530 | |
|
531 | |
|
532 | |
List<File> listFiles() throws MuleException |
533 | |
{ |
534 | |
try |
535 | |
{ |
536 | 0 | List<File> files = new ArrayList<File>(); |
537 | 0 | this.basicListFiles(readDirectory, files); |
538 | 0 | return (files.isEmpty() ? NO_FILES : files); |
539 | |
} |
540 | 0 | catch (Exception e) |
541 | |
{ |
542 | 0 | throw new DefaultMuleException(FileMessages.errorWhileListingFiles(), e); |
543 | |
} |
544 | |
} |
545 | |
|
546 | |
protected void basicListFiles(File currentDirectory, List<File> discoveredFiles) |
547 | |
{ |
548 | |
File[] files; |
549 | 0 | if (fileFilter != null) |
550 | |
{ |
551 | 0 | files = currentDirectory.listFiles(fileFilter); |
552 | |
} |
553 | |
else |
554 | |
{ |
555 | 0 | files = currentDirectory.listFiles(filenameFilter); |
556 | |
} |
557 | |
|
558 | |
|
559 | 0 | if (files == null) |
560 | |
{ |
561 | 0 | return; |
562 | |
} |
563 | |
|
564 | 0 | for (File file : files) |
565 | |
{ |
566 | 0 | if (!file.isDirectory()) |
567 | |
{ |
568 | 0 | discoveredFiles.add(file); |
569 | |
} |
570 | |
else |
571 | |
{ |
572 | 0 | if (((FileConnector) this.getConnector()).isRecursive()) |
573 | |
{ |
574 | 0 | this.basicListFiles(file, discoveredFiles); |
575 | |
} |
576 | |
} |
577 | |
} |
578 | 0 | } |
579 | |
|
580 | |
|
581 | |
|
582 | |
|
583 | |
|
584 | |
|
585 | |
protected void rollbackFileMove(File sourceFile, String destinationFilePath) throws IOException |
586 | |
{ |
587 | |
try |
588 | |
{ |
589 | 0 | FileUtils.moveFile(sourceFile, FileUtils.newFile(destinationFilePath)); |
590 | |
} |
591 | 0 | catch (IOException t) |
592 | |
{ |
593 | 0 | logger.debug("rollback of file move failed: " + t.getMessage()); |
594 | 0 | throw t; |
595 | 0 | } |
596 | 0 | } |
597 | |
|
598 | |
@SuppressWarnings("unchecked") |
599 | |
protected Comparator<File> getComparator() throws Exception |
600 | |
{ |
601 | 0 | Object comparatorClassName = getEndpoint().getProperty(COMPARATOR_CLASS_NAME_PROPERTY); |
602 | 0 | if (comparatorClassName != null) |
603 | |
{ |
604 | 0 | Object reverseProperty = this.getEndpoint().getProperty(COMPARATOR_REVERSE_ORDER_PROPERTY); |
605 | 0 | boolean reverse = false; |
606 | 0 | if (reverseProperty != null) |
607 | |
{ |
608 | 0 | reverse = Boolean.valueOf((String) reverseProperty); |
609 | |
} |
610 | |
|
611 | 0 | Class<?> clazz = Class.forName(comparatorClassName.toString()); |
612 | 0 | Comparator<?> comparator = (Comparator<?>)clazz.newInstance(); |
613 | 0 | return reverse ? new ReverseComparator(comparator) : comparator; |
614 | |
} |
615 | 0 | return null; |
616 | |
} |
617 | |
} |