1
2
3
4
5
6
7 package org.mule.transport.sftp;
8
9 import org.apache.commons.logging.Log;
10 import org.apache.commons.logging.LogFactory;
11 import org.mule.api.endpoint.ImmutableEndpoint;
12 import org.mule.transport.sftp.notification.SftpNotifier;
13 import org.mule.util.FileUtils;
14
15 import java.io.File;
16 import java.io.FilenameFilter;
17 import java.io.IOException;
18 import java.io.InputStream;
19 import java.util.ArrayList;
20 import java.util.List;
21
22
23
24
25
26
27
28 public class SftpReceiverRequesterUtil
29 {
30 private transient Log logger = LogFactory.getLog(getClass());
31
32 private final SftpConnector connector;
33 private final ImmutableEndpoint endpoint;
34 private final FilenameFilter filenameFilter;
35 private final SftpUtil sftpUtil;
36
37 public SftpReceiverRequesterUtil(ImmutableEndpoint endpoint)
38 {
39 this.endpoint = endpoint;
40 this.connector = (SftpConnector) endpoint.getConnector();
41
42 sftpUtil = new SftpUtil(endpoint);
43
44 if (endpoint.getFilter() instanceof FilenameFilter)
45 {
46 this.filenameFilter = (FilenameFilter) endpoint.getFilter();
47 }
48 else
49 {
50 this.filenameFilter = null;
51 }
52
53 }
54
55
56 public String[] getAvailableFiles(boolean onlyGetTheFirstOne) throws Exception
57 {
58
59
60
61
62 if (logger.isDebugEnabled())
63 {
64 logger.debug("Checking files at endpoint " + endpoint.getEndpointURI());
65 }
66
67 SftpClient client = null;
68
69 try
70 {
71 client = connector.createSftpClient(endpoint);
72
73 long fileAge = connector.getFileAge();
74 boolean checkFileAge = connector.getCheckFileAge();
75
76
77 if (endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE) != null)
78 {
79 checkFileAge = true;
80 fileAge = Long.valueOf((String) endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE));
81 }
82
83 logger.debug("fileAge : " + fileAge);
84
85
86 long sizeCheckDelayMs = sftpUtil.getSizeCheckWaitTime();
87
88 String[] files = client.listFiles();
89
90
91
92 List<String> completedFiles = new ArrayList<String>(files.length);
93
94 for (String file : files)
95 {
96
97
98
99
100
101
102
103 if (filenameFilter != null && !filenameFilter.accept(null, file))
104 {
105 continue;
106 }
107
108 if (checkFileAge || sizeCheckDelayMs >= 0)
109 {
110
111
112 if (!hasChanged(file, client, fileAge, sizeCheckDelayMs))
113 {
114
115
116
117
118 completedFiles.add(file);
119 if (onlyGetTheFirstOne)
120 {
121 break;
122 }
123 }
124 }
125 else
126 {
127 completedFiles.add(file);
128 if (onlyGetTheFirstOne)
129 {
130 break;
131 }
132 }
133 }
134 return completedFiles.toArray(new String[completedFiles.size()]);
135 }
136 finally
137 {
138 if (client != null)
139 {
140 connector.releaseClient(endpoint, client);
141 }
142 }
143 }
144
145 public InputStream retrieveFile(String fileName, SftpNotifier notifier) throws Exception
146 {
147
148 SftpClient client = connector.createSftpClient(endpoint, notifier);
149
150
151 String tmpSendingDir = sftpUtil.getTempDirInbound();
152 if (tmpSendingDir != null)
153 {
154
155 boolean addUniqueSuffix = sftpUtil.isUseTempFileTimestampSuffix();
156
157
158 client.createSftpDirIfNotExists(endpoint, tmpSendingDir);
159 String tmpSendingFileName = tmpSendingDir + "/" + fileName;
160
161 if (addUniqueSuffix)
162 {
163 tmpSendingFileName = sftpUtil.createUniqueSuffix(tmpSendingFileName);
164 }
165 String fullTmpSendingPath = endpoint.getEndpointURI().getPath() + "/" + tmpSendingFileName;
166
167 if (logger.isDebugEnabled())
168 {
169 logger.debug("Move " + fileName + " to " + fullTmpSendingPath);
170 }
171 client.rename(fileName, fullTmpSendingPath);
172 fileName = tmpSendingFileName;
173 if (logger.isDebugEnabled())
174 {
175 logger.debug("Move done");
176 }
177 }
178
179
180 String archive = sftpUtil.getArchiveDir();
181
182
183 InputStream fileInputStream = client.retrieveFile(fileName);
184
185 if (!"".equals(archive))
186 {
187 String archiveTmpReceivingDir = sftpUtil.getArchiveTempReceivingDir();
188 String archiveTmpSendingDir = sftpUtil.getArchiveTempSendingDir();
189
190 InputStream is = new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(),
191 endpoint);
192
193
194 int idx = fileName.lastIndexOf('/');
195 String fileNamePart = fileName.substring(idx + 1);
196
197
198 File archiveFile = FileUtils.newFile(archive, fileNamePart);
199
200
201 if ("".equals(archiveTmpReceivingDir) || "".equals(archiveTmpSendingDir))
202 {
203 return archiveFile(is, archiveFile);
204 }
205 else
206 {
207 return archiveFileUsingTempDirs(archive, archiveTmpReceivingDir, archiveTmpSendingDir, is,
208 fileNamePart, archiveFile);
209 }
210 }
211
212
213
214
215 return new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(), endpoint);
216 }
217
218 private InputStream archiveFileUsingTempDirs(String archive,
219 String archiveTmpReceivingDir,
220 String archiveTmpSendingDir,
221 InputStream is,
222 String fileNamePart,
223 File archiveFile) throws IOException
224 {
225
226 File archiveTmpReceivingFolder = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir);
227 File archiveTmpReceivingFile = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir, fileNamePart);
228 if (!archiveTmpReceivingFolder.exists())
229 {
230 if (logger.isInfoEnabled())
231 {
232 logger.info("Creates " + archiveTmpReceivingFolder.getAbsolutePath());
233 }
234 if (!archiveTmpReceivingFolder.mkdirs())
235 throw new IOException("Failed to create archive-tmp-receiving-folder: "
236 + archiveTmpReceivingFolder);
237 }
238
239 File archiveTmpSendingFolder = FileUtils.newFile(archive + '/' + archiveTmpSendingDir);
240 File archiveTmpSendingFile = FileUtils.newFile(archive + '/' + archiveTmpSendingDir, fileNamePart);
241 if (!archiveTmpSendingFolder.exists())
242 {
243 if (logger.isInfoEnabled())
244 {
245 logger.info("Creates " + archiveTmpSendingFolder.getAbsolutePath());
246 }
247 if (!archiveTmpSendingFolder.mkdirs())
248 throw new IOException("Failed to create archive-tmp-sending-folder: "
249 + archiveTmpSendingFolder);
250 }
251
252 if (logger.isInfoEnabled())
253 {
254 logger.info("Copy SftpInputStream to archiveTmpReceivingFile... "
255 + archiveTmpReceivingFile.getAbsolutePath());
256 }
257 sftpUtil.copyStreamToFile(is, archiveTmpReceivingFile);
258
259
260
261 if (logger.isInfoEnabled())
262 {
263 logger.info("Move archiveTmpReceivingFile (" + archiveTmpReceivingFile
264 + ") to archiveTmpSendingFile (" + archiveTmpSendingFile + ")...");
265 }
266 FileUtils.moveFile(archiveTmpReceivingFile, archiveTmpSendingFile);
267
268 if (logger.isDebugEnabled())
269 {
270 logger.debug("Return SftpFileArchiveInputStream for archiveTmpSendingFile ("
271 + archiveTmpSendingFile + ")...");
272 }
273 return new SftpFileArchiveInputStream(archiveTmpSendingFile, archiveFile);
274 }
275
276 private InputStream archiveFile(InputStream is, File archiveFile) throws IOException
277 {
278 File archiveFolder = FileUtils.newFile(archiveFile.getParentFile().getPath());
279 if (!archiveFolder.exists())
280 {
281 if (logger.isInfoEnabled())
282 {
283 logger.info("Creates " + archiveFolder.getAbsolutePath());
284 }
285 if (!archiveFolder.mkdirs())
286 throw new IOException("Failed to create archive-folder: " + archiveFolder);
287 }
288
289 if (logger.isInfoEnabled())
290 {
291 logger.info("Copy SftpInputStream to archiveFile... " + archiveFile.getAbsolutePath());
292 }
293 sftpUtil.copyStreamToFile(is, archiveFile);
294
295 if (logger.isDebugEnabled())
296 {
297 logger.debug("*** Return SftpFileArchiveInputStream for archiveFile...");
298 }
299 return new SftpFileArchiveInputStream(archiveFile);
300 }
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316 private boolean hasChanged(String fileName, SftpClient client, long fileAge, long sizeCheckDelayMs)
317 throws Exception
318 {
319
320
321
322
323 if (fileAge > 0)
324 {
325 long lastModifiedTime = client.getLastModifiedTime(fileName);
326
327 long now = System.currentTimeMillis();
328 long diff = now - lastModifiedTime;
329
330
331 if (diff < fileAge)
332 {
333 if (logger.isDebugEnabled())
334 {
335 logger.debug("The file has not aged enough yet, will return nothing for: " + fileName
336 + ". The file must be " + (fileAge - diff) + "ms older, was " + diff);
337 }
338 return true;
339 }
340 if (logger.isDebugEnabled())
341 {
342 logger.debug("The file " + fileName + " has aged enough. Was " + diff);
343 }
344 }
345
346
347
348
349
350 if (sizeCheckDelayMs > 0)
351 {
352 logger.info("Perform size check with a delay of: " + sizeCheckDelayMs + " ms.");
353 long fileSize1 = client.getSize(fileName);
354 Thread.sleep(sizeCheckDelayMs);
355 long fileSize2 = client.getSize(fileName);
356
357 if (fileSize1 == fileSize2)
358 {
359 logger.info("File is stable (not growing), ready for retrieval: " + fileName);
360 }
361 else
362 {
363 logger.info("File is growing, deferring retrieval: " + fileName);
364 return true;
365 }
366 }
367
368
369 return false;
370 }
371 }