1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
package org.mule.transport.sftp; |
8 | |
|
9 | |
import org.apache.commons.logging.Log; |
10 | |
import org.apache.commons.logging.LogFactory; |
11 | |
import org.mule.api.endpoint.ImmutableEndpoint; |
12 | |
import org.mule.transport.sftp.notification.SftpNotifier; |
13 | |
import org.mule.util.FileUtils; |
14 | |
|
15 | |
import java.io.File; |
16 | |
import java.io.FilenameFilter; |
17 | |
import java.io.IOException; |
18 | |
import java.io.InputStream; |
19 | |
import java.util.ArrayList; |
20 | |
import java.util.List; |
21 | |
|
22 | |
|
23 | |
|
24 | |
|
25 | |
|
26 | |
|
27 | |
|
28 | |
public class SftpReceiverRequesterUtil |
29 | |
{ |
30 | 0 | private transient Log logger = LogFactory.getLog(getClass()); |
31 | |
|
32 | |
private final SftpConnector connector; |
33 | |
private final ImmutableEndpoint endpoint; |
34 | |
private final FilenameFilter filenameFilter; |
35 | |
private final SftpUtil sftpUtil; |
36 | |
|
37 | |
public SftpReceiverRequesterUtil(ImmutableEndpoint endpoint) |
38 | 0 | { |
39 | 0 | this.endpoint = endpoint; |
40 | 0 | this.connector = (SftpConnector) endpoint.getConnector(); |
41 | |
|
42 | 0 | sftpUtil = new SftpUtil(endpoint); |
43 | |
|
44 | 0 | if (endpoint.getFilter() instanceof FilenameFilter) |
45 | |
{ |
46 | 0 | this.filenameFilter = (FilenameFilter) endpoint.getFilter(); |
47 | |
} |
48 | |
else |
49 | |
{ |
50 | 0 | this.filenameFilter = null; |
51 | |
} |
52 | |
|
53 | 0 | } |
54 | |
|
55 | |
|
56 | |
public String[] getAvailableFiles(boolean onlyGetTheFirstOne) throws Exception |
57 | |
{ |
58 | |
|
59 | |
|
60 | |
|
61 | |
|
62 | 0 | if (logger.isDebugEnabled()) |
63 | |
{ |
64 | 0 | logger.debug("Checking files at endpoint " + endpoint.getEndpointURI()); |
65 | |
} |
66 | |
|
67 | 0 | SftpClient client = null; |
68 | |
|
69 | |
try |
70 | |
{ |
71 | 0 | client = connector.createSftpClient(endpoint); |
72 | |
|
73 | 0 | long fileAge = connector.getFileAge(); |
74 | 0 | boolean checkFileAge = connector.getCheckFileAge(); |
75 | |
|
76 | |
|
77 | 0 | if (endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE) != null) |
78 | |
{ |
79 | 0 | checkFileAge = true; |
80 | 0 | fileAge = Long.valueOf((String) endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE)); |
81 | |
} |
82 | |
|
83 | 0 | logger.debug("fileAge : " + fileAge); |
84 | |
|
85 | |
|
86 | 0 | long sizeCheckDelayMs = sftpUtil.getSizeCheckWaitTime(); |
87 | |
|
88 | 0 | String[] files = client.listFiles(); |
89 | |
|
90 | |
|
91 | |
|
92 | 0 | List<String> completedFiles = new ArrayList<String>(files.length); |
93 | |
|
94 | 0 | for (String file : files) |
95 | |
{ |
96 | |
|
97 | |
|
98 | |
|
99 | |
|
100 | |
|
101 | |
|
102 | |
|
103 | 0 | if (filenameFilter != null && !filenameFilter.accept(null, file)) |
104 | |
{ |
105 | 0 | continue; |
106 | |
} |
107 | |
|
108 | 0 | if (checkFileAge || sizeCheckDelayMs >= 0) |
109 | |
{ |
110 | |
|
111 | |
|
112 | 0 | if (!hasChanged(file, client, fileAge, sizeCheckDelayMs)) |
113 | |
{ |
114 | |
|
115 | |
|
116 | |
|
117 | |
|
118 | 0 | completedFiles.add(file); |
119 | 0 | if (onlyGetTheFirstOne) |
120 | |
{ |
121 | 0 | break; |
122 | |
} |
123 | |
} |
124 | |
} |
125 | |
else |
126 | |
{ |
127 | 0 | completedFiles.add(file); |
128 | 0 | if (onlyGetTheFirstOne) |
129 | |
{ |
130 | 0 | break; |
131 | |
} |
132 | |
} |
133 | |
} |
134 | 0 | return completedFiles.toArray(new String[completedFiles.size()]); |
135 | |
} |
136 | |
finally |
137 | |
{ |
138 | 0 | if (client != null) |
139 | |
{ |
140 | 0 | connector.releaseClient(endpoint, client); |
141 | |
} |
142 | |
} |
143 | |
} |
144 | |
|
145 | |
public InputStream retrieveFile(String fileName, SftpNotifier notifier) throws Exception |
146 | |
{ |
147 | |
|
148 | 0 | SftpClient client = connector.createSftpClient(endpoint, notifier); |
149 | |
|
150 | |
|
151 | 0 | String tmpSendingDir = sftpUtil.getTempDirInbound(); |
152 | 0 | if (tmpSendingDir != null) |
153 | |
{ |
154 | |
|
155 | 0 | boolean addUniqueSuffix = sftpUtil.isUseTempFileTimestampSuffix(); |
156 | |
|
157 | |
|
158 | 0 | client.createSftpDirIfNotExists(endpoint, tmpSendingDir); |
159 | 0 | String tmpSendingFileName = tmpSendingDir + "/" + fileName; |
160 | |
|
161 | 0 | if (addUniqueSuffix) |
162 | |
{ |
163 | 0 | tmpSendingFileName = sftpUtil.createUniqueSuffix(tmpSendingFileName); |
164 | |
} |
165 | 0 | String fullTmpSendingPath = endpoint.getEndpointURI().getPath() + "/" + tmpSendingFileName; |
166 | |
|
167 | 0 | if (logger.isDebugEnabled()) |
168 | |
{ |
169 | 0 | logger.debug("Move " + fileName + " to " + fullTmpSendingPath); |
170 | |
} |
171 | 0 | client.rename(fileName, fullTmpSendingPath); |
172 | 0 | fileName = tmpSendingFileName; |
173 | 0 | if (logger.isDebugEnabled()) |
174 | |
{ |
175 | 0 | logger.debug("Move done"); |
176 | |
} |
177 | |
} |
178 | |
|
179 | |
|
180 | 0 | String archive = sftpUtil.getArchiveDir(); |
181 | |
|
182 | |
|
183 | 0 | InputStream fileInputStream = client.retrieveFile(fileName); |
184 | |
|
185 | 0 | if (!"".equals(archive)) |
186 | |
{ |
187 | 0 | String archiveTmpReceivingDir = sftpUtil.getArchiveTempReceivingDir(); |
188 | 0 | String archiveTmpSendingDir = sftpUtil.getArchiveTempSendingDir(); |
189 | |
|
190 | 0 | InputStream is = new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(), |
191 | |
endpoint); |
192 | |
|
193 | |
|
194 | 0 | int idx = fileName.lastIndexOf('/'); |
195 | 0 | String fileNamePart = fileName.substring(idx + 1); |
196 | |
|
197 | |
|
198 | 0 | File archiveFile = FileUtils.newFile(archive, fileNamePart); |
199 | |
|
200 | |
|
201 | 0 | if ("".equals(archiveTmpReceivingDir) || "".equals(archiveTmpSendingDir)) |
202 | |
{ |
203 | 0 | return archiveFile(is, archiveFile); |
204 | |
} |
205 | |
else |
206 | |
{ |
207 | 0 | return archiveFileUsingTempDirs(archive, archiveTmpReceivingDir, archiveTmpSendingDir, is, |
208 | |
fileNamePart, archiveFile); |
209 | |
} |
210 | |
} |
211 | |
|
212 | |
|
213 | |
|
214 | |
|
215 | 0 | return new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(), endpoint); |
216 | |
} |
217 | |
|
218 | |
private InputStream archiveFileUsingTempDirs(String archive, |
219 | |
String archiveTmpReceivingDir, |
220 | |
String archiveTmpSendingDir, |
221 | |
InputStream is, |
222 | |
String fileNamePart, |
223 | |
File archiveFile) throws IOException |
224 | |
{ |
225 | |
|
226 | 0 | File archiveTmpReceivingFolder = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir); |
227 | 0 | File archiveTmpReceivingFile = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir, fileNamePart); |
228 | 0 | if (!archiveTmpReceivingFolder.exists()) |
229 | |
{ |
230 | 0 | if (logger.isInfoEnabled()) |
231 | |
{ |
232 | 0 | logger.info("Creates " + archiveTmpReceivingFolder.getAbsolutePath()); |
233 | |
} |
234 | 0 | if (!archiveTmpReceivingFolder.mkdirs()) |
235 | 0 | throw new IOException("Failed to create archive-tmp-receiving-folder: " |
236 | |
+ archiveTmpReceivingFolder); |
237 | |
} |
238 | |
|
239 | 0 | File archiveTmpSendingFolder = FileUtils.newFile(archive + '/' + archiveTmpSendingDir); |
240 | 0 | File archiveTmpSendingFile = FileUtils.newFile(archive + '/' + archiveTmpSendingDir, fileNamePart); |
241 | 0 | if (!archiveTmpSendingFolder.exists()) |
242 | |
{ |
243 | 0 | if (logger.isInfoEnabled()) |
244 | |
{ |
245 | 0 | logger.info("Creates " + archiveTmpSendingFolder.getAbsolutePath()); |
246 | |
} |
247 | 0 | if (!archiveTmpSendingFolder.mkdirs()) |
248 | 0 | throw new IOException("Failed to create archive-tmp-sending-folder: " |
249 | |
+ archiveTmpSendingFolder); |
250 | |
} |
251 | |
|
252 | 0 | if (logger.isInfoEnabled()) |
253 | |
{ |
254 | 0 | logger.info("Copy SftpInputStream to archiveTmpReceivingFile... " |
255 | |
+ archiveTmpReceivingFile.getAbsolutePath()); |
256 | |
} |
257 | 0 | sftpUtil.copyStreamToFile(is, archiveTmpReceivingFile); |
258 | |
|
259 | |
|
260 | |
|
261 | 0 | if (logger.isInfoEnabled()) |
262 | |
{ |
263 | 0 | logger.info("Move archiveTmpReceivingFile (" + archiveTmpReceivingFile |
264 | |
+ ") to archiveTmpSendingFile (" + archiveTmpSendingFile + ")..."); |
265 | |
} |
266 | 0 | FileUtils.moveFile(archiveTmpReceivingFile, archiveTmpSendingFile); |
267 | |
|
268 | 0 | if (logger.isDebugEnabled()) |
269 | |
{ |
270 | 0 | logger.debug("Return SftpFileArchiveInputStream for archiveTmpSendingFile (" |
271 | |
+ archiveTmpSendingFile + ")..."); |
272 | |
} |
273 | 0 | return new SftpFileArchiveInputStream(archiveTmpSendingFile, archiveFile); |
274 | |
} |
275 | |
|
276 | |
private InputStream archiveFile(InputStream is, File archiveFile) throws IOException |
277 | |
{ |
278 | 0 | File archiveFolder = FileUtils.newFile(archiveFile.getParentFile().getPath()); |
279 | 0 | if (!archiveFolder.exists()) |
280 | |
{ |
281 | 0 | if (logger.isInfoEnabled()) |
282 | |
{ |
283 | 0 | logger.info("Creates " + archiveFolder.getAbsolutePath()); |
284 | |
} |
285 | 0 | if (!archiveFolder.mkdirs()) |
286 | 0 | throw new IOException("Failed to create archive-folder: " + archiveFolder); |
287 | |
} |
288 | |
|
289 | 0 | if (logger.isInfoEnabled()) |
290 | |
{ |
291 | 0 | logger.info("Copy SftpInputStream to archiveFile... " + archiveFile.getAbsolutePath()); |
292 | |
} |
293 | 0 | sftpUtil.copyStreamToFile(is, archiveFile); |
294 | |
|
295 | 0 | if (logger.isDebugEnabled()) |
296 | |
{ |
297 | 0 | logger.debug("*** Return SftpFileArchiveInputStream for archiveFile..."); |
298 | |
} |
299 | 0 | return new SftpFileArchiveInputStream(archiveFile); |
300 | |
} |
301 | |
|
302 | |
|
303 | |
|
304 | |
|
305 | |
|
306 | |
|
307 | |
|
308 | |
|
309 | |
|
310 | |
|
311 | |
|
312 | |
|
313 | |
|
314 | |
|
315 | |
|
316 | |
private boolean hasChanged(String fileName, SftpClient client, long fileAge, long sizeCheckDelayMs) |
317 | |
throws Exception |
318 | |
{ |
319 | |
|
320 | |
|
321 | |
|
322 | |
|
323 | 0 | if (fileAge > 0) |
324 | |
{ |
325 | 0 | long lastModifiedTime = client.getLastModifiedTime(fileName); |
326 | |
|
327 | 0 | long now = System.currentTimeMillis(); |
328 | 0 | long diff = now - lastModifiedTime; |
329 | |
|
330 | |
|
331 | 0 | if (diff < fileAge) |
332 | |
{ |
333 | 0 | if (logger.isDebugEnabled()) |
334 | |
{ |
335 | 0 | logger.debug("The file has not aged enough yet, will return nothing for: " + fileName |
336 | |
+ ". The file must be " + (fileAge - diff) + "ms older, was " + diff); |
337 | |
} |
338 | 0 | return true; |
339 | |
} |
340 | 0 | if (logger.isDebugEnabled()) |
341 | |
{ |
342 | 0 | logger.debug("The file " + fileName + " has aged enough. Was " + diff); |
343 | |
} |
344 | |
} |
345 | |
|
346 | |
|
347 | |
|
348 | |
|
349 | |
|
350 | 0 | if (sizeCheckDelayMs > 0) |
351 | |
{ |
352 | 0 | logger.info("Perform size check with a delay of: " + sizeCheckDelayMs + " ms."); |
353 | 0 | long fileSize1 = client.getSize(fileName); |
354 | 0 | Thread.sleep(sizeCheckDelayMs); |
355 | 0 | long fileSize2 = client.getSize(fileName); |
356 | |
|
357 | 0 | if (fileSize1 == fileSize2) |
358 | |
{ |
359 | 0 | logger.info("File is stable (not growing), ready for retrieval: " + fileName); |
360 | |
} |
361 | |
else |
362 | |
{ |
363 | 0 | logger.info("File is growing, deferring retrieval: " + fileName); |
364 | 0 | return true; |
365 | |
} |
366 | |
} |
367 | |
|
368 | |
|
369 | 0 | return false; |
370 | |
} |
371 | |
} |