1
2
3
4
5
6
7
8
9
10
11 package org.mule.transport.sftp;
12
13 import org.apache.commons.logging.Log;
14 import org.apache.commons.logging.LogFactory;
15 import org.mule.api.endpoint.ImmutableEndpoint;
16 import org.mule.transport.sftp.notification.SftpNotifier;
17 import org.mule.util.FileUtils;
18
19 import java.io.File;
20 import java.io.FilenameFilter;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.util.ArrayList;
24 import java.util.List;
25
26
27
28
29
30
31
32 public class SftpReceiverRequesterUtil
33 {
34 private transient Log logger = LogFactory.getLog(getClass());
35
36 private final SftpConnector connector;
37 private final ImmutableEndpoint endpoint;
38 private final FilenameFilter filenameFilter;
39 private final SftpUtil sftpUtil;
40
41 public SftpReceiverRequesterUtil(ImmutableEndpoint endpoint)
42 {
43 this.endpoint = endpoint;
44 this.connector = (SftpConnector) endpoint.getConnector();
45
46 sftpUtil = new SftpUtil(endpoint);
47
48 if (endpoint.getFilter() instanceof FilenameFilter)
49 {
50 this.filenameFilter = (FilenameFilter) endpoint.getFilter();
51 }
52 else
53 {
54 this.filenameFilter = null;
55 }
56
57 }
58
59
60 public String[] getAvailableFiles(boolean onlyGetTheFirstOne) throws Exception
61 {
62
63
64
65
66 if (logger.isDebugEnabled())
67 {
68 logger.debug("Checking files at endpoint " + endpoint.getEndpointURI());
69 }
70
71 SftpClient client = null;
72
73 try
74 {
75 client = connector.createSftpClient(endpoint);
76
77 long fileAge = connector.getFileAge();
78 boolean checkFileAge = connector.getCheckFileAge();
79
80
81 if (endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE) != null)
82 {
83 checkFileAge = true;
84 fileAge = Long.valueOf((String) endpoint.getProperty(SftpConnector.PROPERTY_FILE_AGE));
85 }
86
87 logger.debug("fileAge : " + fileAge);
88
89
90 long sizeCheckDelayMs = sftpUtil.getSizeCheckWaitTime();
91
92 String[] files = client.listFiles();
93
94
95
96 List<String> completedFiles = new ArrayList<String>(files.length);
97
98 for (String file : files)
99 {
100
101
102
103
104
105
106
107 if (filenameFilter != null && !filenameFilter.accept(null, file))
108 {
109 continue;
110 }
111
112 if (checkFileAge || sizeCheckDelayMs >= 0)
113 {
114
115
116 if (!hasChanged(file, client, fileAge, sizeCheckDelayMs))
117 {
118
119
120
121
122 completedFiles.add(file);
123 if (onlyGetTheFirstOne)
124 {
125 break;
126 }
127 }
128 }
129 else
130 {
131 completedFiles.add(file);
132 if (onlyGetTheFirstOne)
133 {
134 break;
135 }
136 }
137 }
138 return completedFiles.toArray(new String[completedFiles.size()]);
139 }
140 finally
141 {
142 if (client != null)
143 {
144 connector.releaseClient(endpoint, client);
145 }
146 }
147 }
148
149 public InputStream retrieveFile(String fileName, SftpNotifier notifier) throws Exception
150 {
151
152 SftpClient client = connector.createSftpClient(endpoint, notifier);
153
154
155 String tmpSendingDir = sftpUtil.getTempDirInbound();
156 if (tmpSendingDir != null)
157 {
158
159 boolean addUniqueSuffix = sftpUtil.isUseTempFileTimestampSuffix();
160
161
162 client.createSftpDirIfNotExists(endpoint, tmpSendingDir);
163 String tmpSendingFileName = tmpSendingDir + "/" + fileName;
164
165 if (addUniqueSuffix)
166 {
167 tmpSendingFileName = sftpUtil.createUniqueSuffix(tmpSendingFileName);
168 }
169 String fullTmpSendingPath = endpoint.getEndpointURI().getPath() + "/" + tmpSendingFileName;
170
171 if (logger.isDebugEnabled())
172 {
173 logger.debug("Move " + fileName + " to " + fullTmpSendingPath);
174 }
175 client.rename(fileName, fullTmpSendingPath);
176 fileName = tmpSendingFileName;
177 if (logger.isDebugEnabled())
178 {
179 logger.debug("Move done");
180 }
181 }
182
183
184 String archive = sftpUtil.getArchiveDir();
185
186
187 InputStream fileInputStream = client.retrieveFile(fileName);
188
189 if (!"".equals(archive))
190 {
191 String archiveTmpReceivingDir = sftpUtil.getArchiveTempReceivingDir();
192 String archiveTmpSendingDir = sftpUtil.getArchiveTempSendingDir();
193
194 InputStream is = new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(),
195 endpoint);
196
197
198 int idx = fileName.lastIndexOf('/');
199 String fileNamePart = fileName.substring(idx + 1);
200
201
202 File archiveFile = FileUtils.newFile(archive, fileNamePart);
203
204
205 if ("".equals(archiveTmpReceivingDir) || "".equals(archiveTmpSendingDir))
206 {
207 return archiveFile(is, archiveFile);
208 }
209 else
210 {
211 return archiveFileUsingTempDirs(archive, archiveTmpReceivingDir, archiveTmpSendingDir, is,
212 fileNamePart, archiveFile);
213 }
214 }
215
216
217
218
219 return new SftpInputStream(client, fileInputStream, fileName, connector.isAutoDelete(), endpoint);
220 }
221
222 private InputStream archiveFileUsingTempDirs(String archive,
223 String archiveTmpReceivingDir,
224 String archiveTmpSendingDir,
225 InputStream is,
226 String fileNamePart,
227 File archiveFile) throws IOException
228 {
229
230 File archiveTmpReceivingFolder = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir);
231 File archiveTmpReceivingFile = FileUtils.newFile(archive + '/' + archiveTmpReceivingDir, fileNamePart);
232 if (!archiveTmpReceivingFolder.exists())
233 {
234 if (logger.isInfoEnabled())
235 {
236 logger.info("Creates " + archiveTmpReceivingFolder.getAbsolutePath());
237 }
238 if (!archiveTmpReceivingFolder.mkdirs())
239 throw new IOException("Failed to create archive-tmp-receiving-folder: "
240 + archiveTmpReceivingFolder);
241 }
242
243 File archiveTmpSendingFolder = FileUtils.newFile(archive + '/' + archiveTmpSendingDir);
244 File archiveTmpSendingFile = FileUtils.newFile(archive + '/' + archiveTmpSendingDir, fileNamePart);
245 if (!archiveTmpSendingFolder.exists())
246 {
247 if (logger.isInfoEnabled())
248 {
249 logger.info("Creates " + archiveTmpSendingFolder.getAbsolutePath());
250 }
251 if (!archiveTmpSendingFolder.mkdirs())
252 throw new IOException("Failed to create archive-tmp-sending-folder: "
253 + archiveTmpSendingFolder);
254 }
255
256 if (logger.isInfoEnabled())
257 {
258 logger.info("Copy SftpInputStream to archiveTmpReceivingFile... "
259 + archiveTmpReceivingFile.getAbsolutePath());
260 }
261 sftpUtil.copyStreamToFile(is, archiveTmpReceivingFile);
262
263
264
265 if (logger.isInfoEnabled())
266 {
267 logger.info("Move archiveTmpReceivingFile (" + archiveTmpReceivingFile
268 + ") to archiveTmpSendingFile (" + archiveTmpSendingFile + ")...");
269 }
270 FileUtils.moveFile(archiveTmpReceivingFile, archiveTmpSendingFile);
271
272 if (logger.isDebugEnabled())
273 {
274 logger.debug("Return SftpFileArchiveInputStream for archiveTmpSendingFile ("
275 + archiveTmpSendingFile + ")...");
276 }
277 return new SftpFileArchiveInputStream(archiveTmpSendingFile, archiveFile);
278 }
279
280 private InputStream archiveFile(InputStream is, File archiveFile) throws IOException
281 {
282 File archiveFolder = FileUtils.newFile(archiveFile.getParentFile().getPath());
283 if (!archiveFolder.exists())
284 {
285 if (logger.isInfoEnabled())
286 {
287 logger.info("Creates " + archiveFolder.getAbsolutePath());
288 }
289 if (!archiveFolder.mkdirs())
290 throw new IOException("Failed to create archive-folder: " + archiveFolder);
291 }
292
293 if (logger.isInfoEnabled())
294 {
295 logger.info("Copy SftpInputStream to archiveFile... " + archiveFile.getAbsolutePath());
296 }
297 sftpUtil.copyStreamToFile(is, archiveFile);
298
299 if (logger.isDebugEnabled())
300 {
301 logger.debug("*** Return SftpFileArchiveInputStream for archiveFile...");
302 }
303 return new SftpFileArchiveInputStream(archiveFile);
304 }
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320 private boolean hasChanged(String fileName, SftpClient client, long fileAge, long sizeCheckDelayMs)
321 throws Exception
322 {
323
324
325
326
327 if (fileAge > 0)
328 {
329 long lastModifiedTime = client.getLastModifiedTime(fileName);
330
331 long now = System.currentTimeMillis();
332 long diff = now - lastModifiedTime;
333
334
335 if (diff < fileAge)
336 {
337 if (logger.isDebugEnabled())
338 {
339 logger.debug("The file has not aged enough yet, will return nothing for: " + fileName
340 + ". The file must be " + (fileAge - diff) + "ms older, was " + diff);
341 }
342 return true;
343 }
344 if (logger.isDebugEnabled())
345 {
346 logger.debug("The file " + fileName + " has aged enough. Was " + diff);
347 }
348 }
349
350
351
352
353
354 if (sizeCheckDelayMs > 0)
355 {
356 logger.info("Perform size check with a delay of: " + sizeCheckDelayMs + " ms.");
357 long fileSize1 = client.getSize(fileName);
358 Thread.sleep(sizeCheckDelayMs);
359 long fileSize2 = client.getSize(fileName);
360
361 if (fileSize1 == fileSize2)
362 {
363 logger.info("File is stable (not growing), ready for retrieval: " + fileName);
364 }
365 else
366 {
367 logger.info("File is growing, deferring retrieval: " + fileName);
368 return true;
369 }
370 }
371
372
373 return false;
374 }
375 }