Coverage Report - org.mule.transport.sftp.SftpConnector
 
Classes in this File Line Coverage Branch Coverage Complexity
SftpConnector
0%
0/151
0%
0/48
0
 
 1  
 /*
 2  
  * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
 3  
  * The software in this package is published under the terms of the CPAL v1.0
 4  
  * license, a copy of which has been included with this distribution in the
 5  
  * LICENSE.txt file.
 6  
  */
 7  
 package org.mule.transport.sftp;
 8  
 
 9  
 import org.mule.api.MuleContext;
 10  
 import org.mule.api.MuleException;
 11  
 import org.mule.api.construct.FlowConstruct;
 12  
 import org.mule.api.endpoint.EndpointURI;
 13  
 import org.mule.api.endpoint.ImmutableEndpoint;
 14  
 import org.mule.api.endpoint.InboundEndpoint;
 15  
 import org.mule.api.lifecycle.InitialisationException;
 16  
 import org.mule.api.transport.ConnectorException;
 17  
 import org.mule.api.transport.MessageReceiver;
 18  
 import org.mule.config.i18n.CoreMessages;
 19  
 import org.mule.transport.AbstractConnector;
 20  
 import org.mule.transport.file.ExpressionFilenameParser;
 21  
 import org.mule.transport.file.FilenameParser;
 22  
 import org.mule.transport.sftp.notification.SftpNotifier;
 23  
 
 24  
 import java.util.HashMap;
 25  
 import java.util.Map;
 26  
 
 27  
 import org.apache.commons.logging.Log;
 28  
 import org.apache.commons.logging.LogFactory;
 29  
 import org.apache.commons.pool.ObjectPool;
 30  
 import org.apache.commons.pool.impl.GenericObjectPool;
 31  
 
 32  
 /**
 33  
  * <code>SftpConnector</code> sends and receives file messages over sftp using jsch
 34  
  * library Improves on SFTP with VFS Connector in the following ways: 1. Streams
 35  
  * files instead of reading them into memory. The SftpMessageReceiver is a
 36  
  * "non-materializing stream receiver" which does not read the file to memory. The
 37  
  * SftpMessageDispatcher also never materializes the stream and delegates the jsch
 38  
  * library for materialization. 3. Uses jsch library directly instead of using VFS as
 39  
  * middle-man. 3. More explicit connection lifefecyle management. 4. Leverages sftp
 40  
  * stat to determine if a file size changes (simpler and also less memory intensive)
 41  
  */
 42  
 public class SftpConnector extends AbstractConnector
 43  
 {
 44  
 
 45  
     public static final String PROPERTY_POLLING_FREQUENCY = "pollingFrequency";
 46  
     public static final String PROPERTY_DIRECTORY = "directory";
 47  
     public static final String PROPERTY_OUTPUT_PATTERN = "outputPattern";
 48  
     public static final String PROPERTY_FILENAME = "filename";
 49  
     public static final String PROPERTY_ORIGINAL_FILENAME = "originalFilename";
 50  
     public static final String PROPERTY_SELECT_EXPRESSION = "selectExpression";
 51  
     public static final String PROPERTY_FILE_EXTENSION = "fileExtension";
 52  
     public static final String PROPERTY_INCLUDE_SUBFOLDERS = "includeSubfolders";
 53  
     public static final String PROPERTY_IDENTITY_FILE = "identityFile";
 54  
     public static final String PROPERTY_PASS_PHRASE = "passphrase";
 55  
     public static final String PROPERTY_FILE_AGE = "fileAge";
 56  
     public static final String PROPERTY_TEMP_DIR = "tempDir";
 57  
     public static final String PROPERTY_SIZE_CHECK_WAIT_TIME = "sizeCheckWaitTime";
 58  
     public static final String PROPERTY_ARCHIVE_DIR = "archiveDir";
 59  
     public static final String PROPERTY_ARCHIVE_TEMP_RECEIVING_DIR = "archiveTempReceivingDir";
 60  
     public static final String PROPERTY_ARCHIVE_TEMP_SENDING_DIR = "archiveTempSendingDir";
 61  
     public static final String PROPERTY_DUPLICATE_HANDLING = "duplicateHandling";
 62  
     public static final String PROPERTY_USE_TEMP_FILE_TIMESTAMP_SUFFIX = "useTempFileTimestampSuffix";
 63  
     public static final String PROPERTY_DUPLICATE_HANDLING_THROW_EXCEPTION = "throwException";
 64  
     public static final String PROPERTY_DUPLICATE_HANDLING_OVERWRITE = "overwrite";
 65  
     public static final String PROPERTY_DUPLICATE_HANDLING_ASS_SEQ_NO = "addSeqNo";
 66  
     public static final String PROPERTY_MAX_CONNECTION_POOL_SIZE = "maxConnectionPoolSize";
 67  
     public static final String PROPERTY_KEEP_FILE_ON_ERROR = "keepFileOnError";
 68  
 
 69  
     public static final int DEFAULT_POLLING_FREQUENCY = 1000;
 70  
 
 71  
     /**
 72  
      * logger used by this class
 73  
      */
 74  0
     protected final static Log logger = LogFactory.getLog(SftpConnector.class);
 75  
 
 76  0
     private FilenameParser filenameParser = new ExpressionFilenameParser();
 77  
 
 78  
     private long pollingFrequency;
 79  0
     private boolean autoDelete = true;
 80  
     private String outputPattern;
 81  
 
 82  
     private String identityFile;
 83  
     private String passphrase;
 84  
 
 85  0
     private boolean checkFileAge = false;
 86  0
     private long fileAge = 0;
 87  
 
 88  0
     private String tempDirInbound = null;
 89  0
     private String tempDirOutbound = null;
 90  
 
 91  0
     private Map<EndpointURI, GenericObjectPool> pools = new HashMap<EndpointURI, GenericObjectPool>();
 92  
 
 93  0
     private String duplicateHandling = null;
 94  0
     private Boolean useTempFileTimestampSuffix = null;
 95  0
     private Long sizeCheckWaitTime = null;
 96  0
     private String archiveDir = "";
 97  0
     private String archiveTempReceivingDir = "";
 98  0
     private String archiveTempSendingDir = "";
 99  
 
 100  
     /**
 101  
      * Should the file be kept if an error occurs when writing the file on the
 102  
      * outbound endpoint?
 103  
      */
 104  
     private Boolean keepFileOnError;
 105  
 
 106  
     /**
 107  
      * max pool size. 0 for no pool, -1 for no limit, otherwise the specified value
 108  
      */
 109  
     private int maxConnectionPoolSize;
 110  
 
 111  
     /**
 112  
      * Value that can be set via the System property
 113  
      * 'mule.sftp.transport.maxConnectionPoolSize'. If it's set the value is used
 114  
      * instead of <i>maxConnectionPoolSize</i>
 115  
      */
 116  
     private static final Integer overrideMaxConnectionPoolSize;
 117  
 
 118  
     static
 119  
     {
 120  0
         String propValue = System.getProperty("mule.sftp.transport.maxConnectionPoolSize");
 121  0
         if (propValue != null)
 122  
         {
 123  0
             logger.info("Will override the maxConnectionPoolSize to " + propValue
 124  
                         + " from the system property 'mule.sftp.transport.maxConnectionPoolSize'.");
 125  0
             overrideMaxConnectionPoolSize = Integer.parseInt(propValue);
 126  
         }
 127  
         else
 128  
         {
 129  0
             overrideMaxConnectionPoolSize = null;
 130  
         }
 131  0
     }
 132  
 
 133  
     public SftpConnector(MuleContext context)
 134  
     {
 135  0
         super(context);
 136  0
         filenameParser = new ExpressionFilenameParser();
 137  0
     }
 138  
 
 139  
     public String getProtocol()
 140  
     {
 141  0
         return "sftp";
 142  
     }
 143  
 
 144  
     @Override
 145  
     public MessageReceiver createReceiver(FlowConstruct flow, InboundEndpoint endpoint) throws Exception
 146  
     {
 147  0
         long polling = pollingFrequency;
 148  
 
 149  
         // Override properties on the endpoint for the specific endpoint
 150  0
         String tempPolling = (String) endpoint.getProperty(PROPERTY_POLLING_FREQUENCY);
 151  0
         if (tempPolling != null)
 152  
         {
 153  0
             polling = Long.parseLong(tempPolling);
 154  
         }
 155  
 
 156  0
         if (polling <= 0)
 157  
         {
 158  0
             polling = DEFAULT_POLLING_FREQUENCY;
 159  
         }
 160  0
         if (logger.isDebugEnabled())
 161  
         {
 162  0
             logger.debug("Set polling frequency to: " + polling);
 163  
         }
 164  
 
 165  0
         return serviceDescriptor.createMessageReceiver(this, flow, endpoint, new Object[]{polling});
 166  
     }
 167  
 
 168  
     public SftpClient createSftpClient(ImmutableEndpoint endpoint) throws Exception
 169  
     {
 170  0
         return createSftpClient(endpoint, null);
 171  
     }
 172  
 
 173  
     public SftpClient createSftpClient(ImmutableEndpoint endpoint, SftpNotifier notifier) throws Exception
 174  
     {
 175  0
         SftpClient client = null;
 176  0
         boolean ok = false;
 177  
 
 178  
         try
 179  
         {
 180  0
             if (useConnectionPool())
 181  
             {
 182  0
                 ObjectPool pool = getClientPool(endpoint);
 183  0
                 client = (SftpClient) pool.borrowObject();
 184  0
             }
 185  
             else
 186  
             {
 187  0
                 client = SftpConnectionFactory.createClient(endpoint);
 188  
             }
 189  
 
 190  
             // We have to set the working directory before returning
 191  0
             String dir = endpoint.getEndpointURI().getPath();
 192  0
             client.changeWorkingDirectory(dir);
 193  0
             if (logger.isDebugEnabled())
 194  
             {
 195  0
                 logger.debug("Successfully changed working directory to: " + dir);
 196  
             }
 197  
 
 198  
             // TODO ML: Is this always necessary?
 199  0
             client.setNotifier(notifier);
 200  
 
 201  0
             ok = true;
 202  
 
 203  
         }
 204  
         finally
 205  
         {
 206  
             // Release the client if it was created but something failed after that,
 207  
             // otherwise we start to waste ssh-processes...
 208  0
             if (!ok && client != null)
 209  
             {
 210  0
                 releaseClient(endpoint, client);
 211  
             }
 212  
         }
 213  
 
 214  0
         return client;
 215  
     }
 216  
 
 217  
     /**
 218  
      * @return True if connection pooling is used, otherwise false
 219  
      */
 220  
     public boolean useConnectionPool()
 221  
     {
 222  0
         return getMaxConnectionPoolSize() != 0;
 223  
     }
 224  
 
 225  
     public void releaseClient(ImmutableEndpoint endpoint, SftpClient client) throws Exception
 226  
     {
 227  0
         if (useConnectionPool())
 228  
         {
 229  0
             if (getDispatcherFactory().isCreateDispatcherPerRequest())
 230  
             {
 231  0
                 destroyClient(endpoint, client);
 232  
             }
 233  
             else
 234  
             {
 235  0
                 if (client != null && client.isConnected())
 236  
                 {
 237  0
                     ObjectPool pool = getClientPool(endpoint);
 238  0
                     if (logger.isDebugEnabled())
 239  
                     {
 240  0
                         logger.debug("Releasing connection for endpoint " + endpoint.getEndpointURI());
 241  
                     }
 242  0
                     pool.returnObject(client);
 243  0
                 }
 244  
             }
 245  
         }
 246  
         else
 247  
         {
 248  0
             client.disconnect();
 249  
         }
 250  0
     }
 251  
 
 252  
     public void destroyClient(ImmutableEndpoint endpoint, SftpClient client) throws Exception
 253  
     {
 254  0
         if (useConnectionPool())
 255  
         {
 256  0
             if ((client != null) && (client.isConnected()))
 257  
             {
 258  0
                 ObjectPool pool = getClientPool(endpoint);
 259  0
                 pool.invalidateObject(client);
 260  
             }
 261  
         }
 262  0
     }
 263  
 
 264  
     protected synchronized ObjectPool getClientPool(ImmutableEndpoint endpoint)
 265  
     {
 266  0
         GenericObjectPool pool = pools.get(endpoint.getEndpointURI());
 267  
 
 268  0
         if (pool == null)
 269  
         {
 270  0
             if (logger.isDebugEnabled())
 271  
             {
 272  0
                 logger.debug("Pool is null - creating one for endpoint " + endpoint.getEndpointURI()
 273  
                              + " with max size " + getMaxConnectionPoolSize());
 274  
             }
 275  0
             pool = new GenericObjectPool(new SftpConnectionFactory(endpoint), getMaxConnectionPoolSize());
 276  0
             pool.setTestOnBorrow(isValidateConnections());
 277  0
             pools.put(endpoint.getEndpointURI(), pool);
 278  
         }
 279  
         else
 280  
         {
 281  0
             if (logger.isDebugEnabled())
 282  
             {
 283  0
                 logger.debug("Using existing pool for endpoint " + endpoint.getEndpointURI() + ". Active: "
 284  
                              + pool.getNumActive() + ", Idle:" + pool.getNumIdle());
 285  
             }
 286  
         }
 287  
 
 288  0
         return pool;
 289  
     }
 290  
 
 291  
     /*
 292  
      * (non-Javadoc)
 293  
      * @see org.mule.transport.AbstractConnector#doConnect()
 294  
      */
 295  
     protected void doConnect() throws Exception
 296  
     {
 297  
         // Do nothing!
 298  0
     }
 299  
 
 300  
     /*
 301  
      * (non-Javadoc)
 302  
      * @see org.mule.transport.AbstractConnector#doDisconnect()
 303  
      */
 304  
     protected void doDisconnect() throws Exception
 305  
     {
 306  
         // Do nothing!
 307  0
     }
 308  
 
 309  
     /*
 310  
      * (non-Javadoc)
 311  
      * @see org.mule.transport.AbstractConnector#doDispose()
 312  
      */
 313  
     protected void doDispose()
 314  
     {
 315  
         // Do nothing!
 316  0
     }
 317  
 
 318  
     /*
 319  
      * (non-Javadoc)
 320  
      * @see org.mule.transport.AbstractConnector#doInitialise()
 321  
      */
 322  
     protected void doInitialise() throws InitialisationException
 323  
     {
 324  0
         if (filenameParser != null)
 325  
         {
 326  0
             filenameParser.setMuleContext(muleContext);
 327  
         }
 328  0
     }
 329  
 
 330  
     /*
 331  
      * (non-Javadoc)
 332  
      * @see org.mule.transport.AbstractConnector#doStart()
 333  
      */
 334  
     protected void doStart() throws MuleException
 335  
     {
 336  
         // Do nothing!
 337  0
     }
 338  
 
 339  
     /*
 340  
      * (non-Javadoc)
 341  
      * @see org.mule.transport.AbstractConnector#doStop()
 342  
      */
 343  
     protected void doStop() throws MuleException
 344  
     {
 345  0
         if (logger.isDebugEnabled())
 346  
         {
 347  0
             logger.debug("Stopping all pools");
 348  
         }
 349  
         try
 350  
         {
 351  0
             for (ObjectPool pool : pools.values())
 352  
             {
 353  0
                 pool.close();
 354  
             }
 355  
         }
 356  0
         catch (Exception e)
 357  
         {
 358  0
             throw new ConnectorException(CoreMessages.failedToStop("SFTP Connector"), this, e);
 359  
         }
 360  
         finally
 361  
         {
 362  0
             pools.clear();
 363  0
         }
 364  0
     }
 365  
 
 366  
     public long getPollingFrequency()
 367  
     {
 368  0
         return pollingFrequency;
 369  
     }
 370  
 
 371  
     public void setPollingFrequency(long pollingFrequency)
 372  
     {
 373  0
         this.pollingFrequency = pollingFrequency;
 374  0
     }
 375  
 
 376  
     public FilenameParser getFilenameParser()
 377  
     {
 378  0
         return filenameParser;
 379  
     }
 380  
 
 381  
     public void setFilenameParser(FilenameParser filenameParser)
 382  
     {
 383  0
         this.filenameParser = filenameParser;
 384  0
         if (filenameParser != null)
 385  
         {
 386  0
             filenameParser.setMuleContext(muleContext);
 387  
         }
 388  0
     }
 389  
 
 390  
     public String getOutputPattern()
 391  
     {
 392  0
         return outputPattern;
 393  
     }
 394  
 
 395  
     public void setOutputPattern(String outputPattern)
 396  
     {
 397  0
         this.outputPattern = outputPattern;
 398  0
     }
 399  
 
 400  
     public boolean isAutoDelete()
 401  
     {
 402  0
         return autoDelete;
 403  
     }
 404  
 
 405  
     public void setAutoDelete(boolean autoDelete)
 406  
     {
 407  0
         this.autoDelete = autoDelete;
 408  0
     }
 409  
 
 410  
     public String getIdentityFile()
 411  
     {
 412  0
         return identityFile;
 413  
     }
 414  
 
 415  
     public void setIdentityFile(String identityFile)
 416  
     {
 417  0
         this.identityFile = identityFile;
 418  0
     }
 419  
 
 420  
     public String getPassphrase()
 421  
     {
 422  0
         return passphrase;
 423  
     }
 424  
 
 425  
     public void setPassphrase(String passphrase)
 426  
     {
 427  0
         this.passphrase = passphrase;
 428  0
     }
 429  
 
 430  
     /**
 431  
      * Returns the file age.
 432  
      * 
 433  
      * @return Returns the fileAge in milliseconds.
 434  
      */
 435  
     public long getFileAge()
 436  
     {
 437  0
         return fileAge;
 438  
     }
 439  
 
 440  
     /**
 441  
      * Sets the file age.
 442  
      * 
 443  
      * @param fileAge the fileAge in milliseconds to set.
 444  
      */
 445  
     public void setFileAge(long fileAge)
 446  
     {
 447  0
         this.fileAge = fileAge;
 448  0
         this.checkFileAge = true;
 449  0
     }
 450  
 
 451  
     public boolean getCheckFileAge()
 452  
     {
 453  0
         return checkFileAge;
 454  
     }
 455  
 
 456  
     public String getTempDirInbound()
 457  
     {
 458  0
         return tempDirInbound;
 459  
     }
 460  
 
 461  
     public void setTempDirInbound(String pTempDirInbound)
 462  
     {
 463  0
         tempDirInbound = pTempDirInbound;
 464  0
     }
 465  
 
 466  
     public String getTempDirOutbound()
 467  
     {
 468  0
         return tempDirOutbound;
 469  
     }
 470  
 
 471  
     public void setTempDirOutbound(String pTempDirOutbound)
 472  
     {
 473  0
         tempDirOutbound = pTempDirOutbound;
 474  0
     }
 475  
 
 476  
     // Need this method to be public for SftpNotifier
 477  
     @Override
 478  
     public boolean isEnableMessageEvents()
 479  
     {
 480  0
         return super.isEnableMessageEvents();
 481  
     }
 482  
 
 483  
     public void setDuplicateHandling(String duplicateHandling)
 484  
     {
 485  0
         this.duplicateHandling = duplicateHandling;
 486  0
     }
 487  
 
 488  
     public String getDuplicateHandling()
 489  
     {
 490  0
         return duplicateHandling;
 491  
     }
 492  
 
 493  
     public void setUseTempFileTimestampSuffix(Boolean useTempFileTimestampSuffix)
 494  
     {
 495  0
         this.useTempFileTimestampSuffix = useTempFileTimestampSuffix;
 496  0
     }
 497  
 
 498  
     public Boolean isUseTempFileTimestampSuffix()
 499  
     {
 500  0
         return useTempFileTimestampSuffix;
 501  
     }
 502  
 
 503  
     public void setSizeCheckWaitTime(Long sizeCheckWaitTime)
 504  
     {
 505  0
         this.sizeCheckWaitTime = sizeCheckWaitTime;
 506  0
     }
 507  
 
 508  
     public Long getSizeCheckWaitTime()
 509  
     {
 510  0
         return sizeCheckWaitTime;
 511  
     }
 512  
 
 513  
     public void setArchiveDir(String archiveDir)
 514  
     {
 515  0
         this.archiveDir = archiveDir;
 516  0
     }
 517  
 
 518  
     public String getArchiveDir()
 519  
     {
 520  0
         return archiveDir;
 521  
     }
 522  
 
 523  
     public void setArchiveTempReceivingDir(String archiveTempReceivingDir)
 524  
     {
 525  0
         this.archiveTempReceivingDir = archiveTempReceivingDir;
 526  0
     }
 527  
 
 528  
     public String getArchiveTempReceivingDir()
 529  
     {
 530  0
         return archiveTempReceivingDir;
 531  
     }
 532  
 
 533  
     public void setArchiveTempSendingDir(String archiveTempSendingDir)
 534  
     {
 535  0
         this.archiveTempSendingDir = archiveTempSendingDir;
 536  0
     }
 537  
 
 538  
     public String getArchiveTempSendingDir()
 539  
     {
 540  0
         return archiveTempSendingDir;
 541  
     }
 542  
 
 543  
     /**
 544  
      * @see SftpConnector#maxConnectionPoolSize
 545  
      */
 546  
     public void setMaxConnectionPoolSize(int maxConnectionPoolSize)
 547  
     {
 548  0
         this.maxConnectionPoolSize = maxConnectionPoolSize;
 549  0
     }
 550  
 
 551  
     /**
 552  
      * @return the max connection pool size. If the system parameter
 553  
      *         mule.sftp.transport.maxConnectionPoolSize is set, that value will be
 554  
      *         used instead.
 555  
      */
 556  
     public int getMaxConnectionPoolSize()
 557  
     {
 558  0
         if (overrideMaxConnectionPoolSize != null)
 559  
         {
 560  0
             return overrideMaxConnectionPoolSize;
 561  
         }
 562  0
         return maxConnectionPoolSize;
 563  
     }
 564  
 
 565  
     public Boolean isKeepFileOnError()
 566  
     {
 567  0
         return keepFileOnError;
 568  
     }
 569  
 
 570  
     public void setKeepFileOnError(Boolean pKeepFileOnError)
 571  
     {
 572  0
         keepFileOnError = pKeepFileOnError;
 573  0
     }
 574  
 }