View Javadoc
1   /*
2    * Copyright (c) MuleSoft, Inc.  All rights reserved.  http://www.mulesoft.com
3    * The software in this package is published under the terms of the CPAL v1.0
4    * license, a copy of which has been included with this distribution in the
5    * LICENSE.txt file.
6    */
7   package org.mule.routing.correlation;
8   
9   import org.mule.api.MuleContext;
10  import org.mule.api.MuleEvent;
11  import org.mule.api.MuleException;
12  import org.mule.api.MuleMessageCollection;
13  import org.mule.api.construct.FlowConstruct;
14  import org.mule.api.lifecycle.Startable;
15  import org.mule.api.lifecycle.Stoppable;
16  import org.mule.api.processor.MessageProcessor;
17  import org.mule.api.routing.MessageInfoMapping;
18  import org.mule.api.routing.RoutingException;
19  import org.mule.api.service.Service;
20  import org.mule.config.i18n.CoreMessages;
21  import org.mule.context.notification.RoutingNotification;
22  import org.mule.routing.EventGroup;
23  import org.mule.util.StringMessageUtils;
24  import org.mule.util.concurrent.ThreadNameHelper;
25  import org.mule.util.monitor.Expirable;
26  import org.mule.util.monitor.ExpiryMonitor;
27  
28  import java.text.MessageFormat;
29  import java.util.ArrayList;
30  import java.util.List;
31  import java.util.Map;
32  
33  import edu.emory.mathcs.backport.java.util.concurrent.ConcurrentHashMap;
34  import edu.emory.mathcs.backport.java.util.concurrent.ConcurrentMap;
35  import edu.emory.mathcs.backport.java.util.concurrent.TimeUnit;
36  import org.apache.commons.collections.buffer.BoundedFifoBuffer;
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  
40  /**
41   */
42  public class EventCorrelator implements Startable, Stoppable
43  {
44      /**
45       * logger used by this class
46       */
47      protected transient final Log logger = LogFactory.getLog(EventCorrelator.class);
48  
49      public static final String NO_CORRELATION_ID = "no-id";
50  
51      public static final int MAX_PROCESSED_GROUPS = 50000;
52  
53      protected static final long MILLI_TO_NANO_MULTIPLIER = 1000000L;
54  
55      private static final long ONE_DAY_IN_MILLI = 1000 * 60 * 60 * 24;
56  
57      protected long groupTimeToLive = ONE_DAY_IN_MILLI;
58  
59      /**
60       * A map of EventGroup objects. These represent one or more messages to be
61       * agregated, keyed by message id. There will be one response message for every
62       * EventGroup.
63       */
64      protected final ConcurrentMap eventGroups = new ConcurrentHashMap();
65  
66      protected final Object groupsLock = new Object();
67  
68      // @GuardedBy groupsLock
69      protected final BoundedFifoBuffer processedGroups = new BoundedFifoBuffer(MAX_PROCESSED_GROUPS);
70  
71      private long timeout = -1; // undefined
72  
73      private boolean failOnTimeout = true;
74  
75      private MessageInfoMapping messageInfoMapping;
76  
77      private MuleContext muleContext;
78  
79      private EventCorrelatorCallback callback;
80  
81      private MessageProcessor timeoutMessageProcessor;
82  
83      /**
84       * A map with keys = group id and values = group creation time
85       */
86      private Map expiredAndDispatchedGroups = new ConcurrentHashMap();
87  
88      private EventCorrelator.ExpiringGroupMonitoringThread expiringGroupMonitoringThread;
89      private final String name;
90  
91      public EventCorrelator(EventCorrelatorCallback callback, MessageProcessor timeoutMessageProcessor, MessageInfoMapping messageInfoMapping, MuleContext muleContext, String flowConstructName)
92      {
93          if (callback == null)
94          {
95              throw new IllegalArgumentException(CoreMessages.objectIsNull("EventCorrelatorCallback").getMessage());
96          }
97          if (messageInfoMapping == null)
98          {
99              throw new IllegalArgumentException(CoreMessages.objectIsNull("MessageInfoMapping").getMessage());
100         }
101         if (muleContext == null)
102         {
103             throw new IllegalArgumentException(CoreMessages.objectIsNull("MuleContext").getMessage());
104         }
105         this.callback = callback;
106         this.messageInfoMapping = messageInfoMapping;
107         this.muleContext = muleContext;
108         this.timeoutMessageProcessor = timeoutMessageProcessor;
109         name = String.format("%s%s.event.correlator", ThreadNameHelper.getPrefix(muleContext), flowConstructName);
110     }
111 
112     public void forceGroupExpiry(String groupId)
113     {
114         if (eventGroups.get(groupId) != null)
115         {
116             handleGroupExpiry((EventGroup) eventGroups.get(groupId));
117         }
118         else
119         {
120             addProcessedGroup(groupId);
121         }
122     }
123 
124     public MuleEvent process(MuleEvent event) throws RoutingException
125     {
126         // the correlationId of the event's message
127         final String groupId = messageInfoMapping.getCorrelationId(event.getMessage());
128 
129         if (logger.isTraceEnabled())
130         {
131             try
132             {
133                 logger.trace(String.format("Received async reply message for correlationID: %s%n%s%n%s",
134                                            groupId,
135                                            StringMessageUtils.truncate(StringMessageUtils.toString(event.getMessage().getPayload()), 200, false),
136                                            StringMessageUtils.headersToString(event.getMessage())));
137             }
138             catch (Exception e)
139             {
140                 // ignore
141             }
142         }
143         if (groupId == null || groupId.equals("-1"))
144         {
145             throw new RoutingException(CoreMessages.noCorrelationId(), event, timeoutMessageProcessor);
146         }
147 
148         // indicates interleaved EventGroup removal (very rare)
149         boolean lookupMiss = false;
150 
151         // spinloop for the EventGroup lookup
152         while (true)
153         {
154             if (lookupMiss)
155             {
156                 try
157                 {
158                     // recommended over Thread.yield()
159                     Thread.sleep(1);
160                 }
161                 catch (InterruptedException interrupted)
162                 {
163                     Thread.currentThread().interrupt();
164                 }
165             }
166 
167             if (isGroupAlreadyProcessed(groupId))
168             {
169                 if (logger.isDebugEnabled())
170                 {
171                     logger.debug("An event was received for an event group that has already been processed, " +
172                                  "this is probably because the async-reply timed out. Correlation Id is: " + groupId +
173                                  ". Dropping event");
174                 }
175                 //Fire a notification to say we received this message
176                 muleContext.fireNotification(new RoutingNotification(event.getMessage(),
177                                                                      event.getEndpoint().getEndpointURI().toString(),
178                                                                      RoutingNotification.MISSED_AGGREGATION_GROUP_EVENT));
179                 return null;
180             }
181 
182             // check for an existing group first
183             EventGroup group = this.getEventGroup(groupId);
184 
185             // does the group exist?
186             if (group == null)
187             {
188                 // ..apparently not, so create a new one & add it
189                 group = this.addEventGroup(callback.createEventGroup(event, groupId));
190             }
191 
192             // ensure that only one thread at a time evaluates this EventGroup
193             synchronized (groupsLock)
194             {
195                 // make sure no other thread removed the group in the meantime
196                 if (group != this.getEventGroup(groupId))
197                 {
198                     // if that is the (rare) case, spin
199                     lookupMiss = true;
200                     continue;
201                 }
202 
203                 if (logger.isDebugEnabled())
204                 {
205                     logger.debug("Adding event to aggregator group: " + groupId);
206                 }
207 
208                 // add the incoming event to the group
209                 group.addEvent(event);
210 
211                 // check to see if the event group is ready to be aggregated
212                 if (callback.shouldAggregateEvents(group))
213                 {
214                     // create the response event
215                     MuleEvent returnEvent = callback.aggregateEvents(group);
216                     returnEvent.getMessage().setCorrelationId(groupId);
217 
218                     // remove the eventGroup as no further message will be received
219                     // for this group once we aggregate
220                     this.removeEventGroup(group);
221 
222                     return returnEvent;
223                 }
224                 else
225                 {
226                     return null;
227                 }
228             }
229         }
230     }
231 
232     protected EventGroup getEventGroup(String groupId)
233     {
234         return (EventGroup) eventGroups.get(groupId);
235     }
236 
237     protected EventGroup addEventGroup(EventGroup group)
238     {
239         EventGroup previous = (EventGroup) eventGroups.putIfAbsent(group.getGroupId(), group);
240         // a parallel thread might have removed the EventGroup already,
241         // therefore we need to validate our current reference
242         return (previous != null ? previous : group);
243     }
244 
245     protected void removeEventGroup(EventGroup group)
246     {
247         final Object groupId = group.getGroupId();
248         eventGroups.remove(groupId);
249         addProcessedGroup(groupId);
250     }
251 
252     protected void addProcessedGroup(Object id)
253     {
254         synchronized (groupsLock)
255         {
256             if (processedGroups.isFull())
257             {
258                 processedGroups.remove();
259             }
260             processedGroups.add(id);
261         }
262     }
263 
264     protected boolean isGroupAlreadyProcessed(Object id)
265     {
266         synchronized (groupsLock)
267         {
268             return processedGroups.contains(id);
269         }
270     }
271 
272     public boolean isFailOnTimeout()
273     {
274         return failOnTimeout;
275     }
276 
277     public void setFailOnTimeout(boolean failOnTimeout)
278     {
279         this.failOnTimeout = failOnTimeout;
280     }
281 
282     public long getTimeout()
283     {
284         return timeout;
285     }
286 
287     public void setTimeout(long timeout)
288     {
289         this.timeout = timeout;
290     }
291 
292     protected void handleGroupExpiry(EventGroup group)
293     {
294         removeEventGroup(group);
295 
296         final FlowConstruct service = group.toArray()[0].getFlowConstruct();
297 
298         if (isFailOnTimeout())
299         {
300             final MuleMessageCollection messageCollection = group.toMessageCollection();
301             muleContext.fireNotification(new RoutingNotification(messageCollection, null,
302                                                                  RoutingNotification.CORRELATION_TIMEOUT));
303             service.getExceptionListener().handleException(
304                     new CorrelationTimeoutException(CoreMessages.correlationTimedOut(group.getGroupId()),
305                                                     group.getMessageCollectionEvent()), group.getMessageCollectionEvent());
306         }
307         else
308         {
309             if (logger.isDebugEnabled())
310             {
311                 logger.debug(MessageFormat.format(
312                         "Aggregator expired, but ''failOnTimeOut'' is false. Forwarding {0} events out of {1} " +
313                         "total for group ID: {2}", group.size(), group.expectedSize(), group.getGroupId()
314                 ));
315             }
316 
317             try
318             {
319                 if (!(group.getCreated() + groupTimeToLive < System.currentTimeMillis()))
320                 {
321                     MuleEvent newEvent = callback.aggregateEvents(group);
322                     newEvent.getMessage().setCorrelationId(group.getGroupId().toString());
323 
324 
325                     if (!expiredAndDispatchedGroups.containsKey(group.getGroupId()))
326                     {
327                         // TODO which use cases would need a sync reply event returned?
328                         if (timeoutMessageProcessor != null)
329                         {
330                             timeoutMessageProcessor.process(newEvent);
331                         }
332                         else
333                         {
334                             if (!(service instanceof Service))
335                             {
336                                 throw new UnsupportedOperationException("EventAggregator is only supported with Service");
337                             }
338 
339                             ((Service) service).dispatchEvent(newEvent);
340                         }
341                         expiredAndDispatchedGroups.put(group.getGroupId(),
342                                                        group.getCreated());
343                     }
344                     else
345                     {
346                         logger.warn(MessageFormat.format("Discarding group {0}", group.getGroupId()));
347                     }
348                 }
349             }
350             catch (Exception e)
351             {
352                 service.getExceptionListener().handleException(e, group.getMessageCollectionEvent());
353             }
354         }
355     }
356 
357     public void start() throws MuleException
358     {
359         logger.info("Starting event correlator: " + name);
360         if (timeout != 0)
361         {
362             expiringGroupMonitoringThread = new ExpiringGroupMonitoringThread();
363             expiringGroupMonitoringThread.start();
364         }
365     }
366 
367     public void stop() throws MuleException
368     {
369         logger.info("Stopping event correlator: " + name);
370         if (expiringGroupMonitoringThread != null)
371         {
372             expiringGroupMonitoringThread.stopProcessing();
373         }
374     }
375 
376 
377     private final class ExpiringGroupMonitoringThread extends Thread implements Expirable
378     {
379         private ExpiryMonitor expiryMonitor;
380 
381         private volatile boolean stopRequested;
382 
383         public ExpiringGroupMonitoringThread()
384         {
385             setName(name);
386             this.expiryMonitor = new ExpiryMonitor(name, 1000 * 60);
387             //clean up every 30 minutes
388             this.expiryMonitor.addExpirable(1000 * 60 * 30, TimeUnit.MILLISECONDS, this);
389         }
390 
391         /**
392          * Removes the elements in expiredAndDispatchedGroups when groupLife is reached
393          */
394         public void expired()
395         {
396             for (Object o : expiredAndDispatchedGroups.keySet())
397             {
398                 Long time = (Long) expiredAndDispatchedGroups.get(o);
399                 if (time + groupTimeToLive < System.currentTimeMillis())
400                 {
401                     expiredAndDispatchedGroups.remove(o);
402                     logger.warn(MessageFormat.format("Discarding group {0}", o));
403                 }
404             }
405         }
406 
407         public void release()
408         {
409             //no op
410         }
411 
412         public void run()
413         {
414             while (true)
415             {
416                 if (stopRequested)
417                 {
418                     logger.debug("Received request to stop expiring group monitoring");
419                     break;
420                 }
421 
422                 List<EventGroup> expired = new ArrayList<EventGroup>(1);
423                 for (Object o : eventGroups.values())
424                 {
425                     EventGroup group = (EventGroup) o;
426                     if ((group.getCreated() + getTimeout() * MILLI_TO_NANO_MULTIPLIER) < System.nanoTime())
427                     {
428                         expired.add(group);
429                     }
430                 }
431                 if (expired.size() > 0)
432                 {
433                     for (Object anExpired : expired)
434                     {
435                         EventGroup group = (EventGroup) anExpired;
436                         handleGroupExpiry(group);
437                     }
438                 }
439 
440                 try
441                 {
442                     Thread.sleep(100);
443                 }
444                 catch (InterruptedException e)
445                 {
446                     break;
447                 }
448             }
449 
450             logger.debug("Expiring group monitoring fully stopped");
451         }
452 
453         /**
454          * Stops the monitoring of the expired groups.
455          */
456         public void stopProcessing()
457         {
458             logger.debug("Stopping expiring group monitoring");
459             stopRequested = true;
460 
461             try
462             {
463                 this.join();
464             }
465             catch (InterruptedException e)
466             {
467                 // Ignoring
468             }
469         }
470     }
471 }