Menu
Saw a problem recently where all 200 web container threads became hung, meaning none were available to service incoming requests and so the application froze.
Here is a simple web app and JMeter test that I think demonstrates the cause of this problem. The web app consists of two classes, the following servlet:
public class SessionTestServlet extends HttpServlet {
protected static final String SESSION_KEY = "session_key";
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
// set data on session so the listener is invoked
String sessionData = new String("Session data");
request.getSession().setAttribute(SESSION_KEY, sessionData);
PrintWriter writer = response.getWriter();
writer.println("<html><body>OK</body></html>");
writer.flush();
writer.close();
}
}
and the following implementation of HttpSessionListener and HTTPSessionAttributeListener:
public class SessionTestListener implements
HttpSessionListener, HttpSessionAttributeListener {
private static final ConcurrentMap<String, HttpSession> allSessions
= new ConcurrentHashMap<String, HttpSession>();
public void attributeRemoved(HttpSessionBindingEvent hsbe) {}
public void attributeAdded(HttpSessionBindingEvent hsbe) {
System.out.println("Attribute added, " + hsbe.getName()
+ "=" + hsbe.getValue());
int count = 0;
for (HttpSession session : allSessions.values()) {
if (session.getAttribute(SessionTestServlet.SESSION_KEY) != null) {
count++;
}
}
System.out.println(count + " of " + allSessions.size()
+ " sessions have attribute set.");
}
public void attributeReplaced(HttpSessionBindingEvent hsbe) {}
public void sessionCreated(HttpSessionEvent hse) {
allSessions.put(hse.getSession().getId(), session);
}
public void sessionDestroyed(HttpSessionEvent hse) {
allSessions.remove(hse.getSession().getId());
}
}
The JMeter test has 100 requests hit the servlet every second:
<?xml version="1.0" encoding="UTF-8"?>
<jmeterTestPlan version="1.2" properties="2.1">
<hashTree>
<TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="Test Plan" enabled="true">
<stringProp name="TestPlan.comments"></stringProp>
<boolProp name="TestPlan.functional_mode">false</boolProp>
<boolProp name="TestPlan.serialize_threadgroups">false</boolProp>
<elementProp name="TestPlan.user_defined_variables" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true">
<collectionProp name="Arguments.arguments"/>
</elementProp>
<stringProp name="TestPlan.user_define_classpath"></stringProp>
</TestPlan>
<hashTree>
<ThreadGroup guiclass="ThreadGroupGui" testclass="ThreadGroup" testname="Thread Group" enabled="true">
<stringProp name="ThreadGroup.on_sample_error">continue</stringProp>
<elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller" enabled="true">
<boolProp name="LoopController.continue_forever">false</boolProp>
<intProp name="LoopController.loops">-1</intProp>
</elementProp>
<stringProp name="ThreadGroup.num_threads">100</stringProp>
<stringProp name="ThreadGroup.ramp_time">1</stringProp>
<longProp name="ThreadGroup.start_time">1327193422000</longProp>
<longProp name="ThreadGroup.end_time">1327193422000</longProp>
<boolProp name="ThreadGroup.scheduler">false</boolProp>
<stringProp name="ThreadGroup.duration"></stringProp>
<stringProp name="ThreadGroup.delay"></stringProp>
</ThreadGroup>
<hashTree>
<HTTPSamplerProxy guiclass="HttpTestSampleGui" testclass="HTTPSamplerProxy" testname="HTTP Request" enabled="true">
<elementProp name="HTTPsampler.Arguments" elementType="Arguments" guiclass="HTTPArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true">
<collectionProp name="Arguments.arguments"/>
</elementProp>
<stringProp name="HTTPSampler.domain">localhost</stringProp>
<stringProp name="HTTPSampler.port">9080</stringProp>
<stringProp name="HTTPSampler.connect_timeout"></stringProp>
<stringProp name="HTTPSampler.response_timeout"></stringProp>
<stringProp name="HTTPSampler.protocol">http</stringProp>
<stringProp name="HTTPSampler.contentEncoding"></stringProp>
<stringProp name="HTTPSampler.path">/SESSION_TESTWeb/SessionTestServlet</stringProp>
<stringProp name="HTTPSampler.method">GET</stringProp>
<boolProp name="HTTPSampler.follow_redirects">true</boolProp>
<boolProp name="HTTPSampler.auto_redirects">false</boolProp>
<boolProp name="HTTPSampler.use_keepalive">true</boolProp>
<boolProp name="HTTPSampler.DO_MULTIPART_POST">false</boolProp>
<boolProp name="HTTPSampler.monitor">false</boolProp>
<stringProp name="HTTPSampler.embedded_url_re"></stringProp>
</HTTPSamplerProxy>
<hashTree>
<ConstantTimer guiclass="ConstantTimerGui" testclass="ConstantTimer" testname="Constant Timer" enabled="true">
<stringProp name="ConstantTimer.delay">1000</stringProp>
</ConstantTimer>
<hashTree/>
</hashTree>
</hashTree>
<ResultCollector guiclass="ViewResultsFullVisualizer" testclass="ResultCollector" testname="View Results Tree" enabled="true">
<boolProp name="ResultCollector.error_logging">false</boolProp>
<objProp>
<name>saveConfig</name>
<value class="SampleSaveConfiguration">
<time>true</time>
<latency>true</latency>
<timestamp>true</timestamp>
<success>true</success>
<label>true</label>
<code>true</code>
<message>true</message>
<threadName>true</threadName>
<dataType>true</dataType>
<encoding>false</encoding>
<assertions>true</assertions>
<subresults>true</subresults>
<responseData>false</responseData>
<samplerData>false</samplerData>
<xml>true</xml>
<fieldNames>false</fieldNames>
<responseHeaders>false</responseHeaders>
<requestHeaders>false</requestHeaders>
<responseDataOnError>false</responseDataOnError>
<saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
<assertionsResultsToSave>0</assertionsResultsToSave>
<bytes>true</bytes>
</value>
</objProp>
<stringProp name="filename"></stringProp>
</ResultCollector>
<hashTree/>
<ResultCollector guiclass="SummaryReport" testclass="ResultCollector" testname="Summary Report" enabled="true">
<boolProp name="ResultCollector.error_logging">false</boolProp>
<objProp>
<name>saveConfig</name>
<value class="SampleSaveConfiguration">
<time>true</time>
<latency>true</latency>
<timestamp>true</timestamp>
<success>true</success>
<label>true</label>
<code>true</code>
<message>true</message>
<threadName>true</threadName>
<dataType>true</dataType>
<encoding>false</encoding>
<assertions>true</assertions>
<subresults>true</subresults>
<responseData>false</responseData>
<samplerData>false</samplerData>
<xml>true</xml>
<fieldNames>false</fieldNames>
<responseHeaders>false</responseHeaders>
<requestHeaders>false</requestHeaders>
<responseDataOnError>false</responseDataOnError>
<saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
<assertionsResultsToSave>0</assertionsResultsToSave>
<bytes>true</bytes>
</value>
</objProp>
<stringProp name="filename"></stringProp>
</ResultCollector>
<hashTree/>
</hashTree>
</hashTree>
</jmeterTestPlan>
When this test is run against the test web app deployed on WebSphere 7, the application quickly stops responding and a core dump shows this:
1LKDEADLOCK Deadlock detected !!!
NULL ---------------------
NULL
2LKDEADLOCKTHR Thread "WebContainer : 2" (0x000000000225C600)
3LKDEADLOCKWTR is waiting for:
4LKDEADLOCKMON sys_mon_t:0x00000000151938C0 infl_mon_t: 0x0000000015193930:
4LKDEADLOCKOBJ com/ibm/ws/session/store/memory/MemorySession@00000000A38EA0C8/00000000A38EA0D4:
3LKDEADLOCKOWN which is owned by:
2LKDEADLOCKTHR Thread "WebContainer : 1" (0x00000000021FB500)
3LKDEADLOCKWTR which is waiting for:
4LKDEADLOCKMON sys_mon_t:0x0000000015193820 infl_mon_t: 0x0000000015193890:
4LKDEADLOCKOBJ com/ibm/ws/session/store/memory/MemorySession@00000000A14E22C0/00000000A14E22CC:
3LKDEADLOCKOWN which is owned by:
2LKDEADLOCKTHR Thread "WebContainer : 2" (0x000000000225C600)
NULL
It appears that when a thread (T1) executing the servlet's doGet() method calls setAttribute() on the instance of HttpSession implementation (S1), it locks on the monitor of S1. While holding that lock it goes into the iteration of allSessions inside the listener's attributeAdded() method and calls getAttribute(). It looks like inside getAttribute(), WebSphere locks on that instance's monitor (possibly because it is setting a lastUpdateTime field?). So, T1 will in turn lock the monitors of S1, S2, S3, S4, S5... all the while holding the lock on S1 from the setAttribute() call in the servlet.
So if at the same time another thread (T2) is locking on another session (S2)'s monitor in the servlet and then goes into the loop in addAttribute(), the threads deadlock on the S1 and S2 monitors.
I have been unable to find anything explicit in the J2EE specs about this but this part of the Servlet 2.4 spec implies that the container should not be synchronizing on instances of HttpSession implementations:
SRV.7.7.1 Threading Issues
Multiple servlets executing request threads may have active access to a single session object at the same time. The Developer has the responsibility for synchronizing access to session resources as appropriate.
JBoss does not show any deadlocks when we run the test against it. So my questions are:
Thanks
337
public interface HttpSessionProvides a way to identify a user across more than one page request or visit to a Web site and to store information about that user. The servlet container uses this interface to create a session between an HTTP client and an HTTP server.
The session is not thread safe and neither the get not the set methods are guaranteed to be thread safe. In general in a servlet container you should assume to be in a multi threaded environment and no provided tooling is safe. This also goes for the objects you store in the session.
The HttpServletRequest interface provides two methods to get the object of HttpSession: public HttpSession getSession():Returns the current session associated with this request, or if the request does not have a session, creates one.
The HttpSession interface is used to create a session between the client and server. The session is created between an HTTP Client and an HTTP Server by the servlet container using this interface. The Java Web Server keeps track of each user on the site by creating a session object of interface HttpSession.
The Servlet 2.5 MR6 contains a clarification to the part of the Servlet spec quoted in the question:
Clarify SRV 7.7.1 "Threading Issues" (Issue 33)
Change the paragraph which currently is
"Multiple servlets executing request threads may have active access to a single session object at the same time. The Developer has the responsibility for synchronizing access to session resources as appropriate."
to read
"Multiple servlets executing request threads may have active access to the same session object at the same time. The container must ensure that manipulation of internal data structures representing the session attributes is performed in a threadsafe manner. The Developer has the responsibility for threadsafe access to the attribute objects themselves. This will protect the attribute collection inside the HttpSession object from concurrent access, eliminating the opportunity for an application to cause that collection to become corrupted."
This is still current in Servlet 3.0 MR1 and makes WAS's behaviour look more reasonable. However, I would take from it that *set*Attribute might be synchronized but it not that *get*Attribute would be.
So I think the answer is:
(As a side note, changing the test case so that listener.attributeAdded() calls setAttribute instead of getAttribute doesn't cause deadlocks on JBoss 4 or 5.)
190
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With
