Specific couchdb views suddenly start timing out

Question

I have three specific views that suddenly started crashing after working smoothly for two years. Other views on the same database are just fine. This is the error in futon:

Error: os_process_error
OS process timed out.

Couchdb version is 1.2.0.

Here is the crash report logs:

[Sat, 22 Dec 2012 18:28:53 GMT] [error] [<0.1434.0>] OS Process Error <0.1331.0> :: {os_process_error,
                                                 "OS process timed out."}
[Sat, 22 Dec 2012 18:28:53 GMT] [error] [emulator] Error in process <0.1434.0> with exit value: {{nocatch,{os_process_error,"OS process timed out."}},[{couch_os_process,prompt,2,[{file,"/usr/src/build-couchdb/dependencies/couchdb/src/couchdb/couch_os_process.erl"},{line,57}]},{couch_query_servers,map_doc_raw... 


[Sat, 22 Dec 2012 18:28:53 GMT] [error] [<0.1427.0>] ** Generic server <0.1427.0> terminating 
** Last message in was {'EXIT',<0.1431.0>,
                       {{nocatch,
                            {os_process_error,"OS process timed out."}},
                        [{couch_os_process,prompt,2,
                             [{file,
                                  "/usr/src/build-couchdb/dependencies/couchdb/src/couchdb/couch_os_process.erl"},
                              {line,57}]},
                         {couch_query_servers,map_doc_raw,2,
                             [{file,
                                  "/usr/src/build-couchdb/dependencies/couchdb/src/couchdb/couch_query_servers.erl"},
                              {line,88}]},
                         {couch_view_updater,'-do_maps/3-fun-0-',3,
                             [{file,
                                  "/usr/src/build-couchdb/dependencies/couchdb/src/couchdb/couch_view_updater.erl"},
                              {line,174}]},
                         {couch_view_updater,do_maps,3,
                             [{file,
                                  "/usr/src/build-couchdb/dependencies/couchdb/src/couchdb/couch_view_updater.erl"},
                              {line,169}]}]}}
** When Server state == {group_state,undefined,<<"anypy_suggest">>,
                     {"/usr/src/build-couchdb/build/var/lib/couchdb",
                      <<"anypy_suggest">>,
                      {group,
                       <<179,2,154,154,243,188,196,230,224,228,72,35,138,
                         126,255,36>>,
                       nil,<<"_design/message">>,<<"javascript">>,[],
                       [{view,0,0,0,[],
                         <<"function(doc) {
	
	// test if messages shoud be emitted
	var emit_all = function(messages) {
		for(var i in messages) {
			var d = messages[i].when.split(\"-\");
			var day = d[2].split(\"T\");
			emit([d[0]+'-'+d[1]+'-'+day[0], messages[i].group, messages[i].who], 1);
		}
	};


	// vacation plan level messages
	if(doc.doc_type == 'PlanRoute') {

		// route level messages
		if(doc.messages) 
			emit_all(doc.messages);

		// destination level messages
		if(doc.destinations) {
			for(var i in doc.destinations) {
				if(doc.destinations[i].messages) {
					emit_all(doc.destinations[i].messages);
				}
				// service level messages
				if(doc.destinations[i].services) {
					for(var ii in doc.destinations[i].services) {
						if(doc.destinations[i].services[ii].messages) {
							emit_all(doc.destinations[i].services[ii].messages);
						}
					}
				}
			}
		}
	}
}">>,
                         nil,
                         [{<<"by_date">>,
                           <<"function(keys, values) {
	return sum(values)
}">>}],
                         []},
                        {view,1,0,0,
                         [<<"sent_by">>],
                         <<"function(doc) {
	
	var key;

	// test if messages shoud be emitted
	var emit_all = function(level, key, doc, messages) {
		for(var i in messages) {
			do_emit(level, key, doc, messages, i);
		}
	};

	var do_emit = function(level, key, doc, msgs, idx) {
		msg = msgs[idx];
		var context = [];
		for(i=0; i < idx; i++)
		context.push({who:msgs[i].who, message:msgs[i].message});
		var ndoc = {
		message:msg.message,
		when:msg.when,
		who:msg.who,
		group:msg.group,
		viewed_by:msg.viewed_by,
		level: level,
		doc_id: doc._id,
		doc_type: doc.doc_type,
		doc_title: doc.name,
		context: context,
		};

		keys = [msg.who, msg.when];
		emit(keys, ndoc);
	};

	// vacation plan level messages
	if(doc.doc_type == 'PlanRoute') {
		key = doc.staff_id;

		// route level messages
		if(doc.messages) 
			emit_all('', key, doc, doc.messages);

		// destination level messages
		if(doc.destinations) {
			for(var i in doc.destinations) {
				if(doc.destinations[i].messages) {
					var level = 'Destination ('+doc.destinations[i].destination_id+') on VP leg #' + doc.destinations[i].leg;
					emit_all(level, key, doc, doc.destinations[i].messages);
				}
				// service level messages
				if(doc.destinations[i].services && doc.destinations[i].services.length > 0) {
					for(var ii in doc.destinations[i].services) {
						if(doc.destinations[i].services[ii].messages) {
							var level = doc.destinations[i].services[ii].desc;
							emit_all(level, key, doc, doc.destinations[i].services[ii].messages, ii);
						}
					}
				}
			}
		}
	}
}">>,
                         nil,[],[]},
                        {view,2,0,0,
                         [<<"unviewed_by">>],
                         <<"function(doc) {
	
	var key;

	// test if messages shoud be emitted
	var test_and_emit = function(level, group_not, key, doc, messages) {
		for(var i in messages) {
			if(messages[i].group != group_not && messages[i].viewed_by.indexOf(key) == -1) {
				// key not in the list, emit this message and doc details
				do_emit(level, key, doc, messages, i);
			}
		}
	};

	var do_emit = function(level, key, doc, msgs, idx) {
		msg = msgs[idx];
		var context = [];
		for(i=0; i < idx; i++)
		context.push({who:msgs[i].who, message:msgs[i].message});
		var ndoc = {
		message:msg.message,
		when:msg.when,
		who:msg.who,
		group:msg.group,
		viewed_by:msg.viewed_by,
		level: level,
		doc_id: doc._id,
		doc_type: doc.doc_type,
		doc_title: doc.name,
		context: context,
		};
		emit(key, ndoc);
	};

	// suggest level messages
	if(doc.doc_type == 'ReserveSuggest') {
		key = doc.staff_id;
		
		// doc level messages
		if(doc.messages) 
			test_and_emit('','staff', key, doc, doc.messages);
	
		// suggest level messages
		if(doc.suggests) {
			for(var i in doc.suggests) {
				if(doc.suggests[i].messages) {
					var level = doc.suggests[i].desc;
					test_and_emit(level, 'staff', key, doc, doc.suggests[i].messages, ii);
				}
			}
		}
	}

	// vacation plan level messages
	if(doc.doc_type == 'PlanRoute') {
		key = doc.staff_id;

		// route level messages
		if(doc.messages) 
			test_and_emit('','staff', key, doc, doc.messages);

		// destination level messages
		if(doc.destinations) {
			for(var i in doc.destinations) {
				if(doc.destinations[i].messages) {
					var level = 'Destination ('+doc.destinations[i].destination_id+') on VP leg #' + doc.destinations[i].leg;
					test_and_emit(level, 'staff', key, doc, doc.destinations[i].messages);
				}
				// service level messages
				if(doc.destinations[i].services && doc.destinations[i].services.length > 0) {
					for(var ii in doc.destinations[i].services) {
						if(doc.destinations[i].services[ii].messages) {
							var level = 'Service (' + doc.destinations[i].services[ii].type + ')';
							test_and_emit(level, 'staff', key, doc, doc.destinations[i].services[ii].messages, ii);
						}
					}
				}
			}
		}
	}
}">>,
                         nil,[],[]}],
                       {[]},
                       nil,0,0,nil,nil}},
                     {group,
                      <<179,2,154,154,243,188,196,230,224,228,72,35,138,
                        126,255,36>>,
                      <0.1428.0>,<<"_design/message">>,<<"javascript">>,
                      [],
                      [{view,0,503333,0,[],
                        <<"function(doc) {
	
	// test if messages shoud be emitted
	var emit_all = function(messages) {
		for(var i in messages) {
			var d = messages[i].when.split(\"-\");
			var day = d[2].split(\"T\");
			emit([d[0]+'-'+d[1]+'-'+day[0], messages[i].group, messages[i].who], 1);
		}
	};


	// vacation plan level messages
	if(doc.doc_type == 'PlanRoute') {

		// route level messages
		if(doc.messages) 
			emit_all(doc.messages);

		// destination level messages
		if(doc.destinations) {
			for(var i in doc.destinations) {
				if(doc.destinations[i].messages) {
					emit_all(doc.destinations[i].messages);
				}
				// service level messages
				if(doc.destinations[i].services) {
					for(var ii in doc.destinations[i].services) {
						if(doc.destinations[i].services[ii].messages) {
							emit_all(doc.destinations[i].services[ii].messages);
						}
					}
				}
			}
		}
	}
}">>,
                        {btree,<0.1428.0>,
                         {829460268,{128035,[128035]},6593177},
                         #Fun<couch_btree.3.62781489>,
                         #Fun<couch_btree.4.62781489>,
                         #Fun<couch_view.less_json_ids.2>,
                         #Fun<couch_view_group.10.74547581>,snappy},
                        [{<<"by_date">>,
                          <<"function(keys, values) {
	return sum(values)
}">>}],
                        []},
                       {view,1,503333,0,
                        [<<"sent_by">>],
                        <<"function(doc) {
	
	var key;

	// test if messages shoud be emitted
	var emit_all = function(level, key, doc, messages) {
		for(var i in messages) {
			do_emit(level, key, doc, messages, i);
		}
	};

	var do_emit = function(level, key, doc, msgs, idx) {
		msg = msgs[idx];
		var context = [];
		for(i=0; i < idx; i++)
		context.push({who:msgs[i].who, message:msgs[i].message});
		var ndoc = {
		message:msg.message,
		when:msg.when,
		who:msg.who,
		group:msg.group,
		viewed_by:msg.viewed_by,
		level: level,
		doc_id: doc._id,
		doc_type: doc.doc_type,
		doc_title: doc.name,
		context: context,
		};

		keys = [msg.who, msg.when];
		emit(keys, ndoc);
	};

	// vacation plan level messages
	if(doc.doc_type == 'PlanRoute') {
		key = doc.staff_id;

		// route level messages
		if(doc.messages) 
			emit_all('', key, doc, doc.messages);

		// destination level messages
		if(doc.destinations) {
			for(var i in doc.destinations) {
				if(doc.destinations[i].messages) {
					var level = 'Destination ('+doc.destinations[i].destination_id+') on VP leg #' + doc.destinations[i].leg;
					emit_all(level, key, doc, doc.destinations[i].messages);
				}
				// service level messages
				if(doc.destinations[i].services && doc.destinations[i].services.length > 0) {
					for(var ii in doc.destinations[i].services) {
						if(doc.destinations[i].services[ii].messages) {
							var level = doc.destinations[i].services[ii].desc;
							emit_all(level, key, doc, doc.destinations[i].services[ii].messages, ii);
						}
					}
				}
			}
		}
	}
}">>,
                        {btree,<0.1428.0>,
                         {829552398,{108630,[]},204390994},
                         #Fun<couch_btree.3.62781489>,
                         #Fun<couch_btree.4.62781489>,
                         #Fun<couch_view.less_json_ids.2>,
                         #Fun<couch_view_group.10.74547581>,snappy},
                        [],[]},
                       {view,2,503333,0,
                        [<<"unviewed_by">>],
                        <<"function(doc) {
	
	var key;

	// test if messages shoud be emitted
	var test_and_emit = function(level, group_not, key, doc, messages) {
		for(var i in messages) {
			if(messages[i].group != group_not && messages[i].viewed_by.indexOf(key) == -1) {
				// key not in the list, emit this message and doc details
				do_emit(level, key, doc, messages, i);
			}
		}
	};

	var do_emit = function(level, key, doc, msgs, idx) {
		msg = msgs[idx];
		var context = [];
		for(i=0; i < idx; i++)
		context.push({who:msgs[i].who, message:msgs[i].message});
		var ndoc = {
		message:msg.message,
		when:msg.when,
		who:msg.who,
		group:msg.group,
		viewed_by:msg.viewed_by,
		level: level,
		doc_id: doc._id,
		doc_type: doc.doc_type,
		doc_title: doc.name,
		context: context,
		};
		emit(key, ndoc);
	};

	// suggest level messages
	if(doc.doc_type == 'ReserveSuggest') {
		key = doc.staff_id;
		
		// doc level messages
		if(doc.messages) 
			test_and_emit('','staff', key, doc, doc.messages);
	
		// suggest level messages
		if(doc.suggests) {
			for(var i in doc.suggests) {
				if(doc.suggests[i].messages) {
					var level = doc.suggests[i].desc;
					test_and_emit(level, 'staff', key, doc, doc.suggests[i].messages, ii);
				}
			}
		}
	}

	// vacation plan level messages
	if(doc.doc_type == 'PlanRoute') {
		key = doc.staff_id;

		// route level messages
		if(doc.messages) 
			test_and_emit('','staff', key, doc, doc.messages);

		// destination level messages
		if(doc.destinations) {
			for(var i in doc.destinations) {
				if(doc.destinations[i].messages) {
					var level = 'Destination ('+doc.destinations[i].destination_id+') on VP leg #' + doc.destinations[i].leg;
					test_and_emit(level, 'staff', key, doc, doc.destinations[i].messages);
				}
				// service level messages
				if(doc.destinations[i].services && doc.destinations[i].services.length > 0) {
					for(var ii in doc.destinations[i].services) {
						if(doc.destinations[i].services[ii].messages) {
							var level = 'Service (' + doc.destinations[i].services[ii].type + ')';
							test_and_emit(level, 'staff', key, doc, doc.destinations[i].services[ii].messages, ii);
						}
					}
				}
			}
		}
	}
}">>,
                        {btree,<0.1428.0>,
                         {829591489,{139,[]},306902},
                         #Fun<couch_btree.3.62781489>,
                         #Fun<couch_btree.4.62781489>,
                         #Fun<couch_view.less_json_ids.2>,
                         #Fun<couch_view_group.10.74547581>,snappy},
                        [],[]}],
                      {[]},
                      {btree,<0.1428.0>,
                       {829452501,[],6016985},
                       #Fun<couch_btree.3.62781489>,
                       #Fun<couch_btree.4.62781489>,
                       #Fun<couch_btree.5.62781489>,nil,snappy},
                      503333,0,nil,nil},
                     <0.1431.0>,nil,false,
                     [{{<0.130.0>,#Ref<0.0.0.41069>},509735}],
                     <0.1430.0>,false}
** Reason for termination == 
** {os_process_error,"OS process timed out."}

[Sat, 22 Dec 2012 18:28:53 GMT] [error] [<0.1427.0>] {error_report,<0.31.0>,
                  {<0.1427.0>,crash_report,
                   [[{initial_call,
                      {couch_view_group,init,['Argument__1']}},
                     {pid,<0.1427.0>},
                     {registered_name,[]},
                     {error_info,
                      {exit,
                       {os_process_error,"OS process timed out."},
                       [{gen_server,terminate,6,
                         [{file,"gen_server.erl"},{line,737}]},
                        {proc_lib,init_p_do_apply,3,
                         [{file,"proc_lib.erl"},{line,227}]}]}},
                     {ancestors,[<0.1426.0>]},
                     {messages,[]},
                     {links,[<0.1428.0>,<0.122.0>]},
                     {dictionary,[]},
                     {trap_exit,true},
                     {status,running},
                     {heap_size,2584},
                     {stack_size,24},
                     {reductions,545}],
                    []]}}
[Sat, 22 Dec 2012 18:28:53 GMT] [error] [<0.130.0>] Uncaught server error: {os_process_error,
                                        <<"OS process timed out.">>}
[Sat, 22 Dec 2012 18:28:53 GMT] [error] [<0.1428.0>] ** Generic server <0.1428.0> terminating 
** Last message in was {'EXIT',<0.1427.0>,
                           {os_process_error,"OS process timed out."}}
** When Server state == {file,{file_descriptor,prim_file,{#Port<0.2941>,101}},
                          829595830}
** Reason for termination == 
** {os_process_error,"OS process timed out."}

[Sat, 22 Dec 2012 18:28:53 GMT] [error] [<0.1428.0>] {error_report,<0.31.0>,
                  {<0.1428.0>,crash_report,
                   [[{initial_call,{couch_file,init,['Argument__1']}},
                     {pid,<0.1428.0>},
                     {registered_name,[]},
                     {error_info,
                      {exit,
                       {os_process_error,"OS process timed out."},
                       [{gen_server,terminate,6,
                         [{file,"gen_server.erl"},{line,737}]},
                        {proc_lib,init_p_do_apply,3,
                         [{file,"proc_lib.erl"},{line,227}]}]}},
                     {ancestors,[<0.1427.0>,<0.1426.0>]},
                     {messages,[{'EXIT',<0.1430.0>,shutdown}]},
                     {links,[]},
                     {dictionary,[]},
                     {trap_exit,true},
                     {status,running},
                     {heap_size,377},
                     {stack_size,24},
                     {reductions,1125}],
                    []]}}

Also, I am frequently seeing this error in the logs, even though the associated request returns normally:

[Sat, 22 Dec 2012 19:20:28 GMT] [info] [<0.3849.0>] checkpointing view update at seq 232409 for xxxx _design/all_types
[Sat, 22 Dec 2012 19:20:28 GMT] [error] [<0.124.0>] {error_report,<0.31.0>,
                 {<0.124.0>,crash_report,
                  [[{initial_call,
                     {mochiweb_socket_server,init,['Argument__1']}},
                    {pid,<0.124.0>},
                    {registered_name,[]},
                    {error_info,
                     {exit,eaddrinuse,
                      [{gen_server,init_it,6,
                        [{file,"gen_server.erl"},{line,313}]},
                       {proc_lib,init_p_do_apply,3,
                        [{file,"proc_lib.erl"},{line,227}]}]}},
                    {ancestors,
                     [couch_secondary_services,couch_server_sup,<0.32.0>]},
                    {messages,[]},
                    {links,[<0.93.0>]},
                    {dictionary,[]},
                    {trap_exit,true},
                    {status,running},
                    {heap_size,987},
                    {stack_size,24},
                    {reductions,466}],
                   []]}}

Kxepal · Accepted Answer

During view index updation CouchDB runs updates views for specified design document. This also means that timeout error applies for whole this group, not for specific view function. For your case this error means, that your view group now doesn't run as fast as before for some document (single document applies for whole group) - probably, they'd got something big for now.

However, this error is not fatal, it's just a restriction to let query server not run for too long and motivate you to write fast and optimized functions (fast process -> fast response). I see "for" loops in your views - probably it's possible to reduce their complexity for big message list? Anyway, you always may extend OS process timeout by updating CouchDB config:

curl -X PUT http://localhost:5984/_config/couchdb/os_process_timeout -d '"35000"'

Replace 3500 with your number that you like. This value defines amount of milliseconds that query server may process CouchDB commands (view indexation, design document function execution, validation, filters, etc.) before it will be terminated due to timeout. In response of curl request you'll get your old config value.

Edit: replace query_servers_config/os_process_limit with couchdb/os_process_timeout. This is two different options: first one controls amount of query server processes that may be run for single ddoc language while second one specified actual execution time limit.

Specific couchdb views suddenly start timing out

Tags:

couchdb

Tony

1 Answers

Kxepal

Recent Activity

Donate For Us

Specific couchdb views suddenly start timing out

Tags:

couchdb

Tony

1 Answers

Kxepal

Related questions

Recent Activity

Donate For Us