Crashes in riak search enabled bucket (riak_search_kv_hook,precommit,error)

StackOverflow https://stackoverflow.com/questions/19965584

  •  30-07-2022
  •  | 
  •  

Question

I have a riak bucket with search enabled. Mostly it seems to work reasonably well but for some resource it's failing now. I'm not able to see anything exceptional about the resources that fail.

Here's a real exception I'm getting:

>>> key = '/data/v2/search_show/TMS.Person.1485'
>>> item = b.get(key)
>>> item.get_data()
{u'type': u'person', u'expires': u'9999999999', u'subject_name': u'Paul Rodriguez', u'sub_type': u'__None__', u'topic': u'__ref--/data/v2/topic/TMS.Person.1485:r1382637028.194730', u'person': u'__None__', u'searchable_key': u'rodriguez paulrodriguez paul', u'date': u'0000-00-00', u'sport': u'__None__', u'genre': u'__None__', u'id': u'/data/v2/search_show/TMS.Person.1485'}
>>> item.get_siblings()
[]
>>> item.store()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/riak/riak_object.py", line 438, in store
    if_none_match=if_none_match)
  File "/usr/local/lib/python2.7/dist-packages/riak/transports/http.py", line 173, in put
    key=robj.get_key())
  File "/usr/local/lib/python2.7/dist-packages/riak/transports/http.py", line 182, in do_put
    return self.parse_body(response, [200, 201, 300])
  File "/usr/local/lib/python2.7/dist-packages/riak/transports/http.py", line 383, in parse_body
    self.check_http_code(response, expected_statuses)
  File "/usr/local/lib/python2.7/dist-packages/riak/transports/http.py", line 369, in check_http_code
    (expected_statuses, status, response[1]))
Exception: Expected status [200, 201, 300], received 500 : <html><head><title>500 Internal Server Error</title></head><body><h1>Internal Server Error</h1>The server encountered an error while processing this request:<br><pre>{error,
    {error,badarg,
        [{erlang,iolist_to_binary,
             [{hook_crashed,
                  {riak_search_kv_hook,precommit,error,
                      {badmatch,
                          [{{dict,3,16,16,8,80,48,
                                {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                                 []},
                                {{[],[],[],[],[],[],[],[],[],[],
                                  [[<<"X-Riak-VTag">>,51,57,117,77,117,89,54,
                                    81,83,81,89,67,98,52,84,80,86,50,99,115,
                                    55,116]],
                                  [[<<"index">>]],
                                  [],
                                  [[<<"X-Riak-Last-Modified">>|
                                    {1382,637028,329550}]],
                                  [],[]}}},
                            {riak_idx_doc,<<"ctv_tvdata">>,
                                <<"/data/v2/search_show/TMS.Person.1485">>,
                                [{<<"date">>,<<"0000-00-00">>,
                                  [{<<"0000-00-00">>,[0]}]},
                                 {<<"expires">>,<<"9999999999">>,
                                  [{<<"9999999999">>,[0]}]},
                                 {<<"genre">>,<<"__None__">>,
                                  [{<<"__None__">>,[0]}]},
                                 {<<"id">>,
                                  <<"/data/v2/search_show/TMS.Person.1485">>,
                                  [{<<"/data/v2/search_show/TMS.Person.1485">>,
                                    [0]}]},
                                 {<<"person">>,<<"__None__">>,
                                  [{<<"__None__">>,[0]}]},
                                 {<<"searchable_key">>,
                                  <<"rodriguez paulrodriguez paul">>,
                                  [{<<"paulrodriguez">>,[1]},
                                   {<<"paul">>,[2]},
                                   {<<"rodriguez">>,[0]}]},
                                 {<<"sport">>,<<"__None__">>,
                                  [{<<"__None__">>,[0]}]},
                                 {<<"sub_type">>,<<"__None__">>,
                                  [{<<"__None__">>,[0]}]},
                                 {<<"subject_name">>,<<"Paul Rodriguez">>,
                                  [{<<"Paul">>,[0]},{<<"Rodriguez">>,[1]}]},
                                 {<<"topic">>,
                                  <<"__ref--/data/v2/topic/TMS.Person.1485:r1382637028.194730">>,
                                  [{<<"__ref--/data/v2/topic/TMS.Person.1485:r1382637028.194730">>,
                                    [0]}]},
                                 {<<"type">>,<<"person">>,
                                  [{<<"person">>,[0]}]}],
                                [],
                                [{<<"expires">>,<<"9999999999">>,
                                  [<<"9999999999">>]},
                                 {<<"type">>,<<"person">>,[<<"person">>]}],
                                true}},
                           {{dict,3,16,16,8,80,48,
                                {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
                                 []},
                                {{[],[],[],[],[],[],[],[],[],[],
                                  [[<<"X-Riak-VTag">>,113,51,102,55,90,82,113,
                                    56,73,51,90,69,116,53,115,68,51,49,52,75,
                                    115]],
                                  [[<<"index">>]],
                                  [],
                                  [[<<"X-Riak-Last-Modified">>|
                                    {1382,637028,329664}]],
                                  [],[]}}},
                            {riak_idx_doc,<<"ctv_tvdata">>,
                                <<"/data/v2/search_show/TMS.Person.1485">>,
                                [{<<"date">>,<<"0000-00-00">>,
                                  [{<<"0000-00-00">>,[0]}]},
                                 {<<"expires">>,<<"9999999999">>,
                                  [{<<"9999999999">>,[0]}]},
                                 {<<"genre">>,<<"__None__">>,
                                  [{<<"__None__">>,[0]}]},
                                 {<<"id">>,
                                  <<"/data/v2/search_show/TMS.Person.1485">>,
                                  [{<<"/data/v2/search_show/TMS.Person.1485">>,
                                    [0]}]},
                                 {<<"person">>,<<"__None__">>,
                                  [{<<"__None__">>,[0]}]},
                                 {<<"searchable_key">>,
                                  <<"rodriguez paulrodriguez paul">>,
                                  [{<<"paulrodriguez">>,[1]},
                                   {<<"paul">>,[2]},
                                   {<<"rodriguez">>,[0]}]},
                                 {<<"sport">>,<<"__None__">>,
                                  [{<<"__None__">>,[0]}]},
                                 {<<"sub_type">>,<<"__None__">>,
                                  [{<<"__None__">>,[0]}]},
                                 {<<"subject_name">>,<<"Paul Rodriguez">>,
                                  [{<<"Paul">>,[0]},{<<"Rodriguez">>,[1]}]},
                                 {<<"topic">>,
                                  <<"__ref--/data/v2/topic/TMS.Person.1485:r1382637028.194730">>,
                                  [{<<"__ref--/data/v2/topic/TMS.Person.1485:r1382637028.194730">>,
                                    [0]}]},
                                 {<<"type">>,<<"person">>,
                                  [{<<"person">>,[0]}]}],
                                [],
                                [{<<"expires">>,<<"9999999999">>,
                                  [<<"9999999999">>]},
                                 {<<"type">>,<<"person">>,[<<"person">>]}],
                                true}}]}}}],
             []},
         {wrq,append_to_response_body,2,[{file,"src/wrq.erl"},{line,210}]},
         {riak_kv_wm_object,handle_common_error,3,
             [{file,"src/riak_kv_wm_object.erl"},{line,1046}]},
         {webmachine_resource,resource_call,3,
             [{file,"src/webmachine_resource.erl"},{line,186}]},
         {webmachine_resource,do,3,
             [{file,"src/webmachine_resource.erl"},{line,142}]},
         {webmachine_decision_core,resource_call,1,
             [{file,"src/webmachine_decision_core.erl"},{line,48}]},
         {webmachine_decision_core,accept_helper,1,
             [{file,"src/webmachine_decision_core.erl"},{line,615}]},
         {webmachine_decision_core,decision,1,
             [{file,"src/webmachine_decision_core.erl"},{line,518}]}]}}</pre><P><HR><ADDRESS>mochiweb+webmachine web server</ADDRESS></body></html>

Anyone have an idea what could cause something like this? Is it just data corruption?

Was it helpful?

Solution

So I got this solved with help from the RIAK mailing list. It turns out that there is a hidden search bucket that is used and that it's settings are set based on the base bucket.

It was pretty well published that your search bucket must have it's allow_mult property set to false but that alone is not enough since the hidden bucket and the search bucket properties aren't automatically synchronized.

My problem was that I had set allow_mult on my base bucket to false but the hidden search bucket still had it set to true.

So the soltuion is to set allow_mult to false on your bucket and also to set it to false on riaks private search bucket which has the same name as your bucket but prefixed with 'rsid' so for bucket 'foo' you have to set allow_mult to false on both 'foo' and '_rsid_foo'

#The bucket had allow_mult set to true:
$ curl 'http://10.1.2.95:8098/buckets/_rsid_ctv_tvdata/props'
{"props": "name":"_rsid_ctv_tvdata","allow_mult":true,"basic_quorum":false,"big_vclock":50,"chash_keyfun":{"mod":"riak_core_util","fun":"chash_std_keyfun"},"dw":0,"last_write_wins":false,"linkfun":{"mod":"riak_kv_wm_link_walker","fun":"mapreduce_linkfun"},"n_val":3,"notfound_ok":false,"old_vclock":86400,"postcommit":[],"pr":0,"precommit":[],"pw":0,"r":1,"rw":1,"small_vclock":50,"w":1,"young_vclock":20}}

#I fixed it in the python console since that is where I'm more comfortable.  And just to be sure I attempted the same riak PUT both before and after.

>>> import riak
>>> r = riak.RiakClient()
>>> b = r.bucket('ctv_tvdata')
>>> o = b.get('/data/v2/search_show/TMS.Show.9838380')
>>> o.store()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/riak/riak_object.py", line 281, in store
timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/riak/client/transport.py", line 127, in wrapper
    return self._with_retries(pool, thunk)
  File "/usr/local/lib/python2.7/dist-packages/riak/client/transport.py", line 69, in _with_retries
return fn(transport)
  File "/usr/local/lib/python2.7/dist-packages/riak/client/transport.py", line 125, in thunk
return fn(self, transport, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/riak/client/operations.py", line 289, in put
timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/riak/transports/http/transport.py", line 144, in put
return self._parse_body(robj, response, [200, 201, 204, 300])
  File "/usr/local/lib/python2.7/dist-packages/riak/transports/http/codec.py", line 64, in _parse_body
self.check_http_code(status, expected_statuses)
  File "/usr/local/lib/python2.7/dist-packages/riak/transports/http/transport.py", line 446, in check_http_code
    (expected_statuses, status))
Exception: Expected status [200, 201, 204, 300], received 500
>>> b2 = r.bucket('_rsid_ctv_tvdata')
>>> b2.get_properties()
{'old_vclock': 86400, 'pr': 0, 'allow_mult': True, 'big_vclock': 50, 'name': '_rsid_ctv_tvdata', 'chash_keyfun': {'fun': 'chash_std_keyfun', 'mod': 'riak_core_util'}, 'n_val': 3, 'notfound_ok': False, 'linkfun': {'fun': 'mapreduce_linkfun', 'mod': 'riak_kv_wm_link_walker'}, 'pw': 0, 'last_write_wins': False, 'r': 1, 'small_vclock': 50, 'rw': 1, 'basic_quorum': False, 'postcommit': [], 'dw': 0, 'w': 1, 'young_vclock': 20, 'precommit': []}
>>> b2.set_property('allow_mult', False)
>>> o = b.get('/data/v2/search_show/TMS.Show.9838380')
>>> o.store()
<riak.riak_object.RiakObject object at 0x250ca10>
Licensed under: CC-BY-SA with attribution
Not affiliated with StackOverflow
scroll top