System.Runtime.CallbackException in DurableInstancing

Question

Recently, I started getting this exception in my workflow application.

System.Runtime.CallbackException, System.Runtime.DurableInstancing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35

A user callback threw an exception. Check the exception stack and inner exception to determine the callback that failed.

I host my service through a Windows Service that runs the following class:

public class WorkflowHost
{
    private readonly List<string> _services = new List<string>();

    private readonly Dictionary<string, WorkflowServiceHost> _hosts = new Dictionary<string, WorkflowServiceHost>();

    public WorkflowHost(IList<string> serviceNames)
    {
        foreach (string serviceName in serviceNames)
        {
            _services.Add(serviceName);
        } 
    }

    public void Start() 
    { 
        if (_hosts.Count > 0) 
        { 
            Stop(); 
        } 

        foreach(string serviceName in _services) 
        {
            WorkflowService service = LoadService(serviceName);                
            WorkflowServiceHost host = new WorkflowServiceHost(service);

            host.Faulted += new EventHandler(host_Faulted);

            try
            {
                host.Open();
            }
            catch (Exception e)
            {
                Debug.WriteLine(e.Message);
            }

            _hosts.Add(serviceName, host); 
        } 
    }

    void host_Faulted(object sender, EventArgs e)
    {
        Debug.WriteLine("Service Faulted");

        Start();
    }

    public void Stop() 
    { 
        if (_hosts.Count > 0) 
        { 
            foreach (string serviceName in _services) 
            {                    
                IDisposable host = _hosts[serviceName]; 
                host.Dispose(); 
            } 

            _hosts.Clear(); 
        } 
    } 

    private WorkflowService LoadService(String xamlxName)
    {
        string fullFilePath = Path.Combine(@"..\..\..\Services\Sales Funnel Workflow", xamlxName);

        WorkflowService service = XamlServices.Load(fullFilePath) as WorkflowService;

        if (service != null)
        {
            return service;
        }
        else
        {
            throw new NullReferenceException(string.Format("Unable to load service definition from {0}", fullFilePath));
        }
    }
}

Here is the full stack trace of the exception:

<StackTrace>
at System.ServiceModel.Channels.CommunicationObject.OnFaulted()
at System.ServiceModel.Channels.CommunicationObject.Fault()
at System.ServiceModel.Channels.CommunicationObject.Fault(Exception exception)
at  System.ServiceModel.Activities.Dispatcher.DurableInstanceManager.WaitAndHandleStoreEventsCallback(IAsyncResult result)
at System.Runtime.Fx.AsyncThunk.UnhandledExceptionFrame(IAsyncResult result)
at System.Runtime.AsyncResult.Complete(Boolean completedSynchronously)
at System.Runtime.AsyncResult.AsyncCompletionWrapperCallback(IAsyncResult result)
at System.Runtime.Fx.AsyncThunk.UnhandledExceptionFrame(IAsyncResult result)
at System.Runtime.AsyncResult.Complete(Boolean completedSynchronously)
at System.Runtime.DurableInstancing.InstanceHandle.WaitForEventsAsyncResult.Canceled()
at System.Runtime.DurableInstancing.InstanceHandle.Free()
at System.Activities.DurableInstancing.SqlWorkflowInstanceStoreLock.MarkInstanceOwnerLost(Int64 surrogateLockOwnerId)
at System.Activities.DurableInstancing.SqlWorkflowInstanceStoreLock.MarkInstanceOwnerLost(Int64 surrogateLockOwnerId, Boolean hasModificationLock)
at System.Activities.DurableInstancing.LockRenewalTask.HandleError(Exception exception)
at System.Activities.DurableInstancing.PersistenceTask.CommandCompletedCallback(IAsyncResult result)
at System.Runtime.Fx.AsyncThunk.UnhandledExceptionFrame(IAsyncResult result)
at System.Runtime.AsyncResult.Complete(Boolean completedSynchronously)
at System.Runtime.AsyncResult.Complete(Boolean completedSynchronously, Exception exception)
at System.Activities.DurableInstancing.SqlWorkflowInstanceStoreAsyncResult.StartOperation()
at System.Activities.DurableInstancing.SqlWorkflowInstanceStoreAsyncResult.StartOperationCallback(Object state)
at System.Runtime.ActionItem.DefaultActionItem.Invoke()
at System.Runtime.ActionItem.CallbackHelper.InvokeWithoutContext(Object state)
at System.Runtime.IOThreadScheduler.ScheduledOverlapped.IOCallback(UInt32 errorCode, UInt32 numBytes, NativeOverlapped* nativeOverlapped)
at System.Runtime.Fx.IOCompletionThunk.UnhandledExceptionFrame(UInt32 error, UInt32 bytesRead, NativeOverlapped* nativeOverlapped)
at System.Threading._IOCompletionCallback.PerformIOCompletionCallback(UInt32 errorCode, UInt32 numBytes, NativeOverlapped* pOVERLAP)
</StackTrace>
<ExceptionString>System.Runtime.CallbackException: A user callback threw an exception.  Check the exception stack and inner exception to determine the callback that failed. ---&gt; System.ServiceModel.CommunicationObjectFaultedException: The communication object, System.ServiceModel.Activities.WorkflowServiceHost, cannot be used for communication because it is in the Faulted state.
   at System.ServiceModel.Channels.CommunicationObject.Close(TimeSpan timeout)
   at System.ServiceModel.ServiceHostBase.System.IDisposable.Dispose()
   at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.Stop() in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 67
   at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.Start() in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 30
   at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.host_Faulted(Object sender, EventArgs e) in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 57
   at System.EventHandler.Invoke(Object sender, EventArgs e)
   at System.ServiceModel.Channels.CommunicationObject.OnFaulted()
   --- End of inner exception stack trace ---</ExceptionString>
<InnerException>
<ExceptionType>System.ServiceModel.CommunicationObjectFaultedException, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</ExceptionType>
<Message>The communication object, System.ServiceModel.Activities.WorkflowServiceHost, cannot be used for communication because it is in the Faulted state.</Message>
<StackTrace>
at System.ServiceModel.Channels.CommunicationObject.Close(TimeSpan timeout)
at System.ServiceModel.ServiceHostBase.System.IDisposable.Dispose()
at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.Stop() in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 67
at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.Start() in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 30
at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.host_Faulted(Object sender, EventArgs e) in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 57
at System.EventHandler.Invoke(Object sender, EventArgs e)
at System.ServiceModel.Channels.CommunicationObject.OnFaulted()
</StackTrace>
<ExceptionString>System.ServiceModel.CommunicationObjectFaultedException: The communication object, System.ServiceModel.Activities.WorkflowServiceHost, cannot be used for communication because it is in the Faulted state.
   at System.ServiceModel.Channels.CommunicationObject.Close(TimeSpan timeout)
   at System.ServiceModel.ServiceHostBase.System.IDisposable.Dispose()
   at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.Stop() in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 67
   at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.Start() in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 30
   at Zeller.SalesFunnel.WorkFlow.Service.WorkflowHost.host_Faulted(Object sender, EventArgs e) in C:\Projects\TFS\Zeller Corp Toolkit\Main\Toolkit\Services\ZSalesFunnelService\Zeller.SalesFunnel.WorkFlow.Service\WorkflowHost.cs:line 57
   at System.EventHandler.Invoke(Object sender, EventArgs e)
   at System.ServiceModel.Channels.CommunicationObject.OnFaulted()</ExceptionString>
</InnerException>
</Exception>

Does anyone have any ideas why I'd be getting this?

Answer 1

When your host faulted you tried to start it again from the callback to your fault handler. You have to let the fault processing complete and restart your host some other way.

Answer 2

The inner exception says:

The communication object, System.ServiceModel.Activities.WorkflowServiceHost, cannot be used for communication because it is in the Faulted state.

Which is normally the result of a previous exception. So you need to look at what is going on before this happens.