I looked at the Castle Windsor source and from what I can tell the EndPointBehavior needs to be registered first like this:
container
.Register(Component.For<ErrorHandlerBehavior>().Attribute("scope").Eq(WcfExtensionScope.Services),
Component.For<WcfProtoType.IServiceProtoType>()
.ImplementedBy<WcfProtoType.ProtoTypeService>()
.Named("ProtoTypeService")
.AsWcfService( new DefaultServiceModel()
.AddEndpoints(WcfEndpoint
.BoundTo(new BasicHttpBinding(BasicHttpSecurityMode.None))
.At(baseUrl)
).PublishMetadata(o => o.EnableHttpGet())),Component.For<ServiceBase>().ImplementedBy<MyService>());