WCF side
You would have to use Custom Binding and use authenticationMode="SecureConversation" as shown below
<customBinding>
<binding name="CustomWSDualHttpBinding" receiveTimeout="00:10:00" sendTimeout="00:10:00">
<reliableSession inactivityTimeout="00:01:00" maxPendingChannels="16384" maxTransferWindowSize="4096" maxRetryCount="2"/>
<security authenticationMode="SecureConversation" requireDerivedKeys="true">
<secureConversationBootstrap authenticationMode ="UserNameForCertificate"/>
</security>
<compositeDuplex />
<oneWay />
<textMessageEncoding />
<httpTransport />
</binding>
</customBinding>
EDIT: To increase the maximum array length quota and change the buffer size use below binding
<binding name="CustomWSDualHttpBinding" receiveTimeout="00:10:00" sendTimeout="00:10:00">
<reliableSession inactivityTimeout="00:01:00" maxPendingChannels="16384" maxTransferWindowSize="4096" maxRetryCount="2"/>
<binaryMessageEncoding>
<readerQuotas maxDepth="32" maxStringContentLength="2147483647" maxArrayLength="2147483647" maxBytesPerRead="4096" maxNameTableCharCount="16384" />
</binaryMessageEncoding>
<security authenticationMode="SecureConversation" requireDerivedKeys="true">
<secureConversationBootstrap authenticationMode ="UserNameForCertificate"/>
</security>
<compositeDuplex />
<oneWay />
<httpTransport hostNameComparisonMode="StrongWildcard" transferMode="Buffered" maxBufferPoolSize="1073741824" maxBufferSize="1073741824" maxReceivedMessageSize="1073741824" />
</binding>
Include a service certificate and put that in the service behavior
<serviceBehaviors>
<behavior name="passwordValidatorServiceBehavior">
<serviceDebug includeExceptionDetailInFaults="true"/>
<serviceMetadata httpGetEnabled="true"/>
<serviceCredentials>
<userNameAuthentication userNamePasswordValidationMode="Custom" customUserNamePasswordValidatorType="WCFCallbackTry.Custom.CustomUserNameValidator.CustomUserNamePasswordValidator, WCFCallbackTry"/>
<serviceCertificate findValue="9d4c78cde9d2b82d751a5416fd2eb6df98d3b236" storeLocation="LocalMachine" storeName="My" x509FindType="FindByThumbprint"/>
</serviceCredentials>
</behavior>
</serviceBehaviors>
Then expose the endpoints
<services>
<service behaviorConfiguration="passwordValidatorServiceBehavior" name="WCFCallbackTry.Service1">
<endpoint address="http://MachineName:8018/Service1.svc" bindingConfiguration="CustomWSDualHttpBinding" binding="customBinding"
contract="WCFCallbackTry.IService" name="HttpEndPoint" />
<endpoint address="mex" binding="mexHttpBinding" contract="IMetadataExchange" />
<host>
<baseAddresses>
<add baseAddress="http://MachineName:8018/Service1.svc"/>
</baseAddresses>
</host>
</service>
</services>
Client side Call the service as below
ServiceReference1.ServiceClient client = new ServiceReference1.ServiceClient(new System.ServiceModel.InstanceContext(new CallBack()), "HttpEndPoint");
client.ClientCredentials.UserName.UserName = Environment.UserDomainName + @"\" + Environment.UserName;
client.ClientCredentials.UserName.Password = "aWNhdGU/56gfhvYmplY3RD~";
Include the DNS in your code if necessary
EndpointIdentity identity = EndpointIdentity.CreateDnsIdentity("MachineName");
EndpointAddress endpointAddress = new EndpointAddress(uri, identity);
client.Endpoint.Address = endpointAddress;
Hope this helps