Adding the following into ~/.inputrc solves the problem:
set editing-mode vi
set keymap vi-insert
"\C-b": backward-char
"(": "\C-v()\C-b"
"[": "\C-v[]\C-b"
"{": "\C-v{}\C-b"
"\"": "\C-v\"\C-v\"\C-b"
"\047": "\C-v\047\C-v\047\C-b"
or in ~/.bashrc:
set -o vi
bind -m vi-insert '"(" "\C-v()\ei"'
bind -m vi-insert '"[" "\C-v[]\ei"'
bind -m vi-insert '"{" "\C-v{}\ei"'
bind -m vi-insert '"\"" "\C-v\"\C-v\"\ei"'
bind -m vi-insert '"\047" "\C-v\047\C-v\047\ei"'