You'd better write a parser for that, however if you want to try with regexes, this should do the trick:
text = $('textarea').val();
while (text.match(/\[quote.*\[\/quote\]/i) != null) {
//remove the least inside the innermost found quote tags
text = text.replace(/^(.*)\[quote.*?\[\/quote\](.*)$/gmi, '\$1\$2');
}
// now strip anything non-character
text = text.replace(/[^a-z0-9]/gmi, '');