You can't build a literal regex with concatenation, you need to use the RegExp constructor.
I would suggest you do the following:
<input data-filetype="jpg|jpeg|png" id="attachments" type="file">
Then:
function onSelect(e) {
// "this" is the file input
// if you attach the function to an event
// like `$('input:file').change(onSelect)`
var filetypes = $(this).data('filetype');
var re = RegExp('\\.('+ filetypes +')$','i');
if (!re.test(e.files[0].extension)) {
alert('file type is wrong');
e.preventDefault();
}
}