Ok I got it working with the help of others so decided to share here...
Changing button2.onclick
to this works.
button2.onclick =
(
function ()
{
newParagraph.parentNode.removeChild(newParagraph);
button.parentNode.removeChild(button);
button2.parentNode.removeChild(button2);
}
);