Found the solution, it seems that JEditable always adds a little extra when its active and unfortunately that cannot be avoided completely. However the majority of the issue was with a checkbox that was adding height to the row, making that smaller fixed the size jumping issue.
It was also possible to manually set the width and height through the intialisation code.
$('.colClass1').editable('', {
event: "click",
"style": "inherit",
"width": ($('.colClass1').width()-15) + "px",
"height": ($('.colClass1').height() + 8) + "px",
"callback": function (sValue, y) {
var aPos = oTable.fnGetPosition(this);
oTable.fnUpdate(sValue, aPos[0], aPos[1]);
},
"submitdata": function (value, settings) {
return {
"row_id": this.parentNode.getAttribute('AttributeX'),
"column": oTable.fnGetPosition(this)[2]
};
},
"cssclass": "~/Content/themes/header-table/editable.css",
});