Looks like django admin saves the m2m separately which clears the m2m data in my custom save method.
class MyModelAdmin(admmin.ModelAdmin):
def save_related(self, request, form, *args, **kwargs):
super(ArticleAdmin, self).save_related(request, form, *args, **kwargs).
obj = form.instance
obj.sites.add(Site.objects.get(id=1))