A friend of mine, Zoltán Kelemen sent me a solution. He used wrapper classes to encapsulate a program element of a specified language. This way he had eliminated type family applications from the instance heads without more overhead than necessary.
I'm also looking for other solutions.
{-# LANGUAGE TypeFamilies, MultiParamTypeClasses, FlexibleInstances, UndecidableInstances #-}
class Compile a where
comp :: a -> String
-- common elements in all languages
data ElemA l m = ElemA (ElemB l m)
data ElemB l m = ElemB
class Lang l where
-- language-specific elements
data Instructions l :: * -> *
-- wrapper classes for program elements of Lang1
data Lang1A m = WrapperA (ElemA Lang1 m)
data Lang1B m = WrapperB (ElemB Lang1 m)
-- common modes for all languages
class Mode l m where
type MElemA l m :: *
type MElemB l m :: *
-- mode with normal program elements
data SimpleMode
instance Mode l SimpleMode where
type MElemA l SimpleMode = ElemA l SimpleMode
type MElemB l SimpleMode = ElemB l SimpleMode
-- a mode where each program element can be replaced with a string
data NamedMode
instance Mode l NamedMode where
type MElemA l NamedMode = Either String (ElemA l NamedMode)
type MElemB l NamedMode = Either String (ElemB l NamedMode)
-- definition of Lang1 language
data Lang1
instance Lang Lang1 where
data Instructions Lang1 m
= Add (MElemA Lang1 m) (MElemA Lang1 m) (MElemA Lang1 m)
| Mul (MElemA Lang1 m) (MElemA Lang1 m) (MElemA Lang1 m)
-- | ...
-- dumping the source code of Lang1 langauge
-- ILLEGAL TYPE SYNONYM FAMILY APPLICATION HERE
instance Compile (Lang1A m) where
comp (WrapperA e) = "A"
-- AND HERE
instance Compile (Lang1B m) where
comp (WrapperB e) = "B"