When facing a similar problem, I ended up using type eraser. However in my example it requires to use something like reinterpret_cast:
class Buffer {
virtual void Add(const void*) = 0;
};
template <typename T>
class TypedBuffer: Buffer {
virtual void Add(const void* e) { buffer.push_back(*static_cast<T*>(e)); }
};
class Primitive {
virtual void AddVert(const vec3& v) { vertices.Add(reinterpret_cast<const void*>(&v)); }
};