I ended up using a nasty work around which will cause tests to fail if structure changes:
private void assertAtHomeFragment() {
assertTrue(solo.waitForView(ListView.class, 1, 2000));
assertTrue(solo.waitForView(ImageButton.class, 3, 2000));
assertTrue(solo.waitForView(Button.class, 1, 2000));
solo.sleep(1000);
}
private void assertAtLibraryFragment() {
assertTrue(solo.waitForView(ListView.class, 1, 2000));
assertTrue(solo.waitForView(ImageButton.class, 0, 2000));
solo.sleep(1000);
}