Might be its too late to answer your Question , but it'll help the others , those who are facing the same problem as you..
Set Clickable 'true' to your included layouts i.e your left and right drawer. the clickable event consumes the touch.
<?xml version="1.0" encoding="utf-8"?>
<android.support.v4.widget.DrawerLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:id="@+id/drawer_layout"
android:layout_width="match_parent"
android:layout_height="match_parent" >
<FrameLayout
android:id="@+id/fragment_container"
android:layout_width="match_parent"
android:layout_height="match_parent" />
<include
android:id="@+id/left_drawer"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_gravity="start"
android:clickable="true"
layout="@layout/menu_panel" />
<include
android:id="@+id/right_drawer"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_gravity="end"
android:clickable="true"
layout="@layout/search_panel" />
</android.support.v4.widget.DrawerLayout>