It can be done programaticaly with javascript. Remove href from navigation li element. Bind a click event to it that will trigger panel open and popup close state.
Working example: http://jsfiddle.net/Gajotres/2Hv5f/
HTML :
<!DOCTYPE html>
<html>
<head>
<title>jQM Complex Demo</title>
<meta http-equiv='Content-Type' content='text/html; charset=utf-8'/>
<meta name="viewport" content="width=device-width; initial-scale=1.0; maximum-scale=1.0; minimum-scale=1.0; user-scalable=no; target-densityDpi=device-dpi"/>
<link rel="stylesheet" href="http://code.jquery.com/mobile/1.3.1/jquery.mobile-1.3.1.min.css" />
<script src="http://code.jquery.com/mobile/1.3.1/jquery.mobile-1.3.1.min.js"></script>
</head>
<body>
<div data-role="page" id="index">
<!-- CONTENT -->
<ul data-role="listview">
<!--NAVIGATION -->
<a href="#navigation" data-rel="popup">Menu</a>
<div data-role="popup" id="navigation">
<div data-role="collapsible-set">
<div data-role="collapsible">
<h2>Category 1</h2>
<ul data-role="listview">
<li><a href="#" id="popup-panel1">Open Panel 1</a></li>
</ul>
</div><!-- /collapsible -->
</div><!-- /collapsible set -->
</div><!-- /popup -->
<!-- /NAVIGATION -->
<li><a href="#panel1">Open Panel 1</a></li>
</ul>
<!-- /CONTENT -->
<!-- PANEL -->
<div data-role="panel" id="panel1" data-position="left" data-display="overlay" data-position-fixed="true">
<p>content</p>
</div><!-- /panel1-->
<!-- /PANEL -->
</div>
</body>
</html>
Javascript :
$(document).on('pagebeforeshow', '#index', function(){
$(document).on('click', '#popup-panel1', function(){
$('#panel1').panel('open');
$('#navigation').popup('close');
});
});