You can refer to your AppResource like that:
<VisualStateGroup x:Name="CommonStates">
<VisualState x:Name="Normal"/>
<VisualState x:Name="MouseOver">
<Storyboard>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="ListItemBorder" Storyboard.TargetProperty="Background">
<DiscreteObjectKeyFrame KeyTime="0" Value="{StaticResource YourCustomBrush}" />
</ObjectAnimationUsingKeyFrames>
</Storyboard>
</VisualState>
<VisualState x:Name="Disabled"/>
</VisualStateGroup>