I manged to solve this by pausing the Agent when calling him, and unpausing him when call ends, here is the dial plan if anyone interested:
[advice-line]
exten => s,1,noop(new advice line call)
exten => s,n,Answer()
exten => s,n,Set(CHANNEL(language)=he)
exten => s,n,Agi(ivr_start.php,${FROM_DID},${UNIQUEID},${CALLERID(number)})
exten => s,n(hangup),hangup
exten => h,1,noop(hang up advice line)
exten => h,n,gotoif($["${CDR(dstchannel)}" = ""]?skip_unpause)
exten => h,n,Set(AGENT_INTERFACE=${CUT(CDR(dstchannel),,1)})
exten => h,n,noop(Unpause agent after answered call - ${AGENT_INTERFACE})
exten => h,n,UnpauseQueueMember(,${AGENT_INTERFACE})
exten => h,n(skip_unpause),noop()
exten => h,n,Agi(call_finish.php,${FROM_DID},${UNIQUEID},${CALLERID(number)},${EPOCH},${DIALSTATUS},${ANSWEREDTIME})
[advice_line_agent_call]
exten => _X.,1,noop(dialing agent: ${EXTEN})
exten => _X.,n,Set(AGENT_INTERFACE=Local/${EXTEN}@advice_line_agent_call)
exten => _X.,n,PauseQueueMember(,${AGENT_INTERFACE})
exten => _X.,n,Dial(SIP/sip-trunk-out/${EXTEN},15,eg)
exten => h,1,noop(agent call finished, unpause agent if call was not answered, Dial status: ${DIALSTATUS})
exten => h,n,gotoif($["${DIALSTATUS}"="CANCEL"]?unpause:finish)
exten => h,n,noop(Unpause agent after not answered call - ${AGENT_INTERFACE})
exten => h,n(unpause),UnpauseQueueMember(,${AGENT_INTERFACE})
exten => h,n(finish),noop()