User interaction with Leapmotion AS3 library

Question

I can connect the device and attach a custom cursor to one finger, but I can´t use any of the gestures to over/click a button or drag a sprite around, etc.

I´m using Starling in the project. To run this sample just create a Main.as, setup it with Starling and call this class.

My basic code:

package  
{
import com.leapmotion.leap.Controller;
import com.leapmotion.leap.events.LeapEvent;
import com.leapmotion.leap.Finger;
import com.leapmotion.leap.Frame;
import com.leapmotion.leap.Gesture;
import com.leapmotion.leap.Hand;
import com.leapmotion.leap.InteractionBox;
import com.leapmotion.leap.Pointable;
import com.leapmotion.leap.ScreenTapGesture;
import com.leapmotion.leap.Vector3;
import starling.display.Shape;
import starling.display.Sprite;
import starling.events.Event;
import starling.events.TouchEvent;

/**
 * ...
 * @author miau
 */
public class LeapController extends Sprite 
{
    private var _controller:Controller;
    private var _cursor:Shape;
    private var _screenTap:ScreenTapGesture;
    private var _displayWidth:uint = 800;
    private var _displayHeight:uint = 600;

    public function LeapController() 
    {
        addEventListener(Event.ADDED_TO_STAGE, _startController);

    }

    private function _startController(e:Event):void 
    {
        removeEventListener(Event.ADDED_TO_STAGE, _startController);
        //adding controller
        _controller = new Controller();
        _controller.addEventListener( LeapEvent.LEAPMOTION_INIT, onInit );
        _controller.addEventListener( LeapEvent.LEAPMOTION_CONNECTED, onConnect );
        _controller.addEventListener( LeapEvent.LEAPMOTION_DISCONNECTED, onDisconnect );
        _controller.addEventListener( LeapEvent.LEAPMOTION_EXIT, onExit );
        _controller.addEventListener( LeapEvent.LEAPMOTION_FRAME, onFrame );

                    //add test button
        _testButton.x = stage.stageWidth / 2 - _testButton.width / 2;
        _testButton.y = stage.stageHeight / 2 - _testButton.height / 2;
        addChild(_testButton);
        _testButton.touchable = true;
        _testButton.addEventListener(TouchEvent.TOUCH, doSomething);

        //draw ellipse as a cursor
        _cursor = new Shape();
        _cursor.graphics.lineStyle(6, 0xFFE24F);
        _cursor.graphics.drawEllipse(0, 0, 80, 80);
        addChild(_cursor);


    }

    private function onFrame(e:LeapEvent):void 
    {
        trace("ON FRAME STARTED");

        var frame:Frame = e.frame;
        var interactionBox:InteractionBox = frame.interactionBox;

        // Get the first hand
        if(frame.hands.length > 0){
            var hand:Hand = frame.hands[0];
            var numpointables:int = e.frame.pointables.length;
            var pointablesArray:Array = new Array();

            if(frame.pointables.length > 0 && frame.pointables.length < 2){
                //trace("number of pointables: "+frame.pointables[0]);

                for(var j:int = 0; j < frame.pointables.length; j++){

                    //var pointer:DisplayObject = pointablesArray[j];
                        if(j < numpointables){
                            var pointable:Pointable = frame.pointables[j];
                            var normal:Vector3 = pointable.tipPosition;
                            var normalized:Vector3 = interactionBox.normalizePoint(normal);

                            //pointable.isFinger = true;

                            _cursor.x = normalized.x * _displayWidth;
                            _cursor.y = _displayHeight - (normalized.y * _displayHeight);       
                            _cursor.visible = true;

                        }else if (j == 0) {

                            _cursor.visible = false;

                        }
                }

            }           
        }


    }

    private function onExit(e:LeapEvent):void 
    {
        trace("ON EXIT STARTED");
    }

    private function onDisconnect(e:LeapEvent):void 
    {
        trace("ON DISCONNECT STARTED");
    }

    private function onConnect(e:LeapEvent):void 
    {
        trace("ON CONNECT STARTED");
        _controller.enableGesture( Gesture.TYPE_SWIPE );
        _controller.enableGesture( Gesture.TYPE_CIRCLE );
        _controller.enableGesture( Gesture.TYPE_SCREEN_TAP );
        _controller.enableGesture( Gesture.TYPE_KEY_TAP );
    }

    private function onInit(e:LeapEvent):void 
    {
        trace("ON INIT STARTED");
    }

            private function doSomething(e:TouchEvent):void 
    {
        trace("I WAS TOUCHED!!!");
    }

}

}

If a good code Samaritan can update this code to perform a screen tap gesture (or any interacion with any object), I will really appreciate this a lot.

Regards!

Answer 1

controller.enableGesture(Gesture.TYPE_SWIPE);
controller.enableGesture(Gesture.TYPE_SCREEN_TAP);
if(controller.config().setFloat("Gesture.Swipe.MinLength", 200.0) && controller.config().setFloat("Gesture.Swipe.MinVelocity", 500)) controller.config().save();
if(controller.config().setFloat("Gesture.ScreenTap.MinForwardVelocity", 30.0) && controller.config().setFloat("Gesture.ScreenTap.HistorySeconds", .5) && controller.config().setFloat("Gesture.ScreenTap.MinDistance", 1.0)) controller.config().save();
//etc...

Then catch it in the frame event listener:

private function onFrame( event:LeapEvent ):void
    {
        var frame:Frame = event.frame;
        var gestures:Vector.<Gesture> = frame.gestures();
        for ( var i:int = 0; i < gestures.length; i++ )
        {
            var gesture:Gesture = gestures[ i ];

            switch ( gesture.type )
            {
                case Gesture.TYPE_SCREEN_TAP:
                    var screentap:ScreenTapGesture = ScreenTapGesture ( gesture);
                    trace ("ScreenTapGesture-> x: " + Math.round(screentap.position.x ) + ", y: "+ Math.round( screentap.position.y));
                    break;

                case Gesture.TYPE_SWIPE:
                    var screenSwipe:SwipeGesture = SwipeGesture(gesture);
                    if(gesture.state == Gesture.STATE_START) {
                        //
                    }
                    else if(gesture.state == Gesture.STATE_STOP) {
                        //
                        trace("SwipeGesture-> direction: "+screenSwipe.direction + ", duration: " + screenSwipe.duration);
                    }
                    break;
                default:
                    trace( "Unknown gesture type." )
            }
        }
    }

When the event occurs, check the coordinates translated to the stage/screen and whether a hit test returns true.

EDIT: Considering I have no idea how to reliable get the touch point x/y (or better: how to translate them to the correct screen coordinates), I would probably do something like this in my onFrame event:

private function onFrame(event:LeapEvent):void {

    var frame:Frame = event.frame;
    var gestures:Vector.<Gesture> = frame.gestures();
    var posX:Number;
    var posY:Number;
    var s:Shape;

    if(frame.pointables.length > 0) {
        var currentVector:Vector3 = screen.intersectPointable(frame.pointables[0], true); //get normalized vector
        posX = 1920 * currentVector.x - stage.x; //NOTE: I hardcoded the screen res value, you can get it like var w:int = leap.locatedScreens()[0].widthPixels();
        posY = 1080 * ( 1 - currentVector.y ) - stage.y; //NOTE: I hardcoded the screen res value, you can get it like var h:int = leap.locatedScreens()[0].heightPixels(); 
    }

    for(var i:int = 0; i < gestures.length; i++) {
        var gesture:Gesture = gestures[i];
        if(gesture.type == Gesture.TYPE_SCREEN_TAP) {
            if(posX >= _button1.x &&
               posX <= _button1.x + _button1.width &&
               posY >= _button1.y &&
               posY <= _button1.y + _button1.height) {
                s = new Shape();
                s.graphics.beginFill(0x00FF00);
                s.graphics.drawCircle(0, 0, 10);
                s.graphics.endFill();
                s.x = posX;
                s.y = posY;
                stage.addChild(s);
                trace("Lisa tocada!");
               }
            else {
                s = new Shape();
                s.graphics.beginFill(0xFF0000);
                s.graphics.drawCircle(0, 0, 10);
                s.graphics.endFill();
                s.x = posX;
                s.y = posY;
                stage.addChild(s);
                trace("Fallaste! Intentalo otra vez, tiempo: "+new Date().getTime());
            }
        }
    }
}