You should try do something like this:
Remember the starting location in touchesBegan:
UITouch *touch = [touches anyObject];
self.startTouchPoint = [touch locationInView:self];
Move the view by the difference in position in touchesMoved :
CGPoint movedPoint = [touch locationInView:self];
CGFloat deltaX = movedPoint.x - _startTouchPoint.x;
CGFloat deltaY = movedPoint.y - _startTouchPoint.y;
CGPoint center = self.center;
center.x += x;
center.y += y;
self.center = center;