How to receive socket information if you don't know how much data is coming through?

Question

I am working on a GUI interface based on Python to interact with a robot running Python and an Arduino Mega as a motor controller and sensor controller.

Originally, I was going to use a remote desktop to load my GUI from the robot. This turned out to be very slow because of the remote desktop. I decided that a server and client would be better.

I have a sketch running on my Arduino, that catches motor commands, and performs them. It also waits for a "Ping" command to come through, at which time it should check the ultrasonic sensor in three different positions, then write this information back to the server, which should catch this data and in turn pass it on to the client GUI. I have gotten most all of it to work, but I can't seem to get the data from the server back to the client. I had thought that a simple "client.recv()" would accomplish this, but it's not.

How can I receive that data, if I don't know exactly how much data is coming back?

The Arduino sends the data as "dist1,dist2,dist3 \n".

Here is my Arduino code:

#include <LiquidCrystal.h>
#include <Ping.h>
#include <Servo.h>
// Parallax Ping Unit for distance sensing.
Ping sonic(22);

// Setup LCD pins.
LiquidCrystal lcd(7, 8, 9, 10, 11, 12);

// Servo for Ping unit sweeping.
Servo  PingServo;

// Setup Ping distance setting variables.
int pingDisCent;
int pingDisLeft;
int pingDisRight;

// Variable to keep commands in.
char MsgRcvd;

// Motor setup information.

int LF[] = {23,24};
int LR[] = {25,26};
int RF[] = {27,28};
int RR[] = {29,30};

// Set Debugging here
// 1 - Debug on - Motors don't turn when commands are sent.
// 0 - Debug off - Motors turn when commands are sent.
int debug = 1;

//Variables for speed
int SpdPin = 22;
int Speed = 255;

void setup()
{
  Serial.begin(9600);  // start serial communications

  // Setup motors for output.
  int i;
  for(i = 0; i < 2; i++){
    pinMode(LF[i], OUTPUT);
    pinMode(LR[i], OUTPUT);
    pinMode(RF[i], OUTPUT);
    pinMode(RR[i], OUTPUT);
  }

  // Setup servo to sweep.
  PingServo.attach(6);
  PingServo.write(90);

  // Set up the LCD's number of rows and columns:
  lcd.begin(16, 2);

  // Print a message to the LCD.
  lcd.print("Waiting...");

  // Setup speed pin.
  pinMode(SpdPin, OUTPUT);
}

void loop()
{
  if (Serial.available() > 0)    //Check to see if a command is available.
  {
    MsgRcvd = Serial.read();    // If a command is there, see what it is.
    switch (MsgRcvd)
    {
      case '0':
        Stop();
        break;
      case '1':
        MoveForward();
        break;
      case '2':
        MoveLeft();
        break;
      case '3':
        MoveRight();
        break;
      case '4':
        MoveBackward();
        break;
      case '~':
        active_ir();
        break;
      case 'M':                    // Check to see if we have a connection from the GUI - if so spit out information to the LCD.
        lcd.clear();
        lcd.print("Connected");
        lcd.setCursor(0,1);
        lcd.print("waiting..");
        break;
      case 'D':
        lcd.setCursor(0,1);
        lcd.print("Disconnected"); // Client disconnected - spit out a disconnect to the LCD.
        break;
    }
  }
  delay(100);
}

//  ===================================
//  =====    Ping Ultrasonic      =====
//  ===================================
void active_ir()
{
  // Read to the right.
  PingServo.write(30);
  delay(300);
  pingDisRight = sonic.inch();
  delay(500);

  // Read to the front.
  PingServo.write(90);
  delay(300);
  pingDisCent = sonic.inch();
  delay(500);
  //  Read to the left.
  PingServo.write(150);
  delay(300);
  pingDisLeft = sonic.inch();
  delay(500);
  Serial.print(pingDisLeft);
  Serial.print(',');
  Serial.print(pingDisCent);
  Serial.print(',');
  Serial.println(pingDisRight);
  return;
}



//  ==========================================
//  ======        MOTOR CONTROL      =========
//  ==========================================

void MoveForward()
{
  lcd.clear();
  lcd.setCursor(0,0);
  lcd.print("Forward");
  if (debug == 0){

    digitalWrite(LF[0], HIGH);
    digitalWrite(LF[1], LOW);
    digitalWrite(LR[0], HIGH);
    digitalWrite(LR[1], LOW);
    digitalWrite(RF[0], HIGH);
    digitalWrite(RF[1], LOW);
    digitalWrite(RR[0], HIGH);
    digitalWrite(RR[1], LOW);
  }
}

void MoveBackward()
{
  lcd.clear();
  lcd.setCursor(0,0);
  lcd.print("Reverse");
  if (debug == 0){
    analogWrite(SpdPin, Speed);
    digitalWrite(LF[0], LOW);
    digitalWrite(LF[1], HIGH);
    digitalWrite(LR[0], LOW);
    digitalWrite(LR[1], HIGH);
    digitalWrite(RF[0], LOW);
    digitalWrite(RF[1], HIGH);
    digitalWrite(RR[0], LOW);
    digitalWrite(RR[1], HIGH);
  }
}

void MoveLeft()
{
  lcd.clear();
  lcd.setCursor(0,0);
  lcd.print("Left");
  if (debug == 0){
    analogWrite(SpdPin, Speed);
    digitalWrite(LF[0], LOW);
    digitalWrite(LF[1], HIGH);
    digitalWrite(LR[0], LOW);
    digitalWrite(LR[1], HIGH);
    digitalWrite(RF[0], HIGH);
    digitalWrite(RF[1], LOW);
    digitalWrite(RR[0], HIGH);
    digitalWrite(RR[1], LOW);
  }
}

void MoveRight()
{
  lcd.clear();
  lcd.setCursor(0,0);
  lcd.print("Right");
  if (debug == 0) {
    analogWrite(SpdPin, Speed);
    digitalWrite(LF[0], HIGH);
    digitalWrite(LF[1], LOW);
    digitalWrite(LR[0], HIGH);
    digitalWrite(LR[1], LOW);
    digitalWrite(RF[0], LOW);
    digitalWrite(RF[1], HIGH);
    digitalWrite(RR[0], LOW);
    digitalWrite(RR[1], HIGH);
  }
}

void Stop()
{
  lcd.clear();
  lcd.setCursor(0,0);
  lcd.print("Stopping");
  if (debug == 0){
    digitalWrite(LF[0], LOW);
    digitalWrite(LF[1], LOW);
    digitalWrite(LR[0], LOW);
    digitalWrite(LR[1], LOW);
    digitalWrite(RF[0], LOW);
    digitalWrite(RF[1], LOW);
    digitalWrite(RR[0], LOW);
    digitalWrite(RR[1], LOW);
  }
}

Here is my Python server code:

import serial
import socket

Serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
Serv.bind(('', 9000))
Serv.listen(1)
print "Listening on TCP 9000"
motor = serial.Serial('/dev/ttyUSB0', 9600, timeout=1)
print "Connected to Motor Controller: /dev/ttyUSB0"

while(1):
    print "Waiting For Connection..."
    connection, addr = Serv.accept()
    connection.setblocking(0)
    print "Connected by", addr[0]
    while(1):
        try:
            Servdata = connection.recv(1)
            break
        except:
            pass
    if (Servdata == 'M'):
        print "Entering Manual Mode"
        motor.write(Servdata)
        while(Servdata != 'X'):
            Servdata = '9'
            try:
                Servdata = connection.recv(1)
            except:
                pass
            if Servdata == 'X':
                print "Exiting"
                break
            if Servdata == '0':
                print "Stopping"
                motor.write(Servdata)
            if Servdata == '1':
                print "Forward"
                motor.write(Servdata)
            if Servdata == '2':
                print "Left"
                motor.write(Servdata)
            if Servdata == '3':
                print "Right"
                motor.write(Servdata)
            if Servdata == '4':
                motor.write(Servdata)
                print "Backwards"
            if Servdata == '~':
                motor.write(Servdata)
                retval = motor.readline()
                Serv.send(retval)
            else:
                pass
        motor.write('0')
        connection.close()
        print addr[0], "Closed Manual Mode"

And last but not least, the client GUI code (and this is also where I think my problems lie...):

from socket import *
from PythonCard import model
HOST = ''
PORT = 9000
ADDR = (HOST,PORT)
BUFSIZE = 4096
Client = socket (AF_INET,SOCK_STREAM)
Client.connect((ADDR))
Client.send('M')
class MainWindow(model.Background):
    def on_SetSpdBtn_mouseClick(self, event):
        spd = self.components.SpdSpn.value
    def on_FwdBtn_mouseClick(self, event):
        spd = self.components.SpdSpn.value
        Client.send('1')
    def on_LftBtn_mouseClick(self, event):
        spd = self.components.SpdSpn.value
        Client.send('2')
    def on_RitBtn_mouseClick(self, event):
        spd = self.components.SpdSpn.value
        Client.send('3')
    def on_RevBtn_mouseClick(self, event):
        spd = self.components.SpdSpn.value
        Client.send('4')
    def on_StpBtn_mouseClick(self, event):
        spd = self.components.SpdSpn.value
        Client.send('0')
    def on_GetPing_mouseClick(self, event):
        Client.send('~')
        retval = Client.recv()
        ping_data = retval.strip() # Strip out the newline, if you read an entire line.
        split_data = ping_data.split(',')
        L_Ping = split_data[0]
        R_Ping = split_data[1]
        self.components.PingLeft.text = str(L_Ping)
        self.components.PingRight.text = str(R_Ping)
app = model.Application(MainWindow)
app.MainLoop()

Answer 1

I think I found three problems in this code; the first is wasteful, and the second is probably why you came here today, and the third is why you think you came in today. :)

Busy Waiting

This code is busy waiting for data to come in on the connection:

connection.setblocking(0)
print "Connected by", addr[0]
while(1):
    try:
        Servdata = connection.recv(1)
        break
    except:
        pass

And again here:

    while(Servdata != 'X'):
        Servdata = '9'
        try:
            Servdata = connection.recv(1)
        except:
            pass
        # ...
        else:
            pass

This burns CPU cycles like crazy; better hope you're not running on battery power. It also doesn't buy you anything; you might as well call a blocking recv(). Let the CPU go to sleep while it waits for the next input byte. (If you were actually using non-blocking for anything then the busy waiting would make more sense, but you're not. If you want to limit the time that the server will block for input, there's always settimeout(). But don't just blindly use that, either, because this code would benefit most from just switching to blocking recv().)

Not sending data to the client

        if Servdata == '~':
            motor.write(Servdata)
            retval = motor.readline()
            Serv.send(retval)

I don't think this block of code has been executed yet :) Serv isn't connected to anything, it is a listening socket. You probably meant connection.send(retval) here; I can't find any other lines in the server that would actually send data to the client, so presumably this was supposed to be it.

All at once

This code in the client is a little brittle, but will probably never break:

def on_GetPing_mouseClick(self, event):
    Client.send('~')
    retval = Client.recv()
    ping_data = retval.strip() # strip out the newline, if you read an entire line
    split_data = ping_data.split(',')
    L_Ping = split_data[0]
    R_Ping = split_data[1]
    self.components.PingLeft.text = str(L_Ping)
    self.components.PingRight.text = str(R_Ping)

This code presumes that a recv() call will return exactly one protocol message. TCP streams don't work that way, the peers are free to send outgoing data in whatever sizes they darn well please. (TCP/IP stacks combine multiple application-level messages into a single TCP packet all the time. They also send packets smaller than requested, to avoid fragmentation.)

What would be far safer would be filling a queue with the contents received from the remote peer, and then parsing the queue for your commands / messages. You might find ten commands in the queue, you might find only part of a command -- but your code needs to be prepared to push partial messages on the queue, and consume complete messages from the queue when they are available.

It's a bit of extra work, but required for safe operation in less-than-ideal situations. You might never run into the problem on your LAN, but experience trouble when you go wireless or route over larger networks.