pycurl subprocess in a separate Thread

Question

I need help to get the output from pycurl, that I'm trying to run in subprocess. This output I'm trying to put in a queue and than pull this queue out in a different class.

unfortunately, Right now I have no output =(

import threading
import random
import time
import Queue
import urllib2
import sys
import simplejson, pycurl
import sys, signal

queue = Queue.Queue()
keep_running = True
user = "username"
pswd = "pass"

class MyThread(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        curl_path = '/usr/bin/curl'
        curl_list = [curl_path]
        args = ('curl', 'https://stream.twitter.com/1/statuses/filter.json?track=java', '-u', 'user:pass')
        for arg in args:
            curl_list.append(arg)
            child = subprocess.Popen(
                         curl_list,
                         shell=False,
                         #stdout=subprocess.PIPE)
                         stderr=subprocess.PIPE)
            try:
                out += child.communicate()
                c_out.write(out)
                self.queue.put(c_out)
                self.queue.task_done()
            except KeyboardInterrupt:
                child.kill()


class Starter():
    def __init__(self):
        self.queue = queue
        t = MyThread(self.queue)
        t.daemon=True
        t.start()
        self.next()

    def next(self):
        while True:
            time.sleep(0.5)
            if not self.queue.empty():
                line = self.queue.get(timeout=0.2)
                print '\n\nIM IN STARTER %s' % line
            else:
                print 'waiting for queue'

def main():  
    try:
        Starter()     
    except KeyboardInterrupt, e:
        print 'Stopping'
        raise

main() 

Answer 1

You seem to be confusing your arguments to subprocess quite a bit... the args list should be all of the different pieces of the command that you would be using for curl, you are currently putting them all together in a fashion that is not going to work with subprocess. Your curl_list should look more like this...

curl_path = '/usr/bin/curl'
curl_list = [curl_path, 'https://stream.twitter.com/1/statuses/filter.json?track=java', '-u', 'user:pass']

You are also using an unnecessary for at the moment... you don't want to loop over that list you just want to pass it to subprocess which will handle it appropriately. And you are also going to want stdout to get the results from that, so you need to include the pipe there as well.

I.E, the entire thing should be...

def run(self):
    curl_path = '/usr/bin/curl'
    curl_list = [curl_path, 'https://stream.twitter.com/1/statuses/filter.json?track=java', '-u', 'user:pass']

    child = subprocess.Popen(curl_list,
                             shell=False,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
    try:
        out += child.communicate()[0]
        c_out.write(out)
        self.queue.put(c_out)
        self.queue.task_done()
        except KeyboardInterrupt:
            child.kill()

Might want to take another look at the subprocess documentation to better understand the changes above. I haven't actually run this through an interpreter so it may not be perfect but it should get you going in the right direction... good luck!