“overloading” doesn't work with stderr

Question

/*---------------stdcallbk_module.h---------------------*/
#ifndef STDCALLBK_MODULE_H
#include <Python.h>
#define STDCALLBK_MODULE_H


// Type definition for the callback function.
typedef void __stdcall(CALLBK_FUNC_T)(const char* p);


#ifdef __cplusplus
extern "C" {
#endif
  void register_callback(CALLBK_FUNC_T* callback);
  void import_stdcallbk(void);
#ifdef __cplusplus
}
#endif


#endif    /* #define STDCALLBK_MODULE_H */


/*---------------stdcallbk_module.c---------------------*/
#include "stdcallbk_module.h"

static CALLBK_FUNC_T *callback_write_func = NULL;
static FILE *fp;    /*for debugging*/
static PyObject* g_stdout;

typedef struct
{
    PyObject_HEAD
    CALLBK_FUNC_T *write;
} Stdout;


static PyObject* write2(PyObject* self, PyObject* args)
{
    const char* p;

    if (!PyArg_ParseTuple(args, "s", &p))
        return NULL;

    fputs(p, fp);    fflush(fp);
    if(callback_write_func)
        callback_write_func(p);
    else
        printf("----%s----", p);

    return PyLong_FromLong(strlen(p));
}

static PyObject* flush2(PyObject* self, PyObject* args)
{
    // no-op
    return Py_BuildValue("");
}


static PyMethodDef Stdout_methods[] =
{
    {"write", write2, METH_VARARGS, "sys-stdout-write"},
    {"flush", flush2, METH_VARARGS, "sys-stdout-write"},
    {NULL, NULL, 0, NULL}
};



static PyTypeObject StdoutType =
{
    PyVarObject_HEAD_INIT(0, 0)
    "stdcallbk.StdoutType", /* tp_name */
    sizeof(Stdout), /* tp_basicsize */
    0, /* tp_itemsize */
    0, /* tp_dealloc */
    0, /* tp_print */
    0, /* tp_getattr */
    0, /* tp_setattr */
    0, /* tp_reserved */
    0, /* tp_repr */
    0, /* tp_as_number */
    0, /* tp_as_sequence */
    0, /* tp_as_mapping */
    0, /* tp_hash */
    0, /* tp_call */
    0, /* tp_str */
    0, /* tp_getattro */
    0, /* tp_setattro */
    0, /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT, /* tp_flags */
    "stdcallbk.Stdout objects", /* tp_doc */
    0, /* tp_traverse */
    0, /* tp_clear */
    0, /* tp_richcompare */
    0, /* tp_weaklistoffset */
    0, /* tp_iter */
    0, /* tp_iternext */
    Stdout_methods, /* tp_methods */
    0, /* tp_members */
    0, /* tp_getset */
    0, /* tp_base */
    0, /* tp_dict */
    0, /* tp_descr_get */
    0, /* tp_descr_set */
    0, /* tp_dictoffset */
    0, /* tp_init */
    0, /* tp_alloc */
    0, /* tp_new */
};


static struct PyModuleDef stdcallbkmodule = {
    PyModuleDef_HEAD_INIT,
    "stdcallbk",   /* name of module */
    NULL, /* module documentation, may be NULL */
    -1,       /* size of per-interpreter state of the module,
                or -1 if the module keeps state in global variables. */
    NULL,
    NULL,
    NULL,
    NULL,
    NULL
};


PyMODINIT_FUNC PyInit_stdcallbk(void)
{
    PyObject* m;

    fp = fopen("debuggered.txt", "wt");
    fputs("got to here Vers 12\n", fp);  fflush(fp);

    StdoutType.tp_new = PyType_GenericNew;
    if (PyType_Ready(&StdoutType) < 0)
        return 0;

    m = PyModule_Create(&stdcallbkmodule);
    if (m)
    {
        Py_INCREF(&StdoutType);
        PyModule_AddObject(m, "Stdout", (PyObject*) &StdoutType);
        fputs("PyModule_AddObject() Called\n", fp); fflush(fp);
    }
    g_stdout = StdoutType.tp_new(&StdoutType, 0, 0);

    /*PySys_SetObject("stdout", g_stdout)*/
    fprintf(fp, "PySys_SetObject(stdout) returned %d\n", PySys_SetObject("stdout", g_stdout));
    fflush(fp);

    /*PySys_SetObject("stderr", g_stdout)*/
    fprintf(fp, "PySys_SetObject(stderr) returned %d\n", PySys_SetObject("stderr", g_stdout));
    fflush(fp);

    return m;
}




/* called by cpp exe _after_ Py_Initialize(); */
void __declspec(dllexport) import_stdcallbk(void)
{
    PyImport_ImportModule("stdcallbk");
}



/* called by cpp exe _before_ Py_Initialize(); */
void __declspec(dllexport) register_callback(CALLBK_FUNC_T* callback)
{
    PyImport_AppendInittab("stdcallbk", &PyInit_stdcallbk);
    callback_write_func = callback;
}

/*------------- Embarcadero C++ Builder exe ---------------------*/
#include "/py_module/stdcallbk_module.h"



void __stdcall callback_write_func(const char* p)
{
    ShowMessage(p);
}
//---------------------------------------------------------------------------
void __fastcall TForm1::Button1Click(TObject *Sender)
{
    PyObject *pName, *pModule;

    register_callback(callback_write_func);

    Py_Initialize();
    import_stdcallbk();
    //PyRun_SimpleString("import stdcallbk\n");

    pName = PyUnicode_FromString("hello_emb.py");
    pModule = PyImport_Import(pName);
    Py_Finalize();

}
//---------------------------------------------------------------------------



-------------- python script - hello_emb.py ----------------
#import stdcallbk

print("HELLO FRED !")


assert 1==2


------------------------------------------------------------

The code above is borrowed from here: How to redirect stderr in Python? Via Python C API?

When run from the cpp exe, I get a messagebox "HELLO FRED !" followed by an empty message box which I assume is the carriage return. All good so far, but when the exception is raised, I get nothing. Nothing is written to the text file either.

But... if I uncomment the "import stdcallbk" from the python script and run it from the command line, I get this:

D:\projects\embed_python3\Win32\Debug>hello_emb.py
----HELLO FRED !--------
--------Traceback (most recent call last):
--------  File "D:\projects\embed_python3\Win32\Debug\hello_emb.py", line 7, in <module>
--------    --------assert 1==2--------
--------AssertionError----------------
----

and this:

D:\projects\embed_python3\Win32\Debug>type debuggered.txt
got to here Vers 12
PyModule_AddObject() Called
PySys_SetObject(stdout) returned 0
PySys_SetObject(stderr) returned 0
HELLO FRED !
Traceback (most recent call last):
  File "D:\projects\embed_python3\Win32\Debug\hello_emb.py", line 7, in <module>
    assert 1==2
AssertionError

So it DOES work, just not when it is run from the cpp exe.

Anybody have any idea what on earth is going on here?

Cheers

Answer 1

Your assert will cause hello_emb import to fail. The body of the module is executed when a module is imported. In your case this happens inside the PyImport_Import() call. It will print "HELLO FRED!" just fine but then it will get to the assert which will raise an exception.

PyImport_Import() will return NULL to indicate that. You have to check for that and take appropriate action (which may not always be to print the traceback, that's why it's left up to you).

If you want to print the error to stderr, you have to call PyErr_Print().

pName = PyUnicode_FromString("hello_emb.py");
pModule = PyImport_Import(pName);
if (pModule == NULL)
    PyErr_Print();

Unrelated problems with the code:

That's not the only place that is missing checks for errors (but only the above one is the reason for missing traceback). Calls to fopen or pretty much any Python C/API call can fail and you should handle that.

Also, there's a reference leak in PyInit_stdcallbk. Right after you create the stdout object, its reference count is 1. Then you set it to sys.stdout and sys.stderr which makes it 3. After that you should Py_DECREF it to make it 2 so that if/when it is removed from stdout and stderr, it will be freed from memory. Right now it will never be freed.