Fails to load Python module with Python 3

Question

#include <Python.h>
#include <fstream>
#include <iostream>
#include <string>
#include <filesystem>
#include <sys/types.h>

#include <dirent.h>


static const char * sPythonCode =
"class Test :\n"
"   def __init__(self) : \n"
"     self.Disc_ = 0. \n"
"   def getset(self) : \n"
"     self.Disc_ = 7. \n"
"     return self.Disc_ \n";


std::string writeFile()
{
   static int iFile = 0;
   std::string sFileName(std::string("test") + std::to_string(iFile));
   std::ofstream out("py/" + sFileName + ".py");
   out << sPythonCode;
   out.flush();
   out.close();
   iFile++;
   return sFileName;
}

static bool bPythonOpen = false;
#define PYTHONPATHLEN 501

static void _PyInit()
{
   if (!Py_IsInitialized())
   {
      Py_InitializeEx(0);
   }
}

void openPython(void)
{
   if (!bPythonOpen)
   {
      const size_t szBufferN = 1000;
      char acLoadPath[szBufferN];
      const char *pypath = "./py";

      _PyInit();

      PyRun_SimpleString("import sys");
      PyRun_SimpleString("print('python (%d.%d.%d) initialized' % (sys.version_info.major, sys.version_info.minor, sys.version_info.micro))");
      PyRun_SimpleString("print('--------------------------')");


      snprintf(acLoadPath, szBufferN, "sys.path.append('%s')", pypath);
      PyRun_SimpleString(acLoadPath);

      bPythonOpen = true;
   }
}

PyObject *loadPythonModule(const char *acModule)
{
   PyObject *pyModule = NULL;

   if (bPythonOpen && acModule && strcmp(acModule, ""))
   {
      printf("%s\n", acModule);
      pyModule = PyImport_ImportModule(acModule);
      if (!pyModule)
      {
         PyErr_Print();
      }
   }
   return pyModule;
}

void loadPython()
{

   std::string sFileName = writeFile();

   openPython();
   //sleep(1);
   PyObject *pPythonModule = loadPythonModule(sFileName.c_str());
   
   if (pPythonModule)
      PyDict_DelItemString(PyImport_GetModuleDict(), PyModule_GetName((PyObject *)pPythonModule));
}

int main(int argc, char **argv)
{
   for (int i = 0; i < 10; i++)
   {     
      loadPython();
   }
}

My working env:

• gcc version 8.3.1 20190311 (Red Hat 8.3.1-3) (GCC)
• Red Hat Enterprise Linux Server release 7.6 (Maipo)
• problem with python 3.6.10 / 3.8.3

Command to compile:

g++ pythontest.cpp -I/opt/python/python3.6.10/include/python3.6m -L/opt/python/python3.6.10/lib -lpython3.6m

create py directory:

mkdir py

When I run this code I have random error on different test file that I load.

Example of output:

python (3.6.10) initialized
--------------------------
test0
test1
test2
test3
ModuleNotFoundError: No module named 'test3'
test4
test5
ModuleNotFoundError: No module named 'test5'
test6
test7
ModuleNotFoundError: No module named 'test7'
test8
test9
ModuleNotFoundError: No module named 'test9'

Good to know:

• If I uncomment the line with the sleep it works well
• If I remove the iFile++, it works also as it used after an already created file
• If I relaunch a second without rm -rf py directory it works also
• If I erase file after each run in the loadPython function and remove iFile++ it works also
• If I use strace to launch the executable I don't see the problem

For the moment it seems that the Python loader does not see the file on disk, however in case of failure if I print what I have in the directory thanks to dirent I see the testx.py

Please note that we reproduce the error on different Linux servers (not a hardware problem and even on Windows), with Python 2.7.x it works perfectly well.

Answer 1

You should call __import__('importlib').invalidate_caches() each time you modify modules folders to let C Python knows it must read directories again.