User defined job type not available during calculation via SLURM

Question

I am trying to setup a pyiron calculation (version 0.3.6.). I want to execute a non-python script on a computer cluster via SLURM. I have written an own OwnProgramJob class, which inherits from the GenericJob class. Everything is running smoothly on my local computer. When running on the cluster, however, my own class is not available in pyiron:

...    
File "/beegfs-home/users/fufl/.local/project/lib/python3.8/site-packages/pyiron_base/generic/hdfio.py", line 1251, in import_class
        return getattr(
AttributeError: module '__main__' has no attribute 'OwnProgramJob'

How can I make my own class available for pyiron on the cluster?

I suppose one way would be to add my own class directly into the pyiron source code and to modify the JOB_CLASS_DICT, following the suggestion from https://github.com/pyiron/pyiron/issues/973#issuecomment-694347111. Is there another way without modifying the pyiron source code?

My source code can be found below for reference.

Thank you very much,

Florian

---

Jupyter notebook:

import pyiron
from pathlib import Path

pr = pyiron.Project(path=f"{str(Path.home())}/pyiron/projects/example")

from pyiron_base import GenericJob
import os

class OwnProgramJob(GenericJob):
    def __init__(self, project, job_name):
        super().__init__(project, job_name)
        self.input = OwnProgramInput()
        self.executable = "cat input.in > output.out"
    
    def write_input(self):
        with open(os.path.join(self.working_directory, "input.in"), 'w') as infile:
            infile.write("asd 100")
    
    def collect_output(self):
        file = os.path.join(self.working_directory, "output.out")
        with open(file) as f:
            line = f.readlines()[0]
            energy = float(line.split()[1])
        with self.project_hdf5.open("output/generic") as h5out:
            h5out["energy_tot"] = energy
    
    
class OwnProgramInput(GenericParameters):
    def __init__(self, input_file_name=None):
        super(OwnProgramInput, self).__init__(
            input_file_name=input_file_name,
            table_name="input")
        
    def load_default(self):
        self.load_string("input_energy 100")

job = pr.create_job(job_type=OwnProgramJob, job_name="test", delete_existing_job=True)

job.server.queue = 'cpu'

job.run()

pr.job_table()

SLURM jobfile:

#SBATCH --workdir={{working_directory}}
#SBATCH --get-user-env=L
#SBATCH --partition=cpu                                                                                                                                                                                                                  
{%- if run_time_max %}
#SBATCH --time={{run_time_max // 60}}
{%- endif %}
{%- if memory_max %}
#SBATCH --mem={{memory_max}}
{%- endif %}
#SBATCH --cpus-per-task={{cores}} 
    
{{command}}

Answer 1

For the job class to be available when submitting to the queuing system it is necessary that it is included in the python path. So I suggest splitting the class definition in a separate python module named ownprogramjob.py:

import os
from pyiron_base import GenericJob, GenericParameters


class OwnProgramJob(GenericJob):
    def __init__(self, project, job_name):
        super().__init__(project, job_name)
        self.input = OwnProgramInput()
        self.executable = "cat input.in > output.out"
    
    def write_input(self):
        with open(os.path.join(self.working_directory, "input.in"), 'w') as infile:
            infile.write("asd 100")
    
    def collect_output(self):
        file = os.path.join(self.working_directory, "output.out")
        with open(file) as f:
            line = f.readlines()[0]
            energy = float(line.split()[1])
        with self.project_hdf5.open("output/generic") as h5out:
            h5out["energy_tot"] = energy
    
    
class OwnProgramInput(GenericParameters):
    def __init__(self, input_file_name=None):
        super(OwnProgramInput, self).__init__(
            input_file_name=input_file_name,
            table_name="input")
        
    def load_default(self):
        self.load_string("input_energy 100")

Then you can submit it using:

from pyiron import Project
from ownprogramjob import OwnProgramJob


pr = Project("test")
job = pr.create_job(job_type=OwnProgramJob, job_name="test", delete_existing_job=True)

job.server.queue = 'cpu'

job.run()

pr.job_table()

Best,

Jan