pythontensorflowtensorboard
Generating multiple Tensorboard directories
I'm trying to run the same training and Test in a loop, while changing some parameters. Each time the training is done, Tensorflow sould generate a new Tensorboard directory.
And that's exactly the problem I have:
........................
def generatingTBFolder(index):
global TB_Folder
TB_Folder ='TB_Graphs_'+str(index)
with tf.device('/cpu:0'):
for k in range(2,5):## here is the problem
generatingTBFolder(k)
train(k,numberOFclasses)
I don't use a loop like :
with tf.device('/cpu:0'):
generatingTBFolder(3)
train(3,numberOFclasses)
the program works fine and generate a TB folder with the result. but using the loop delivers the following :
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1038 try:
-> 1039 return fn(*args)
1040 except errors.OpError as e:
c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\client\session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
1020 feed_dict, fetch_list, target_list,
-> 1021 status, run_metadata)
1022
c:\users\engine\appdata\local\programs\python\python35\lib\contextlib.py in __exit__(self, type, value, traceback)
65 try:
---> 66 next(self.gen)
67 except StopIteration:
c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\framework\errors_impl.py in raise_exception_on_not_ok_status()
465 compat.as_text(pywrap_tensorflow.TF_Message(status)),
--> 466 pywrap_tensorflow.TF_GetCode(status))
467 finally:
InvalidArgumentError: You must feed a value for placeholder tensor 'input/Features_values' with dtype float
[[Node: input/Features_values = Placeholder[dtype=DT_FLOAT, shape=[], _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
During handling of the above exception, another exception occurred:
InvalidArgumentError Traceback (most recent call last)
<ipython-input-2-54a526dfc682> in <module>()
238 for k in range(2,5):
239 generatingTBFolder(k)
--> 240 train(k,numberOFclasses)
<ipython-input-2-54a526dfc682> in train(numberOfFeatures, numberOFclasses)
208 for i in range(max_steps):
209 if i%5 ==0: # Record summarie and Test-set accruracy
--> 210 summary, acc = sess.run([merged,accuracy], feed_dict=feed_dict(False))
211 #summary, acc = sess.run([merged,accuracy], feed_dict={x:np.reshape(Test_Frame.values[:,indices[0:numberOfFeatures]],[len(Test_Frame),numberOfFeatures]),y_:np.asarray(Test_Frame.iloc[:,-1]),keep_prob:0.9})
212 test_writer.add_summary(summary,i)
c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
776 try:
777 result = self._run(None, fetches, feed_dict, options_ptr,
--> 778 run_metadata_ptr)
779 if run_metadata:
780 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
980 if final_fetches or final_targets:
981 results = self._do_run(handle, final_targets, final_fetches,
--> 982 feed_dict_string, options, run_metadata)
983 else:
984 results = []
c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1030 if handle is None:
1031 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
-> 1032 target_list, options, run_metadata)
1033 else:
1034 return self._do_call(_prun_fn, self._session, handle, feed_dict,
c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1050 except KeyError:
1051 pass
-> 1052 raise type(e)(node_def, op, message)
1053
1054 def _extend_graph(self):
InvalidArgumentError: You must feed a value for placeholder tensor 'input/Features_values' with dtype float
[[Node: input/Features_values = Placeholder[dtype=DT_FLOAT, shape=[], _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Caused by op 'input/Features_values', defined at:
File "c:\users\engine\appdata\local\programs\python\python35\lib\runpy.py", line 184, in _run_module_as_main
"__main__", mod_spec)
File "c:\users\engine\appdata\local\programs\python\python35\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\ipykernel\__main__.py", line 3, in <module>
app.launch_new_instance()
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
app.start()
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\ipykernel\kernelapp.py", line 474, in start
ioloop.IOLoop.instance().start()
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tornado\ioloop.py", line 887, in start
handler_func(fd_obj, events)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tornado\stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tornado\stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\ipykernel\kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\ipykernel\kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\ipykernel\kernelbase.py", line 390, in execute_request
user_expressions, allow_stdin)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\ipykernel\ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\ipykernel\zmqshell.py", line 501, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\IPython\core\interactiveshell.py", line 2717, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\IPython\core\interactiveshell.py", line 2821, in run_ast_nodes
if self.run_code(code, result):
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\IPython\core\interactiveshell.py", line 2881, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-54a526dfc682>", line 240, in <module>
train(k,numberOFclasses)
File "<ipython-input-1-54a526dfc682>", line 90, in train
x =tf.placeholder(tf.float32,[None,numberOfFeatures],name='Features_values')
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\ops\array_ops.py", line 1507, in placeholder
name=name)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 1997, in _placeholder
name=name)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "c:\users\engine\appdata\local\programs\python\python35\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'input/Features_values' with dtype float
[[Node: input/Features_values = Placeholder[dtype=DT_FLOAT, shape=[], _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
I've already seen this error, when I start a Tensorflow program and it crashes because of shape error or any non syntax error, starting the same program without restarting the python kernel delivers that the same error message. The problem no is I really need to do that in a loop ( 500 is the actual limit) so there is no way to do it manually.
Any idea how may I solve this ?
Solution
I found a workaround to this issue, by starting the whole script using another one, that uses a loop:
if __name__ == "__main__" :
with tf.device('/cpu:0'):
print("Got ", int(sys.argv[1]))
generatingTBFolder(int(sys.argv[1]))
train(int(sys.argv[1]),numberOFclasses)
And here is the script:
import os
for i in range(2,456):
command="python TensorCode.py "+str(i)
os.system(command)
the result looks good in Tensorboard.
hope this may helps someone out there !
