diff --git a/Godot/icon.svg b/Godot/icon.svg index f1189e6..fcfa04a 100644 --- a/Godot/icon.svg +++ b/Godot/icon.svg @@ -1,66 +1,43 @@ - - - - - - - - - - - - - - + + + +Created with Fabric.js 5.2.4 + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Godot/logs/cleanrl/clean_rl_example__1__1715886195/events.out.tfevents.1715886195.valanixos.440226.0 b/Godot/logs/cleanrl/clean_rl_example__1__1715886195/events.out.tfevents.1715886195.valanixos.440226.0 deleted file mode 100644 index 0d682e9..0000000 Binary files a/Godot/logs/cleanrl/clean_rl_example__1__1715886195/events.out.tfevents.1715886195.valanixos.440226.0 and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_08-21-40/.validate_storage_marker b/Godot/logs/rllib/PPO_2024-05-15_08-21-40/.validate_storage_marker deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/PPO_2024-05-15_08-21-40/basic-variant-state-2024-05-15_08-21-40.json b/Godot/logs/rllib/PPO_2024-05-15_08-21-40/basic-variant-state-2024-05-15_08-21-40.json deleted file mode 100644 index 87c03d9..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_08-21-40/basic-variant-state-2024-05-15_08-21-40.json and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_08-21-40/experiment_state-2024-05-15_08-21-40.json b/Godot/logs/rllib/PPO_2024-05-15_08-21-40/experiment_state-2024-05-15_08-21-40.json deleted file mode 100644 index 2eb2189..0000000 --- a/Godot/logs/rllib/PPO_2024-05-15_08-21-40/experiment_state-2024-05-15_08-21-40.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"649d1_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30382d32312d3430948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f36343964315f30303030305f305f323032342d30352d31355f30382d32312d3430948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30382d32312d34309475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": null,\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": null,\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_649d1_00000_0_2024-05-15_08-21-40\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 30, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715754100.9918525, "_session_str": "2024-05-15_08-21-40", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715754100.9918525}} \ No newline at end of file diff --git a/Godot/logs/rllib/PPO_2024-05-15_08-21-40/tuner.pkl b/Godot/logs/rllib/PPO_2024-05-15_08-21-40/tuner.pkl deleted file mode 100644 index 76e3977..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_08-21-40/tuner.pkl and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_08-22-50/.validate_storage_marker b/Godot/logs/rllib/PPO_2024-05-15_08-22-50/.validate_storage_marker deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/PPO_2024-05-15_08-22-50/basic-variant-state-2024-05-15_08-22-50.json b/Godot/logs/rllib/PPO_2024-05-15_08-22-50/basic-variant-state-2024-05-15_08-22-50.json deleted file mode 100644 index e94b9c7..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_08-22-50/basic-variant-state-2024-05-15_08-22-50.json and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_08-22-50/experiment_state-2024-05-15_08-22-50.json b/Godot/logs/rllib/PPO_2024-05-15_08-22-50/experiment_state-2024-05-15_08-22-50.json deleted file mode 100644 index 08a9b7b..0000000 --- a/Godot/logs/rllib/PPO_2024-05-15_08-22-50/experiment_state-2024-05-15_08-22-50.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"8dccf_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30382d32322d3530948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f38646363665f30303030305f305f323032342d30352d31355f30382d32322d3530948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30382d32322d35309475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": null,\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": null,\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_8dccf_00000_0_2024-05-15_08-22-50\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 370, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715754170.0899887, "_session_str": "2024-05-15_08-22-50", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715754170.0899887}} \ No newline at end of file diff --git a/Godot/logs/rllib/PPO_2024-05-15_08-22-50/tuner.pkl b/Godot/logs/rllib/PPO_2024-05-15_08-22-50/tuner.pkl deleted file mode 100644 index ec9c9bd..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_08-22-50/tuner.pkl and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/.validate_storage_marker b/Godot/logs/rllib/PPO_2024-05-15_09-05-02/.validate_storage_marker deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.pkl b/Godot/logs/rllib/PPO_2024-05-15_09-05-02/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.pkl deleted file mode 100644 index 0d14ffa..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.pkl and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.txt b/Godot/logs/rllib/PPO_2024-05-15_09-05-02/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.txt deleted file mode 100644 index 68aefdb..0000000 --- a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.txt +++ /dev/null @@ -1,51 +0,0 @@ -Failure # 1 (occurred at 2024-05-15_09-06-07) -The actor died because of an error raised in its creation task, ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 239, in _setup - self.add_workers( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 754, in add_workers - raise result.get() - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/utils/actor_manager.py", line 497, in _fetch_result - result = ray.get(r) - ^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -During handling of the above exception, another exception occurred: - -ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 554, in __init__ - super().__init__( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__ - self.setup(copy.deepcopy(self.config)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup - self.workers = EnvRunnerGroup( - ^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__ - raise e.args[0].args[2] -TimeoutError: timed out diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/basic-variant-state-2024-05-15_09-05-02.json b/Godot/logs/rllib/PPO_2024-05-15_09-05-02/basic-variant-state-2024-05-15_09-05-02.json deleted file mode 100644 index 3e0daac..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/basic-variant-state-2024-05-15_09-05-02.json and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/experiment_state-2024-05-15_09-05-02.json b/Godot/logs/rllib/PPO_2024-05-15_09-05-02/experiment_state-2024-05-15_09-05-02.json deleted file mode 100644 index 361e495..0000000 --- a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/experiment_state-2024-05-15_09-05-02.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"734af_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30392d30352d3032948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f37333461665f30303030305f305f323032342d30352d31355f30392d30352d3032948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30392d30352d30329475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"ERROR\",\n \"relative_logdir\": \"PPO_godot_734af_00000_0_2024-05-15_09-05-02\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": 1715756767.1916943,\n \"num_failures\": 1,\n \"num_failures_after_restore\": 0,\n \"error_filename\": \"error.txt\",\n \"pickled_error_filename\": \"error.pkl\",\n \"last_result\": {\n \"trial_id\": \"734af_00000\"\n },\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 644, "_has_errored": true, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715756702.603119, "_session_str": "2024-05-15_09-05-02", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715756702.603119}} \ No newline at end of file diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/tuner.pkl b/Godot/logs/rllib/PPO_2024-05-15_09-05-02/tuner.pkl deleted file mode 100644 index f9517a3..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-05-02/tuner.pkl and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-10-12/.validate_storage_marker b/Godot/logs/rllib/PPO_2024-05-15_09-10-12/.validate_storage_marker deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-10-12/basic-variant-state-2024-05-15_09-10-12.json b/Godot/logs/rllib/PPO_2024-05-15_09-10-12/basic-variant-state-2024-05-15_09-10-12.json deleted file mode 100644 index e0081cb..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-10-12/basic-variant-state-2024-05-15_09-10-12.json and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-10-12/experiment_state-2024-05-15_09-10-12.json b/Godot/logs/rllib/PPO_2024-05-15_09-10-12/experiment_state-2024-05-15_09-10-12.json deleted file mode 100644 index c56f8f6..0000000 --- a/Godot/logs/rllib/PPO_2024-05-15_09-10-12/experiment_state-2024-05-15_09-10-12.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"2c223_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30392d31302d3132948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f32633232335f30303030305f305f323032342d30352d31355f30392d31302d3132948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30392d31302d31329475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_2c223_00000_0_2024-05-15_09-10-12\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 107, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715757012.708886, "_session_str": "2024-05-15_09-10-12", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715757012.708886}} \ No newline at end of file diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-10-12/tuner.pkl b/Godot/logs/rllib/PPO_2024-05-15_09-10-12/tuner.pkl deleted file mode 100644 index 86c44d4..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-10-12/tuner.pkl and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-19-31/.validate_storage_marker b/Godot/logs/rllib/PPO_2024-05-15_09-19-31/.validate_storage_marker deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-19-31/basic-variant-state-2024-05-15_09-19-31.json b/Godot/logs/rllib/PPO_2024-05-15_09-19-31/basic-variant-state-2024-05-15_09-19-31.json deleted file mode 100644 index 536e3bb..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-19-31/basic-variant-state-2024-05-15_09-19-31.json and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-19-31/experiment_state-2024-05-15_09-19-31.json b/Godot/logs/rllib/PPO_2024-05-15_09-19-31/experiment_state-2024-05-15_09-19-31.json deleted file mode 100644 index 4bda095..0000000 --- a/Godot/logs/rllib/PPO_2024-05-15_09-19-31/experiment_state-2024-05-15_09-19-31.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"79228_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30392d31392d3331948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f37393232385f30303030305f305f323032342d30352d31355f30392d31392d3331948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30392d31392d33319475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_79228_00000_0_2024-05-15_09-19-31\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 256, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715757571.3920052, "_session_str": "2024-05-15_09-19-31", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715757571.3920052}} \ No newline at end of file diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-19-31/tuner.pkl b/Godot/logs/rllib/PPO_2024-05-15_09-19-31/tuner.pkl deleted file mode 100644 index 93f778a..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-19-31/tuner.pkl and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-23-16/.validate_storage_marker b/Godot/logs/rllib/PPO_2024-05-15_09-23-16/.validate_storage_marker deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-23-16/basic-variant-state-2024-05-15_09-23-16.json b/Godot/logs/rllib/PPO_2024-05-15_09-23-16/basic-variant-state-2024-05-15_09-23-16.json deleted file mode 100644 index 8dfa826..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-23-16/basic-variant-state-2024-05-15_09-23-16.json and /dev/null differ diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-23-16/experiment_state-2024-05-15_09-23-16.json b/Godot/logs/rllib/PPO_2024-05-15_09-23-16/experiment_state-2024-05-15_09-23-16.json deleted file mode 100644 index 3d1dc6b..0000000 --- a/Godot/logs/rllib/PPO_2024-05-15_09-23-16/experiment_state-2024-05-15_09-23-16.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"ff92f_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30392d32332d3136948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f66663932665f30303030305f305f323032342d30352d31355f30392d32332d3136948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30392d32332d31369475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_ff92f_00000_0_2024-05-15_09-23-16\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 70, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715757796.9438848, "_session_str": "2024-05-15_09-23-16", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715757796.9438848}} \ No newline at end of file diff --git a/Godot/logs/rllib/PPO_2024-05-15_09-23-16/tuner.pkl b/Godot/logs/rllib/PPO_2024-05-15_09-23-16/tuner.pkl deleted file mode 100644 index a5dde74..0000000 Binary files a/Godot/logs/rllib/PPO_2024-05-15_09-23-16/tuner.pkl and /dev/null differ diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/artifacts/2024-05-15_08-21-40/PPO_2024-05-15_08-21-40/driver_artifacts/basic-variant-state-2024-05-15_08-21-40.json b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/artifacts/2024-05-15_08-21-40/PPO_2024-05-15_08-21-40/driver_artifacts/basic-variant-state-2024-05-15_08-21-40.json deleted file mode 100644 index 87c03d9..0000000 Binary files a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/artifacts/2024-05-15_08-21-40/PPO_2024-05-15_08-21-40/driver_artifacts/basic-variant-state-2024-05-15_08-21-40.json and /dev/null differ diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/artifacts/2024-05-15_08-21-40/PPO_2024-05-15_08-21-40/driver_artifacts/experiment_state-2024-05-15_08-21-40.json b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/artifacts/2024-05-15_08-21-40/PPO_2024-05-15_08-21-40/driver_artifacts/experiment_state-2024-05-15_08-21-40.json deleted file mode 100644 index 2eb2189..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/artifacts/2024-05-15_08-21-40/PPO_2024-05-15_08-21-40/driver_artifacts/experiment_state-2024-05-15_08-21-40.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"649d1_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30382d32312d3430948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f36343964315f30303030305f305f323032342d30352d31355f30382d32312d3430948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30382d32312d34309475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": null,\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": null,\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_649d1_00000_0_2024-05-15_08-21-40\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 30, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715754100.9918525, "_session_str": "2024-05-15_08-21-40", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715754100.9918525}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/agent-424238335.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/agent-424238335.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/agent-424238335.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/agent-424238335.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/dashboard.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/dashboard.err deleted file mode 100644 index 2dbdcf3..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/dashboard.err +++ /dev/null @@ -1,6 +0,0 @@ -Traceback (most recent call last): - File "python/ray/_raylet.pyx", line 3270, in ray._raylet.check_health - File "python/ray/_raylet.pyx", line 583, in ray._raylet.check_status -ray.exceptions.RpcError: failed to connect to all addresses; last error: UNKNOWN: ipv4:192.168.1.56:56287: Failed to connect to remote host: Connection refused -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/dashboard/dashboard.py:236: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead - logger.warn("Exiting with SIGTERM immediately...") diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/debug_state.txt b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/debug_state.txt deleted file mode 100644 index 3d6cc54..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/debug_state.txt +++ /dev/null @@ -1,208 +0,0 @@ -NodeManager: -Node ID: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -Node name: 192.168.1.56 -InitialConfigResources: {node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 177374392320000, CPU: 120000, memory: 354748784640000} -ClusterTaskManager: -========== Node: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a ================= -Infeasible queue length: 0 -Schedule queue length: 0 -Dispatch queue length: 0 -num_waiting_for_resource: 0 -num_waiting_for_plasma_memory: 0 -num_waiting_for_remote_node_resources: 0 -num_worker_not_started_by_job_config_not_exist: 0 -num_worker_not_started_by_registration_timeout: 0 -num_tasks_waiting_for_workers: 0 -num_cancelled_tasks: 0 -cluster_resource_scheduler state: -Local id: 3934636312908726399 Local resources: {"total":{object_store_memory: [177374392320000], bundle_group_1_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_e13059e3372a28e5a1b0393dba1401000000: [50000], bundle_group_3_e13059e3372a28e5a1b0393dba1401000000: [10000000], bundle_group_4_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_2_e13059e3372a28e5a1b0393dba1401000000: [10000], memory: [354748784640000], bundle_group_e13059e3372a28e5a1b0393dba1401000000: [50000000], bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_0_e13059e3372a28e5a1b0393dba1401000000: [10000], CPU_group_3_e13059e3372a28e5a1b0393dba1401000000: [10000], node:__internal_head__: [10000], CPU: [120000], node:192.168.1.56: [10000], CPU_group_4_e13059e3372a28e5a1b0393dba1401000000: [10000], bundle_group_2_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_1_e13059e3372a28e5a1b0393dba1401000000: [10000]}}, "available": {object_store_memory: [177374392320000], bundle_group_1_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_e13059e3372a28e5a1b0393dba1401000000: [0], bundle_group_3_e13059e3372a28e5a1b0393dba1401000000: [10000000], bundle_group_4_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_2_e13059e3372a28e5a1b0393dba1401000000: [10000], memory: [354748784640000], bundle_group_e13059e3372a28e5a1b0393dba1401000000: [49999950], bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: [9999990], CPU_group_0_e13059e3372a28e5a1b0393dba1401000000: [0], CPU_group_3_e13059e3372a28e5a1b0393dba1401000000: [10000], node:__internal_head__: [10000], CPU: [70000], node:192.168.1.56: [10000], CPU_group_4_e13059e3372a28e5a1b0393dba1401000000: [10000], bundle_group_2_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_1_e13059e3372a28e5a1b0393dba1401000000: [10000]}}, "labels":{"ray.io/node_id":"5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a",} is_draining: 0 is_idle: 0 Cluster resources: node id: 3934636312908726399{"total":{bundle_group_2_e13059e3372a28e5a1b0393dba1401000000: 10000000, CPU_group_2_e13059e3372a28e5a1b0393dba1401000000: 10000, CPU_group_1_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_1_e13059e3372a28e5a1b0393dba1401000000: 10000000, object_store_memory: 177374392320000, node:192.168.1.56: 10000, CPU_group_4_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 50000000, memory: 354748784640000, CPU_group_0_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: 10000000, node:__internal_head__: 10000, CPU_group_3_e13059e3372a28e5a1b0393dba1401000000: 10000, CPU: 120000, bundle_group_4_e13059e3372a28e5a1b0393dba1401000000: 10000000, CPU_group_e13059e3372a28e5a1b0393dba1401000000: 50000, bundle_group_3_e13059e3372a28e5a1b0393dba1401000000: 10000000}}, "available": {bundle_group_2_e13059e3372a28e5a1b0393dba1401000000: 10000000, CPU_group_2_e13059e3372a28e5a1b0393dba1401000000: 10000, CPU_group_1_e13059e3372a28e5a1b0393dba1401000000: 10000, object_store_memory: 177374392320000, bundle_group_1_e13059e3372a28e5a1b0393dba1401000000: 10000000, node:192.168.1.56: 10000, CPU_group_4_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 49999950, memory: 354748784640000, bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: 9999990, node:__internal_head__: 10000, CPU_group_3_e13059e3372a28e5a1b0393dba1401000000: 10000, CPU: 70000, bundle_group_4_e13059e3372a28e5a1b0393dba1401000000: 10000000, bundle_group_3_e13059e3372a28e5a1b0393dba1401000000: 10000000}}, "labels":{"ray.io/node_id":"5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -Waiting tasks size: 0 -Number of executing tasks: 5 -Number of pinned task arguments: 0 -Number of total spilled tasks: 0 -Number of spilled waiting tasks: 0 -Number of spilled unschedulable tasks: 0 -Resource usage { - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=257429): {CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=257432): {CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=257431): {CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=257430): {bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10, CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000} - - (language=PYTHON actor_or_task=PPO.__init__ pid=257357): {bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10, CPU_group_0_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: 10, CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000} -} -Running tasks by scheduling class: - - {depth=1 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.algorithms.ppo.ppo, class_name=PPO, function_name=__init__, function_hash=81e047a6019e40b6ac8e70bc624f8725} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "\3410Y\3437*(\345\241\2609=\272\024\001\000\000\000" - placement_group_capture_child_tasks: true -} - resource_set={bundle_group_e13059e3372a28e5a1b0393dba1401000000 : 0.001, bundle_group_0_e13059e3372a28e5a1b0393dba1401000000 : 0.001, CPU_group_e13059e3372a28e5a1b0393dba1401000000 : 1, CPU_group_0_e13059e3372a28e5a1b0393dba1401000000 : 1, }}: 1/18446744073709551615 - - {depth=2 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.evaluation.rollout_worker, class_name=RolloutWorker, function_name=__init__, function_hash=2934c96b687748fb9b4a35f16299e44e} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "\3410Y\3437*(\345\241\2609=\272\024\001\000\000\000" - placement_group_bundle_index: -1 - placement_group_capture_child_tasks: true -} - resource_set={CPU_group_e13059e3372a28e5a1b0393dba1401000000 : 1, bundle_group_e13059e3372a28e5a1b0393dba1401000000 : 0.001, }}: 4/18446744073709551615 -================================================== - -ClusterResources: -LocalObjectManager: -- num pinned objects: 0 -- pinned objects size: 0 -- num objects pending restore: 0 -- num objects pending spill: 0 -- num bytes pending spill: 0 -- num bytes currently spilled: 0 -- cumulative spill requests: 0 -- cumulative restore requests: 0 -- spilled objects pending delete: 0 - -ObjectManager: -- num local objects: 0 -- num unfulfilled push requests: 0 -- num object pull requests: 0 -- num chunks received total: 0 -- num chunks received failed (all): 0 -- num chunks received failed / cancelled: 0 -- num chunks received failed / plasma error: 0 -Event stats: -Global stats: 0 total (0 active) -Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -Execution time: mean = -nan s, total = 0.000 s -Event stats: -PushManager: -- num pushes in flight: 0 -- num chunks in flight: 0 -- num chunks remaining: 0 -- max chunks allowed: 409 -OwnershipBasedObjectDirectory: -- num listeners: 0 -- cumulative location updates: 0 -- num location updates per second: 0.000 -- num location lookups per second: 0.000 -- num locations added per second: 0.000 -- num locations removed per second: 0.000 -BufferPool: -- create buffer state map size: 0 -PullManager: -- num bytes available for pulled objects: 17737439232 -- num bytes being pulled (all): 0 -- num bytes being pulled / pinned: 0 -- get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- first get request bundle: N/A -- first wait request bundle: N/A -- first task request bundle: N/A -- num objects queued: 0 -- num objects actively pulled (all): 0 -- num objects actively pulled / pinned: 0 -- num bundles being pulled: 0 -- num pull retries: 0 -- max timeout seconds: 0 -- max timeout request is already processed. No entry. - -WorkerPool: -- registered jobs: 1 -- process_failed_job_config_missing: 0 -- process_failed_rate_limited: 0 -- process_failed_pending_registration: 0 -- process_failed_runtime_env_setup_failed: 0 -- num PYTHON workers: 12 -- num PYTHON drivers: 1 -- num object spill callbacks queued: 0 -- num object restore queued: 0 -- num util functions queued: 0 -- num idle workers: 7 -TaskDependencyManager: -- task deps map size: 0 -- get req map size: 0 -- wait req map size: 0 -- local objects map size: 0 -WaitManager: -- num active wait requests: 0 -Subscriber: -Channel WORKER_OBJECT_LOCATIONS_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_REF_REMOVED_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_OBJECT_EVICTION -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -num async plasma notifications: 0 -Remote node managers: -Event stats: -Global stats: 1047 total (27 active) -Queueing time: mean = 6.266 ms, max = 1.316 s, min = 80.000 ns, total = 6.561 s -Execution time: mean = 3.897 ms, total = 4.080 s -Event stats: - NodeManagerService.grpc_server.ReportWorkerBacklog - 127 total (0 active), Execution time: mean = 95.444 us, total = 12.121 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.ReportWorkerBacklog.HandleRequestImpl - 127 total (0 active), Execution time: mean = 10.989 us, total = 1.396 ms, Queueing time: mean = 11.534 us, max = 90.610 us, min = 1.950 us, total = 1.465 ms - ObjectManager.UpdateAvailableMemory - 100 total (0 active), Execution time: mean = 1.307 us, total = 130.690 us, Queueing time: mean = 7.801 us, max = 14.780 us, min = 4.940 us, total = 780.089 us - NodeManager.CheckGC - 100 total (1 active), Execution time: mean = 947.400 ns, total = 94.740 us, Queueing time: mean = 42.552 us, max = 1.551 ms, min = 4.834 us, total = 4.255 ms - RaySyncer.OnDemandBroadcasting - 100 total (1 active), Execution time: mean = 8.816 us, total = 881.598 us, Queueing time: mean = 35.004 us, max = 1.541 ms, min = 5.143 us, total = 3.500 ms - ClientConnection.async_read.ProcessMessageHeader - 83 total (13 active), Execution time: mean = 2.709 us, total = 224.870 us, Queueing time: mean = 78.468 ms, max = 1.316 s, min = 8.730 us, total = 6.513 s - ClientConnection.async_read.ProcessMessage - 70 total (0 active), Execution time: mean = 384.076 us, total = 26.885 ms, Queueing time: mean = 12.194 us, max = 459.398 us, min = 2.170 us, total = 853.597 us - RayletWorkerPool.deadline_timer.kill_idle_workers - 50 total (1 active), Execution time: mean = 29.997 us, total = 1.500 ms, Queueing time: mean = 84.166 us, max = 3.581 ms, min = 6.447 us, total = 4.208 ms - MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 40 total (1 active), Execution time: mean = 84.302 us, total = 3.372 ms, Queueing time: mean = 13.585 us, max = 35.718 us, min = 4.118 us, total = 543.386 us - ClientConnection.async_write.DoAsyncWrites - 19 total (0 active), Execution time: mean = 454.737 ns, total = 8.640 us, Queueing time: mean = 10.924 us, max = 37.650 us, min = 5.510 us, total = 207.560 us - ObjectManager.ObjectDeleted - 18 total (0 active), Execution time: mean = 6.061 us, total = 109.090 us, Queueing time: mean = 29.991 us, max = 143.339 us, min = 14.230 us, total = 539.839 us - NodeManagerService.grpc_server.GetSystemConfig.HandleRequestImpl - 18 total (0 active), Execution time: mean = 31.315 us, total = 563.669 us, Queueing time: mean = 10.577 us, max = 22.040 us, min = 5.000 us, total = 190.380 us - ObjectManager.ObjectAdded - 18 total (0 active), Execution time: mean = 5.523 us, total = 99.410 us, Queueing time: mean = 12.605 us, max = 30.460 us, min = 4.340 us, total = 226.890 us - NodeManagerService.grpc_server.GetSystemConfig - 18 total (0 active), Execution time: mean = 149.376 us, total = 2.689 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 13 total (0 active), Execution time: mean = 90.959 us, total = 1.182 ms, Queueing time: mean = 1.958 ms, max = 6.687 ms, min = 17.970 us, total = 25.448 ms - NodeManager.ScheduleAndDispatchTasks - 11 total (1 active), Execution time: mean = 5.335 us, total = 58.680 us, Queueing time: mean = 9.394 us, max = 17.863 us, min = 5.637 us, total = 103.337 us - NodeManager.deadline_timer.flush_free_objects - 10 total (1 active), Execution time: mean = 2.068 us, total = 20.680 us, Queueing time: mean = 55.932 us, max = 483.012 us, min = 6.183 us, total = 559.321 us - NodeManagerService.grpc_server.GetResourceLoad - 10 total (0 active), Execution time: mean = 139.640 us, total = 1.396 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.deadline_timer.spill_objects_when_over_threshold - 10 total (1 active), Execution time: mean = 827.000 ns, total = 8.270 us, Queueing time: mean = 57.125 us, max = 481.982 us, min = 7.613 us, total = 571.251 us - NodeManagerService.grpc_server.GetResourceLoad.HandleRequestImpl - 10 total (0 active), Execution time: mean = 52.351 us, total = 523.509 us, Queueing time: mean = 10.185 us, max = 16.690 us, min = 7.210 us, total = 101.849 us - NodeManagerService.grpc_server.RequestWorkerLease - 6 total (0 active), Execution time: mean = 364.345 ms, total = 2.186 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.RequestWorkerLease.HandleRequestImpl - 6 total (0 active), Execution time: mean = 113.930 us, total = 683.579 us, Queueing time: mean = 239.866 us, max = 751.998 us, min = 6.810 us, total = 1.439 ms - CoreWorkerService.grpc_client.Exit.OnReplyReceived - 5 total (0 active), Execution time: mean = 12.074 us, total = 60.370 us, Queueing time: mean = 10.302 us, max = 15.150 us, min = 7.200 us, total = 51.509 us - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 5 total (1 active), Execution time: mean = 362.904 ms, total = 1.815 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RaySyncer.BroadcastMessage - 5 total (0 active), Execution time: mean = 92.108 us, total = 460.540 us, Queueing time: mean = 130.000 ns, max = 180.000 ns, min = 80.000 ns, total = 650.000 ns - WorkerInfoGcsService.grpc_client.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 1.079 ms, total = 5.393 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - Subscriber.HandlePublishedMessage_GCS_WORKER_DELTA_CHANNEL - 5 total (0 active), Execution time: mean = 1.728 us, total = 8.640 us, Queueing time: mean = 61.160 us, max = 72.490 us, min = 56.720 us, total = 305.799 us - - 5 total (0 active), Execution time: mean = 132.000 ns, total = 660.000 ns, Queueing time: mean = 8.438 us, max = 10.689 us, min = 7.000 us, total = 42.189 us - WorkerInfoGcsService.grpc_client.ReportWorkerFailure.OnReplyReceived - 5 total (0 active), Execution time: mean = 9.860 us, total = 49.300 us, Queueing time: mean = 149.821 us, max = 249.979 us, min = 8.350 us, total = 749.107 us - CoreWorkerService.grpc_client.Exit - 5 total (0 active), Execution time: mean = 842.498 us, total = 4.212 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll.OnReplyReceived - 4 total (0 active), Execution time: mean = 69.392 us, total = 277.568 us, Queueing time: mean = 55.710 us, max = 177.129 us, min = 7.700 us, total = 222.839 us - ClusterResourceManager.ResetRemoteNodeView - 4 total (1 active), Execution time: mean = 1.492 us, total = 5.970 us, Queueing time: mean = 8.781 us, max = 17.279 us, min = 8.823 us, total = 35.124 us - NodeInfoGcsService.grpc_client.CheckAlive - 2 total (0 active), Execution time: mean = 299.624 us, total = 599.248 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RaySyncerRegister - 2 total (0 active), Execution time: mean = 845.000 ns, total = 1.690 us, Queueing time: mean = 235.000 ns, max = 390.000 ns, min = 80.000 ns, total = 470.000 ns - NodeManager.GcsCheckAlive - 2 total (1 active), Execution time: mean = 60.120 us, total = 120.240 us, Queueing time: mean = 149.742 us, max = 299.484 us, min = 299.484 us, total = 299.484 us - NodeManager.deadline_timer.record_metrics - 2 total (1 active), Execution time: mean = 257.514 us, total = 515.028 us, Queueing time: mean = 8.553 us, max = 17.105 us, min = 17.105 us, total = 17.105 us - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 2 total (0 active), Execution time: mean = 239.190 us, total = 478.379 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.CheckAlive.OnReplyReceived - 2 total (0 active), Execution time: mean = 13.740 us, total = 27.480 us, Queueing time: mean = 10.925 us, max = 11.030 us, min = 10.820 us, total = 21.850 us - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 2 total (0 active), Execution time: mean = 75.605 us, total = 151.210 us, Queueing time: mean = 500.413 us, max = 913.367 us, min = 87.460 us, total = 1.001 ms - NodeManager.deadline_timer.debug_state_dump - 1 total (1 active, 1 running), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.ReturnWorker - 1 total (0 active), Execution time: mean = 127.369 us, total = 127.369 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.GetAllJobInfo - 1 total (0 active), Execution time: mean = 301.119 us, total = 301.119 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - Subscriber.HandlePublishedMessage_GCS_JOB_CHANNEL - 1 total (0 active), Execution time: mean = 22.890 us, total = 22.890 us, Queueing time: mean = 83.339 us, max = 83.339 us, min = 83.339 us, total = 83.339 us - NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 10.159 ms, total = 10.159 ms, Queueing time: mean = 6.700 us, max = 6.700 us, min = 6.700 us, total = 6.700 us - JobInfoGcsService.grpc_client.GetAllJobInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 7.430 us, total = 7.430 us, Queueing time: mean = 4.300 us, max = 4.300 us, min = 4.300 us, total = 4.300 us - NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 595.718 us, total = 595.718 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetAllNodeInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 83.820 us, total = 83.820 us, Queueing time: mean = 5.360 us, max = 5.360 us, min = 5.360 us, total = 5.360 us - NodeManagerService.grpc_server.PrepareBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 216.439 us, total = 216.439 us, Queueing time: mean = 8.990 us, max = 8.990 us, min = 8.990 us, total = 8.990 us - NodeManager.deadline_timer.print_event_loop_stats - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 359.519 us, total = 359.519 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetAllNodeInfo - 1 total (0 active), Execution time: mean = 278.129 us, total = 278.129 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 129.890 us, total = 129.890 us, Queueing time: mean = 5.080 us, max = 5.080 us, min = 5.080 us, total = 5.080 us - NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.ReturnWorker.HandleRequestImpl - 1 total (0 active), Execution time: mean = 42.810 us, total = 42.810 us, Queueing time: mean = 11.950 us, max = 11.950 us, min = 11.950 us, total = 11.950 us - NodeManagerService.grpc_server.PrepareBundleResources - 1 total (0 active), Execution time: mean = 279.579 us, total = 279.579 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.CommitBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 173.739 us, total = 173.739 us, Queueing time: mean = 6.420 us, max = 6.420 us, min = 6.420 us, total = 6.420 us - JobInfoGcsService.grpc_client.AddJob.OnReplyReceived - 1 total (0 active), Execution time: mean = 15.700 us, total = 15.700 us, Queueing time: mean = 69.289 us, max = 69.289 us, min = 69.289 us, total = 69.289 us - WorkerPool.PopWorkerCallback - 1 total (0 active), Execution time: mean = 11.110 us, total = 11.110 us, Queueing time: mean = 5.349 us, max = 5.349 us, min = 5.349 us, total = 5.349 us - NodeManagerService.grpc_server.CommitBundleResources - 1 total (0 active), Execution time: mean = 223.729 us, total = 223.729 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.AddJob - 1 total (0 active), Execution time: mean = 353.939 us, total = 353.939 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -DebugString() time ms: 0 \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/debug_state_gcs.txt b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/debug_state_gcs.txt deleted file mode 100644 index c1416c1..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/debug_state_gcs.txt +++ /dev/null @@ -1,150 +0,0 @@ -GcsNodeManager: -- RegisterNode request count: 1 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 26 -- GetInternalConfig request count: 1 - -GcsActorManager: -- RegisterActor request count: 5 -- CreateActor request count: 5 -- GetActorInfo request count: 5 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 5 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 1 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 5 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 2 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 1 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 5 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 1 -- Registered placement groups count: 1 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: -RAY_LOG_CHANNEL -- cumulative published messages: 7 -- cumulative published bytes: 4754 -- current buffered bytes: 378 -GCS_NODE_INFO_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 559 -- current buffered bytes: 0 -GCS_WORKER_DELTA_CHANNEL -- cumulative published messages: 5 -- cumulative published bytes: 485 -- current buffered bytes: 97 -GCS_ACTOR_CHANNEL -- cumulative published messages: 10 -- cumulative published bytes: 5864 -- current buffered bytes: 665 -GCS_JOB_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 280 -- current buffered bytes: 280 - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 25 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 13 --Total num of actor creation tasks: 5 --Total num of actor tasks: 6 --Total num of normal tasks: 1 --Total num of driver tasks: 1 - - - - -Global stats: 736 total (19 active) -Queueing time: mean = 121.367 us, max = 26.350 ms, min = 200.000 ns, total = 89.326 ms -Execution time: mean = 3.099 ms, total = 2.281 s -Event stats: - GcsInMemoryStore.Put - 111 total (0 active), Execution time: mean = 263.758 us, total = 29.277 ms, Queueing time: mean = 252.445 us, max = 26.181 ms, min = 600.000 ns, total = 28.021 ms - GcsInMemoryStore.Get - 75 total (0 active), Execution time: mean = 35.806 us, total = 2.685 ms, Queueing time: mean = 6.298 us, max = 296.869 us, min = 560.000 ns, total = 472.329 us - InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 68 total (0 active), Execution time: mean = 8.690 us, total = 590.939 us, Queueing time: mean = 13.208 us, max = 106.509 us, min = 3.160 us, total = 898.166 us - InternalKVGcsService.grpc_server.InternalKVGet - 68 total (0 active), Execution time: mean = 120.321 us, total = 8.182 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVPut - 65 total (0 active), Execution time: mean = 126.788 us, total = 8.241 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 65 total (0 active), Execution time: mean = 7.386 us, total = 480.069 us, Queueing time: mean = 33.550 us, max = 206.769 us, min = 3.520 us, total = 2.181 ms - NodeInfoGcsService.grpc_server.GetAllNodeInfo - 26 total (0 active), Execution time: mean = 147.213 us, total = 3.828 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 26 total (0 active), Execution time: mean = 19.258 us, total = 500.709 us, Queueing time: mean = 13.521 us, max = 55.990 us, min = 3.980 us, total = 351.549 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 159.297 us, total = 2.867 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 14.164 us, total = 254.960 us, Queueing time: mean = 29.422 us, max = 151.780 us, min = 5.450 us, total = 529.589 us - InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 12 total (0 active), Execution time: mean = 3.667 us, total = 44.000 us, Queueing time: mean = 12.629 us, max = 27.090 us, min = 2.110 us, total = 151.550 us - InternalKVGcsService.grpc_client.InternalKVPut - 12 total (0 active), Execution time: mean = 663.662 us, total = 7.964 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RayletLoadPulled - 10 total (1 active), Execution time: mean = 57.438 us, total = 574.377 us, Queueing time: mean = 33.745 us, max = 275.254 us, min = 5.826 us, total = 337.454 us - NodeManagerService.grpc_client.GetResourceLoad - 8 total (0 active), Execution time: mean = 358.958 us, total = 2.872 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 9.004 us, total = 72.029 us, Queueing time: mean = 165.235 us, max = 1.232 ms, min = 7.310 us, total = 1.322 ms - NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 1.371 ms, total = 10.967 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 8 total (0 active), Execution time: mean = 20.915 us, total = 167.320 us, Queueing time: mean = 8.619 us, max = 11.950 us, min = 6.340 us, total = 68.950 us - GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 5.349 us, total = 37.440 us, Queueing time: mean = 13.557 us, max = 19.810 us, min = 1.770 us, total = 94.900 us - ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 98.212 us, total = 491.059 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 173.023 us, total = 865.117 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 6.072 us, total = 30.360 us, Queueing time: mean = 29.798 us, max = 89.750 us, min = 6.250 us, total = 148.990 us - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 9.720 us, total = 48.599 us, Queueing time: mean = 108.618 us, max = 268.199 us, min = 5.670 us, total = 543.088 us - WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 891.008 us, total = 4.455 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 19.798 us, total = 98.990 us, Queueing time: mean = 122.620 us, max = 557.449 us, min = 10.130 us, total = 613.099 us - CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 297.391 us, total = 1.487 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 137.746 us, total = 688.728 us, Queueing time: mean = 77.952 us, max = 185.980 us, min = 7.420 us, total = 389.759 us - CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsResourceManager::Update - 5 total (0 active), Execution time: mean = 41.756 us, total = 208.780 us, Queueing time: mean = 9.402 us, max = 14.300 us, min = 6.420 us, total = 47.008 us - ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 202.172 us, total = 1.011 ms, Queueing time: mean = 7.404 us, max = 9.850 us, min = 6.490 us, total = 37.020 us - ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 115.754 us, total = 578.769 us, Queueing time: mean = 7.054 us, max = 9.110 us, min = 6.150 us, total = 35.270 us - NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 437.626 ms, total = 2.188 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ClusterResourceManager.ResetRemoteNodeView - 4 total (1 active), Execution time: mean = 2.167 us, total = 8.670 us, Queueing time: mean = 6.939 us, max = 13.297 us, min = 5.866 us, total = 27.754 us - NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 4 total (0 active), Execution time: mean = 6.803 us, total = 27.210 us, Queueing time: mean = 9.760 us, max = 13.870 us, min = 5.530 us, total = 39.040 us - NodeInfoGcsService.grpc_server.CheckAlive - 4 total (0 active), Execution time: mean = 94.700 us, total = 378.799 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 65.217 us, total = 260.869 us, Queueing time: mean = 13.203 ms, max = 26.350 ms, min = 11.020 us, total = 52.810 ms - HealthCheck - 2 total (0 active), Execution time: mean = 2.305 us, total = 4.610 us, Queueing time: mean = 11.560 us, max = 11.900 us, min = 11.220 us, total = 23.120 us - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 2 total (0 active), Execution time: mean = 18.790 us, total = 37.580 us, Queueing time: mean = 11.985 us, max = 13.820 us, min = 10.150 us, total = 23.970 us - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 2 total (0 active), Execution time: mean = 111.195 us, total = 222.390 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 4.210 us, total = 4.210 us, Queueing time: mean = 2.350 us, max = 2.350 us, min = 2.350 us, total = 2.350 us - NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 45.140 us, total = 45.140 us, Queueing time: mean = 6.580 us, max = 6.580 us, min = 6.580 us, total = 6.580 us - NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 275.589 us, total = 275.589 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 78.419 us, total = 78.419 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 94.919 us, total = 94.919 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 73.759 us, total = 73.759 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 70.000 ns, total = 70.000 ns, Queueing time: mean = 76.150 us, max = 76.150 us, min = 76.150 us, total = 76.150 us - InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 21.909 us, total = 21.909 us, Queueing time: mean = 8.590 us, max = 8.590 us, min = 8.590 us, total = 8.590 us - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 38.550 us, total = 38.550 us, Queueing time: mean = 8.370 us, max = 8.370 us, min = 8.370 us, total = 8.370 us - JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 212.909 us, total = 212.909 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 46.590 us, total = 46.590 us, Queueing time: mean = 7.540 us, max = 7.540 us, min = 7.540 us, total = 7.540 us - NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 53.830 us, total = 53.830 us, Queueing time: mean = 8.540 us, max = 8.540 us, min = 8.540 us, total = 8.540 us - NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 39.100 us, total = 39.100 us, Queueing time: mean = 5.740 us, max = 5.740 us, min = 5.740 us, total = 5.740 us - NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 749.138 us, total = 749.138 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 7.509 us, total = 7.509 us, Queueing time: mean = 12.700 us, max = 12.700 us, min = 12.700 us, total = 12.700 us - NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 6.690 us, total = 6.690 us, Queueing time: mean = 8.720 us, max = 8.720 us, min = 8.720 us, total = 8.720 us - GCSServer.deadline_timer.debug_state_dump - 1 total (1 active, 1 running), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 338.789 us, total = 338.789 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 104.050 us, total = 104.050 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 1.970 us, total = 1.970 us, Queueing time: mean = 1.020 us, max = 1.020 us, min = 1.020 us, total = 1.020 us - JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 4.780 us, total = 4.780 us, Queueing time: mean = 7.830 us, max = 7.830 us, min = 7.830 us, total = 7.830 us - InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 55.380 us, total = 55.380 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 6.250 us, total = 6.250 us, Queueing time: mean = 200.000 ns, max = 200.000 ns, min = 200.000 ns, total = 200.000 ns - InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.780 us, total = 9.780 us, Queueing time: mean = 4.750 us, max = 4.750 us, min = 4.750 us, total = 4.750 us - NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 98.280 us, total = 98.280 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/gcs_server.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/gcs_server.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/gcs_server.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/gcs_server.out deleted file mode 100644 index c373fc2..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/gcs_server.out +++ /dev/null @@ -1,184 +0,0 @@ -[2024-05-15 08:21:38,980 I 256391 256391] (gcs_server) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 08:21:38,981 I 256391 256391] (gcs_server) event.cc:234: Set ray event level to warning -[2024-05-15 08:21:38,981 I 256391 256391] (gcs_server) event.cc:342: Ray Event initialized for GCS -[2024-05-15 08:21:38,981 I 256391 256391] (gcs_server) gcs_server.cc:74: GCS storage type is StorageType::IN_MEMORY -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:42: Loading job table data. -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:54: Loading node table data. -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:80: Loading actor table data. -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:93: Loading actor task spec table data. -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:66: Loading placement group table data. -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:46: Finished loading job table data, size = 0 -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:58: Finished loading node table data, size = 0 -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:84: Finished loading actor table data, size = 0 -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:97: Finished loading actor task spec table data, size = 0 -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:71: Finished loading placement group table data, size = 0 -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_server.cc:162: No existing server cluster ID found. Generating new ID: 535ba0eef614eb34440db6e780ab29a9f7318b794e022f2f96a3d065 -[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_server.cc:648: Autoscaler V2 enabled: 0 -[2024-05-15 08:21:38,983 I 256391 256391] (gcs_server) grpc_server.cc:134: GcsServer server started, listening on port 56287. -[2024-05-15 08:21:39,008 I 256391 256391] (gcs_server) gcs_server.cc:250: GcsNodeManager: -- RegisterNode request count: 0 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 0 -- GetInternalConfig request count: 0 - -GcsActorManager: -- RegisterActor request count: 0 -- CreateActor request count: 0 -- GetActorInfo request count: 0 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 0 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 0 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 0 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 0 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 0 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 0 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 0 -- Registered placement groups count: 0 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 0 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 0 --Total num of actor creation tasks: 0 --Total num of actor tasks: 0 --Total num of normal tasks: 0 --Total num of driver tasks: 0 - - -[2024-05-15 08:21:39,008 I 256391 256391] (gcs_server) gcs_server.cc:844: Event stats: - - -Global stats: 21 total (10 active) -Queueing time: mean = 3.759 ms, max = 26.350 ms, min = 1.000 us, total = 78.947 ms -Execution time: mean = 1.257 ms, total = 26.400 ms -Event stats: - InternalKVGcsService.grpc_client.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.GetAll - 5 total (0 active), Execution time: mean = 4.548 us, total = 22.740 us, Queueing time: mean = 18.250 us, max = 19.810 us, min = 17.030 us, total = 91.250 us - PeriodicalRunner.RunFnPeriodically - 4 total (2 active, 1 running), Execution time: mean = 645.000 ns, total = 2.580 us, Queueing time: mean = 13.168 ms, max = 26.350 ms, min = 26.320 ms, total = 52.670 ms - GcsInMemoryStore.Put - 3 total (0 active), Execution time: mean = 8.790 ms, total = 26.369 ms, Queueing time: mean = 8.728 ms, max = 26.181 ms, min = 1.000 us, total = 26.184 ms - GcsInMemoryStore.Get - 1 total (0 active), Execution time: mean = 5.480 us, total = 5.480 us, Queueing time: mean = 1.290 us, max = 1.290 us, min = 1.290 us, total = 1.290 us - ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RayletLoadPulled - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 08:21:39,008 I 256391 256391] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats: - - -Global stats: 2 total (1 active) -Queueing time: mean = 31.364 us, max = 62.729 us, min = 62.729 us, total = 62.729 us -Execution time: mean = 10.225 us, total = 20.450 us -Event stats: - PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 20.450 us, total = 20.450 us, Queueing time: mean = 62.729 us, max = 62.729 us, min = 62.729 us, total = 62.729 us - GcsTaskManager.GcJobSummary - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 08:21:40,379 I 256391 256391] (gcs_server) gcs_node_manager.cc:55: Registering node info, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 08:21:40,379 I 256391 256391] (gcs_server) gcs_node_manager.cc:61: Finished registering node info, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 08:21:40,379 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:797: A new node: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a registered, will try to reschedule all the infeasible placement groups. -[2024-05-15 08:21:40,963 I 256391 256391] (gcs_server) gcs_job_manager.cc:42: Adding job, job id = 01000000, driver pid = 256306 -[2024-05-15 08:21:40,963 I 256391 256391] (gcs_server) gcs_job_manager.cc:57: Finished adding job, job id = 01000000, driver pid = 256306 -[2024-05-15 08:21:40,999 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:432: Registering placement group, placement group id = e13059e3372a28e5a1b0393dba1401000000, name = , strategy = 0 -[2024-05-15 08:21:40,999 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:436: Finished registering placement group, placement group id = e13059e3372a28e5a1b0393dba1401000000, name = , strategy = 0 -[2024-05-15 08:21:40,999 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:185: Preparing resource from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for bundles: {placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4}}, -[2024-05-15 08:21:41,000 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:195: Finished leasing resource from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for bundles: {placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4}}, -[2024-05-15 08:21:41,000 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:213: Committing resource to a node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for bundles: {placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2}}, -[2024-05-15 08:21:41,000 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:220: Finished committing resource to 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for bundles: {placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2}}, -[2024-05-15 08:21:41,000 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:321: Successfully created placement group , id: e13059e3372a28e5a1b0393dba1401000000 -[2024-05-15 08:21:41,175 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = fdf624732d0a9f104204059a01000000 -[2024-05-15 08:21:41,176 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = fdf624732d0a9f104204059a01000000 -[2024-05-15 08:21:41,177 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = fdf624732d0a9f104204059a01000000 -[2024-05-15 08:21:41,177 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor fdf624732d0a9f104204059a01000000, job id = 01000000 -[2024-05-15 08:21:41,580 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor fdf624732d0a9f104204059a01000000, job id = 01000000 -[2024-05-15 08:21:41,580 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor fdf624732d0a9f104204059a01000000 on worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000 -[2024-05-15 08:21:43,096 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = ba75f4cef86c74fb1b900c5a01000000 -[2024-05-15 08:21:43,096 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = ba75f4cef86c74fb1b900c5a01000000 -[2024-05-15 08:21:43,096 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = ba75f4cef86c74fb1b900c5a01000000 -[2024-05-15 08:21:43,096 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ba75f4cef86c74fb1b900c5a01000000, job id = 01000000 -[2024-05-15 08:21:43,097 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = ad0947e6b294e1fc2aa241ba01000000 -[2024-05-15 08:21:43,097 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = ad0947e6b294e1fc2aa241ba01000000 -[2024-05-15 08:21:43,097 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = ad0947e6b294e1fc2aa241ba01000000 -[2024-05-15 08:21:43,097 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ad0947e6b294e1fc2aa241ba01000000, job id = 01000000 -[2024-05-15 08:21:43,098 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 28c480d27043ee6d59fd4ecb01000000 -[2024-05-15 08:21:43,098 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 28c480d27043ee6d59fd4ecb01000000 -[2024-05-15 08:21:43,098 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 28c480d27043ee6d59fd4ecb01000000 -[2024-05-15 08:21:43,098 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 28c480d27043ee6d59fd4ecb01000000, job id = 01000000 -[2024-05-15 08:21:43,099 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 1c936e2ca7df5e9888d17b2701000000 -[2024-05-15 08:21:43,099 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 1c936e2ca7df5e9888d17b2701000000 -[2024-05-15 08:21:43,099 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 1c936e2ca7df5e9888d17b2701000000 -[2024-05-15 08:21:43,099 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 1c936e2ca7df5e9888d17b2701000000, job id = 01000000 -[2024-05-15 08:21:43,518 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ad0947e6b294e1fc2aa241ba01000000, job id = 01000000 -[2024-05-15 08:21:43,518 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 28c480d27043ee6d59fd4ecb01000000, job id = 01000000 -[2024-05-15 08:21:43,518 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor ad0947e6b294e1fc2aa241ba01000000 on worker 4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000 -[2024-05-15 08:21:43,518 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 28c480d27043ee6d59fd4ecb01000000 on worker 8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000 -[2024-05-15 08:21:43,571 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 1c936e2ca7df5e9888d17b2701000000, job id = 01000000 -[2024-05-15 08:21:43,571 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ba75f4cef86c74fb1b900c5a01000000, job id = 01000000 -[2024-05-15 08:21:43,571 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 1c936e2ca7df5e9888d17b2701000000 on worker 1c55cb5b210853988205f9bc82cfcfa1dc3961d0edbf43051c4c3d5f at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000 -[2024-05-15 08:21:43,571 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor ba75f4cef86c74fb1b900c5a01000000 on worker 3a95f67a1d239748b37bd2874e2ec5ee7411384a73ed36570ee1d355 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000 -[2024-05-15 08:21:48,989 W 256391 256412] (gcs_server) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:56844: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 08:21:54,092 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = fdf624732d0a9f104204059a01000000, job id = 01000000 -[2024-05-15 08:21:54,092 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = fdf624732d0a9f104204059a01000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:21:54,092 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:607: Cancelling all committed bundles of a placement group, id is e13059e3372a28e5a1b0393dba1401000000 -[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:458: Placement group of an id, e13059e3372a28e5a1b0393dba1401000000 is removed successfully. -[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_manager.cc:1167: Actor ad0947e6b294e1fc2aa241ba01000000 is failed on worker 4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000 -[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ad0947e6b294e1fc2aa241ba01000000, job id = 01000000 -[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_manager.cc:1167: Actor 28c480d27043ee6d59fd4ecb01000000 is failed on worker 8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000 -[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 28c480d27043ee6d59fd4ecb01000000, job id = 01000000 -[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = ba75f4cef86c74fb1b900c5a01000000, job id = 01000000 -[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = ba75f4cef86c74fb1b900c5a01000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 28c480d27043ee6d59fd4ecb01000000, job id = 01000000 -[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 28c480d27043ee6d59fd4ecb01000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 1c936e2ca7df5e9888d17b2701000000, job id = 01000000 -[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 1c936e2ca7df5e9888d17b2701000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = ad0947e6b294e1fc2aa241ba01000000, job id = 01000000 -[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = ad0947e6b294e1fc2aa241ba01000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_actor_manager.cc:798: Actor fdf624732d0a9f104204059a01000000 is out of scope, destroying actor, job id = 01000000 -[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:54,129 I 256391 256391] (gcs_server) gcs_actor_manager.cc:794: Worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 failed, destroying actor child, job id = 01000000 -[2024-05-15 08:21:54,130 I 256391 256391] (gcs_server) gcs_actor_manager.cc:794: Worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 failed, destroying actor child, job id = 01000000 -[2024-05-15 08:21:54,130 I 256391 256391] (gcs_server) gcs_actor_manager.cc:794: Worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 failed, destroying actor child, job id = 01000000 -[2024-05-15 08:21:54,130 I 256391 256391] (gcs_server) gcs_actor_manager.cc:794: Worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 failed, destroying actor child, job id = 01000000 -[2024-05-15 08:21:54,602 I 256391 256391] (gcs_server) gcs_job_manager.cc:87: Finished marking job state, job id = 01000000 -[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_node_manager.cc:128: Shutting down raylet, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_node_manager.cc:252: Removing node, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, node name = 192.168.1.56 -[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:767: Node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a failed, rescheduling the placement groups on the dead node. -[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_actor_manager.cc:1048: Node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a failed, reconstructing actors. -[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_job_manager.cc:302: Node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a failed, mark all jobs from this node as finished -[2024-05-15 08:21:54,985 I 256391 256391] (gcs_server) gcs_node_manager.cc:170: Raylet 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a is drained. Status GrpcUnavailable: RPC Error message: Cancelling all calls; RPC Error details: . The information will be published to the cluster. -[2024-05-15 08:21:54,997 I 256391 256439] (gcs_server) ray_syncer-inl.h:318: Failed to read the message from: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:55,010 I 256391 256391] (gcs_server) gcs_server_main.cc:111: GCS server received SIGTERM, shutting down... -[2024-05-15 08:21:55,010 I 256391 256391] (gcs_server) gcs_server.cc:272: Stopping GCS server. -[2024-05-15 08:21:55,015 I 256391 256391] (gcs_server) gcs_server.cc:292: GCS server stopped. -[2024-05-15 08:21:55,015 I 256391 256391] (gcs_server) io_service_pool.cc:47: IOServicePool is stopped. -[2024-05-15 08:21:55,092 I 256391 256391] (gcs_server) stats.h:120: Stats module has shutdown. diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/log_monitor.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/log_monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/monitor.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/monitor.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/monitor.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/raylet.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/raylet.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/raylet.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/raylet.out deleted file mode 100644 index 32bb8e9..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/raylet.out +++ /dev/null @@ -1,255 +0,0 @@ -[2024-05-15 08:21:40,367 I 256534 256534] (raylet) main.cc:179: Setting cluster ID to: 535ba0eef614eb34440db6e780ab29a9f7318b794e022f2f96a3d065 -[2024-05-15 08:21:40,369 I 256534 256534] (raylet) main.cc:242: Raylet is not set to kill unknown children. -[2024-05-15 08:21:40,369 I 256534 256534] (raylet) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 08:21:40,369 I 256534 256534] (raylet) main.cc:371: Setting node ID to: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:40,369 I 256534 256534] (raylet) store_runner.cc:32: Allowing the Plasma store to use up to 17.7374GB of memory. -[2024-05-15 08:21:40,369 I 256534 256534] (raylet) store_runner.cc:48: Starting object store with directory /dev/shm, fallback /home/valapeos/Projects/pneumarl/logs/rllib, and huge page support disabled -[2024-05-15 08:21:40,370 I 256534 256562] (raylet) dlmalloc.cc:154: create_and_mmap_buffer(17737449480, /dev/shm/plasmaXXXXXX) -[2024-05-15 08:21:40,370 I 256534 256562] (raylet) store.cc:564: ========== Plasma store: ================= -Current usage: 0 / 17.7374 GB -- num bytes created total: 0 -0 pending objects of total size 0MB -- objects spillable: 0 -- bytes spillable: 0 -- objects unsealed: 0 -- bytes unsealed: 0 -- objects in use: 0 -- bytes in use: 0 -- objects evictable: 0 -- bytes evictable: 0 - -- objects created by worker: 0 -- bytes created by worker: 0 -- objects restored: 0 -- bytes restored: 0 -- objects received: 0 -- bytes received: 0 -- objects errored: 0 -- bytes errored: 0 - -[2024-05-15 08:21:40,371 I 256534 256534] (raylet) grpc_server.cc:134: ObjectManager server started, listening on port 35145. -[2024-05-15 08:21:40,372 I 256534 256534] (raylet) worker_killing_policy.cc:101: Running GroupByOwner policy. -[2024-05-15 08:21:40,373 W 256534 256534] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:21:40,373 I 256534 256534] (raylet) memory_monitor.cc:47: MemoryMonitor initialized with usage threshold at 63982133248 bytes (0.95 system memory), total system memory bytes: 67349614592 -[2024-05-15 08:21:40,373 I 256534 256534] (raylet) node_manager.cc:285: Initializing NodeManager with ID 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:40,373 I 256534 256534] (raylet) grpc_server.cc:134: NodeManager server started, listening on port 45647. -[2024-05-15 08:21:40,379 I 256534 256596] (raylet) agent_manager.cc:78: Monitor agent process with name dashboard_agent/424238335 -[2024-05-15 08:21:40,379 I 256534 256598] (raylet) agent_manager.cc:78: Monitor agent process with name runtime_env_agent -[2024-05-15 08:21:40,379 I 256534 256534] (raylet) event.cc:234: Set ray event level to warning -[2024-05-15 08:21:40,379 I 256534 256534] (raylet) event.cc:342: Ray Event initialized for RAYLET -[2024-05-15 08:21:40,380 I 256534 256534] (raylet) raylet.cc:128: Raylet of id, 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a started. Raylet consists of node_manager and object_manager. node_manager address: 192.168.1.56:45647 object_manager address: 192.168.1.56:35145 hostname: valanixos -[2024-05-15 08:21:40,381 I 256534 256534] (raylet) node_manager.cc:521: [state-dump] NodeManager: -[state-dump] Node ID: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[state-dump] Node name: 192.168.1.56 -[state-dump] InitialConfigResources: {node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 177374392320000, CPU: 120000, memory: 354748784640000} -[state-dump] ClusterTaskManager: -[state-dump] ========== Node: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a ================= -[state-dump] Infeasible queue length: 0 -[state-dump] Schedule queue length: 0 -[state-dump] Dispatch queue length: 0 -[state-dump] num_waiting_for_resource: 0 -[state-dump] num_waiting_for_plasma_memory: 0 -[state-dump] num_waiting_for_remote_node_resources: 0 -[state-dump] num_worker_not_started_by_job_config_not_exist: 0 -[state-dump] num_worker_not_started_by_registration_timeout: 0 -[state-dump] num_tasks_waiting_for_workers: 0 -[state-dump] num_cancelled_tasks: 0 -[state-dump] cluster_resource_scheduler state: -[state-dump] Local id: 3934636312908726399 Local resources: {"total":{CPU: [120000], memory: [354748784640000], node:192.168.1.56: [10000], node:__internal_head__: [10000], object_store_memory: [177374392320000]}}, "available": {CPU: [120000], memory: [354748784640000], node:192.168.1.56: [10000], node:__internal_head__: [10000], object_store_memory: [177374392320000]}}, "labels":{"ray.io/node_id":"5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a",} is_draining: 0 is_idle: 1 Cluster resources: node id: 3934636312908726399{"total":{node:__internal_head__: 10000, memory: 354748784640000, CPU: 120000, object_store_memory: 177374392320000, node:192.168.1.56: 10000}}, "available": {node:__internal_head__: 10000, memory: 354748784640000, CPU: 120000, object_store_memory: 177374392320000, node:192.168.1.56: 10000}}, "labels":{"ray.io/node_id":"5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -[state-dump] Waiting tasks size: 0 -[state-dump] Number of executing tasks: 0 -[state-dump] Number of pinned task arguments: 0 -[state-dump] Number of total spilled tasks: 0 -[state-dump] Number of spilled waiting tasks: 0 -[state-dump] Number of spilled unschedulable tasks: 0 -[state-dump] Resource usage { -[state-dump] } -[state-dump] Running tasks by scheduling class: -[state-dump] ================================================== -[state-dump] -[state-dump] ClusterResources: -[state-dump] LocalObjectManager: -[state-dump] - num pinned objects: 0 -[state-dump] - pinned objects size: 0 -[state-dump] - num objects pending restore: 0 -[state-dump] - num objects pending spill: 0 -[state-dump] - num bytes pending spill: 0 -[state-dump] - num bytes currently spilled: 0 -[state-dump] - cumulative spill requests: 0 -[state-dump] - cumulative restore requests: 0 -[state-dump] - spilled objects pending delete: 0 -[state-dump] -[state-dump] ObjectManager: -[state-dump] - num local objects: 0 -[state-dump] - num unfulfilled push requests: 0 -[state-dump] - num object pull requests: 0 -[state-dump] - num chunks received total: 0 -[state-dump] - num chunks received failed (all): 0 -[state-dump] - num chunks received failed / cancelled: 0 -[state-dump] - num chunks received failed / plasma error: 0 -[state-dump] Event stats: -[state-dump] Global stats: 0 total (0 active) -[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] Execution time: mean = -nan s, total = 0.000 s -[state-dump] Event stats: -[state-dump] PushManager: -[state-dump] - num pushes in flight: 0 -[state-dump] - num chunks in flight: 0 -[state-dump] - num chunks remaining: 0 -[state-dump] - max chunks allowed: 409 -[state-dump] OwnershipBasedObjectDirectory: -[state-dump] - num listeners: 0 -[state-dump] - cumulative location updates: 0 -[state-dump] - num location updates per second: 0.000 -[state-dump] - num location lookups per second: 0.000 -[state-dump] - num locations added per second: 0.000 -[state-dump] - num locations removed per second: 0.000 -[state-dump] BufferPool: -[state-dump] - create buffer state map size: 0 -[state-dump] PullManager: -[state-dump] - num bytes available for pulled objects: 17737439232 -[state-dump] - num bytes being pulled (all): 0 -[state-dump] - num bytes being pulled / pinned: 0 -[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - first get request bundle: N/A -[state-dump] - first wait request bundle: N/A -[state-dump] - first task request bundle: N/A -[state-dump] - num objects queued: 0 -[state-dump] - num objects actively pulled (all): 0 -[state-dump] - num objects actively pulled / pinned: 0 -[state-dump] - num bundles being pulled: 0 -[state-dump] - num pull retries: 0 -[state-dump] - max timeout seconds: 0 -[state-dump] - max timeout request is already processed. No entry. -[state-dump] -[state-dump] WorkerPool: -[state-dump] - registered jobs: 0 -[state-dump] - process_failed_job_config_missing: 0 -[state-dump] - process_failed_rate_limited: 0 -[state-dump] - process_failed_pending_registration: 0 -[state-dump] - process_failed_runtime_env_setup_failed: 0 -[state-dump] - num PYTHON workers: 0 -[state-dump] - num PYTHON drivers: 0 -[state-dump] - num object spill callbacks queued: 0 -[state-dump] - num object restore queued: 0 -[state-dump] - num util functions queued: 0 -[state-dump] - num idle workers: 0 -[state-dump] TaskDependencyManager: -[state-dump] - task deps map size: 0 -[state-dump] - get req map size: 0 -[state-dump] - wait req map size: 0 -[state-dump] - local objects map size: 0 -[state-dump] WaitManager: -[state-dump] - num active wait requests: 0 -[state-dump] Subscriber: -[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_REF_REMOVED_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_OBJECT_EVICTION -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] num async plasma notifications: 0 -[state-dump] Remote node managers: -[state-dump] Event stats: -[state-dump] Global stats: 27 total (13 active) -[state-dump] Queueing time: mean = 866.145 us, max = 6.687 ms, min = 5.080 us, total = 23.386 ms -[state-dump] Execution time: mean = 458.777 us, total = 12.387 ms -[state-dump] Event stats: -[state-dump] PeriodicalRunner.RunFnPeriodically - 11 total (2 active, 1 running), Execution time: mean = 80.443 us, total = 884.877 us, Queueing time: mean = 2.125 ms, max = 6.687 ms, min = 17.970 us, total = 23.374 ms -[state-dump] NodeManager.deadline_timer.debug_state_dump - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.flush_free_objects - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 10.159 ms, total = 10.159 ms, Queueing time: mean = 6.700 us, max = 6.700 us, min = 6.700 us, total = 6.700 us -[state-dump] NodeManager.deadline_timer.record_metrics - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 595.718 us, total = 595.718 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 359.519 us, total = 359.519 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 258.210 us, total = 258.210 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 129.890 us, total = 129.890 us, Queueing time: mean = 5.080 us, max = 5.080 us, min = 5.080 us, total = 5.080 us -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] DebugString() time ms: 0 -[state-dump] -[state-dump] -[2024-05-15 08:21:40,381 I 256534 256534] (raylet) accessor.cc:627: Received notification for node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, IsAlive = 1 -[2024-05-15 08:21:40,486 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256618, the token is 0 -[2024-05-15 08:21:40,488 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256619, the token is 1 -[2024-05-15 08:21:40,489 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256620, the token is 2 -[2024-05-15 08:21:40,491 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256621, the token is 3 -[2024-05-15 08:21:40,493 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256622, the token is 4 -[2024-05-15 08:21:40,495 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256623, the token is 5 -[2024-05-15 08:21:40,497 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256624, the token is 6 -[2024-05-15 08:21:40,499 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256625, the token is 7 -[2024-05-15 08:21:40,501 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256626, the token is 8 -[2024-05-15 08:21:40,503 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256627, the token is 9 -[2024-05-15 08:21:40,505 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256628, the token is 10 -[2024-05-15 08:21:40,507 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256629, the token is 11 -[2024-05-15 08:21:40,844 I 256534 256562] (raylet) object_store.cc:35: Object store current usage 8e-09 / 17.7374 GB. -[2024-05-15 08:21:40,964 I 256534 256534] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 256306 is dead: 0 driver address: 192.168.1.56 -[2024-05-15 08:21:40,964 I 256534 256534] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 08:21:41,179 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:21:41,181 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257357, the token is 12 -[2024-05-15 08:21:41,991 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:41,991 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:41,992 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:41,992 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:42,194 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:43,098 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:21:43,100 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257429, the token is 13 -[2024-05-15 08:21:43,100 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:21:43,102 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257430, the token is 14 -[2024-05-15 08:21:43,103 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:21:43,104 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257431, the token is 15 -[2024-05-15 08:21:43,104 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:21:43,106 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257432, the token is 16 -[2024-05-15 08:21:45,381 W 256534 256534] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:21:50,379 W 256534 256556] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:56844: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 08:21:50,383 W 256534 256534] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:21:54,095 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:21:54,095 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:21:54,602 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=3, has creation task exception = false -[2024-05-15 08:21:54,602 I 256534 256534] (raylet) node_manager.cc:1541: Driver (pid=256306) is disconnected. job_id: 01000000 -[2024-05-15 08:21:54,603 I 256534 256534] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 256306 is dead: 1 driver address: 192.168.1.56 -[2024-05-15 08:21:54,603 I 256534 256534] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 08:21:54,785 I 256534 256534] (raylet) worker_pool.cc:1107: Force exiting worker whose job has exited a424db48d9263a3a0f640c7168dc3404402e2f6e7d613ff172defb0c -[2024-05-15 08:21:54,795 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:21:54,797 I 256534 256534] (raylet) main.cc:413: Raylet received SIGTERM, shutting down... -[2024-05-15 08:21:54,797 I 256534 256534] (raylet) accessor.cc:451: Unregistering node info, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a -[2024-05-15 08:21:54,797 I 256534 256534] (raylet) agent_manager.cc:109: Killing agent dashboard_agent/424238335, pid 256593. -[2024-05-15 08:21:54,802 I 256534 256596] (raylet) agent_manager.cc:80: Agent process with name dashboard_agent/424238335 exited, exit code 0. -[2024-05-15 08:21:54,802 I 256534 256534] (raylet) agent_manager.cc:109: Killing agent runtime_env_agent, pid 256597. -[2024-05-15 08:21:54,806 I 256534 256598] (raylet) agent_manager.cc:80: Agent process with name runtime_env_agent exited, exit code 0. -[2024-05-15 08:21:54,807 I 256534 256534] (raylet) io_service_pool.cc:47: IOServicePool is stopped. -[2024-05-15 08:21:54,982 I 256534 256534] (raylet) stats.h:120: Stats module has shutdown. -[2024-05-15 08:21:54,982 I 256534 256600] (raylet) experimental_mutable_object_provider.cc:200: Core worker channel io service stopped. diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/runtime_env_agent.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/runtime_env_agent.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/runtime_env_agent.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/runtime_env_agent.out deleted file mode 100644 index 69cef6d..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/runtime_env_agent.out +++ /dev/null @@ -1,2 +0,0 @@ -======== Running on http://0.0.0.0:58296 ======== -(Press CTRL+C to quit) diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-088c8dd8ad9f3344d35f9d6ca2fab223b5c066286566df5e2814a716-ffffffff-256622.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-088c8dd8ad9f3344d35f9d6ca2fab223b5c066286566df5e2814a716-ffffffff-256622.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-088c8dd8ad9f3344d35f9d6ca2fab223b5c066286566df5e2814a716-ffffffff-256622.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-088c8dd8ad9f3344d35f9d6ca2fab223b5c066286566df5e2814a716-ffffffff-256622.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-0ba47e56fc39de11cf4eca1339c6c204bac7e0a2d015f4d53b685e7d-ffffffff-256618.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-0ba47e56fc39de11cf4eca1339c6c204bac7e0a2d015f4d53b685e7d-ffffffff-256618.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-0ba47e56fc39de11cf4eca1339c6c204bac7e0a2d015f4d53b685e7d-ffffffff-256618.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-0ba47e56fc39de11cf4eca1339c6c204bac7e0a2d015f4d53b685e7d-ffffffff-256618.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-11c6225956c5b66ed8a16a8ea47a0738c92a140a6bb167abd090228a-ffffffff-256623.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-11c6225956c5b66ed8a16a8ea47a0738c92a140a6bb167abd090228a-ffffffff-256623.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-11c6225956c5b66ed8a16a8ea47a0738c92a140a6bb167abd090228a-ffffffff-256623.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-11c6225956c5b66ed8a16a8ea47a0738c92a140a6bb167abd090228a-ffffffff-256623.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-1c55cb5b210853988205f9bc82cfcfa1dc3961d0edbf43051c4c3d5f-01000000-257432.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-1c55cb5b210853988205f9bc82cfcfa1dc3961d0edbf43051c4c3d5f-01000000-257432.err deleted file mode 100644 index c2fc33c..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-1c55cb5b210853988205f9bc82cfcfa1dc3961d0edbf43051c4c3d5f-01000000-257432.err +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-1c55cb5b210853988205f9bc82cfcfa1dc3961d0edbf43051c4c3d5f-01000000-257432.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-1c55cb5b210853988205f9bc82cfcfa1dc3961d0edbf43051c4c3d5f-01000000-257432.out deleted file mode 100644 index d6a807e..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-1c55cb5b210853988205f9bc82cfcfa1dc3961d0edbf43051c4c3d5f-01000000-257432.out +++ /dev/null @@ -1,4 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -No game binary has been provided, please press PLAY in the Godot editor -waiting for remote GODOT connection on port 11012 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-28dc8bd40dbba67b6976632683843077211e76dce4524fa909714882-ffffffff-256629.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-28dc8bd40dbba67b6976632683843077211e76dce4524fa909714882-ffffffff-256629.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-28dc8bd40dbba67b6976632683843077211e76dce4524fa909714882-ffffffff-256629.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-28dc8bd40dbba67b6976632683843077211e76dce4524fa909714882-ffffffff-256629.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-3a95f67a1d239748b37bd2874e2ec5ee7411384a73ed36570ee1d355-01000000-257429.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-3a95f67a1d239748b37bd2874e2ec5ee7411384a73ed36570ee1d355-01000000-257429.err deleted file mode 100644 index c2fc33c..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-3a95f67a1d239748b37bd2874e2ec5ee7411384a73ed36570ee1d355-01000000-257429.err +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-3a95f67a1d239748b37bd2874e2ec5ee7411384a73ed36570ee1d355-01000000-257429.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-3a95f67a1d239748b37bd2874e2ec5ee7411384a73ed36570ee1d355-01000000-257429.out deleted file mode 100644 index 05bdbb6..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-3a95f67a1d239748b37bd2874e2ec5ee7411384a73ed36570ee1d355-01000000-257429.out +++ /dev/null @@ -1,4 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -No game binary has been provided, please press PLAY in the Godot editor -waiting for remote GODOT connection on port 11009 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3-01000000-257430.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3-01000000-257430.err deleted file mode 100644 index c9c058a..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3-01000000-257430.err +++ /dev/null @@ -1,34 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -[2024-05-15 08:21:54,138 C 257430 257430] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe -*** StackTrace Information *** -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7ff3f600dbba] ray::operator<<() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7ff3f600f477] ray::SpdLogMessage::Flush() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7ff3f600f917] ray::RayLog::~RayLog() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7ff3f57d2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7ff3f57d319a] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7ff3f57ea0de] ray::core::InboundRequest::Accept() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7ff3f57bc440] ray::core::NormalSchedulingQueue::ScheduleRequests() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7ff3f5a4e71e] EventTracker::RecordExecution() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7ff3f5a47b0e] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7ff3f5a47f86] boost::asio::detail::completion_handler<>::do_complete() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7ff3f60f78eb] boost::asio::detail::scheduler::do_run_one() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7ff3f60f9269] boost::asio::detail::scheduler::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7ff3f60f9972] boost::asio::io_context::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7ff3f57713fd] ray::core::CoreWorker::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7ff3f57b5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7ff3f57b60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7ff3f55bd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop() -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7ff3f74413c2] method_vectorcall_NOARGS -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7ff3f745ad93] PyObject_Vectorcall -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7ff3f72f7c5c] _PyEval_EvalFrameDefault -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7ff3f752bc9c] _PyEval_Vector.constprop.0 -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7ff3f752be3a] PyEval_EvalCode -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7ff3f7556a10] run_mod -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7ff3f7576752] _PyRun_SimpleFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7ff3f7576ff1] _PyRun_AnyFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7ff3f757990f] Py_RunMain -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7ff3f703d10e] __libc_start_call_main -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7ff3f703d1c9] __libc_start_main@GLIBC_2.2.5 -ray::RolloutWorker(_start+0x25) [0x401075] _start - diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3-01000000-257430.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3-01000000-257430.out deleted file mode 100644 index 705be67..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3-01000000-257430.out +++ /dev/null @@ -1,4 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -No game binary has been provided, please press PLAY in the Godot editor -waiting for remote GODOT connection on port 11010 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4ef9856839b7d000545063738652593137cd3c80ee4d1f2be6eeb23b-ffffffff-256619.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4ef9856839b7d000545063738652593137cd3c80ee4d1f2be6eeb23b-ffffffff-256619.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4ef9856839b7d000545063738652593137cd3c80ee4d1f2be6eeb23b-ffffffff-256619.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-4ef9856839b7d000545063738652593137cd3c80ee4d1f2be6eeb23b-ffffffff-256619.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0-01000000-257431.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0-01000000-257431.err deleted file mode 100644 index feb8217..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0-01000000-257431.err +++ /dev/null @@ -1,34 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -[2024-05-15 08:21:54,138 C 257431 257431] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe -*** StackTrace Information *** -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7f0e8860dbba] ray::operator<<() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7f0e8860f477] ray::SpdLogMessage::Flush() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7f0e8860f917] ray::RayLog::~RayLog() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7f0e87dd2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7f0e87dd319a] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7f0e87dea0de] ray::core::InboundRequest::Accept() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7f0e87dbc440] ray::core::NormalSchedulingQueue::ScheduleRequests() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7f0e8804e71e] EventTracker::RecordExecution() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7f0e88047b0e] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7f0e88047f86] boost::asio::detail::completion_handler<>::do_complete() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7f0e886f78eb] boost::asio::detail::scheduler::do_run_one() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7f0e886f9269] boost::asio::detail::scheduler::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7f0e886f9972] boost::asio::io_context::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7f0e87d713fd] ray::core::CoreWorker::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7f0e87db5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7f0e87db60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7f0e87bbd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop() -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7f0e89c413c2] method_vectorcall_NOARGS -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7f0e89c5ad93] PyObject_Vectorcall -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7f0e89af7c5c] _PyEval_EvalFrameDefault -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7f0e89d2bc9c] _PyEval_Vector.constprop.0 -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7f0e89d2be3a] PyEval_EvalCode -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7f0e89d56a10] run_mod -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7f0e89d76752] _PyRun_SimpleFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7f0e89d76ff1] _PyRun_AnyFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7f0e89d7990f] Py_RunMain -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7f0e8975a10e] __libc_start_call_main -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7f0e8975a1c9] __libc_start_main@GLIBC_2.2.5 -ray::RolloutWorker(_start+0x25) [0x401075] _start - diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0-01000000-257431.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0-01000000-257431.out deleted file mode 100644 index 14280b7..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0-01000000-257431.out +++ /dev/null @@ -1,4 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -No game binary has been provided, please press PLAY in the Godot editor -waiting for remote GODOT connection on port 11011 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-a424db48d9263a3a0f640c7168dc3404402e2f6e7d613ff172defb0c-ffffffff-256627.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-a424db48d9263a3a0f640c7168dc3404402e2f6e7d613ff172defb0c-ffffffff-256627.err deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-a424db48d9263a3a0f640c7168dc3404402e2f6e7d613ff172defb0c-ffffffff-256627.err +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-a424db48d9263a3a0f640c7168dc3404402e2f6e7d613ff172defb0c-ffffffff-256627.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-a424db48d9263a3a0f640c7168dc3404402e2f6e7d613ff172defb0c-ffffffff-256627.out deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-a424db48d9263a3a0f640c7168dc3404402e2f6e7d613ff172defb0c-ffffffff-256627.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ae6b1b2b43a0d413c653821fae1a8169b4fb9936ecd34d7413a45594-ffffffff-256621.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ae6b1b2b43a0d413c653821fae1a8169b4fb9936ecd34d7413a45594-ffffffff-256621.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ae6b1b2b43a0d413c653821fae1a8169b4fb9936ecd34d7413a45594-ffffffff-256621.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ae6b1b2b43a0d413c653821fae1a8169b4fb9936ecd34d7413a45594-ffffffff-256621.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-aff5b59bc62ec1463abd674b780a1d6de27dbe081edffa3dd5ac5492-ffffffff-256626.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-aff5b59bc62ec1463abd674b780a1d6de27dbe081edffa3dd5ac5492-ffffffff-256626.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-aff5b59bc62ec1463abd674b780a1d6de27dbe081edffa3dd5ac5492-ffffffff-256626.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-aff5b59bc62ec1463abd674b780a1d6de27dbe081edffa3dd5ac5492-ffffffff-256626.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-b94b35eebae2968a829b2e957c84f07aeaafb523b7723a6a3ddce46a-ffffffff-256624.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-b94b35eebae2968a829b2e957c84f07aeaafb523b7723a6a3ddce46a-ffffffff-256624.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-b94b35eebae2968a829b2e957c84f07aeaafb523b7723a6a3ddce46a-ffffffff-256624.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-b94b35eebae2968a829b2e957c84f07aeaafb523b7723a6a3ddce46a-ffffffff-256624.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-bd0b32b6433da96732001a5183d4230f62c26f360460b8ba80aa4052-ffffffff-256628.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-bd0b32b6433da96732001a5183d4230f62c26f360460b8ba80aa4052-ffffffff-256628.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-bd0b32b6433da96732001a5183d4230f62c26f360460b8ba80aa4052-ffffffff-256628.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-bd0b32b6433da96732001a5183d4230f62c26f360460b8ba80aa4052-ffffffff-256628.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ccd6ae108e600e0a0b7bdb7a43d68319d1243616edd76dfa6a35c093-ffffffff-256620.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ccd6ae108e600e0a0b7bdb7a43d68319d1243616edd76dfa6a35c093-ffffffff-256620.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ccd6ae108e600e0a0b7bdb7a43d68319d1243616edd76dfa6a35c093-ffffffff-256620.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ccd6ae108e600e0a0b7bdb7a43d68319d1243616edd76dfa6a35c093-ffffffff-256620.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ec204b681caad43733b5c2c9125317ac287675b06166d22c1d6440aa-ffffffff-256625.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ec204b681caad43733b5c2c9125317ac287675b06166d22c1d6440aa-ffffffff-256625.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ec204b681caad43733b5c2c9125317ac287675b06166d22c1d6440aa-ffffffff-256625.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-ec204b681caad43733b5c2c9125317ac287675b06166d22c1d6440aa-ffffffff-256625.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17-01000000-257357.err b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17-01000000-257357.err deleted file mode 100644 index 92c2840..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17-01000000-257357.err +++ /dev/null @@ -1,5 +0,0 @@ -:job_id:01000000 -:actor_name:PPO -2024-05-15 08:21:43,045 WARNING deprecation.py:50 -- DeprecationWarning: `AlgorithmConfig.num_envs_per_worker` has been deprecated. Use `AlgorithmConfig.num_envs_per_env_runner` instead. This will raise an error in the future! -2024-05-15 08:21:43,051 WARNING deprecation.py:50 -- DeprecationWarning: `WorkerSet(num_workers=... OR local_worker=...)` has been deprecated. Use `EnvRunnerGroup(num_env_runners=... AND local_env_runner=...)` instead. This will raise an error in the future! -2024-05-15 08:21:43,051 WARNING deprecation.py:50 -- DeprecationWarning: `max_num_worker_restarts` has been deprecated. Use `AlgorithmConfig.max_num_env_runner_restarts` instead. This will raise an error in the future! diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17-01000000-257357.out b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17-01000000-257357.out deleted file mode 100644 index 1124c85..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/logs/worker-f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17-01000000-257357.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:actor_name:PPO diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/node_ip_address.json b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/node_ip_address.json deleted file mode 100644 index 8d998d8..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/node_ip_address.json +++ /dev/null @@ -1 +0,0 @@ -{"node_ip_address": "192.168.1.56"} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/node_ip_address.json.lock b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/node_ip_address.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/ports_by_node.json b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/ports_by_node.json deleted file mode 100644 index 5eda1ed..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/ports_by_node.json +++ /dev/null @@ -1 +0,0 @@ -{"192.168.1.56:/home/valapeos/Projects/pneumarl/logs/rllib/session_2024-05-15_08-21-38_956751_256306/sockets/plasma_store": {"metrics_agent_port": 56844, "metrics_export_port": 42241, "dashboard_agent_listen_port": 52365, "runtime_env_agent_port": 58296, "gcs_server_port": 56287}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/ports_by_node.json.lock b/Godot/logs/rllib/session_2024-05-15_08-21-38_956751_256306/ports_by_node.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/artifacts/2024-05-15_08-22-50/PPO_2024-05-15_08-22-50/driver_artifacts/basic-variant-state-2024-05-15_08-22-50.json b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/artifacts/2024-05-15_08-22-50/PPO_2024-05-15_08-22-50/driver_artifacts/basic-variant-state-2024-05-15_08-22-50.json deleted file mode 100644 index e94b9c7..0000000 Binary files a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/artifacts/2024-05-15_08-22-50/PPO_2024-05-15_08-22-50/driver_artifacts/basic-variant-state-2024-05-15_08-22-50.json and /dev/null differ diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/artifacts/2024-05-15_08-22-50/PPO_2024-05-15_08-22-50/driver_artifacts/experiment_state-2024-05-15_08-22-50.json b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/artifacts/2024-05-15_08-22-50/PPO_2024-05-15_08-22-50/driver_artifacts/experiment_state-2024-05-15_08-22-50.json deleted file mode 100644 index 08a9b7b..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/artifacts/2024-05-15_08-22-50/PPO_2024-05-15_08-22-50/driver_artifacts/experiment_state-2024-05-15_08-22-50.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"8dccf_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30382d32322d3530948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f38646363665f30303030305f305f323032342d30352d31355f30382d32322d3530948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30382d32322d35309475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": null,\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": null,\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_8dccf_00000_0_2024-05-15_08-22-50\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 370, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715754170.0899887, "_session_str": "2024-05-15_08-22-50", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715754170.0899887}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/agent-424238335.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/agent-424238335.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/agent-424238335.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/agent-424238335.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/dashboard.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/dashboard.err deleted file mode 100644 index 7479327..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/dashboard.err +++ /dev/null @@ -1,6 +0,0 @@ -Traceback (most recent call last): - File "python/ray/_raylet.pyx", line 3270, in ray._raylet.check_health - File "python/ray/_raylet.pyx", line 583, in ray._raylet.check_status -ray.exceptions.RpcError: failed to connect to all addresses; last error: UNKNOWN: ipv4:192.168.1.56:47881: Failed to connect to remote host: Connection refused -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/dashboard/dashboard.py:236: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead - logger.warn("Exiting with SIGTERM immediately...") diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/debug_state.txt b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/debug_state.txt deleted file mode 100644 index 4c523ab..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/debug_state.txt +++ /dev/null @@ -1,208 +0,0 @@ -NodeManager: -Node ID: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -Node name: 192.168.1.56 -InitialConfigResources: {CPU: 120000, memory: 353902559240000, node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 176951279610000} -ClusterTaskManager: -========== Node: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 ================= -Infeasible queue length: 0 -Schedule queue length: 0 -Dispatch queue length: 0 -num_waiting_for_resource: 0 -num_waiting_for_plasma_memory: 0 -num_waiting_for_remote_node_resources: 0 -num_worker_not_started_by_job_config_not_exist: 0 -num_worker_not_started_by_registration_timeout: 0 -num_tasks_waiting_for_workers: 0 -num_cancelled_tasks: 0 -cluster_resource_scheduler state: -Local id: -1885364415256342489 Local resources: {"total":{node:__internal_head__: [10000], bundle_group_81c6c20970b7e7ed4f550507158301000000: [50000000], bundle_group_4_81c6c20970b7e7ed4f550507158301000000: [10000000], bundle_group_3_81c6c20970b7e7ed4f550507158301000000: [10000000], bundle_group_2_81c6c20970b7e7ed4f550507158301000000: [10000000], CPU: [120000], CPU_group_2_81c6c20970b7e7ed4f550507158301000000: [10000], CPU_group_3_81c6c20970b7e7ed4f550507158301000000: [10000], bundle_group_1_81c6c20970b7e7ed4f550507158301000000: [10000000], CPU_group_0_81c6c20970b7e7ed4f550507158301000000: [10000], bundle_group_0_81c6c20970b7e7ed4f550507158301000000: [10000000], memory: [353902559240000], CPU_group_4_81c6c20970b7e7ed4f550507158301000000: [10000], CPU_group_81c6c20970b7e7ed4f550507158301000000: [50000], CPU_group_1_81c6c20970b7e7ed4f550507158301000000: [10000], object_store_memory: [176951279610000], node:192.168.1.56: [10000]}}, "available": {node:__internal_head__: [10000], bundle_group_81c6c20970b7e7ed4f550507158301000000: [49999950], bundle_group_4_81c6c20970b7e7ed4f550507158301000000: [10000000], bundle_group_3_81c6c20970b7e7ed4f550507158301000000: [10000000], bundle_group_2_81c6c20970b7e7ed4f550507158301000000: [10000000], CPU: [70000], CPU_group_2_81c6c20970b7e7ed4f550507158301000000: [10000], CPU_group_3_81c6c20970b7e7ed4f550507158301000000: [10000], bundle_group_1_81c6c20970b7e7ed4f550507158301000000: [10000000], CPU_group_0_81c6c20970b7e7ed4f550507158301000000: [0], bundle_group_0_81c6c20970b7e7ed4f550507158301000000: [9999990], memory: [353902559240000], CPU_group_4_81c6c20970b7e7ed4f550507158301000000: [10000], CPU_group_81c6c20970b7e7ed4f550507158301000000: [0], CPU_group_1_81c6c20970b7e7ed4f550507158301000000: [10000], object_store_memory: [176951279610000], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450",} is_draining: 0 is_idle: 0 Cluster resources: node id: -1885364415256342489{"total":{memory: 353902559240000, CPU_group_4_81c6c20970b7e7ed4f550507158301000000: 10000, CPU_group_0_81c6c20970b7e7ed4f550507158301000000: 10000, CPU_group_1_81c6c20970b7e7ed4f550507158301000000: 10000, CPU_group_81c6c20970b7e7ed4f550507158301000000: 50000, node:192.168.1.56: 10000, object_store_memory: 176951279610000, node:__internal_head__: 10000, bundle_group_81c6c20970b7e7ed4f550507158301000000: 50000000, CPU_group_3_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_3_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_4_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_2_81c6c20970b7e7ed4f550507158301000000: 10000000, CPU: 120000, CPU_group_2_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_1_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_0_81c6c20970b7e7ed4f550507158301000000: 10000000}}, "available": {object_store_memory: 176951279610000, node:192.168.1.56: 10000, CPU_group_1_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_0_81c6c20970b7e7ed4f550507158301000000: 9999990, CPU_group_4_81c6c20970b7e7ed4f550507158301000000: 10000, memory: 353902559240000, bundle_group_1_81c6c20970b7e7ed4f550507158301000000: 10000000, CPU_group_2_81c6c20970b7e7ed4f550507158301000000: 10000, CPU: 70000, bundle_group_2_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_3_81c6c20970b7e7ed4f550507158301000000: 10000000, CPU_group_3_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_4_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_81c6c20970b7e7ed4f550507158301000000: 49999950, node:__internal_head__: 10000}}, "labels":{"ray.io/node_id":"a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -Waiting tasks size: 0 -Number of executing tasks: 5 -Number of pinned task arguments: 0 -Number of total spilled tasks: 0 -Number of spilled waiting tasks: 0 -Number of spilled unschedulable tasks: 0 -Resource usage { - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=259038): {bundle_group_81c6c20970b7e7ed4f550507158301000000: 10, CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=259036): {CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_81c6c20970b7e7ed4f550507158301000000: 10} - - (language=PYTHON actor_or_task=PPO.__init__ pid=258963): {CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000, CPU_group_0_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_81c6c20970b7e7ed4f550507158301000000: 10, bundle_group_0_81c6c20970b7e7ed4f550507158301000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=259039): {bundle_group_81c6c20970b7e7ed4f550507158301000000: 10, CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=259037): {bundle_group_81c6c20970b7e7ed4f550507158301000000: 10, CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000} -} -Running tasks by scheduling class: - - {depth=1 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.algorithms.ppo.ppo, class_name=PPO, function_name=__init__, function_hash=af33c904067c44ebb8a2cc1b903cfde9} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "\201\306\302\tp\267\347\355OU\005\007\025\203\001\000\000\000" - placement_group_capture_child_tasks: true -} - resource_set={bundle_group_81c6c20970b7e7ed4f550507158301000000 : 0.001, CPU_group_81c6c20970b7e7ed4f550507158301000000 : 1, CPU_group_0_81c6c20970b7e7ed4f550507158301000000 : 1, bundle_group_0_81c6c20970b7e7ed4f550507158301000000 : 0.001, }}: 1/18446744073709551615 - - {depth=2 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.evaluation.rollout_worker, class_name=RolloutWorker, function_name=__init__, function_hash=3e0ea47c3fc64ebd817dc76e776d34cf} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "\201\306\302\tp\267\347\355OU\005\007\025\203\001\000\000\000" - placement_group_bundle_index: -1 - placement_group_capture_child_tasks: true -} - resource_set={bundle_group_81c6c20970b7e7ed4f550507158301000000 : 0.001, CPU_group_81c6c20970b7e7ed4f550507158301000000 : 1, }}: 4/18446744073709551615 -================================================== - -ClusterResources: -LocalObjectManager: -- num pinned objects: 0 -- pinned objects size: 0 -- num objects pending restore: 0 -- num objects pending spill: 0 -- num bytes pending spill: 0 -- num bytes currently spilled: 0 -- cumulative spill requests: 0 -- cumulative restore requests: 0 -- spilled objects pending delete: 0 - -ObjectManager: -- num local objects: 0 -- num unfulfilled push requests: 0 -- num object pull requests: 0 -- num chunks received total: 0 -- num chunks received failed (all): 0 -- num chunks received failed / cancelled: 0 -- num chunks received failed / plasma error: 0 -Event stats: -Global stats: 0 total (0 active) -Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -Execution time: mean = -nan s, total = 0.000 s -Event stats: -PushManager: -- num pushes in flight: 0 -- num chunks in flight: 0 -- num chunks remaining: 0 -- max chunks allowed: 409 -OwnershipBasedObjectDirectory: -- num listeners: 0 -- cumulative location updates: 0 -- num location updates per second: 0.000 -- num location lookups per second: 0.000 -- num locations added per second: 0.000 -- num locations removed per second: 0.000 -BufferPool: -- create buffer state map size: 0 -PullManager: -- num bytes available for pulled objects: 17695127961 -- num bytes being pulled (all): 0 -- num bytes being pulled / pinned: 0 -- get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- first get request bundle: N/A -- first wait request bundle: N/A -- first task request bundle: N/A -- num objects queued: 0 -- num objects actively pulled (all): 0 -- num objects actively pulled / pinned: 0 -- num bundles being pulled: 0 -- num pull retries: 0 -- max timeout seconds: 0 -- max timeout request is already processed. No entry. - -WorkerPool: -- registered jobs: 1 -- process_failed_job_config_missing: 0 -- process_failed_rate_limited: 0 -- process_failed_pending_registration: 0 -- process_failed_runtime_env_setup_failed: 0 -- num PYTHON workers: 12 -- num PYTHON drivers: 1 -- num object spill callbacks queued: 0 -- num object restore queued: 0 -- num util functions queued: 0 -- num idle workers: 7 -TaskDependencyManager: -- task deps map size: 0 -- get req map size: 0 -- wait req map size: 0 -- local objects map size: 0 -WaitManager: -- num active wait requests: 0 -Subscriber: -Channel WORKER_OBJECT_LOCATIONS_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_OBJECT_EVICTION -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_REF_REMOVED_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -num async plasma notifications: 0 -Remote node managers: -Event stats: -Global stats: 2474 total (27 active) -Queueing time: mean = 2.790 ms, max = 1.347 s, min = 70.000 ns, total = 6.903 s -Execution time: mean = 1.664 ms, total = 4.117 s -Event stats: - NodeManagerService.grpc_server.ReportWorkerBacklog.HandleRequestImpl - 387 total (0 active), Execution time: mean = 9.061 us, total = 3.507 ms, Queueing time: mean = 10.254 us, max = 414.788 us, min = 4.500 us, total = 3.968 ms - NodeManagerService.grpc_server.ReportWorkerBacklog - 387 total (0 active), Execution time: mean = 87.039 us, total = 33.684 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RaySyncer.OnDemandBroadcasting - 300 total (1 active), Execution time: mean = 5.138 us, total = 1.541 ms, Queueing time: mean = 10.532 us, max = 61.158 us, min = 5.749 us, total = 3.160 ms - ObjectManager.UpdateAvailableMemory - 300 total (0 active), Execution time: mean = 978.530 ns, total = 293.559 us, Queueing time: mean = 6.798 us, max = 27.920 us, min = 4.160 us, total = 2.040 ms - NodeManager.CheckGC - 300 total (1 active), Execution time: mean = 687.800 ns, total = 206.340 us, Queueing time: mean = 14.601 us, max = 225.787 us, min = 5.938 us, total = 4.380 ms - RayletWorkerPool.deadline_timer.kill_idle_workers - 150 total (1 active), Execution time: mean = 11.656 us, total = 1.748 ms, Queueing time: mean = 11.804 us, max = 247.415 us, min = 5.048 us, total = 1.771 ms - MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 120 total (1 active), Execution time: mean = 80.284 us, total = 9.634 ms, Queueing time: mean = 10.150 us, max = 25.878 us, min = 3.129 us, total = 1.218 ms - ClientConnection.async_read.ProcessMessageHeader - 83 total (13 active), Execution time: mean = 2.708 us, total = 224.740 us, Queueing time: mean = 82.469 ms, max = 1.347 s, min = 9.280 us, total = 6.845 s - ClientConnection.async_read.ProcessMessage - 70 total (0 active), Execution time: mean = 443.031 us, total = 31.012 ms, Queueing time: mean = 53.817 us, max = 3.130 ms, min = 1.950 us, total = 3.767 ms - NodeManager.ScheduleAndDispatchTasks - 31 total (1 active), Execution time: mean = 4.692 us, total = 145.450 us, Queueing time: mean = 8.759 us, max = 18.074 us, min = 6.457 us, total = 271.529 us - NodeManagerService.grpc_server.GetResourceLoad.HandleRequestImpl - 30 total (0 active), Execution time: mean = 50.454 us, total = 1.514 ms, Queueing time: mean = 9.796 us, max = 16.609 us, min = 6.390 us, total = 293.868 us - NodeManager.deadline_timer.flush_free_objects - 30 total (1 active), Execution time: mean = 1.674 us, total = 50.230 us, Queueing time: mean = 40.812 us, max = 324.394 us, min = 2.845 us, total = 1.224 ms - NodeManagerService.grpc_server.GetResourceLoad - 30 total (0 active), Execution time: mean = 136.638 us, total = 4.099 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.deadline_timer.spill_objects_when_over_threshold - 30 total (1 active), Execution time: mean = 1.005 us, total = 30.150 us, Queueing time: mean = 41.323 us, max = 323.594 us, min = 3.375 us, total = 1.240 ms - ClientConnection.async_write.DoAsyncWrites - 19 total (0 active), Execution time: mean = 681.053 ns, total = 12.940 us, Queueing time: mean = 10.481 us, max = 15.320 us, min = 6.630 us, total = 199.130 us - ObjectManager.ObjectDeleted - 18 total (0 active), Execution time: mean = 8.087 us, total = 145.560 us, Queueing time: mean = 47.910 us, max = 424.198 us, min = 12.300 us, total = 862.376 us - NodeManagerService.grpc_server.GetSystemConfig.HandleRequestImpl - 18 total (0 active), Execution time: mean = 25.750 us, total = 463.508 us, Queueing time: mean = 10.899 us, max = 16.410 us, min = 6.660 us, total = 196.189 us - ObjectManager.ObjectAdded - 18 total (0 active), Execution time: mean = 7.946 us, total = 143.020 us, Queueing time: mean = 36.346 us, max = 359.769 us, min = 5.730 us, total = 654.237 us - NodeManagerService.grpc_server.GetSystemConfig - 18 total (0 active), Execution time: mean = 138.977 us, total = 2.502 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 13 total (0 active), Execution time: mean = 108.665 us, total = 1.413 ms, Queueing time: mean = 2.209 ms, max = 7.494 ms, min = 18.500 us, total = 28.713 ms - ClusterResourceManager.ResetRemoteNodeView - 11 total (1 active), Execution time: mean = 1.784 us, total = 19.620 us, Queueing time: mean = 8.919 us, max = 11.237 us, min = 7.955 us, total = 98.109 us - RaySyncer.BroadcastMessage - 6 total (0 active), Execution time: mean = 111.837 us, total = 671.020 us, Queueing time: mean = 171.667 ns, max = 210.000 ns, min = 100.000 ns, total = 1.030 us - - 6 total (0 active), Execution time: mean = 103.333 ns, total = 620.000 ns, Queueing time: mean = 8.868 us, max = 11.030 us, min = 7.790 us, total = 53.210 us - NodeManager.GcsCheckAlive - 6 total (1 active), Execution time: mean = 84.400 us, total = 506.398 us, Queueing time: mean = 100.684 us, max = 158.511 us, min = 42.914 us, total = 604.106 us - NodeManager.deadline_timer.record_metrics - 6 total (1 active), Execution time: mean = 191.353 us, total = 1.148 ms, Queueing time: mean = 13.328 us, max = 45.615 us, min = 6.792 us, total = 79.968 us - NodeManagerService.grpc_server.RequestWorkerLease.HandleRequestImpl - 6 total (0 active), Execution time: mean = 122.608 us, total = 735.649 us, Queueing time: mean = 260.461 us, max = 829.938 us, min = 5.530 us, total = 1.563 ms - NodeManagerService.grpc_server.RequestWorkerLease - 6 total (0 active), Execution time: mean = 364.173 ms, total = 2.185 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.CheckAlive.OnReplyReceived - 6 total (0 active), Execution time: mean = 11.150 us, total = 66.900 us, Queueing time: mean = 8.055 us, max = 9.550 us, min = 5.810 us, total = 48.330 us - NodeInfoGcsService.grpc_client.CheckAlive - 6 total (0 active), Execution time: mean = 314.641 us, total = 1.888 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - Subscriber.HandlePublishedMessage_GCS_WORKER_DELTA_CHANNEL - 5 total (0 active), Execution time: mean = 2.344 us, total = 11.720 us, Queueing time: mean = 51.394 us, max = 66.610 us, min = 45.690 us, total = 256.970 us - CoreWorkerService.grpc_client.Exit.OnReplyReceived - 5 total (0 active), Execution time: mean = 11.538 us, total = 57.690 us, Queueing time: mean = 9.718 us, max = 20.000 us, min = 5.040 us, total = 48.590 us - CoreWorkerService.grpc_client.Exit - 5 total (0 active), Execution time: mean = 800.304 us, total = 4.002 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_client.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 1.124 ms, total = 5.622 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_client.ReportWorkerFailure.OnReplyReceived - 5 total (0 active), Execution time: mean = 7.830 us, total = 39.149 us, Queueing time: mean = 46.599 us, max = 72.909 us, min = 9.690 us, total = 232.997 us - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 5 total (1 active), Execution time: mean = 361.620 ms, total = 1.808 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll.OnReplyReceived - 4 total (0 active), Execution time: mean = 60.217 us, total = 240.869 us, Queueing time: mean = 7.112 us, max = 10.609 us, min = 4.280 us, total = 28.449 us - NodeManager.deadline_timer.debug_state_dump - 3 total (1 active, 1 running), Execution time: mean = 398.066 us, total = 1.194 ms, Queueing time: mean = 5.335 us, max = 10.340 us, min = 5.666 us, total = 16.006 us - RaySyncerRegister - 2 total (0 active), Execution time: mean = 815.000 ns, total = 1.630 us, Queueing time: mean = 165.000 ns, max = 260.000 ns, min = 70.000 ns, total = 330.000 ns - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 2 total (0 active), Execution time: mean = 276.159 us, total = 552.319 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 2 total (0 active), Execution time: mean = 68.675 us, total = 137.350 us, Queueing time: mean = 540.198 us, max = 1.004 ms, min = 76.220 us, total = 1.080 ms - NodeManager.deadline_timer.print_event_loop_stats - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.AddJob - 1 total (0 active), Execution time: mean = 377.469 us, total = 377.469 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetAllNodeInfo - 1 total (0 active), Execution time: mean = 301.139 us, total = 301.139 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.GetAllJobInfo - 1 total (0 active), Execution time: mean = 289.119 us, total = 289.119 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 147.790 us, total = 147.790 us, Queueing time: mean = 8.099 us, max = 8.099 us, min = 8.099 us, total = 8.099 us - NodeManagerService.grpc_server.CommitBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 219.390 us, total = 219.390 us, Queueing time: mean = 6.870 us, max = 6.870 us, min = 6.870 us, total = 6.870 us - Subscriber.HandlePublishedMessage_GCS_JOB_CHANNEL - 1 total (0 active), Execution time: mean = 23.820 us, total = 23.820 us, Queueing time: mean = 56.110 us, max = 56.110 us, min = 56.110 us, total = 56.110 us - NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 730.548 us, total = 730.548 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 11.090 ms, total = 11.090 ms, Queueing time: mean = 8.630 us, max = 8.630 us, min = 8.630 us, total = 8.630 us - JobInfoGcsService.grpc_client.GetAllJobInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 5.820 us, total = 5.820 us, Queueing time: mean = 4.820 us, max = 4.820 us, min = 4.820 us, total = 4.820 us - NodeManagerService.grpc_server.PrepareBundleResources - 1 total (0 active), Execution time: mean = 293.789 us, total = 293.789 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetAllNodeInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 73.390 us, total = 73.390 us, Queueing time: mean = 5.250 us, max = 5.250 us, min = 5.250 us, total = 5.250 us - NodeManagerService.grpc_server.ReturnWorker - 1 total (0 active), Execution time: mean = 121.460 us, total = 121.460 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerPool.PopWorkerCallback - 1 total (0 active), Execution time: mean = 12.860 us, total = 12.860 us, Queueing time: mean = 6.460 us, max = 6.460 us, min = 6.460 us, total = 6.460 us - NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 429.879 us, total = 429.879 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.CommitBundleResources - 1 total (0 active), Execution time: mean = 275.420 us, total = 275.420 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.PrepareBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 234.479 us, total = 234.479 us, Queueing time: mean = 6.390 us, max = 6.390 us, min = 6.390 us, total = 6.390 us - JobInfoGcsService.grpc_client.AddJob.OnReplyReceived - 1 total (0 active), Execution time: mean = 16.820 us, total = 16.820 us, Queueing time: mean = 4.910 us, max = 4.910 us, min = 4.910 us, total = 4.910 us - NodeManagerService.grpc_server.ReturnWorker.HandleRequestImpl - 1 total (0 active), Execution time: mean = 37.550 us, total = 37.550 us, Queueing time: mean = 9.240 us, max = 9.240 us, min = 9.240 us, total = 9.240 us -DebugString() time ms: 0 \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/debug_state_gcs.txt b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/debug_state_gcs.txt deleted file mode 100644 index be04100..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/debug_state_gcs.txt +++ /dev/null @@ -1,150 +0,0 @@ -GcsNodeManager: -- RegisterNode request count: 1 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 28 -- GetInternalConfig request count: 1 - -GcsActorManager: -- RegisterActor request count: 5 -- CreateActor request count: 5 -- GetActorInfo request count: 5 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 5 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 1 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 5 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 8 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 1 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 5 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 1 -- Registered placement groups count: 1 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: -RAY_LOG_CHANNEL -- cumulative published messages: 13 -- cumulative published bytes: 9872 -- current buffered bytes: 853 -GCS_JOB_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 280 -- current buffered bytes: 280 -GCS_ACTOR_CHANNEL -- cumulative published messages: 10 -- cumulative published bytes: 5864 -- current buffered bytes: 1328 -GCS_WORKER_DELTA_CHANNEL -- cumulative published messages: 5 -- cumulative published bytes: 485 -- current buffered bytes: 97 -GCS_NODE_INFO_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 559 -- current buffered bytes: 0 - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 25 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 13 --Total num of actor creation tasks: 5 --Total num of actor tasks: 6 --Total num of normal tasks: 1 --Total num of driver tasks: 1 - - - - -Global stats: 1124 total (19 active) -Queueing time: mean = 82.405 us, max = 26.288 ms, min = 230.000 ns, total = 92.623 ms -Execution time: mean = 2.055 ms, total = 2.310 s -Event stats: - GcsInMemoryStore.Put - 159 total (0 active), Execution time: mean = 185.082 us, total = 29.428 ms, Queueing time: mean = 173.789 us, max = 26.115 ms, min = 680.000 ns, total = 27.632 ms - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 113 total (0 active), Execution time: mean = 6.158 us, total = 695.898 us, Queueing time: mean = 35.882 us, max = 229.260 us, min = 2.960 us, total = 4.055 ms - InternalKVGcsService.grpc_server.InternalKVPut - 113 total (0 active), Execution time: mean = 122.098 us, total = 13.797 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Get - 81 total (0 active), Execution time: mean = 18.103 us, total = 1.466 ms, Queueing time: mean = 11.716 us, max = 272.030 us, min = 620.000 ns, total = 948.999 us - InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 74 total (0 active), Execution time: mean = 10.299 us, total = 762.127 us, Queueing time: mean = 9.189 us, max = 33.660 us, min = 4.360 us, total = 679.978 us - InternalKVGcsService.grpc_server.InternalKVGet - 74 total (0 active), Execution time: mean = 148.802 us, total = 11.011 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 48 total (0 active), Execution time: mean = 3.526 us, total = 169.250 us, Queueing time: mean = 12.323 us, max = 27.350 us, min = 4.810 us, total = 591.518 us - InternalKVGcsService.grpc_client.InternalKVPut - 48 total (0 active), Execution time: mean = 383.844 us, total = 18.425 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RayletLoadPulled - 40 total (1 active), Execution time: mean = 74.947 us, total = 2.998 ms, Queueing time: mean = 18.572 us, max = 277.794 us, min = 4.442 us, total = 742.873 us - NodeManagerService.grpc_client.GetResourceLoad - 38 total (0 active), Execution time: mean = 337.670 us, total = 12.831 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 38 total (0 active), Execution time: mean = 25.848 us, total = 982.226 us, Queueing time: mean = 8.813 us, max = 21.630 us, min = 5.180 us, total = 334.899 us - NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 28 total (0 active), Execution time: mean = 12.195 us, total = 341.460 us, Queueing time: mean = 20.317 us, max = 361.159 us, min = 5.360 us, total = 568.879 us - NodeInfoGcsService.grpc_server.GetAllNodeInfo - 28 total (0 active), Execution time: mean = 100.779 us, total = 2.822 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 18.321 us, total = 329.770 us, Queueing time: mean = 36.146 us, max = 275.469 us, min = 6.010 us, total = 650.628 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 157.753 us, total = 2.840 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.CheckAlive - 16 total (0 active), Execution time: mean = 88.642 us, total = 1.418 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 16 total (0 active), Execution time: mean = 6.994 us, total = 111.910 us, Queueing time: mean = 9.734 us, max = 16.280 us, min = 6.620 us, total = 155.739 us - ClusterResourceManager.ResetRemoteNodeView - 14 total (1 active), Execution time: mean = 2.224 us, total = 31.130 us, Queueing time: mean = 6.328 us, max = 11.648 us, min = 4.711 us, total = 88.591 us - HealthCheck - 12 total (0 active), Execution time: mean = 2.049 us, total = 24.590 us, Queueing time: mean = 9.494 us, max = 18.440 us, min = 7.090 us, total = 113.929 us - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 8 total (0 active), Execution time: mean = 115.109 us, total = 920.869 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 8 total (0 active), Execution time: mean = 25.117 us, total = 200.940 us, Queueing time: mean = 10.002 us, max = 11.780 us, min = 7.730 us, total = 80.020 us - NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 14.996 us, total = 119.970 us, Queueing time: mean = 65.355 us, max = 436.549 us, min = 4.770 us, total = 522.839 us - NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 712.702 us, total = 5.702 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 5.966 us, total = 41.760 us, Queueing time: mean = 12.703 us, max = 17.850 us, min = 1.920 us, total = 88.920 us - GcsResourceManager::Update - 6 total (0 active), Execution time: mean = 42.716 us, total = 256.298 us, Queueing time: mean = 9.708 us, max = 13.170 us, min = 5.790 us, total = 58.249 us - WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 17.160 us, total = 85.799 us, Queueing time: mean = 282.695 us, max = 692.049 us, min = 10.030 us, total = 1.413 ms - ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 126.632 us, total = 633.159 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 325.337 us, total = 1.627 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 166.314 us, total = 831.568 us, Queueing time: mean = 31.304 us, max = 105.160 us, min = 4.820 us, total = 156.520 us - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 191.969 us, total = 959.847 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 6.290 us, total = 31.450 us, Queueing time: mean = 31.650 us, max = 75.050 us, min = 6.040 us, total = 158.250 us - NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 437.588 ms, total = 2.188 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 833.848 us, total = 4.169 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 131.430 us, total = 657.148 us, Queueing time: mean = 7.282 us, max = 11.420 us, min = 5.470 us, total = 36.410 us - CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 10.648 us, total = 53.240 us, Queueing time: mean = 121.932 us, max = 289.339 us, min = 14.550 us, total = 609.658 us - ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 217.513 us, total = 1.088 ms, Queueing time: mean = 7.596 us, max = 8.430 us, min = 6.900 us, total = 37.980 us - PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 53.635 us, total = 214.540 us, Queueing time: mean = 13.173 ms, max = 26.288 ms, min = 14.480 us, total = 52.692 ms - GCSServer.deadline_timer.debug_state_dump - 4 total (1 active, 1 running), Execution time: mean = 338.224 us, total = 1.353 ms, Queueing time: mean = 4.808 us, max = 6.686 us, min = 6.045 us, total = 19.233 us - InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.290 us, total = 9.290 us, Queueing time: mean = 4.890 us, max = 4.890 us, min = 4.890 us, total = 4.890 us - NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 113.319 us, total = 113.319 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 100.969 us, total = 100.969 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 68.430 us, total = 68.430 us, Queueing time: mean = 17.470 us, max = 17.470 us, min = 17.470 us, total = 17.470 us - NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 407.498 us, total = 407.498 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 11.320 us, total = 11.320 us, Queueing time: mean = 13.690 us, max = 13.690 us, min = 13.690 us, total = 13.690 us - NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 42.040 us, total = 42.040 us, Queueing time: mean = 7.230 us, max = 7.230 us, min = 7.230 us, total = 7.230 us - JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 47.930 us, total = 47.930 us, Queueing time: mean = 8.830 us, max = 8.830 us, min = 8.830 us, total = 8.830 us - JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 4.790 us, total = 4.790 us, Queueing time: mean = 7.410 us, max = 7.410 us, min = 7.410 us, total = 7.410 us - InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 78.379 us, total = 78.379 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 42.060 us, total = 42.060 us, Queueing time: mean = 6.720 us, max = 6.720 us, min = 6.720 us, total = 6.720 us - GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 2.030 us, total = 2.030 us, Queueing time: mean = 1.070 us, max = 1.070 us, min = 1.070 us, total = 1.070 us - JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 7.970 us, total = 7.970 us, Queueing time: mean = 6.560 us, max = 6.560 us, min = 6.560 us, total = 6.560 us - GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 2.930 us, total = 2.930 us, Queueing time: mean = 1.850 us, max = 1.850 us, min = 1.850 us, total = 1.850 us - NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 793.648 us, total = 793.648 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 80.000 ns, total = 80.000 ns, Queueing time: mean = 95.950 us, max = 95.950 us, min = 95.950 us, total = 95.950 us - InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 49.420 us, total = 49.420 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 95.100 us, total = 95.100 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 70.760 us, total = 70.760 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 53.540 us, total = 53.540 us, Queueing time: mean = 7.710 us, max = 7.710 us, min = 7.710 us, total = 7.710 us - GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 6.850 us, total = 6.850 us, Queueing time: mean = 230.000 ns, max = 230.000 ns, min = 230.000 ns, total = 230.000 ns - GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 329.699 us, total = 329.699 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 141.810 us, total = 141.810 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 16.150 us, total = 16.150 us, Queueing time: mean = 6.570 us, max = 6.570 us, min = 6.570 us, total = 6.570 us \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/gcs_server.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/gcs_server.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/gcs_server.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/gcs_server.out deleted file mode 100644 index ee47275..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/gcs_server.out +++ /dev/null @@ -1,190 +0,0 @@ -[2024-05-15 08:22:48,114 I 257996 257996] (gcs_server) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) event.cc:234: Set ray event level to warning -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) event.cc:342: Ray Event initialized for GCS -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_server.cc:74: GCS storage type is StorageType::IN_MEMORY -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:42: Loading job table data. -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:54: Loading node table data. -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:80: Loading actor table data. -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:93: Loading actor task spec table data. -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:66: Loading placement group table data. -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:46: Finished loading job table data, size = 0 -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:58: Finished loading node table data, size = 0 -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:84: Finished loading actor table data, size = 0 -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:97: Finished loading actor task spec table data, size = 0 -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:71: Finished loading placement group table data, size = 0 -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_server.cc:162: No existing server cluster ID found. Generating new ID: 08d8dd0a3ae9d38657267cfdd0aaa4f146ec1f5e86c8c03b04920aec -[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_server.cc:648: Autoscaler V2 enabled: 0 -[2024-05-15 08:22:48,116 I 257996 257996] (gcs_server) grpc_server.cc:134: GcsServer server started, listening on port 47881. -[2024-05-15 08:22:48,142 I 257996 257996] (gcs_server) gcs_server.cc:250: GcsNodeManager: -- RegisterNode request count: 0 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 0 -- GetInternalConfig request count: 0 - -GcsActorManager: -- RegisterActor request count: 0 -- CreateActor request count: 0 -- GetActorInfo request count: 0 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 0 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 0 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 0 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 0 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 0 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 0 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 0 -- Registered placement groups count: 0 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 0 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 0 --Total num of actor creation tasks: 0 --Total num of actor tasks: 0 --Total num of normal tasks: 0 --Total num of driver tasks: 0 - - -[2024-05-15 08:22:48,142 I 257996 257996] (gcs_server) gcs_server.cc:844: Event stats: - - -Global stats: 29 total (18 active) -Queueing time: mean = 2.716 ms, max = 26.288 ms, min = 1.000 us, total = 78.756 ms -Execution time: mean = 907.843 us, total = 26.327 ms -Event stats: - InternalKVGcsService.grpc_client.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.GetAll - 5 total (0 active), Execution time: mean = 3.798 us, total = 18.990 us, Queueing time: mean = 16.800 us, max = 17.850 us, min = 15.510 us, total = 84.000 us - PeriodicalRunner.RunFnPeriodically - 4 total (2 active, 1 running), Execution time: mean = 1.025 us, total = 4.100 us, Queueing time: mean = 13.138 ms, max = 26.288 ms, min = 26.264 ms, total = 52.552 ms - InternalKVGcsService.grpc_server.InternalKVPut - 4 total (4 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 4 total (4 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Put - 3 total (0 active), Execution time: mean = 8.766 ms, total = 26.299 ms, Queueing time: mean = 8.706 ms, max = 26.115 ms, min = 1.000 us, total = 26.118 ms - GcsInMemoryStore.Get - 1 total (0 active), Execution time: mean = 5.370 us, total = 5.370 us, Queueing time: mean = 1.240 us, max = 1.240 us, min = 1.240 us, total = 1.240 us - ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RayletLoadPulled - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 08:22:48,142 I 257996 257996] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats: - - -Global stats: 2 total (1 active) -Queueing time: mean = 26.755 us, max = 53.509 us, min = 53.509 us, total = 53.509 us -Execution time: mean = 10.900 us, total = 21.800 us -Event stats: - PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 21.800 us, total = 21.800 us, Queueing time: mean = 53.509 us, max = 53.509 us, min = 53.509 us, total = 53.509 us - GcsTaskManager.GcJobSummary - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 08:22:49,503 I 257996 257996] (gcs_server) gcs_node_manager.cc:55: Registering node info, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 08:22:49,503 I 257996 257996] (gcs_server) gcs_node_manager.cc:61: Finished registering node info, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 08:22:49,503 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:797: A new node: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 registered, will try to reschedule all the infeasible placement groups. -[2024-05-15 08:22:50,064 I 257996 257996] (gcs_server) gcs_job_manager.cc:42: Adding job, job id = 01000000, driver pid = 257884 -[2024-05-15 08:22:50,064 I 257996 257996] (gcs_server) gcs_job_manager.cc:57: Finished adding job, job id = 01000000, driver pid = 257884 -[2024-05-15 08:22:50,097 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:432: Registering placement group, placement group id = 81c6c20970b7e7ed4f550507158301000000, name = , strategy = 0 -[2024-05-15 08:22:50,097 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:436: Finished registering placement group, placement group id = 81c6c20970b7e7ed4f550507158301000000, name = , strategy = 0 -[2024-05-15 08:22:50,097 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:185: Preparing resource from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for bundles: {placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4}}, -[2024-05-15 08:22:50,098 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:195: Finished leasing resource from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for bundles: {placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4}}, -[2024-05-15 08:22:50,098 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:213: Committing resource to a node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for bundles: {placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3}}, -[2024-05-15 08:22:50,098 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:220: Finished committing resource to a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for bundles: {placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3}}, -[2024-05-15 08:22:50,098 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:321: Successfully created placement group , id: 81c6c20970b7e7ed4f550507158301000000 -[2024-05-15 08:22:50,280 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = bdd59c451e6d1ad3c5498e6901000000 -[2024-05-15 08:22:50,281 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = bdd59c451e6d1ad3c5498e6901000000 -[2024-05-15 08:22:50,281 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = bdd59c451e6d1ad3c5498e6901000000 -[2024-05-15 08:22:50,281 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor bdd59c451e6d1ad3c5498e6901000000, job id = 01000000 -[2024-05-15 08:22:50,690 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor bdd59c451e6d1ad3c5498e6901000000, job id = 01000000 -[2024-05-15 08:22:50,690 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor bdd59c451e6d1ad3c5498e6901000000 on worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000 -[2024-05-15 08:22:52,262 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = f4ac3f8db7fdd2b856cf9a4d01000000 -[2024-05-15 08:22:52,262 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = f4ac3f8db7fdd2b856cf9a4d01000000 -[2024-05-15 08:22:52,263 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = f4ac3f8db7fdd2b856cf9a4d01000000 -[2024-05-15 08:22:52,263 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor f4ac3f8db7fdd2b856cf9a4d01000000, job id = 01000000 -[2024-05-15 08:22:52,263 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 81bf39d29ed6424b18900e4501000000 -[2024-05-15 08:22:52,263 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 81bf39d29ed6424b18900e4501000000 -[2024-05-15 08:22:52,264 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 81bf39d29ed6424b18900e4501000000 -[2024-05-15 08:22:52,264 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 81bf39d29ed6424b18900e4501000000, job id = 01000000 -[2024-05-15 08:22:52,264 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 228710b5ac6babc91120ca1101000000 -[2024-05-15 08:22:52,264 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 228710b5ac6babc91120ca1101000000 -[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 228710b5ac6babc91120ca1101000000 -[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 228710b5ac6babc91120ca1101000000, job id = 01000000 -[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 0206158a753b8236d33a875801000000 -[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 0206158a753b8236d33a875801000000 -[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 0206158a753b8236d33a875801000000 -[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 0206158a753b8236d33a875801000000, job id = 01000000 -[2024-05-15 08:22:52,695 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor f4ac3f8db7fdd2b856cf9a4d01000000, job id = 01000000 -[2024-05-15 08:22:52,695 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor f4ac3f8db7fdd2b856cf9a4d01000000 on worker 6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000 -[2024-05-15 08:22:52,697 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 0206158a753b8236d33a875801000000, job id = 01000000 -[2024-05-15 08:22:52,697 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 0206158a753b8236d33a875801000000 on worker d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000 -[2024-05-15 08:22:52,698 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 228710b5ac6babc91120ca1101000000, job id = 01000000 -[2024-05-15 08:22:52,698 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 228710b5ac6babc91120ca1101000000 on worker 15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000 -[2024-05-15 08:22:52,747 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 81bf39d29ed6424b18900e4501000000, job id = 01000000 -[2024-05-15 08:22:52,747 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 81bf39d29ed6424b18900e4501000000 on worker 7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000 -[2024-05-15 08:22:58,128 W 257996 258017] (gcs_server) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:59951: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 08:23:28,795 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = bdd59c451e6d1ad3c5498e6901000000, job id = 01000000 -[2024-05-15 08:23:28,795 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = bdd59c451e6d1ad3c5498e6901000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:23:28,795 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 01000000ffffffffffffffffffffffffffffffffffffffffffffffff failed, destroying actor child, job id = 01000000 -[2024-05-15 08:23:28,795 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:607: Cancelling all committed bundles of a placement group, id is 81c6c20970b7e7ed4f550507158301000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_job_manager.cc:87: Finished marking job state, job id = 01000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:830: Placement group of an id, 81c6c20970b7e7ed4f550507158301000000 is successfully removed because the job died. -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1167: Actor f4ac3f8db7fdd2b856cf9a4d01000000 is failed on worker 6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor f4ac3f8db7fdd2b856cf9a4d01000000, job id = 01000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1167: Actor 228710b5ac6babc91120ca1101000000 is failed on worker 15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 228710b5ac6babc91120ca1101000000, job id = 01000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1167: Actor 0206158a753b8236d33a875801000000 is failed on worker d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 0206158a753b8236d33a875801000000, job id = 01000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1167: Actor 81bf39d29ed6424b18900e4501000000 is failed on worker 7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000 -[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 81bf39d29ed6424b18900e4501000000, job id = 01000000 -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = f4ac3f8db7fdd2b856cf9a4d01000000, job id = 01000000 -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = f4ac3f8db7fdd2b856cf9a4d01000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 0206158a753b8236d33a875801000000, job id = 01000000 -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 0206158a753b8236d33a875801000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 228710b5ac6babc91120ca1101000000, job id = 01000000 -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 228710b5ac6babc91120ca1101000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 81bf39d29ed6424b18900e4501000000, job id = 01000000 -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 81bf39d29ed6424b18900e4501000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:23:28,828 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 failed, destroying actor child, job id = 01000000 -[2024-05-15 08:23:28,828 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 failed, destroying actor child, job id = 01000000 -[2024-05-15 08:23:28,828 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 failed, destroying actor child, job id = 01000000 -[2024-05-15 08:23:28,828 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 failed, destroying actor child, job id = 01000000 -[2024-05-15 08:23:28,838 I 257996 257996] (gcs_server) gcs_node_manager.cc:128: Shutting down raylet, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:23:28,839 I 257996 257996] (gcs_server) gcs_node_manager.cc:252: Removing node, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, node name = 192.168.1.56 -[2024-05-15 08:23:28,839 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:767: Node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 failed, rescheduling the placement groups on the dead node. -[2024-05-15 08:23:28,839 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1048: Node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 failed, reconstructing actors. -[2024-05-15 08:23:28,839 I 257996 257996] (gcs_server) gcs_job_manager.cc:302: Node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 failed, mark all jobs from this node as finished -[2024-05-15 08:23:28,921 I 257996 257996] (gcs_server) gcs_node_manager.cc:170: Raylet a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 is drained. Status GrpcUnavailable: RPC Error message: Cancelling all calls; RPC Error details: . The information will be published to the cluster. -[2024-05-15 08:23:28,934 I 257996 258044] (gcs_server) ray_syncer-inl.h:318: Failed to read the message from: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:23:28,952 I 257996 257996] (gcs_server) gcs_server_main.cc:111: GCS server received SIGTERM, shutting down... -[2024-05-15 08:23:28,952 I 257996 257996] (gcs_server) gcs_server.cc:272: Stopping GCS server. -[2024-05-15 08:23:28,957 I 257996 257996] (gcs_server) gcs_server.cc:292: GCS server stopped. -[2024-05-15 08:23:28,957 I 257996 257996] (gcs_server) io_service_pool.cc:47: IOServicePool is stopped. -[2024-05-15 08:23:29,046 I 257996 257996] (gcs_server) stats.h:120: Stats module has shutdown. diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/log_monitor.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/log_monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/monitor.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/monitor.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/monitor.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/raylet.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/raylet.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/raylet.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/raylet.out deleted file mode 100644 index 82e3158..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/raylet.out +++ /dev/null @@ -1,259 +0,0 @@ -[2024-05-15 08:22:49,490 I 258144 258144] (raylet) main.cc:179: Setting cluster ID to: 08d8dd0a3ae9d38657267cfdd0aaa4f146ec1f5e86c8c03b04920aec -[2024-05-15 08:22:49,492 I 258144 258144] (raylet) main.cc:242: Raylet is not set to kill unknown children. -[2024-05-15 08:22:49,492 I 258144 258144] (raylet) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 08:22:49,492 I 258144 258144] (raylet) main.cc:371: Setting node ID to: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:22:49,492 I 258144 258144] (raylet) store_runner.cc:32: Allowing the Plasma store to use up to 17.6951GB of memory. -[2024-05-15 08:22:49,492 I 258144 258144] (raylet) store_runner.cc:48: Starting object store with directory /dev/shm, fallback /home/valapeos/Projects/pneumarl/logs/rllib, and huge page support disabled -[2024-05-15 08:22:49,492 I 258144 258172] (raylet) dlmalloc.cc:154: create_and_mmap_buffer(17695244296, /dev/shm/plasmaXXXXXX) -[2024-05-15 08:22:49,492 I 258144 258172] (raylet) store.cc:564: ========== Plasma store: ================= -Current usage: 0 / 17.6951 GB -- num bytes created total: 0 -0 pending objects of total size 0MB -- objects spillable: 0 -- bytes spillable: 0 -- objects unsealed: 0 -- bytes unsealed: 0 -- objects in use: 0 -- bytes in use: 0 -- objects evictable: 0 -- bytes evictable: 0 - -- objects created by worker: 0 -- bytes created by worker: 0 -- objects restored: 0 -- bytes restored: 0 -- objects received: 0 -- bytes received: 0 -- objects errored: 0 -- bytes errored: 0 - -[2024-05-15 08:22:49,494 I 258144 258144] (raylet) grpc_server.cc:134: ObjectManager server started, listening on port 35889. -[2024-05-15 08:22:49,495 I 258144 258144] (raylet) worker_killing_policy.cc:101: Running GroupByOwner policy. -[2024-05-15 08:22:49,495 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:22:49,495 I 258144 258144] (raylet) memory_monitor.cc:47: MemoryMonitor initialized with usage threshold at 63982133248 bytes (0.95 system memory), total system memory bytes: 67349614592 -[2024-05-15 08:22:49,495 I 258144 258144] (raylet) node_manager.cc:285: Initializing NodeManager with ID a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:22:49,495 I 258144 258144] (raylet) grpc_server.cc:134: NodeManager server started, listening on port 46465. -[2024-05-15 08:22:49,501 I 258144 258204] (raylet) agent_manager.cc:78: Monitor agent process with name dashboard_agent/424238335 -[2024-05-15 08:22:49,502 I 258144 258206] (raylet) agent_manager.cc:78: Monitor agent process with name runtime_env_agent -[2024-05-15 08:22:49,502 I 258144 258144] (raylet) event.cc:234: Set ray event level to warning -[2024-05-15 08:22:49,502 I 258144 258144] (raylet) event.cc:342: Ray Event initialized for RAYLET -[2024-05-15 08:22:49,503 I 258144 258144] (raylet) raylet.cc:128: Raylet of id, a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 started. Raylet consists of node_manager and object_manager. node_manager address: 192.168.1.56:46465 object_manager address: 192.168.1.56:35889 hostname: valanixos -[2024-05-15 08:22:49,504 I 258144 258144] (raylet) node_manager.cc:521: [state-dump] NodeManager: -[state-dump] Node ID: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[state-dump] Node name: 192.168.1.56 -[state-dump] InitialConfigResources: {CPU: 120000, memory: 353902559240000, node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 176951279610000} -[state-dump] ClusterTaskManager: -[state-dump] ========== Node: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 ================= -[state-dump] Infeasible queue length: 0 -[state-dump] Schedule queue length: 0 -[state-dump] Dispatch queue length: 0 -[state-dump] num_waiting_for_resource: 0 -[state-dump] num_waiting_for_plasma_memory: 0 -[state-dump] num_waiting_for_remote_node_resources: 0 -[state-dump] num_worker_not_started_by_job_config_not_exist: 0 -[state-dump] num_worker_not_started_by_registration_timeout: 0 -[state-dump] num_tasks_waiting_for_workers: 0 -[state-dump] num_cancelled_tasks: 0 -[state-dump] cluster_resource_scheduler state: -[state-dump] Local id: -1885364415256342489 Local resources: {"total":{node:__internal_head__: [10000], CPU: [120000], memory: [353902559240000], object_store_memory: [176951279610000], node:192.168.1.56: [10000]}}, "available": {node:__internal_head__: [10000], CPU: [120000], memory: [353902559240000], object_store_memory: [176951279610000], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450",} is_draining: 0 is_idle: 1 Cluster resources: node id: -1885364415256342489{"total":{node:192.168.1.56: 10000, node:__internal_head__: 10000, memory: 353902559240000, object_store_memory: 176951279610000, CPU: 120000}}, "available": {node:192.168.1.56: 10000, node:__internal_head__: 10000, memory: 353902559240000, object_store_memory: 176951279610000, CPU: 120000}}, "labels":{"ray.io/node_id":"a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -[state-dump] Waiting tasks size: 0 -[state-dump] Number of executing tasks: 0 -[state-dump] Number of pinned task arguments: 0 -[state-dump] Number of total spilled tasks: 0 -[state-dump] Number of spilled waiting tasks: 0 -[state-dump] Number of spilled unschedulable tasks: 0 -[state-dump] Resource usage { -[state-dump] } -[state-dump] Running tasks by scheduling class: -[state-dump] ================================================== -[state-dump] -[state-dump] ClusterResources: -[state-dump] LocalObjectManager: -[state-dump] - num pinned objects: 0 -[state-dump] - pinned objects size: 0 -[state-dump] - num objects pending restore: 0 -[state-dump] - num objects pending spill: 0 -[state-dump] - num bytes pending spill: 0 -[state-dump] - num bytes currently spilled: 0 -[state-dump] - cumulative spill requests: 0 -[state-dump] - cumulative restore requests: 0 -[state-dump] - spilled objects pending delete: 0 -[state-dump] -[state-dump] ObjectManager: -[state-dump] - num local objects: 0 -[state-dump] - num unfulfilled push requests: 0 -[state-dump] - num object pull requests: 0 -[state-dump] - num chunks received total: 0 -[state-dump] - num chunks received failed (all): 0 -[state-dump] - num chunks received failed / cancelled: 0 -[state-dump] - num chunks received failed / plasma error: 0 -[state-dump] Event stats: -[state-dump] Global stats: 0 total (0 active) -[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] Execution time: mean = -nan s, total = 0.000 s -[state-dump] Event stats: -[state-dump] PushManager: -[state-dump] - num pushes in flight: 0 -[state-dump] - num chunks in flight: 0 -[state-dump] - num chunks remaining: 0 -[state-dump] - max chunks allowed: 409 -[state-dump] OwnershipBasedObjectDirectory: -[state-dump] - num listeners: 0 -[state-dump] - cumulative location updates: 0 -[state-dump] - num location updates per second: 0.000 -[state-dump] - num location lookups per second: 0.000 -[state-dump] - num locations added per second: 0.000 -[state-dump] - num locations removed per second: 0.000 -[state-dump] BufferPool: -[state-dump] - create buffer state map size: 0 -[state-dump] PullManager: -[state-dump] - num bytes available for pulled objects: 17695127961 -[state-dump] - num bytes being pulled (all): 0 -[state-dump] - num bytes being pulled / pinned: 0 -[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - first get request bundle: N/A -[state-dump] - first wait request bundle: N/A -[state-dump] - first task request bundle: N/A -[state-dump] - num objects queued: 0 -[state-dump] - num objects actively pulled (all): 0 -[state-dump] - num objects actively pulled / pinned: 0 -[state-dump] - num bundles being pulled: 0 -[state-dump] - num pull retries: 0 -[state-dump] - max timeout seconds: 0 -[state-dump] - max timeout request is already processed. No entry. -[state-dump] -[state-dump] WorkerPool: -[state-dump] - registered jobs: 0 -[state-dump] - process_failed_job_config_missing: 0 -[state-dump] - process_failed_rate_limited: 0 -[state-dump] - process_failed_pending_registration: 0 -[state-dump] - process_failed_runtime_env_setup_failed: 0 -[state-dump] - num PYTHON workers: 0 -[state-dump] - num PYTHON drivers: 0 -[state-dump] - num object spill callbacks queued: 0 -[state-dump] - num object restore queued: 0 -[state-dump] - num util functions queued: 0 -[state-dump] - num idle workers: 0 -[state-dump] TaskDependencyManager: -[state-dump] - task deps map size: 0 -[state-dump] - get req map size: 0 -[state-dump] - wait req map size: 0 -[state-dump] - local objects map size: 0 -[state-dump] WaitManager: -[state-dump] - num active wait requests: 0 -[state-dump] Subscriber: -[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_OBJECT_EVICTION -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_REF_REMOVED_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] num async plasma notifications: 0 -[state-dump] Remote node managers: -[state-dump] Event stats: -[state-dump] Global stats: 27 total (13 active) -[state-dump] Queueing time: mean = 976.153 us, max = 7.494 ms, min = 8.099 us, total = 26.356 ms -[state-dump] Execution time: mean = 506.838 us, total = 13.685 ms -[state-dump] Event stats: -[state-dump] PeriodicalRunner.RunFnPeriodically - 11 total (2 active, 1 running), Execution time: mean = 90.688 us, total = 997.568 us, Queueing time: mean = 2.394 ms, max = 7.494 ms, min = 18.500 us, total = 26.339 ms -[state-dump] NodeManager.deadline_timer.flush_free_objects - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 730.548 us, total = 730.548 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.debug_state_dump - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 288.960 us, total = 288.960 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 11.090 ms, total = 11.090 ms, Queueing time: mean = 8.630 us, max = 8.630 us, min = 8.630 us, total = 8.630 us -[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 429.879 us, total = 429.879 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 147.790 us, total = 147.790 us, Queueing time: mean = 8.099 us, max = 8.099 us, min = 8.099 us, total = 8.099 us -[state-dump] NodeManager.deadline_timer.record_metrics - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] DebugString() time ms: 0 -[state-dump] -[state-dump] -[2024-05-15 08:22:49,505 I 258144 258144] (raylet) accessor.cc:627: Received notification for node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, IsAlive = 1 -[2024-05-15 08:22:49,514 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258226, the token is 0 -[2024-05-15 08:22:49,516 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258227, the token is 1 -[2024-05-15 08:22:49,517 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258228, the token is 2 -[2024-05-15 08:22:49,519 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258229, the token is 3 -[2024-05-15 08:22:49,521 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258230, the token is 4 -[2024-05-15 08:22:49,523 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258231, the token is 5 -[2024-05-15 08:22:49,525 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258232, the token is 6 -[2024-05-15 08:22:49,526 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258233, the token is 7 -[2024-05-15 08:22:49,528 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258234, the token is 8 -[2024-05-15 08:22:49,530 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258235, the token is 9 -[2024-05-15 08:22:49,532 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258236, the token is 10 -[2024-05-15 08:22:49,535 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258237, the token is 11 -[2024-05-15 08:22:49,883 I 258144 258172] (raylet) object_store.cc:35: Object store current usage 8e-09 / 17.6951 GB. -[2024-05-15 08:22:50,065 I 258144 258144] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 257884 is dead: 0 driver address: 192.168.1.56 -[2024-05-15 08:22:50,065 I 258144 258144] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 08:22:50,284 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:22:50,285 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258963, the token is 12 -[2024-05-15 08:22:51,110 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:22:51,111 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:22:51,111 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:22:51,111 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:22:51,311 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:22:52,264 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:22:52,267 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 259036, the token is 13 -[2024-05-15 08:22:52,267 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:22:52,269 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 259037, the token is 14 -[2024-05-15 08:22:52,270 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:22:52,271 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 259038, the token is 15 -[2024-05-15 08:22:52,271 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 08:22:52,273 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 259039, the token is 16 -[2024-05-15 08:22:54,505 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:22:59,502 W 258144 258166] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:59951: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 08:22:59,507 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:23:04,508 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:23:09,510 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:23:14,512 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:23:19,513 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:23:24,515 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 08:23:28,795 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=3, has creation task exception = false -[2024-05-15 08:23:28,795 I 258144 258144] (raylet) node_manager.cc:1541: Driver (pid=257884) is disconnected. job_id: 01000000 -[2024-05-15 08:23:28,795 I 258144 258144] (raylet) node_manager.cc:1066: The leased worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 is killed because the owner process 01000000ffffffffffffffffffffffffffffffffffffffffffffffff died. -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 257884 is dead: 1 driver address: 192.168.1.56 -[2024-05-15 08:23:28,797 I 258144 258144] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 08:23:28,838 I 258144 258144] (raylet) main.cc:413: Raylet received SIGTERM, shutting down... -[2024-05-15 08:23:28,838 I 258144 258144] (raylet) accessor.cc:451: Unregistering node info, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 -[2024-05-15 08:23:28,839 I 258144 258144] (raylet) agent_manager.cc:109: Killing agent dashboard_agent/424238335, pid 258203. -[2024-05-15 08:23:28,848 I 258144 258204] (raylet) agent_manager.cc:80: Agent process with name dashboard_agent/424238335 exited, exit code 0. -[2024-05-15 08:23:28,848 I 258144 258144] (raylet) agent_manager.cc:109: Killing agent runtime_env_agent, pid 258205. -[2024-05-15 08:23:28,853 I 258144 258206] (raylet) agent_manager.cc:80: Agent process with name runtime_env_agent exited, exit code 0. -[2024-05-15 08:23:28,853 I 258144 258144] (raylet) io_service_pool.cc:47: IOServicePool is stopped. -[2024-05-15 08:23:28,919 I 258144 258144] (raylet) stats.h:120: Stats module has shutdown. -[2024-05-15 08:23:28,919 I 258144 258208] (raylet) experimental_mutable_object_provider.cc:200: Core worker channel io service stopped. diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/runtime_env_agent.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/runtime_env_agent.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/runtime_env_agent.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/runtime_env_agent.out deleted file mode 100644 index d70a16e..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/runtime_env_agent.out +++ /dev/null @@ -1,2 +0,0 @@ -======== Running on http://0.0.0.0:43711 ======== -(Press CTRL+C to quit) diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01-01000000-258963.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01-01000000-258963.err deleted file mode 100644 index 4a4cf37..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01-01000000-258963.err +++ /dev/null @@ -1,5 +0,0 @@ -:job_id:01000000 -:actor_name:PPO -2024-05-15 08:22:52,212 WARNING deprecation.py:50 -- DeprecationWarning: `AlgorithmConfig.num_envs_per_worker` has been deprecated. Use `AlgorithmConfig.num_envs_per_env_runner` instead. This will raise an error in the future! -2024-05-15 08:22:52,217 WARNING deprecation.py:50 -- DeprecationWarning: `WorkerSet(num_workers=... OR local_worker=...)` has been deprecated. Use `EnvRunnerGroup(num_env_runners=... AND local_env_runner=...)` instead. This will raise an error in the future! -2024-05-15 08:22:52,217 WARNING deprecation.py:50 -- DeprecationWarning: `max_num_worker_restarts` has been deprecated. Use `AlgorithmConfig.max_num_env_runner_restarts` instead. This will raise an error in the future! diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01-01000000-258963.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01-01000000-258963.out deleted file mode 100644 index 1124c85..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01-01000000-258963.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:actor_name:PPO diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60-01000000-259038.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60-01000000-259038.err deleted file mode 100644 index 704ee4a..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60-01000000-259038.err +++ /dev/null @@ -1,34 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -[2024-05-15 08:23:28,835 C 259038 259038] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe -*** StackTrace Information *** -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7f3f0780dbba] ray::operator<<() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7f3f0780f477] ray::SpdLogMessage::Flush() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7f3f0780f917] ray::RayLog::~RayLog() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7f3f06fd2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7f3f06fd319a] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7f3f06fea0de] ray::core::InboundRequest::Accept() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7f3f06fbc440] ray::core::NormalSchedulingQueue::ScheduleRequests() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7f3f0724e71e] EventTracker::RecordExecution() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7f3f07247b0e] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7f3f07247f86] boost::asio::detail::completion_handler<>::do_complete() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7f3f078f78eb] boost::asio::detail::scheduler::do_run_one() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7f3f078f9269] boost::asio::detail::scheduler::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7f3f078f9972] boost::asio::io_context::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7f3f06f713fd] ray::core::CoreWorker::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7f3f06fb5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7f3f06fb60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7f3f06dbd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop() -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7f3f08c413c2] method_vectorcall_NOARGS -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7f3f08c5ad93] PyObject_Vectorcall -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7f3f08af7c5c] _PyEval_EvalFrameDefault -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7f3f08d2bc9c] _PyEval_Vector.constprop.0 -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7f3f08d2be3a] PyEval_EvalCode -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7f3f08d56a10] run_mod -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7f3f08d76752] _PyRun_SimpleFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7f3f08d76ff1] _PyRun_AnyFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7f3f08d7990f] Py_RunMain -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7f3f0875a10e] __libc_start_call_main -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7f3f0875a1c9] __libc_start_main@GLIBC_2.2.5 -ray::RolloutWorker(_start+0x25) [0x401075] _start - diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60-01000000-259038.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60-01000000-259038.out deleted file mode 100644 index 14280b7..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60-01000000-259038.out +++ /dev/null @@ -1,4 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -No game binary has been provided, please press PLAY in the Godot editor -waiting for remote GODOT connection on port 11011 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-23ae23584aeed2d6f6b43647109ffe7509bb5041622bdc4ba14da1e2-ffffffff-258231.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-23ae23584aeed2d6f6b43647109ffe7509bb5041622bdc4ba14da1e2-ffffffff-258231.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-23ae23584aeed2d6f6b43647109ffe7509bb5041622bdc4ba14da1e2-ffffffff-258231.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-23ae23584aeed2d6f6b43647109ffe7509bb5041622bdc4ba14da1e2-ffffffff-258231.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-453601d934db8aa23a43e7841dd1e5d29190bff6d7e0381fb6eaab4c-ffffffff-258228.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-453601d934db8aa23a43e7841dd1e5d29190bff6d7e0381fb6eaab4c-ffffffff-258228.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-453601d934db8aa23a43e7841dd1e5d29190bff6d7e0381fb6eaab4c-ffffffff-258228.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-453601d934db8aa23a43e7841dd1e5d29190bff6d7e0381fb6eaab4c-ffffffff-258228.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-4b3bc6362909c62b6ba3310603dfeeb112f3350b47e847e47a0806a4-ffffffff-258233.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-4b3bc6362909c62b6ba3310603dfeeb112f3350b47e847e47a0806a4-ffffffff-258233.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-4b3bc6362909c62b6ba3310603dfeeb112f3350b47e847e47a0806a4-ffffffff-258233.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-4b3bc6362909c62b6ba3310603dfeeb112f3350b47e847e47a0806a4-ffffffff-258233.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-5b8eb02e89f5b811154eedc6863d3cf28b213dbb03a8bf91784a4b9f-ffffffff-258235.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-5b8eb02e89f5b811154eedc6863d3cf28b213dbb03a8bf91784a4b9f-ffffffff-258235.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-5b8eb02e89f5b811154eedc6863d3cf28b213dbb03a8bf91784a4b9f-ffffffff-258235.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-5b8eb02e89f5b811154eedc6863d3cf28b213dbb03a8bf91784a4b9f-ffffffff-258235.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986-01000000-259036.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986-01000000-259036.err deleted file mode 100644 index ef946dd..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986-01000000-259036.err +++ /dev/null @@ -1,34 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -[2024-05-15 08:23:28,835 C 259036 259036] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe -*** StackTrace Information *** -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7f4617c0dbba] ray::operator<<() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7f4617c0f477] ray::SpdLogMessage::Flush() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7f4617c0f917] ray::RayLog::~RayLog() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7f46173d2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7f46173d319a] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7f46173ea0de] ray::core::InboundRequest::Accept() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7f46173bc440] ray::core::NormalSchedulingQueue::ScheduleRequests() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7f461764e71e] EventTracker::RecordExecution() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7f4617647b0e] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7f4617647f86] boost::asio::detail::completion_handler<>::do_complete() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7f4617cf78eb] boost::asio::detail::scheduler::do_run_one() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7f4617cf9269] boost::asio::detail::scheduler::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7f4617cf9972] boost::asio::io_context::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7f46173713fd] ray::core::CoreWorker::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7f46173b5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7f46173b60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7f46171bd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop() -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7f46190413c2] method_vectorcall_NOARGS -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7f461905ad93] PyObject_Vectorcall -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7f4618ef7c5c] _PyEval_EvalFrameDefault -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7f461912bc9c] _PyEval_Vector.constprop.0 -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7f461912be3a] PyEval_EvalCode -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7f4619156a10] run_mod -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7f4619176752] _PyRun_SimpleFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7f4619176ff1] _PyRun_AnyFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7f461917990f] Py_RunMain -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7f4618c3d10e] __libc_start_call_main -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7f4618c3d1c9] __libc_start_main@GLIBC_2.2.5 -ray::RolloutWorker(_start+0x25) [0x401075] _start - diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986-01000000-259036.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986-01000000-259036.out deleted file mode 100644 index 05bdbb6..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986-01000000-259036.out +++ /dev/null @@ -1,4 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -No game binary has been provided, please press PLAY in the Godot editor -waiting for remote GODOT connection on port 11009 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79-01000000-259037.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79-01000000-259037.err deleted file mode 100644 index 30c28db..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79-01000000-259037.err +++ /dev/null @@ -1,34 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -[2024-05-15 08:23:28,836 C 259037 259037] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe -*** StackTrace Information *** -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7f452600dbba] ray::operator<<() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7f452600f477] ray::SpdLogMessage::Flush() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7f452600f917] ray::RayLog::~RayLog() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7f45257d2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7f45257d319a] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7f45257ea0de] ray::core::InboundRequest::Accept() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7f45257bc440] ray::core::NormalSchedulingQueue::ScheduleRequests() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7f4525a4e71e] EventTracker::RecordExecution() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7f4525a47b0e] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7f4525a47f86] boost::asio::detail::completion_handler<>::do_complete() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7f45260f78eb] boost::asio::detail::scheduler::do_run_one() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7f45260f9269] boost::asio::detail::scheduler::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7f45260f9972] boost::asio::io_context::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7f45257713fd] ray::core::CoreWorker::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7f45257b5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7f45257b60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7f45255bd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop() -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7f45274413c2] method_vectorcall_NOARGS -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7f452745ad93] PyObject_Vectorcall -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7f45272f7c5c] _PyEval_EvalFrameDefault -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7f452752bc9c] _PyEval_Vector.constprop.0 -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7f452752be3a] PyEval_EvalCode -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7f4527556a10] run_mod -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7f4527576752] _PyRun_SimpleFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7f4527576ff1] _PyRun_AnyFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7f452757990f] Py_RunMain -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7f452701810e] __libc_start_call_main -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7f45270181c9] __libc_start_main@GLIBC_2.2.5 -ray::RolloutWorker(_start+0x25) [0x401075] _start - diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79-01000000-259037.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79-01000000-259037.out deleted file mode 100644 index 705be67..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79-01000000-259037.out +++ /dev/null @@ -1,4 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -No game binary has been provided, please press PLAY in the Godot editor -waiting for remote GODOT connection on port 11010 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-a26d4872a9517f31748ef85ef4b61295fa277994245a88cc1115f9b5-ffffffff-258226.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-a26d4872a9517f31748ef85ef4b61295fa277994245a88cc1115f9b5-ffffffff-258226.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-a26d4872a9517f31748ef85ef4b61295fa277994245a88cc1115f9b5-ffffffff-258226.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-a26d4872a9517f31748ef85ef4b61295fa277994245a88cc1115f9b5-ffffffff-258226.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-bec713c6aa138b33cc7cdeac2d164931c7fa69982ab633ab3bb012a8-ffffffff-258237.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-bec713c6aa138b33cc7cdeac2d164931c7fa69982ab633ab3bb012a8-ffffffff-258237.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-bec713c6aa138b33cc7cdeac2d164931c7fa69982ab633ab3bb012a8-ffffffff-258237.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-bec713c6aa138b33cc7cdeac2d164931c7fa69982ab633ab3bb012a8-ffffffff-258237.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d01c561a322e85170aa8beea7fc30ee4c96cb702e538db553b722f7e-ffffffff-258229.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d01c561a322e85170aa8beea7fc30ee4c96cb702e538db553b722f7e-ffffffff-258229.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d01c561a322e85170aa8beea7fc30ee4c96cb702e538db553b722f7e-ffffffff-258229.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d01c561a322e85170aa8beea7fc30ee4c96cb702e538db553b722f7e-ffffffff-258229.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad-01000000-259039.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad-01000000-259039.err deleted file mode 100644 index 83bf3cc..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad-01000000-259039.err +++ /dev/null @@ -1,34 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -[2024-05-15 08:23:28,835 C 259039 259039] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe -*** StackTrace Information *** -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7f8a3aa0dbba] ray::operator<<() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7f8a3aa0f477] ray::SpdLogMessage::Flush() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7f8a3aa0f917] ray::RayLog::~RayLog() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7f8a3a1d2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7f8a3a1d319a] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7f8a3a1ea0de] ray::core::InboundRequest::Accept() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7f8a3a1bc440] ray::core::NormalSchedulingQueue::ScheduleRequests() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7f8a3a44e71e] EventTracker::RecordExecution() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7f8a3a447b0e] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7f8a3a447f86] boost::asio::detail::completion_handler<>::do_complete() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7f8a3aaf78eb] boost::asio::detail::scheduler::do_run_one() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7f8a3aaf9269] boost::asio::detail::scheduler::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7f8a3aaf9972] boost::asio::io_context::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7f8a3a1713fd] ray::core::CoreWorker::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7f8a3a1b5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7f8a3a1b60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7f8a39fbd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop() -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7f8a3be413c2] method_vectorcall_NOARGS -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7f8a3be5ad93] PyObject_Vectorcall -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7f8a3bcf7c5c] _PyEval_EvalFrameDefault -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7f8a3bf2bc9c] _PyEval_Vector.constprop.0 -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7f8a3bf2be3a] PyEval_EvalCode -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7f8a3bf56a10] run_mod -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7f8a3bf76752] _PyRun_SimpleFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7f8a3bf76ff1] _PyRun_AnyFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7f8a3bf7990f] Py_RunMain -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7f8a3ba3d10e] __libc_start_call_main -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7f8a3ba3d1c9] __libc_start_main@GLIBC_2.2.5 -ray::RolloutWorker(_start+0x25) [0x401075] _start - diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad-01000000-259039.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad-01000000-259039.out deleted file mode 100644 index d6a807e..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad-01000000-259039.out +++ /dev/null @@ -1,4 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -No game binary has been provided, please press PLAY in the Godot editor -waiting for remote GODOT connection on port 11012 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d914e37ecbefc02af00ecaba7f91eee0853600fb3202d470d4fcd287-ffffffff-258234.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d914e37ecbefc02af00ecaba7f91eee0853600fb3202d470d4fcd287-ffffffff-258234.err deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d914e37ecbefc02af00ecaba7f91eee0853600fb3202d470d4fcd287-ffffffff-258234.err +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d914e37ecbefc02af00ecaba7f91eee0853600fb3202d470d4fcd287-ffffffff-258234.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d914e37ecbefc02af00ecaba7f91eee0853600fb3202d470d4fcd287-ffffffff-258234.out deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-d914e37ecbefc02af00ecaba7f91eee0853600fb3202d470d4fcd287-ffffffff-258234.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-e1c767d88b12894f8d25744f2d6eb40b08f4a359ac233593928c7ec6-ffffffff-258230.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-e1c767d88b12894f8d25744f2d6eb40b08f4a359ac233593928c7ec6-ffffffff-258230.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-e1c767d88b12894f8d25744f2d6eb40b08f4a359ac233593928c7ec6-ffffffff-258230.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-e1c767d88b12894f8d25744f2d6eb40b08f4a359ac233593928c7ec6-ffffffff-258230.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-f5767192327306c54b685733a986d54a00809284e4316fe367f3a9d9-ffffffff-258236.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-f5767192327306c54b685733a986d54a00809284e4316fe367f3a9d9-ffffffff-258236.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-f5767192327306c54b685733a986d54a00809284e4316fe367f3a9d9-ffffffff-258236.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-f5767192327306c54b685733a986d54a00809284e4316fe367f3a9d9-ffffffff-258236.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-f961bb5950fd3f0a8e5476d0dd37bf1e3f14e399441e3c0f11a1ce28-ffffffff-258232.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-f961bb5950fd3f0a8e5476d0dd37bf1e3f14e399441e3c0f11a1ce28-ffffffff-258232.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-f961bb5950fd3f0a8e5476d0dd37bf1e3f14e399441e3c0f11a1ce28-ffffffff-258232.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-f961bb5950fd3f0a8e5476d0dd37bf1e3f14e399441e3c0f11a1ce28-ffffffff-258232.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-fca3ddf3cb0eb3102354a008e38158e58da3b4ea937ac4c7eee3c46d-ffffffff-258227.err b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-fca3ddf3cb0eb3102354a008e38158e58da3b4ea937ac4c7eee3c46d-ffffffff-258227.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-fca3ddf3cb0eb3102354a008e38158e58da3b4ea937ac4c7eee3c46d-ffffffff-258227.out b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/logs/worker-fca3ddf3cb0eb3102354a008e38158e58da3b4ea937ac4c7eee3c46d-ffffffff-258227.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/node_ip_address.json b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/node_ip_address.json deleted file mode 100644 index 8d998d8..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/node_ip_address.json +++ /dev/null @@ -1 +0,0 @@ -{"node_ip_address": "192.168.1.56"} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/node_ip_address.json.lock b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/node_ip_address.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/ports_by_node.json b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/ports_by_node.json deleted file mode 100644 index e69343f..0000000 --- a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/ports_by_node.json +++ /dev/null @@ -1 +0,0 @@ -{"192.168.1.56:/home/valapeos/Projects/pneumarl/logs/rllib/session_2024-05-15_08-22-48_088144_257884/sockets/plasma_store": {"metrics_agent_port": 59951, "metrics_export_port": 58005, "dashboard_agent_listen_port": 52365, "runtime_env_agent_port": 43711, "gcs_server_port": 47881}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/ports_by_node.json.lock b/Godot/logs/rllib/session_2024-05-15_08-22-48_088144_257884/ports_by_node.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.pkl b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.pkl deleted file mode 100644 index 0d14ffa..0000000 Binary files a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.pkl and /dev/null differ diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.txt b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.txt deleted file mode 100644 index 68aefdb..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/PPO_godot_734af_00000_0_2024-05-15_09-05-02/error.txt +++ /dev/null @@ -1,51 +0,0 @@ -Failure # 1 (occurred at 2024-05-15_09-06-07) -The actor died because of an error raised in its creation task, ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 239, in _setup - self.add_workers( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 754, in add_workers - raise result.get() - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/utils/actor_manager.py", line 497, in _fetch_result - result = ray.get(r) - ^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -During handling of the above exception, another exception occurred: - -ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 554, in __init__ - super().__init__( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__ - self.setup(copy.deepcopy(self.config)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup - self.workers = EnvRunnerGroup( - ^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__ - raise e.args[0].args[2] -TimeoutError: timed out diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/basic-variant-state-2024-05-15_09-05-02.json b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/basic-variant-state-2024-05-15_09-05-02.json deleted file mode 100644 index 3e0daac..0000000 Binary files a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/basic-variant-state-2024-05-15_09-05-02.json and /dev/null differ diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/experiment_state-2024-05-15_09-05-02.json b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/experiment_state-2024-05-15_09-05-02.json deleted file mode 100644 index 361e495..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/artifacts/2024-05-15_09-05-02/PPO_2024-05-15_09-05-02/driver_artifacts/experiment_state-2024-05-15_09-05-02.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"734af_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30392d30352d3032948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f37333461665f30303030305f305f323032342d30352d31355f30392d30352d3032948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30392d30352d30329475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"ERROR\",\n \"relative_logdir\": \"PPO_godot_734af_00000_0_2024-05-15_09-05-02\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": 1715756767.1916943,\n \"num_failures\": 1,\n \"num_failures_after_restore\": 0,\n \"error_filename\": \"error.txt\",\n \"pickled_error_filename\": \"error.pkl\",\n \"last_result\": {\n \"trial_id\": \"734af_00000\"\n },\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 644, "_has_errored": true, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715756702.603119, "_session_str": "2024-05-15_09-05-02", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715756702.603119}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/agent-424238335.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/agent-424238335.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/agent-424238335.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/agent-424238335.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/dashboard.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/dashboard.err deleted file mode 100644 index c88b3c0..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/dashboard.err +++ /dev/null @@ -1,2 +0,0 @@ -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/dashboard/dashboard.py:236: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead - logger.warn("Exiting with SIGTERM immediately...") diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/debug_state.txt b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/debug_state.txt deleted file mode 100644 index 9ff4ce2..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/debug_state.txt +++ /dev/null @@ -1,208 +0,0 @@ -NodeManager: -Node ID: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -Node name: 192.168.1.56 -InitialConfigResources: {CPU: 120000, memory: 365508034560000, node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 182754017280000} -ClusterTaskManager: -========== Node: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 ================= -Infeasible queue length: 0 -Schedule queue length: 0 -Dispatch queue length: 0 -num_waiting_for_resource: 0 -num_waiting_for_plasma_memory: 0 -num_waiting_for_remote_node_resources: 0 -num_worker_not_started_by_job_config_not_exist: 0 -num_worker_not_started_by_registration_timeout: 0 -num_tasks_waiting_for_workers: 0 -num_cancelled_tasks: 0 -cluster_resource_scheduler state: -Local id: 4706454582974575632 Local resources: {"total":{object_store_memory: [182754017280000], node:192.168.1.56: [10000], CPU_group_1_656f9c54c57685dae4b6a63a81d401000000: [10000], bundle_group_1_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU_group_3_656f9c54c57685dae4b6a63a81d401000000: [10000], bundle_group_3_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU_group_656f9c54c57685dae4b6a63a81d401000000: [50000], bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: [10000000], bundle_group_2_656f9c54c57685dae4b6a63a81d401000000: [10000000], bundle_group_4_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU: [120000], bundle_group_656f9c54c57685dae4b6a63a81d401000000: [50000000], CPU_group_2_656f9c54c57685dae4b6a63a81d401000000: [10000], memory: [365508034560000], node:__internal_head__: [10000], CPU_group_0_656f9c54c57685dae4b6a63a81d401000000: [10000], CPU_group_4_656f9c54c57685dae4b6a63a81d401000000: [10000]}}, "available": {object_store_memory: [182754017280000], node:192.168.1.56: [10000], CPU_group_1_656f9c54c57685dae4b6a63a81d401000000: [10000], bundle_group_1_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU_group_3_656f9c54c57685dae4b6a63a81d401000000: [10000], bundle_group_3_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU_group_656f9c54c57685dae4b6a63a81d401000000: [0], bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: [9999990], bundle_group_2_656f9c54c57685dae4b6a63a81d401000000: [10000000], bundle_group_4_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU: [70000], bundle_group_656f9c54c57685dae4b6a63a81d401000000: [49999950], CPU_group_2_656f9c54c57685dae4b6a63a81d401000000: [10000], memory: [365508034560000], node:__internal_head__: [10000], CPU_group_0_656f9c54c57685dae4b6a63a81d401000000: [0], CPU_group_4_656f9c54c57685dae4b6a63a81d401000000: [10000]}}, "labels":{"ray.io/node_id":"2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691",} is_draining: 0 is_idle: 0 Cluster resources: node id: 4706454582974575632{"total":{CPU_group_656f9c54c57685dae4b6a63a81d401000000: 50000, bundle_group_2_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_4_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_3_656f9c54c57685dae4b6a63a81d401000000: 10000000, node:192.168.1.56: 10000, object_store_memory: 182754017280000, bundle_group_1_656f9c54c57685dae4b6a63a81d401000000: 10000000, CPU_group_3_656f9c54c57685dae4b6a63a81d401000000: 10000, node:__internal_head__: 10000, CPU_group_1_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_4_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_0_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 50000000, CPU: 120000, memory: 365508034560000, CPU_group_2_656f9c54c57685dae4b6a63a81d401000000: 10000}}, "available": {CPU_group_4_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_1_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_3_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_3_656f9c54c57685dae4b6a63a81d401000000: 10000000, node:192.168.1.56: 10000, object_store_memory: 182754017280000, node:__internal_head__: 10000, memory: 365508034560000, CPU_group_2_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 49999950, CPU: 70000, bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: 9999990, bundle_group_4_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_2_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_1_656f9c54c57685dae4b6a63a81d401000000: 10000000}}, "labels":{"ray.io/node_id":"2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -Waiting tasks size: 0 -Number of executing tasks: 5 -Number of pinned task arguments: 0 -Number of total spilled tasks: 0 -Number of spilled waiting tasks: 0 -Number of spilled unschedulable tasks: 0 -Resource usage { - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=13722): {CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=13723): {CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10} - - (language=PYTHON actor_or_task=PPO.__init__ pid=13649): {bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10, CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_0_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=13721): {CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=13724): {CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10} -} -Running tasks by scheduling class: - - {depth=1 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.algorithms.ppo.ppo, class_name=PPO, function_name=__init__, function_hash=1023ce2155184297951346509793ac63} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "eo\234T\305v\205\332\344\266\246:\201\324\001\000\000\000" - placement_group_capture_child_tasks: true -} - resource_set={bundle_group_0_656f9c54c57685dae4b6a63a81d401000000 : 0.001, CPU_group_656f9c54c57685dae4b6a63a81d401000000 : 1, CPU_group_0_656f9c54c57685dae4b6a63a81d401000000 : 1, bundle_group_656f9c54c57685dae4b6a63a81d401000000 : 0.001, }}: 1/18446744073709551615 - - {depth=2 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.evaluation.rollout_worker, class_name=RolloutWorker, function_name=__init__, function_hash=b8267bd017c84e5881fa6c6f47c3e4c0} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "eo\234T\305v\205\332\344\266\246:\201\324\001\000\000\000" - placement_group_bundle_index: -1 - placement_group_capture_child_tasks: true -} - resource_set={CPU_group_656f9c54c57685dae4b6a63a81d401000000 : 1, bundle_group_656f9c54c57685dae4b6a63a81d401000000 : 0.001, }}: 4/18446744073709551615 -================================================== - -ClusterResources: -LocalObjectManager: -- num pinned objects: 0 -- pinned objects size: 0 -- num objects pending restore: 0 -- num objects pending spill: 0 -- num bytes pending spill: 0 -- num bytes currently spilled: 0 -- cumulative spill requests: 0 -- cumulative restore requests: 0 -- spilled objects pending delete: 0 - -ObjectManager: -- num local objects: 0 -- num unfulfilled push requests: 0 -- num object pull requests: 0 -- num chunks received total: 0 -- num chunks received failed (all): 0 -- num chunks received failed / cancelled: 0 -- num chunks received failed / plasma error: 0 -Event stats: -Global stats: 0 total (0 active) -Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -Execution time: mean = -nan s, total = 0.000 s -Event stats: -PushManager: -- num pushes in flight: 0 -- num chunks in flight: 0 -- num chunks remaining: 0 -- max chunks allowed: 409 -OwnershipBasedObjectDirectory: -- num listeners: 0 -- cumulative location updates: 0 -- num location updates per second: 0.000 -- num location lookups per second: 0.000 -- num locations added per second: 0.000 -- num locations removed per second: 0.000 -BufferPool: -- create buffer state map size: 0 -PullManager: -- num bytes available for pulled objects: 18275401728 -- num bytes being pulled (all): 0 -- num bytes being pulled / pinned: 0 -- get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- first get request bundle: N/A -- first wait request bundle: N/A -- first task request bundle: N/A -- num objects queued: 0 -- num objects actively pulled (all): 0 -- num objects actively pulled / pinned: 0 -- num bundles being pulled: 0 -- num pull retries: 0 -- max timeout seconds: 0 -- max timeout request is already processed. No entry. - -WorkerPool: -- registered jobs: 1 -- process_failed_job_config_missing: 0 -- process_failed_rate_limited: 0 -- process_failed_pending_registration: 0 -- process_failed_runtime_env_setup_failed: 0 -- num PYTHON workers: 12 -- num PYTHON drivers: 1 -- num object spill callbacks queued: 0 -- num object restore queued: 0 -- num util functions queued: 0 -- num idle workers: 7 -TaskDependencyManager: -- task deps map size: 0 -- get req map size: 0 -- wait req map size: 0 -- local objects map size: 0 -WaitManager: -- num active wait requests: 0 -Subscriber: -Channel WORKER_OBJECT_LOCATIONS_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_OBJECT_EVICTION -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_REF_REMOVED_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -num async plasma notifications: 0 -Remote node managers: -Event stats: -Global stats: 4610 total (27 active) -Queueing time: mean = 1.447 ms, max = 1.315 s, min = 70.000 ns, total = 6.673 s -Execution time: mean = 926.474 us, total = 4.271 s -Event stats: - NodeManagerService.grpc_server.ReportWorkerBacklog.HandleRequestImpl - 777 total (0 active), Execution time: mean = 9.890 us, total = 7.684 ms, Queueing time: mean = 11.345 us, max = 260.564 us, min = 5.370 us, total = 8.815 ms - NodeManagerService.grpc_server.ReportWorkerBacklog - 777 total (0 active), Execution time: mean = 95.817 us, total = 74.450 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.CheckGC - 600 total (1 active), Execution time: mean = 862.498 ns, total = 517.499 us, Queueing time: mean = 51.458 us, max = 20.877 ms, min = 6.798 us, total = 30.875 ms - RaySyncer.OnDemandBroadcasting - 600 total (1 active), Execution time: mean = 4.655 us, total = 2.793 ms, Queueing time: mean = 48.281 us, max = 20.869 ms, min = 7.100 us, total = 28.969 ms - ObjectManager.UpdateAvailableMemory - 600 total (0 active), Execution time: mean = 1.120 us, total = 671.870 us, Queueing time: mean = 8.347 us, max = 17.989 us, min = 4.399 us, total = 5.008 ms - RayletWorkerPool.deadline_timer.kill_idle_workers - 300 total (1 active), Execution time: mean = 9.071 us, total = 2.721 ms, Queueing time: mean = 24.985 us, max = 3.787 ms, min = 4.986 us, total = 7.495 ms - MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 240 total (1 active), Execution time: mean = 93.805 us, total = 22.513 ms, Queueing time: mean = 21.520 us, max = 1.959 ms, min = 4.529 us, total = 5.165 ms - ClientConnection.async_read.ProcessMessageHeader - 83 total (13 active), Execution time: mean = 2.637 us, total = 218.895 us, Queueing time: mean = 78.589 ms, max = 1.315 s, min = 10.660 us, total = 6.523 s - ClientConnection.async_read.ProcessMessage - 70 total (0 active), Execution time: mean = 556.123 us, total = 38.929 ms, Queueing time: mean = 20.546 us, max = 519.328 us, min = 1.760 us, total = 1.438 ms - NodeManager.ScheduleAndDispatchTasks - 61 total (1 active), Execution time: mean = 4.952 us, total = 302.072 us, Queueing time: mean = 10.238 us, max = 28.640 us, min = 5.442 us, total = 624.532 us - NodeManager.deadline_timer.spill_objects_when_over_threshold - 60 total (1 active), Execution time: mean = 1.159 us, total = 69.518 us, Queueing time: mean = 124.476 us, max = 4.077 ms, min = 6.857 us, total = 7.469 ms - NodeManagerService.grpc_server.GetResourceLoad - 60 total (0 active), Execution time: mean = 156.631 us, total = 9.398 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.deadline_timer.flush_free_objects - 60 total (1 active), Execution time: mean = 2.495 us, total = 149.680 us, Queueing time: mean = 123.346 us, max = 4.077 ms, min = 4.140 us, total = 7.401 ms - NodeManagerService.grpc_server.GetResourceLoad.HandleRequestImpl - 60 total (0 active), Execution time: mean = 53.331 us, total = 3.200 ms, Queueing time: mean = 13.716 us, max = 193.985 us, min = 6.280 us, total = 822.984 us - ClusterResourceManager.ResetRemoteNodeView - 21 total (1 active), Execution time: mean = 1.950 us, total = 40.949 us, Queueing time: mean = 11.393 us, max = 23.973 us, min = 7.953 us, total = 239.256 us - ClientConnection.async_write.DoAsyncWrites - 19 total (0 active), Execution time: mean = 595.789 ns, total = 11.320 us, Queueing time: mean = 12.061 us, max = 39.439 us, min = 6.350 us, total = 229.155 us - ObjectManager.ObjectAdded - 18 total (0 active), Execution time: mean = 6.807 us, total = 122.527 us, Queueing time: mean = 12.869 us, max = 27.039 us, min = 5.780 us, total = 231.644 us - NodeManagerService.grpc_server.GetSystemConfig.HandleRequestImpl - 18 total (0 active), Execution time: mean = 23.248 us, total = 418.471 us, Queueing time: mean = 10.128 us, max = 17.150 us, min = 6.550 us, total = 182.299 us - ObjectManager.ObjectDeleted - 18 total (0 active), Execution time: mean = 7.564 us, total = 136.158 us, Queueing time: mean = 29.838 us, max = 161.227 us, min = 11.850 us, total = 537.091 us - NodeManagerService.grpc_server.GetSystemConfig - 18 total (0 active), Execution time: mean = 114.145 us, total = 2.055 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 13 total (0 active), Execution time: mean = 126.892 us, total = 1.650 ms, Queueing time: mean = 2.277 ms, max = 7.175 ms, min = 18.609 us, total = 29.601 ms - NodeManager.deadline_timer.record_metrics - 12 total (1 active), Execution time: mean = 236.378 us, total = 2.837 ms, Queueing time: mean = 376.783 us, max = 3.826 ms, min = 9.922 us, total = 4.521 ms - NodeInfoGcsService.grpc_client.CheckAlive - 12 total (0 active), Execution time: mean = 296.571 us, total = 3.559 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.GcsCheckAlive - 12 total (1 active), Execution time: mean = 96.280 us, total = 1.155 ms, Queueing time: mean = 504.905 us, max = 4.043 ms, min = 53.416 us, total = 6.059 ms - NodeInfoGcsService.grpc_client.CheckAlive.OnReplyReceived - 12 total (0 active), Execution time: mean = 13.821 us, total = 165.847 us, Queueing time: mean = 10.679 us, max = 16.709 us, min = 7.860 us, total = 128.146 us - NodeManagerService.grpc_server.RequestWorkerLease.HandleRequestImpl - 6 total (0 active), Execution time: mean = 129.099 us, total = 774.591 us, Queueing time: mean = 289.732 us, max = 937.888 us, min = 7.350 us, total = 1.738 ms - NodeManagerService.grpc_server.RequestWorkerLease - 6 total (0 active), Execution time: mean = 375.071 ms, total = 2.250 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.deadline_timer.debug_state_dump - 6 total (1 active, 1 running), Execution time: mean = 1.167 ms, total = 7.003 ms, Queueing time: mean = 8.847 us, max = 13.645 us, min = 8.547 us, total = 53.084 us - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 5 total (1 active), Execution time: mean = 361.638 ms, total = 1.808 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.Exit - 5 total (0 active), Execution time: mean = 913.509 us, total = 4.568 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.Exit.OnReplyReceived - 5 total (0 active), Execution time: mean = 11.781 us, total = 58.907 us, Queueing time: mean = 14.184 us, max = 38.179 us, min = 5.980 us, total = 70.919 us - - 5 total (0 active), Execution time: mean = 134.000 ns, total = 670.000 ns, Queueing time: mean = 8.842 us, max = 9.990 us, min = 8.019 us, total = 44.208 us - WorkerInfoGcsService.grpc_client.ReportWorkerFailure.OnReplyReceived - 5 total (0 active), Execution time: mean = 17.188 us, total = 85.938 us, Queueing time: mean = 42.755 us, max = 68.518 us, min = 5.860 us, total = 213.773 us - Subscriber.HandlePublishedMessage_GCS_WORKER_DELTA_CHANNEL - 5 total (0 active), Execution time: mean = 1.738 us, total = 8.689 us, Queueing time: mean = 93.382 us, max = 101.948 us, min = 69.249 us, total = 466.911 us - WorkerInfoGcsService.grpc_client.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 1.060 ms, total = 5.299 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RaySyncer.BroadcastMessage - 5 total (0 active), Execution time: mean = 99.532 us, total = 497.658 us, Queueing time: mean = 352.000 ns, max = 1.100 us, min = 90.000 ns, total = 1.760 us - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll.OnReplyReceived - 4 total (0 active), Execution time: mean = 93.540 us, total = 374.162 us, Queueing time: mean = 9.925 us, max = 11.770 us, min = 6.350 us, total = 39.700 us - NodeManager.deadline_timer.print_event_loop_stats - 2 total (1 active), Execution time: mean = 440.882 us, total = 881.764 us, Queueing time: mean = 4.390 us, max = 8.780 us, min = 8.780 us, total = 8.780 us - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 2 total (0 active), Execution time: mean = 70.573 us, total = 141.147 us, Queueing time: mean = 652.500 us, max = 1.244 ms, min = 60.769 us, total = 1.305 ms - RaySyncerRegister - 2 total (0 active), Execution time: mean = 800.000 ns, total = 1.600 us, Queueing time: mean = 130.000 ns, max = 190.000 ns, min = 70.000 ns, total = 260.000 ns - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 2 total (0 active), Execution time: mean = 241.449 us, total = 482.898 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.CommitBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 182.946 us, total = 182.946 us, Queueing time: mean = 5.770 us, max = 5.770 us, min = 5.770 us, total = 5.770 us - NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 522.368 us, total = 522.368 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerPool.PopWorkerCallback - 1 total (0 active), Execution time: mean = 18.460 us, total = 18.460 us, Queueing time: mean = 6.200 us, max = 6.200 us, min = 6.200 us, total = 6.200 us - NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 863.500 us, total = 863.500 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.CommitBundleResources - 1 total (0 active), Execution time: mean = 236.855 us, total = 236.855 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - Subscriber.HandlePublishedMessage_GCS_JOB_CHANNEL - 1 total (0 active), Execution time: mean = 25.699 us, total = 25.699 us, Queueing time: mean = 94.678 us, max = 94.678 us, min = 94.678 us, total = 94.678 us - JobInfoGcsService.grpc_client.GetAllJobInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 7.480 us, total = 7.480 us, Queueing time: mean = 10.389 us, max = 10.389 us, min = 10.389 us, total = 10.389 us - NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 140.966 us, total = 140.966 us, Queueing time: mean = 6.160 us, max = 6.160 us, min = 6.160 us, total = 6.160 us - JobInfoGcsService.grpc_client.GetAllJobInfo - 1 total (0 active), Execution time: mean = 236.415 us, total = 236.415 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.ReturnWorker - 1 total (0 active), Execution time: mean = 148.266 us, total = 148.266 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.AddJob - 1 total (0 active), Execution time: mean = 544.817 us, total = 544.817 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.AddJob.OnReplyReceived - 1 total (0 active), Execution time: mean = 14.470 us, total = 14.470 us, Queueing time: mean = 92.518 us, max = 92.518 us, min = 92.518 us, total = 92.518 us - NodeManagerService.grpc_server.PrepareBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 197.545 us, total = 197.545 us, Queueing time: mean = 8.130 us, max = 8.130 us, min = 8.130 us, total = 8.130 us - NodeInfoGcsService.grpc_client.GetAllNodeInfo - 1 total (0 active), Execution time: mean = 257.524 us, total = 257.524 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 12.673 ms, total = 12.673 ms, Queueing time: mean = 5.610 us, max = 5.610 us, min = 5.610 us, total = 5.610 us - NodeManagerService.grpc_server.PrepareBundleResources - 1 total (0 active), Execution time: mean = 282.294 us, total = 282.294 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetAllNodeInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 80.008 us, total = 80.008 us, Queueing time: mean = 7.170 us, max = 7.170 us, min = 7.170 us, total = 7.170 us - NodeManagerService.grpc_server.ReturnWorker.HandleRequestImpl - 1 total (0 active), Execution time: mean = 47.639 us, total = 47.639 us, Queueing time: mean = 11.740 us, max = 11.740 us, min = 11.740 us, total = 11.740 us -DebugString() time ms: 0 \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/debug_state_gcs.txt b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/debug_state_gcs.txt deleted file mode 100644 index ba03aa5..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/debug_state_gcs.txt +++ /dev/null @@ -1,150 +0,0 @@ -GcsNodeManager: -- RegisterNode request count: 1 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 31 -- GetInternalConfig request count: 1 - -GcsActorManager: -- RegisterActor request count: 5 -- CreateActor request count: 5 -- GetActorInfo request count: 5 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 5 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 1 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 5 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 12 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 1 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 5 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 1 -- Registered placement groups count: 1 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: -GCS_NODE_INFO_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 557 -- current buffered bytes: 0 -RAY_LOG_CHANNEL -- cumulative published messages: 21 -- cumulative published bytes: 23269 -- current buffered bytes: 853 -GCS_ACTOR_CHANNEL -- cumulative published messages: 10 -- cumulative published bytes: 5864 -- current buffered bytes: 1328 -GCS_WORKER_DELTA_CHANNEL -- cumulative published messages: 5 -- cumulative published bytes: 485 -- current buffered bytes: 97 -GCS_JOB_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 279 -- current buffered bytes: 279 - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 23 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 12 --Total num of actor creation tasks: 5 --Total num of actor tasks: 5 --Total num of normal tasks: 1 --Total num of driver tasks: 1 - - - - -Global stats: 1385 total (19 active) -Queueing time: mean = 75.379 us, max = 29.578 ms, min = -0.000 s, total = 104.400 ms -Execution time: mean = 1.744 ms, total = 2.416 s -Event stats: - GcsInMemoryStore.Put - 191 total (0 active), Execution time: mean = 175.900 us, total = 33.597 ms, Queueing time: mean = 158.811 us, max = 28.625 ms, min = 540.000 ns, total = 30.333 ms - InternalKVGcsService.grpc_server.InternalKVPut - 145 total (0 active), Execution time: mean = 150.208 us, total = 21.780 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 145 total (0 active), Execution time: mean = 7.502 us, total = 1.088 ms, Queueing time: mean = 48.142 us, max = 322.193 us, min = 900.000 ns, total = 6.981 ms - GcsInMemoryStore.Get - 85 total (0 active), Execution time: mean = 21.017 us, total = 1.786 ms, Queueing time: mean = 5.391 us, max = 274.798 us, min = 570.000 ns, total = 458.205 us - InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 78 total (0 active), Execution time: mean = 10.044 us, total = 783.409 us, Queueing time: mean = 10.235 us, max = 87.590 us, min = 4.380 us, total = 798.305 us - InternalKVGcsService.grpc_server.InternalKVGet - 78 total (0 active), Execution time: mean = 116.124 us, total = 9.058 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut - 72 total (0 active), Execution time: mean = 439.657 us, total = 31.655 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 72 total (0 active), Execution time: mean = 4.300 us, total = 309.574 us, Queueing time: mean = 12.588 us, max = 32.699 us, min = 4.990 us, total = 906.331 us - RayletLoadPulled - 60 total (1 active), Execution time: mean = 85.206 us, total = 5.112 ms, Queueing time: mean = 21.618 us, max = 673.207 us, min = -0.000 s, total = 1.297 ms - NodeManagerService.grpc_client.GetResourceLoad - 58 total (0 active), Execution time: mean = 382.663 us, total = 22.194 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 58 total (0 active), Execution time: mean = 26.139 us, total = 1.516 ms, Queueing time: mean = 10.960 us, max = 52.640 us, min = 4.640 us, total = 635.677 us - NodeInfoGcsService.grpc_server.GetAllNodeInfo - 31 total (0 active), Execution time: mean = 97.298 us, total = 3.016 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 31 total (0 active), Execution time: mean = 12.556 us, total = 389.246 us, Queueing time: mean = 10.092 us, max = 54.680 us, min = 4.710 us, total = 312.857 us - NodeInfoGcsService.grpc_server.CheckAlive - 24 total (0 active), Execution time: mean = 102.376 us, total = 2.457 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 24 total (0 active), Execution time: mean = 6.262 us, total = 150.297 us, Queueing time: mean = 11.301 us, max = 18.500 us, min = 7.710 us, total = 271.215 us - ClusterResourceManager.ResetRemoteNodeView - 21 total (1 active), Execution time: mean = 2.899 us, total = 60.889 us, Queueing time: mean = 31.392 us, max = 449.736 us, min = -0.000 s, total = 659.225 us - HealthCheck - 18 total (0 active), Execution time: mean = 2.805 us, total = 50.490 us, Queueing time: mean = 11.451 us, max = 23.970 us, min = 7.740 us, total = 206.125 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 14.958 us, total = 269.238 us, Queueing time: mean = 12.243 us, max = 49.120 us, min = 5.200 us, total = 220.378 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 106.928 us, total = 1.925 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 12 total (0 active), Execution time: mean = 152.279 us, total = 1.827 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 12 total (0 active), Execution time: mean = 31.195 us, total = 374.341 us, Queueing time: mean = 14.813 us, max = 38.549 us, min = 6.700 us, total = 177.758 us - NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 930.015 us, total = 7.440 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 20.247 us, total = 161.980 us, Queueing time: mean = 19.156 us, max = 72.530 us, min = 8.320 us, total = 153.250 us - GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 5.057 us, total = 35.400 us, Queueing time: mean = 15.167 us, max = 39.821 us, min = 1.470 us, total = 106.172 us - GCSServer.deadline_timer.debug_state_dump - 6 total (1 active, 1 running), Execution time: mean = 467.784 us, total = 2.807 ms, Queueing time: mean = 8.311 us, max = 13.255 us, min = 6.554 us, total = 49.865 us - ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 340.253 us, total = 1.701 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 10.690 us, total = 53.448 us, Queueing time: mean = 157.395 us, max = 414.269 us, min = 5.390 us, total = 786.974 us - GcsResourceManager::Update - 5 total (0 active), Execution time: mean = 43.813 us, total = 219.067 us, Queueing time: mean = 8.998 us, max = 11.619 us, min = 7.240 us, total = 44.989 us - ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 228.000 us, total = 1.140 ms, Queueing time: mean = 8.000 us, max = 9.150 us, min = 7.050 us, total = 39.999 us - ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 5.532 us, total = 27.659 us, Queueing time: mean = 28.326 us, max = 88.800 us, min = 4.760 us, total = 141.628 us - ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 700.382 us, total = 3.502 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 154.424 us, total = 772.120 us, Queueing time: mean = 7.308 us, max = 8.910 us, min = 6.120 us, total = 36.540 us - ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 88.440 us, total = 442.199 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 450.525 ms, total = 2.253 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 136.665 us, total = 683.327 us, Queueing time: mean = 7.626 us, max = 10.129 us, min = 5.900 us, total = 38.129 us - WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 21.929 us, total = 109.646 us, Queueing time: mean = 124.327 us, max = 514.508 us, min = 10.880 us, total = 621.633 us - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 219.332 us, total = 1.097 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 56.263 us, total = 225.050 us, Queueing time: mean = 14.740 ms, max = 29.578 ms, min = 12.341 us, total = 58.962 ms - GCSServer.deadline_timer.debug_state_event_stats_print - 2 total (1 active), Execution time: mean = 286.014 us, total = 572.029 us, Queueing time: mean = 4.987 us, max = 9.975 us, min = 9.975 us, total = 9.975 us - NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 84.119 us, total = 84.119 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 103.350 us, total = 103.350 us, Queueing time: mean = 7.670 us, max = 7.670 us, min = 7.670 us, total = 7.670 us - NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 48.040 us, total = 48.040 us, Queueing time: mean = 6.410 us, max = 6.410 us, min = 6.410 us, total = 6.410 us - NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 6.580 us, total = 6.580 us, Queueing time: mean = 10.119 us, max = 10.119 us, min = 10.119 us, total = 10.119 us - InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 11.950 us, total = 11.950 us, Queueing time: mean = 4.570 us, max = 4.570 us, min = 4.570 us, total = 4.570 us - InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.770 us, total = 9.770 us, Queueing time: mean = 8.310 us, max = 8.310 us, min = 8.310 us, total = 8.310 us - InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 50.110 us, total = 50.110 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 80.000 ns, total = 80.000 ns, Queueing time: mean = 73.960 us, max = 73.960 us, min = 73.960 us, total = 73.960 us - InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 66.770 us, total = 66.770 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 890.288 us, total = 890.288 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 2.410 us, total = 2.410 us, Queueing time: mean = 1.000 us, max = 1.000 us, min = 1.000 us, total = 1.000 us - JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 73.200 us, total = 73.200 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 192.409 us, total = 192.409 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 158.430 us, total = 158.430 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 52.330 us, total = 52.330 us, Queueing time: mean = 8.230 us, max = 8.230 us, min = 8.230 us, total = 8.230 us - NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 373.380 us, total = 373.380 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 5.530 us, total = 5.530 us, Queueing time: mean = 5.890 us, max = 5.890 us, min = 5.890 us, total = 5.890 us - GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 5.460 us, total = 5.460 us, Queueing time: mean = 140.000 ns, max = 140.000 ns, min = 140.000 ns, total = 140.000 ns - NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 40.400 us, total = 40.400 us, Queueing time: mean = 6.230 us, max = 6.230 us, min = 6.230 us, total = 6.230 us - NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 68.850 us, total = 68.850 us, Queueing time: mean = 11.870 us, max = 11.870 us, min = 11.870 us, total = 11.870 us - NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 297.979 us, total = 297.979 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 2.120 us, total = 2.120 us, Queueing time: mean = 1.060 us, max = 1.060 us, min = 1.060 us, total = 1.060 us - JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 108.050 us, total = 108.050 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 5.860 us, total = 5.860 us, Queueing time: mean = 7.190 us, max = 7.190 us, min = 7.190 us, total = 7.190 us \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/gcs_server.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/gcs_server.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/gcs_server.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/gcs_server.out deleted file mode 100644 index 8e1ccb8..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/gcs_server.out +++ /dev/null @@ -1,696 +0,0 @@ -[2024-05-15 09:05:00,574 I 12692 12692] (gcs_server) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 09:05:00,579 I 12692 12692] (gcs_server) event.cc:234: Set ray event level to warning -[2024-05-15 09:05:00,579 I 12692 12692] (gcs_server) event.cc:342: Ray Event initialized for GCS -[2024-05-15 09:05:00,579 I 12692 12692] (gcs_server) gcs_server.cc:74: GCS storage type is StorageType::IN_MEMORY -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:42: Loading job table data. -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:54: Loading node table data. -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:80: Loading actor table data. -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:93: Loading actor task spec table data. -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:66: Loading placement group table data. -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:46: Finished loading job table data, size = 0 -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:58: Finished loading node table data, size = 0 -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:84: Finished loading actor table data, size = 0 -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:97: Finished loading actor task spec table data, size = 0 -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_init_data.cc:71: Finished loading placement group table data, size = 0 -[2024-05-15 09:05:00,581 I 12692 12692] (gcs_server) gcs_server.cc:162: No existing server cluster ID found. Generating new ID: f80091d542af94b49f72839a1f9e8e9b93d17c7c67f37f1c21fb29b2 -[2024-05-15 09:05:00,582 I 12692 12692] (gcs_server) gcs_server.cc:648: Autoscaler V2 enabled: 0 -[2024-05-15 09:05:00,583 I 12692 12692] (gcs_server) grpc_server.cc:134: GcsServer server started, listening on port 51730. -[2024-05-15 09:05:00,610 I 12692 12692] (gcs_server) gcs_server.cc:250: GcsNodeManager: -- RegisterNode request count: 0 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 0 -- GetInternalConfig request count: 0 - -GcsActorManager: -- RegisterActor request count: 0 -- CreateActor request count: 0 -- GetActorInfo request count: 0 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 0 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 0 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 0 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 0 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 0 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 0 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 0 -- Registered placement groups count: 0 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 0 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 0 --Total num of actor creation tasks: 0 --Total num of actor tasks: 0 --Total num of normal tasks: 0 --Total num of driver tasks: 0 - - -[2024-05-15 09:05:00,610 I 12692 12692] (gcs_server) gcs_server.cc:844: Event stats: - - -Global stats: 21 total (10 active) -Queueing time: mean = 4.170 ms, max = 29.578 ms, min = 1.000 us, total = 87.561 ms -Execution time: mean = 1.419 ms, total = 29.795 ms -Event stats: - InternalKVGcsService.grpc_client.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.GetAll - 5 total (0 active), Execution time: mean = 4.124 us, total = 20.620 us, Queueing time: mean = 20.528 us, max = 39.821 us, min = 15.100 us, total = 102.642 us - PeriodicalRunner.RunFnPeriodically - 4 total (2 active, 1 running), Execution time: mean = 710.000 ns, total = 2.840 us, Queueing time: mean = 14.707 ms, max = 29.578 ms, min = 29.251 ms, total = 58.829 ms - GcsInMemoryStore.Put - 3 total (0 active), Execution time: mean = 9.922 ms, total = 29.767 ms, Queueing time: mean = 9.543 ms, max = 28.625 ms, min = 1.000 us, total = 28.628 ms - GcsInMemoryStore.Get - 1 total (0 active), Execution time: mean = 5.240 us, total = 5.240 us, Queueing time: mean = 1.300 us, max = 1.300 us, min = 1.300 us, total = 1.300 us - RayletLoadPulled - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 09:05:00,610 I 12692 12692] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats: - - -Global stats: 2 total (1 active) -Queueing time: mean = 32.815 us, max = 65.630 us, min = 65.630 us, total = 65.630 us -Execution time: mean = 11.830 us, total = 23.660 us -Event stats: - PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 23.660 us, total = 23.660 us, Queueing time: mean = 65.630 us, max = 65.630 us, min = 65.630 us, total = 65.630 us - GcsTaskManager.GcJobSummary - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 09:05:01,983 I 12692 12692] (gcs_server) gcs_node_manager.cc:55: Registering node info, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 09:05:01,983 I 12692 12692] (gcs_server) gcs_node_manager.cc:61: Finished registering node info, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 09:05:01,983 I 12692 12692] (gcs_server) gcs_placement_group_manager.cc:797: A new node: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 registered, will try to reschedule all the infeasible placement groups. -[2024-05-15 09:05:02,568 I 12692 12692] (gcs_server) gcs_job_manager.cc:42: Adding job, job id = 01000000, driver pid = 12647 -[2024-05-15 09:05:02,568 I 12692 12692] (gcs_server) gcs_job_manager.cc:57: Finished adding job, job id = 01000000, driver pid = 12647 -[2024-05-15 09:05:02,610 I 12692 12692] (gcs_server) gcs_placement_group_manager.cc:432: Registering placement group, placement group id = 656f9c54c57685dae4b6a63a81d401000000, name = , strategy = 0 -[2024-05-15 09:05:02,610 I 12692 12692] (gcs_server) gcs_placement_group_manager.cc:436: Finished registering placement group, placement group id = 656f9c54c57685dae4b6a63a81d401000000, name = , strategy = 0 -[2024-05-15 09:05:02,611 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:185: Preparing resource from node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for bundles: {placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={0}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={1}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={2}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={3}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={4}}, -[2024-05-15 09:05:02,612 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:195: Finished leasing resource from 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for bundles: {placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={0}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={1}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={2}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={3}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={4}}, -[2024-05-15 09:05:02,612 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:213: Committing resource to a node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for bundles: {placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={0}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={1}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={2}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={3}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={4}}, -[2024-05-15 09:05:02,612 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:220: Finished committing resource to 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for bundles: {placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={0}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={1}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={2}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={3}},{placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={4}}, -[2024-05-15 09:05:02,612 I 12692 12692] (gcs_server) gcs_placement_group_manager.cc:321: Successfully created placement group , id: 656f9c54c57685dae4b6a63a81d401000000 -[2024-05-15 09:05:02,795 I 12692 12692] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 0b0450d90a301440d0afa9f001000000 -[2024-05-15 09:05:02,795 I 12692 12692] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 0b0450d90a301440d0afa9f001000000 -[2024-05-15 09:05:02,796 I 12692 12692] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 0b0450d90a301440d0afa9f001000000 -[2024-05-15 09:05:02,796 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor 0b0450d90a301440d0afa9f001000000, job id = 01000000 -[2024-05-15 09:05:03,202 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor 0b0450d90a301440d0afa9f001000000, job id = 01000000 -[2024-05-15 09:05:03,202 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 0b0450d90a301440d0afa9f001000000 on worker 88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:05:04,755 I 12692 12692] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = cf76be01afb9954787ceb84801000000 -[2024-05-15 09:05:04,756 I 12692 12692] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = cf76be01afb9954787ceb84801000000 -[2024-05-15 09:05:04,756 I 12692 12692] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = cf76be01afb9954787ceb84801000000 -[2024-05-15 09:05:04,756 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor cf76be01afb9954787ceb84801000000, job id = 01000000 -[2024-05-15 09:05:04,756 I 12692 12692] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 64110d8601f56bf14e9797c501000000 -[2024-05-15 09:05:04,757 I 12692 12692] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 64110d8601f56bf14e9797c501000000 -[2024-05-15 09:05:04,757 I 12692 12692] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 64110d8601f56bf14e9797c501000000 -[2024-05-15 09:05:04,757 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor 64110d8601f56bf14e9797c501000000, job id = 01000000 -[2024-05-15 09:05:04,758 I 12692 12692] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 6c1e2f8e5911cf8ba4dcac5b01000000 -[2024-05-15 09:05:04,758 I 12692 12692] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 6c1e2f8e5911cf8ba4dcac5b01000000 -[2024-05-15 09:05:04,758 I 12692 12692] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 6c1e2f8e5911cf8ba4dcac5b01000000 -[2024-05-15 09:05:04,758 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor 6c1e2f8e5911cf8ba4dcac5b01000000, job id = 01000000 -[2024-05-15 09:05:04,759 I 12692 12692] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 79760d3ce8052092d662e89401000000 -[2024-05-15 09:05:04,759 I 12692 12692] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 79760d3ce8052092d662e89401000000 -[2024-05-15 09:05:04,759 I 12692 12692] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 79760d3ce8052092d662e89401000000 -[2024-05-15 09:05:04,759 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor 79760d3ce8052092d662e89401000000, job id = 01000000 -[2024-05-15 09:05:05,180 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor cf76be01afb9954787ceb84801000000, job id = 01000000 -[2024-05-15 09:05:05,181 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor cf76be01afb9954787ceb84801000000 on worker 6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:05:05,183 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor 6c1e2f8e5911cf8ba4dcac5b01000000, job id = 01000000 -[2024-05-15 09:05:05,183 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 6c1e2f8e5911cf8ba4dcac5b01000000 on worker cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:05:05,231 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor 79760d3ce8052092d662e89401000000, job id = 01000000 -[2024-05-15 09:05:05,232 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 79760d3ce8052092d662e89401000000 on worker 5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:05:05,282 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 for actor 64110d8601f56bf14e9797c501000000, job id = 01000000 -[2024-05-15 09:05:05,282 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 64110d8601f56bf14e9797c501000000 on worker 7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:05:10,591 W 12692 12713] (gcs_server) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:62660: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 09:06:00,611 I 12692 12692] (gcs_server) gcs_server.cc:250: GcsNodeManager: -- RegisterNode request count: 1 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 31 -- GetInternalConfig request count: 1 - -GcsActorManager: -- RegisterActor request count: 5 -- CreateActor request count: 5 -- GetActorInfo request count: 5 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 5 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 1 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 5 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 12 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 1 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 5 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 1 -- Registered placement groups count: 1 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: -GCS_NODE_INFO_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 557 -- current buffered bytes: 0 -RAY_LOG_CHANNEL -- cumulative published messages: 21 -- cumulative published bytes: 23269 -- current buffered bytes: 853 -GCS_ACTOR_CHANNEL -- cumulative published messages: 10 -- cumulative published bytes: 5864 -- current buffered bytes: 1328 -GCS_WORKER_DELTA_CHANNEL -- cumulative published messages: 5 -- cumulative published bytes: 485 -- current buffered bytes: 97 -GCS_JOB_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 279 -- current buffered bytes: 279 - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 23 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 12 --Total num of actor creation tasks: 5 --Total num of actor tasks: 5 --Total num of normal tasks: 1 --Total num of driver tasks: 1 - - -[2024-05-15 09:06:00,611 I 12692 12692] (gcs_server) gcs_server.cc:844: Event stats: - - -Global stats: 1383 total (19 active) -Queueing time: mean = 75.156 us, max = 29.578 ms, min = -0.000 s, total = 103.940 ms -Execution time: mean = 1.746 ms, total = 2.415 s -Event stats: - GcsInMemoryStore.Put - 191 total (0 active), Execution time: mean = 175.900 us, total = 33.597 ms, Queueing time: mean = 158.811 us, max = 28.625 ms, min = 540.000 ns, total = 30.333 ms - InternalKVGcsService.grpc_server.InternalKVPut - 145 total (0 active), Execution time: mean = 150.208 us, total = 21.780 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 145 total (0 active), Execution time: mean = 7.502 us, total = 1.088 ms, Queueing time: mean = 48.142 us, max = 322.193 us, min = 900.000 ns, total = 6.981 ms - GcsInMemoryStore.Get - 85 total (0 active), Execution time: mean = 21.017 us, total = 1.786 ms, Queueing time: mean = 5.391 us, max = 274.798 us, min = 570.000 ns, total = 458.205 us - InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 78 total (0 active), Execution time: mean = 10.044 us, total = 783.409 us, Queueing time: mean = 10.235 us, max = 87.590 us, min = 4.380 us, total = 798.305 us - InternalKVGcsService.grpc_server.InternalKVGet - 78 total (0 active), Execution time: mean = 116.124 us, total = 9.058 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut - 72 total (0 active), Execution time: mean = 439.657 us, total = 31.655 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 72 total (0 active), Execution time: mean = 4.300 us, total = 309.574 us, Queueing time: mean = 12.588 us, max = 32.699 us, min = 4.990 us, total = 906.331 us - RayletLoadPulled - 60 total (1 active), Execution time: mean = 85.206 us, total = 5.112 ms, Queueing time: mean = 21.618 us, max = 673.207 us, min = -0.000 s, total = 1.297 ms - NodeManagerService.grpc_client.GetResourceLoad - 58 total (0 active), Execution time: mean = 382.663 us, total = 22.194 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 58 total (0 active), Execution time: mean = 26.139 us, total = 1.516 ms, Queueing time: mean = 10.960 us, max = 52.640 us, min = 4.640 us, total = 635.677 us - NodeInfoGcsService.grpc_server.GetAllNodeInfo - 31 total (0 active), Execution time: mean = 97.298 us, total = 3.016 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 31 total (0 active), Execution time: mean = 12.556 us, total = 389.246 us, Queueing time: mean = 10.092 us, max = 54.680 us, min = 4.710 us, total = 312.857 us - NodeInfoGcsService.grpc_server.CheckAlive - 24 total (0 active), Execution time: mean = 102.376 us, total = 2.457 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 24 total (0 active), Execution time: mean = 6.262 us, total = 150.297 us, Queueing time: mean = 11.301 us, max = 18.500 us, min = 7.710 us, total = 271.215 us - ClusterResourceManager.ResetRemoteNodeView - 20 total (1 active), Execution time: mean = 2.986 us, total = 59.729 us, Queueing time: mean = 10.474 us, max = 44.818 us, min = -0.000 s, total = 209.489 us - HealthCheck - 18 total (0 active), Execution time: mean = 2.805 us, total = 50.490 us, Queueing time: mean = 11.451 us, max = 23.970 us, min = 7.740 us, total = 206.125 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 14.958 us, total = 269.238 us, Queueing time: mean = 12.243 us, max = 49.120 us, min = 5.200 us, total = 220.378 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 106.928 us, total = 1.925 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 12 total (0 active), Execution time: mean = 152.279 us, total = 1.827 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 12 total (0 active), Execution time: mean = 31.195 us, total = 374.341 us, Queueing time: mean = 14.813 us, max = 38.549 us, min = 6.700 us, total = 177.758 us - NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 930.015 us, total = 7.440 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 20.247 us, total = 161.980 us, Queueing time: mean = 19.156 us, max = 72.530 us, min = 8.320 us, total = 153.250 us - GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 5.057 us, total = 35.400 us, Queueing time: mean = 15.167 us, max = 39.821 us, min = 1.470 us, total = 106.172 us - GCSServer.deadline_timer.debug_state_dump - 6 total (1 active), Execution time: mean = 467.784 us, total = 2.807 ms, Queueing time: mean = 8.311 us, max = 13.255 us, min = 6.554 us, total = 49.865 us - ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 340.253 us, total = 1.701 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 10.690 us, total = 53.448 us, Queueing time: mean = 157.395 us, max = 414.269 us, min = 5.390 us, total = 786.974 us - GcsResourceManager::Update - 5 total (0 active), Execution time: mean = 43.813 us, total = 219.067 us, Queueing time: mean = 8.998 us, max = 11.619 us, min = 7.240 us, total = 44.989 us - ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 228.000 us, total = 1.140 ms, Queueing time: mean = 8.000 us, max = 9.150 us, min = 7.050 us, total = 39.999 us - ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 5.532 us, total = 27.659 us, Queueing time: mean = 28.326 us, max = 88.800 us, min = 4.760 us, total = 141.628 us - ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 700.382 us, total = 3.502 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 154.424 us, total = 772.120 us, Queueing time: mean = 7.308 us, max = 8.910 us, min = 6.120 us, total = 36.540 us - ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 88.440 us, total = 442.199 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 450.525 ms, total = 2.253 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 136.665 us, total = 683.327 us, Queueing time: mean = 7.626 us, max = 10.129 us, min = 5.900 us, total = 38.129 us - WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 21.929 us, total = 109.646 us, Queueing time: mean = 124.327 us, max = 514.508 us, min = 10.880 us, total = 621.633 us - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 219.332 us, total = 1.097 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 56.263 us, total = 225.050 us, Queueing time: mean = 14.740 ms, max = 29.578 ms, min = 12.341 us, total = 58.962 ms - NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 84.119 us, total = 84.119 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 103.350 us, total = 103.350 us, Queueing time: mean = 7.670 us, max = 7.670 us, min = 7.670 us, total = 7.670 us - NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 48.040 us, total = 48.040 us, Queueing time: mean = 6.410 us, max = 6.410 us, min = 6.410 us, total = 6.410 us - NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 6.580 us, total = 6.580 us, Queueing time: mean = 10.119 us, max = 10.119 us, min = 10.119 us, total = 10.119 us - InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 11.950 us, total = 11.950 us, Queueing time: mean = 4.570 us, max = 4.570 us, min = 4.570 us, total = 4.570 us - InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.770 us, total = 9.770 us, Queueing time: mean = 8.310 us, max = 8.310 us, min = 8.310 us, total = 8.310 us - InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 50.110 us, total = 50.110 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 80.000 ns, total = 80.000 ns, Queueing time: mean = 73.960 us, max = 73.960 us, min = 73.960 us, total = 73.960 us - InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 66.770 us, total = 66.770 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active, 1 running), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 890.288 us, total = 890.288 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 2.410 us, total = 2.410 us, Queueing time: mean = 1.000 us, max = 1.000 us, min = 1.000 us, total = 1.000 us - JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 73.200 us, total = 73.200 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 192.409 us, total = 192.409 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 158.430 us, total = 158.430 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 52.330 us, total = 52.330 us, Queueing time: mean = 8.230 us, max = 8.230 us, min = 8.230 us, total = 8.230 us - NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 373.380 us, total = 373.380 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 5.530 us, total = 5.530 us, Queueing time: mean = 5.890 us, max = 5.890 us, min = 5.890 us, total = 5.890 us - GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 5.460 us, total = 5.460 us, Queueing time: mean = 140.000 ns, max = 140.000 ns, min = 140.000 ns, total = 140.000 ns - NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 40.400 us, total = 40.400 us, Queueing time: mean = 6.230 us, max = 6.230 us, min = 6.230 us, total = 6.230 us - NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 68.850 us, total = 68.850 us, Queueing time: mean = 11.870 us, max = 11.870 us, min = 11.870 us, total = 11.870 us - NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 297.979 us, total = 297.979 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 2.120 us, total = 2.120 us, Queueing time: mean = 1.060 us, max = 1.060 us, min = 1.060 us, total = 1.060 us - JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 108.050 us, total = 108.050 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 5.860 us, total = 5.860 us, Queueing time: mean = 7.190 us, max = 7.190 us, min = 7.190 us, total = 7.190 us - - -[2024-05-15 09:06:00,611 I 12692 12692] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats: - - -Global stats: 1552 total (1 active) -Queueing time: mean = 6.006 us, max = 65.630 us, min = 5.550 us, total = 9.321 ms -Execution time: mean = 51.059 us, total = 79.243 ms -Event stats: - TaskInfoGcsService.grpc_server.AddTaskEventData.HandleRequestImpl - 769 total (0 active), Execution time: mean = 7.755 us, total = 5.964 ms, Queueing time: mean = 11.815 us, max = 63.708 us, min = 5.550 us, total = 9.086 ms - TaskInfoGcsService.grpc_server.AddTaskEventData - 769 total (0 active), Execution time: mean = 95.184 us, total = 73.197 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsTaskManager.GcJobSummary - 13 total (1 active), Execution time: mean = 4.561 us, total = 59.297 us, Queueing time: mean = 13.041 us, max = 17.003 us, min = 8.947 us, total = 169.534 us - PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 23.660 us, total = 23.660 us, Queueing time: mean = 65.630 us, max = 65.630 us, min = 65.630 us, total = 65.630 us - - -[2024-05-15 09:06:06,921 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:484: Finished actor creation task for actor cf76be01afb9954787ceb84801000000 on worker 6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:06:06,921 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1280: Failed to create an actor due to the application failure, actor id = cf76be01afb9954787ceb84801000000, job id = 01000000 -[2024-05-15 09:06:06,921 I 12692 12692] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = cf76be01afb9954787ceb84801000000, status = CreationTaskError: User exception: -er() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -System error: -CreationTaskError: Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -[2024-05-15 09:06:06,928 W 12692 12692] (gcs_server) gcs_worker_manager.cc:55: Reporting worker exit, worker id = 6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, address = 192.168.1.56, exit_type = USER_ERROR, exit_detail = Worker exits because there was an exception in the initialization method (e.g., __init__). Fix the exceptions from the initialization to resolve the issue. Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out. Unintentional worker failures have been reported. If there are lots of this logs, that might indicate there are unexpected failures in the cluster. -[2024-05-15 09:06:06,928 W 12692 12692] (gcs_server) gcs_actor_manager.cc:970: Worker 6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a on node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 exits, type=USER_ERROR, has creation_task_exception = 1 -[2024-05-15 09:06:06,929 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1167: Actor cf76be01afb9954787ceb84801000000 is failed on worker 6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, need_reschedule = 0, death context type = CreationTaskFailureContext, remaining_restarts = 0, job id = 01000000 -[2024-05-15 09:06:06,933 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:484: Finished actor creation task for actor 6c1e2f8e5911cf8ba4dcac5b01000000 on worker cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:06:06,933 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1280: Failed to create an actor due to the application failure, actor id = 6c1e2f8e5911cf8ba4dcac5b01000000, job id = 01000000 -[2024-05-15 09:06:06,933 I 12692 12692] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 6c1e2f8e5911cf8ba4dcac5b01000000, status = CreationTaskError: User exception: -er() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -System error: -CreationTaskError: Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13723, ip=192.168.1.56, actor_id=6c1e2f8e5911cf8ba4dcac5b01000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -[2024-05-15 09:06:06,938 W 12692 12692] (gcs_server) gcs_worker_manager.cc:55: Reporting worker exit, worker id = cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, address = 192.168.1.56, exit_type = USER_ERROR, exit_detail = Worker exits because there was an exception in the initialization method (e.g., __init__). Fix the exceptions from the initialization to resolve the issue. Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13723, ip=192.168.1.56, actor_id=6c1e2f8e5911cf8ba4dcac5b01000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out. Unintentional worker failures have been reported. If there are lots of this logs, that might indicate there are unexpected failures in the cluster. -[2024-05-15 09:06:06,938 W 12692 12692] (gcs_server) gcs_actor_manager.cc:970: Worker cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91 on node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 exits, type=USER_ERROR, has creation_task_exception = 1 -[2024-05-15 09:06:06,938 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1167: Actor 6c1e2f8e5911cf8ba4dcac5b01000000 is failed on worker cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, need_reschedule = 0, death context type = CreationTaskFailureContext, remaining_restarts = 0, job id = 01000000 -[2024-05-15 09:06:06,967 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:484: Finished actor creation task for actor 79760d3ce8052092d662e89401000000 on worker 5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:06:06,967 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1280: Failed to create an actor due to the application failure, actor id = 79760d3ce8052092d662e89401000000, job id = 01000000 -[2024-05-15 09:06:06,967 I 12692 12692] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 79760d3ce8052092d662e89401000000, status = CreationTaskError: User exception: -er() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -System error: -CreationTaskError: Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13724, ip=192.168.1.56, actor_id=79760d3ce8052092d662e89401000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -[2024-05-15 09:06:06,972 W 12692 12692] (gcs_server) gcs_worker_manager.cc:55: Reporting worker exit, worker id = 5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, address = 192.168.1.56, exit_type = USER_ERROR, exit_detail = Worker exits because there was an exception in the initialization method (e.g., __init__). Fix the exceptions from the initialization to resolve the issue. Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13724, ip=192.168.1.56, actor_id=79760d3ce8052092d662e89401000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out. Unintentional worker failures have been reported. If there are lots of this logs, that might indicate there are unexpected failures in the cluster. -[2024-05-15 09:06:06,972 W 12692 12692] (gcs_server) gcs_actor_manager.cc:970: Worker 5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38 on node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 exits, type=USER_ERROR, has creation_task_exception = 1 -[2024-05-15 09:06:06,972 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1167: Actor 79760d3ce8052092d662e89401000000 is failed on worker 5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, need_reschedule = 0, death context type = CreationTaskFailureContext, remaining_restarts = 0, job id = 01000000 -[2024-05-15 09:06:07,036 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:484: Finished actor creation task for actor 64110d8601f56bf14e9797c501000000 on worker 7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:06:07,036 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1280: Failed to create an actor due to the application failure, actor id = 64110d8601f56bf14e9797c501000000, job id = 01000000 -[2024-05-15 09:06:07,036 I 12692 12692] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 64110d8601f56bf14e9797c501000000, status = CreationTaskError: User exception: -er() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -System error: -CreationTaskError: Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13722, ip=192.168.1.56, actor_id=64110d8601f56bf14e9797c501000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -[2024-05-15 09:06:07,044 W 12692 12692] (gcs_server) gcs_worker_manager.cc:55: Reporting worker exit, worker id = 7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, address = 192.168.1.56, exit_type = USER_ERROR, exit_detail = Worker exits because there was an exception in the initialization method (e.g., __init__). Fix the exceptions from the initialization to resolve the issue. Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13722, ip=192.168.1.56, actor_id=64110d8601f56bf14e9797c501000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out. Unintentional worker failures have been reported. If there are lots of this logs, that might indicate there are unexpected failures in the cluster. -[2024-05-15 09:06:07,044 W 12692 12692] (gcs_server) gcs_actor_manager.cc:970: Worker 7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed on node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 exits, type=USER_ERROR, has creation_task_exception = 1 -[2024-05-15 09:06:07,044 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1167: Actor 64110d8601f56bf14e9797c501000000 is failed on worker 7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, need_reschedule = 0, death context type = CreationTaskFailureContext, remaining_restarts = 0, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:798: Actor 64110d8601f56bf14e9797c501000000 is out of scope, destroying actor, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 64110d8601f56bf14e9797c501000000, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:798: Actor 6c1e2f8e5911cf8ba4dcac5b01000000 is out of scope, destroying actor, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 6c1e2f8e5911cf8ba4dcac5b01000000, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:798: Actor 79760d3ce8052092d662e89401000000 is out of scope, destroying actor, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 79760d3ce8052092d662e89401000000, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:798: Actor cf76be01afb9954787ceb84801000000 is out of scope, destroying actor, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = cf76be01afb9954787ceb84801000000, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_scheduler.cc:484: Finished actor creation task for actor 0b0450d90a301440d0afa9f001000000 on worker 88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1280: Failed to create an actor due to the application failure, actor id = 0b0450d90a301440d0afa9f001000000, job id = 01000000 -[2024-05-15 09:06:07,180 I 12692 12692] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 0b0450d90a301440d0afa9f001000000, status = CreationTaskError: User exception: -elf.config)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup - self.workers = EnvRunnerGroup( - ^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__ - raise e.args[0].args[2] -TimeoutError: timed out - -System error: -CreationTaskError: Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 239, in _setup - self.add_workers( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 754, in add_workers - raise result.get() - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/utils/actor_manager.py", line 497, in _fetch_result - result = ray.get(r) - ^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -During handling of the above exception, another exception occurred: - -ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 554, in __init__ - super().__init__( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__ - self.setup(copy.deepcopy(self.config)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup - self.workers = EnvRunnerGroup( - ^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__ - raise e.args[0].args[2] -TimeoutError: timed out -[2024-05-15 09:06:07,189 W 12692 12692] (gcs_server) gcs_worker_manager.cc:55: Reporting worker exit, worker id = 88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, address = 192.168.1.56, exit_type = USER_ERROR, exit_detail = Worker exits because there was an exception in the initialization method (e.g., __init__). Fix the exceptions from the initialization to resolve the issue. Exception raised from an actor init method. Traceback: The actor died because of an error raised in its creation task, ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 239, in _setup - self.add_workers( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 754, in add_workers - raise result.get() - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/utils/actor_manager.py", line 497, in _fetch_result - result = ray.get(r) - ^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -During handling of the above exception, another exception occurred: - -ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 554, in __init__ - super().__init__( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__ - self.setup(copy.deepcopy(self.config)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup - self.workers = EnvRunnerGroup( - ^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__ - raise e.args[0].args[2] -TimeoutError: timed out. Unintentional worker failures have been reported. If there are lots of this logs, that might indicate there are unexpected failures in the cluster. -[2024-05-15 09:06:07,189 W 12692 12692] (gcs_server) gcs_actor_manager.cc:970: Worker 88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76 on node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 exits, type=USER_ERROR, has creation_task_exception = 1 -[2024-05-15 09:06:07,189 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1167: Actor 0b0450d90a301440d0afa9f001000000 is failed on worker 88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76 at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, need_reschedule = 0, death context type = CreationTaskFailureContext, remaining_restarts = 0, job id = 01000000 -[2024-05-15 09:06:07,190 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:607: Cancelling all committed bundles of a placement group, id is 656f9c54c57685dae4b6a63a81d401000000 -[2024-05-15 09:06:07,191 I 12692 12692] (gcs_server) gcs_placement_group_manager.cc:458: Placement group of an id, 656f9c54c57685dae4b6a63a81d401000000 is removed successfully. -[2024-05-15 09:06:07,191 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={0} at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:06:07,191 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={1} at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:06:07,191 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={2} at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:06:07,191 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={3} at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:06:07,191 I 12692 12692] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={656f9c54c57685dae4b6a63a81d401000000}, bundle index={4} at node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:06:07,191 I 12692 12692] (gcs_server) gcs_actor_manager.cc:798: Actor 0b0450d90a301440d0afa9f001000000 is out of scope, destroying actor, job id = 01000000 -[2024-05-15 09:06:07,191 I 12692 12692] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 0b0450d90a301440d0afa9f001000000, job id = 01000000 -[2024-05-15 09:06:07,704 I 12692 12692] (gcs_server) gcs_job_manager.cc:87: Finished marking job state, job id = 01000000 -[2024-05-15 09:06:07,715 I 12692 12692] (gcs_server) gcs_node_manager.cc:128: Shutting down raylet, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:06:07,715 I 12692 12692] (gcs_server) gcs_node_manager.cc:252: Removing node, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, node name = 192.168.1.56 -[2024-05-15 09:06:07,715 I 12692 12692] (gcs_server) gcs_placement_group_manager.cc:767: Node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 failed, rescheduling the placement groups on the dead node. -[2024-05-15 09:06:07,715 I 12692 12692] (gcs_server) gcs_actor_manager.cc:1048: Node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 failed, reconstructing actors. -[2024-05-15 09:06:07,715 I 12692 12692] (gcs_server) gcs_job_manager.cc:302: Node 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 failed, mark all jobs from this node as finished -[2024-05-15 09:06:07,817 I 12692 12692] (gcs_server) gcs_node_manager.cc:170: Raylet 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 is drained. Status GrpcUnavailable: RPC Error message: Cancelling all calls; RPC Error details: . The information will be published to the cluster. -[2024-05-15 09:06:07,830 I 12692 12740] (gcs_server) ray_syncer-inl.h:318: Failed to read the message from: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:06:07,878 I 12692 12692] (gcs_server) gcs_server_main.cc:111: GCS server received SIGTERM, shutting down... -[2024-05-15 09:06:07,878 I 12692 12692] (gcs_server) gcs_server.cc:272: Stopping GCS server. -[2024-05-15 09:06:07,884 I 12692 12692] (gcs_server) gcs_server.cc:292: GCS server stopped. -[2024-05-15 09:06:07,884 I 12692 12692] (gcs_server) io_service_pool.cc:47: IOServicePool is stopped. -[2024-05-15 09:06:07,927 I 12692 12692] (gcs_server) stats.h:120: Stats module has shutdown. diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/log_monitor.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/log_monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/monitor.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/monitor.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/monitor.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/raylet.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/raylet.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/raylet.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/raylet.out deleted file mode 100644 index 15b95ba..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/raylet.out +++ /dev/null @@ -1,989 +0,0 @@ -[2024-05-15 09:05:01,963 I 12828 12828] (raylet) main.cc:179: Setting cluster ID to: f80091d542af94b49f72839a1f9e8e9b93d17c7c67f37f1c21fb29b2 -[2024-05-15 09:05:01,971 I 12828 12828] (raylet) main.cc:242: Raylet is not set to kill unknown children. -[2024-05-15 09:05:01,971 I 12828 12828] (raylet) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 09:05:01,971 I 12828 12828] (raylet) main.cc:371: Setting node ID to: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:05:01,972 I 12828 12828] (raylet) store_runner.cc:32: Allowing the Plasma store to use up to 18.2754GB of memory. -[2024-05-15 09:05:01,972 I 12828 12828] (raylet) store_runner.cc:48: Starting object store with directory /dev/shm, fallback /home/valapeos/Projects/pneumarl/logs/rllib, and huge page support disabled -[2024-05-15 09:05:01,972 I 12828 12859] (raylet) dlmalloc.cc:154: create_and_mmap_buffer(18275500040, /dev/shm/plasmaXXXXXX) -[2024-05-15 09:05:01,973 I 12828 12859] (raylet) store.cc:564: ========== Plasma store: ================= -Current usage: 0 / 18.2754 GB -- num bytes created total: 0 -0 pending objects of total size 0MB -- objects spillable: 0 -- bytes spillable: 0 -- objects unsealed: 0 -- bytes unsealed: 0 -- objects in use: 0 -- bytes in use: 0 -- objects evictable: 0 -- bytes evictable: 0 - -- objects created by worker: 0 -- bytes created by worker: 0 -- objects restored: 0 -- bytes restored: 0 -- objects received: 0 -- bytes received: 0 -- objects errored: 0 -- bytes errored: 0 - -[2024-05-15 09:05:01,974 I 12828 12828] (raylet) grpc_server.cc:134: ObjectManager server started, listening on port 34323. -[2024-05-15 09:05:01,976 I 12828 12828] (raylet) worker_killing_policy.cc:101: Running GroupByOwner policy. -[2024-05-15 09:05:01,976 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:01,976 I 12828 12828] (raylet) memory_monitor.cc:47: MemoryMonitor initialized with usage threshold at 63982133248 bytes (0.95 system memory), total system memory bytes: 67349614592 -[2024-05-15 09:05:01,976 I 12828 12828] (raylet) node_manager.cc:285: Initializing NodeManager with ID 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:05:01,976 I 12828 12828] (raylet) grpc_server.cc:134: NodeManager server started, listening on port 33813. -[2024-05-15 09:05:01,981 I 12828 12890] (raylet) agent_manager.cc:78: Monitor agent process with name dashboard_agent/424238335 -[2024-05-15 09:05:01,982 I 12828 12892] (raylet) agent_manager.cc:78: Monitor agent process with name runtime_env_agent -[2024-05-15 09:05:01,982 I 12828 12828] (raylet) event.cc:234: Set ray event level to warning -[2024-05-15 09:05:01,982 I 12828 12828] (raylet) event.cc:342: Ray Event initialized for RAYLET -[2024-05-15 09:05:01,983 I 12828 12828] (raylet) raylet.cc:128: Raylet of id, 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 started. Raylet consists of node_manager and object_manager. node_manager address: 192.168.1.56:33813 object_manager address: 192.168.1.56:34323 hostname: valanixos -[2024-05-15 09:05:01,985 I 12828 12828] (raylet) node_manager.cc:521: [state-dump] NodeManager: -[state-dump] Node ID: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[state-dump] Node name: 192.168.1.56 -[state-dump] InitialConfigResources: {CPU: 120000, memory: 365508034560000, node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 182754017280000} -[state-dump] ClusterTaskManager: -[state-dump] ========== Node: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 ================= -[state-dump] Infeasible queue length: 0 -[state-dump] Schedule queue length: 0 -[state-dump] Dispatch queue length: 0 -[state-dump] num_waiting_for_resource: 0 -[state-dump] num_waiting_for_plasma_memory: 0 -[state-dump] num_waiting_for_remote_node_resources: 0 -[state-dump] num_worker_not_started_by_job_config_not_exist: 0 -[state-dump] num_worker_not_started_by_registration_timeout: 0 -[state-dump] num_tasks_waiting_for_workers: 0 -[state-dump] num_cancelled_tasks: 0 -[state-dump] cluster_resource_scheduler state: -[state-dump] Local id: 4706454582974575632 Local resources: {"total":{node:__internal_head__: [10000], CPU: [120000], memory: [365508034560000], object_store_memory: [182754017280000], node:192.168.1.56: [10000]}}, "available": {node:__internal_head__: [10000], CPU: [120000], memory: [365508034560000], object_store_memory: [182754017280000], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691",} is_draining: 0 is_idle: 1 Cluster resources: node id: 4706454582974575632{"total":{object_store_memory: 182754017280000, CPU: 120000, memory: 365508034560000, node:__internal_head__: 10000, node:192.168.1.56: 10000}}, "available": {object_store_memory: 182754017280000, CPU: 120000, memory: 365508034560000, node:__internal_head__: 10000, node:192.168.1.56: 10000}}, "labels":{"ray.io/node_id":"2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -[state-dump] Waiting tasks size: 0 -[state-dump] Number of executing tasks: 0 -[state-dump] Number of pinned task arguments: 0 -[state-dump] Number of total spilled tasks: 0 -[state-dump] Number of spilled waiting tasks: 0 -[state-dump] Number of spilled unschedulable tasks: 0 -[state-dump] Resource usage { -[state-dump] } -[state-dump] Running tasks by scheduling class: -[state-dump] ================================================== -[state-dump] -[state-dump] ClusterResources: -[state-dump] LocalObjectManager: -[state-dump] - num pinned objects: 0 -[state-dump] - pinned objects size: 0 -[state-dump] - num objects pending restore: 0 -[state-dump] - num objects pending spill: 0 -[state-dump] - num bytes pending spill: 0 -[state-dump] - num bytes currently spilled: 0 -[state-dump] - cumulative spill requests: 0 -[state-dump] - cumulative restore requests: 0 -[state-dump] - spilled objects pending delete: 0 -[state-dump] -[state-dump] ObjectManager: -[state-dump] - num local objects: 0 -[state-dump] - num unfulfilled push requests: 0 -[state-dump] - num object pull requests: 0 -[state-dump] - num chunks received total: 0 -[state-dump] - num chunks received failed (all): 0 -[state-dump] - num chunks received failed / cancelled: 0 -[state-dump] - num chunks received failed / plasma error: 0 -[state-dump] Event stats: -[state-dump] Global stats: 0 total (0 active) -[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] Execution time: mean = -nan s, total = 0.000 s -[state-dump] Event stats: -[state-dump] PushManager: -[state-dump] - num pushes in flight: 0 -[state-dump] - num chunks in flight: 0 -[state-dump] - num chunks remaining: 0 -[state-dump] - max chunks allowed: 409 -[state-dump] OwnershipBasedObjectDirectory: -[state-dump] - num listeners: 0 -[state-dump] - cumulative location updates: 0 -[state-dump] - num location updates per second: 0.000 -[state-dump] - num location lookups per second: 0.000 -[state-dump] - num locations added per second: 0.000 -[state-dump] - num locations removed per second: 0.000 -[state-dump] BufferPool: -[state-dump] - create buffer state map size: 0 -[state-dump] PullManager: -[state-dump] - num bytes available for pulled objects: 18275401728 -[state-dump] - num bytes being pulled (all): 0 -[state-dump] - num bytes being pulled / pinned: 0 -[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - first get request bundle: N/A -[state-dump] - first wait request bundle: N/A -[state-dump] - first task request bundle: N/A -[state-dump] - num objects queued: 0 -[state-dump] - num objects actively pulled (all): 0 -[state-dump] - num objects actively pulled / pinned: 0 -[state-dump] - num bundles being pulled: 0 -[state-dump] - num pull retries: 0 -[state-dump] - max timeout seconds: 0 -[state-dump] - max timeout request is already processed. No entry. -[state-dump] -[state-dump] WorkerPool: -[state-dump] - registered jobs: 0 -[state-dump] - process_failed_job_config_missing: 0 -[state-dump] - process_failed_rate_limited: 0 -[state-dump] - process_failed_pending_registration: 0 -[state-dump] - process_failed_runtime_env_setup_failed: 0 -[state-dump] - num PYTHON workers: 0 -[state-dump] - num PYTHON drivers: 0 -[state-dump] - num object spill callbacks queued: 0 -[state-dump] - num object restore queued: 0 -[state-dump] - num util functions queued: 0 -[state-dump] - num idle workers: 0 -[state-dump] TaskDependencyManager: -[state-dump] - task deps map size: 0 -[state-dump] - get req map size: 0 -[state-dump] - wait req map size: 0 -[state-dump] - local objects map size: 0 -[state-dump] WaitManager: -[state-dump] - num active wait requests: 0 -[state-dump] Subscriber: -[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_OBJECT_EVICTION -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_REF_REMOVED_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] num async plasma notifications: 0 -[state-dump] Remote node managers: -[state-dump] Event stats: -[state-dump] Global stats: 27 total (13 active) -[state-dump] Queueing time: mean = 998.145 us, max = 7.175 ms, min = 5.610 us, total = 26.950 ms -[state-dump] Execution time: mean = 585.054 us, total = 15.796 ms -[state-dump] Event stats: -[state-dump] PeriodicalRunner.RunFnPeriodically - 11 total (2 active, 1 running), Execution time: mean = 122.294 us, total = 1.345 ms, Queueing time: mean = 2.449 ms, max = 7.175 ms, min = 18.609 us, total = 26.938 ms -[state-dump] NodeManager.deadline_timer.flush_free_objects - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 863.500 us, total = 863.500 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.debug_state_dump - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.record_metrics - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 522.368 us, total = 522.368 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 12.673 ms, total = 12.673 ms, Queueing time: mean = 5.610 us, max = 5.610 us, min = 5.610 us, total = 5.610 us -[state-dump] NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 251.054 us, total = 251.054 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 140.966 us, total = 140.966 us, Queueing time: mean = 6.160 us, max = 6.160 us, min = 6.160 us, total = 6.160 us -[state-dump] DebugString() time ms: 0 -[state-dump] -[state-dump] -[2024-05-15 09:05:01,985 I 12828 12828] (raylet) accessor.cc:627: Received notification for node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691, IsAlive = 1 -[2024-05-15 09:05:02,075 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12910, the token is 0 -[2024-05-15 09:05:02,077 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12911, the token is 1 -[2024-05-15 09:05:02,079 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12912, the token is 2 -[2024-05-15 09:05:02,080 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12913, the token is 3 -[2024-05-15 09:05:02,082 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12914, the token is 4 -[2024-05-15 09:05:02,084 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12915, the token is 5 -[2024-05-15 09:05:02,086 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12916, the token is 6 -[2024-05-15 09:05:02,087 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12917, the token is 7 -[2024-05-15 09:05:02,089 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12918, the token is 8 -[2024-05-15 09:05:02,095 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12919, the token is 9 -[2024-05-15 09:05:02,101 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12920, the token is 10 -[2024-05-15 09:05:02,103 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 12921, the token is 11 -[2024-05-15 09:05:02,430 I 12828 12859] (raylet) object_store.cc:35: Object store current usage 8e-09 / 18.2754 GB. -[2024-05-15 09:05:02,568 I 12828 12828] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 12647 is dead: 0 driver address: 192.168.1.56 -[2024-05-15 09:05:02,568 I 12828 12828] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 09:05:02,801 I 12828 12828] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:05:02,803 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 13649, the token is 12 -[2024-05-15 09:05:03,591 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:05:03,591 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:05:03,592 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:05:03,592 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:05:03,791 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:05:04,757 I 12828 12828] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:05:04,760 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 13721, the token is 13 -[2024-05-15 09:05:04,760 I 12828 12828] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:05:04,762 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 13722, the token is 14 -[2024-05-15 09:05:04,763 I 12828 12828] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:05:04,765 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 13723, the token is 15 -[2024-05-15 09:05:04,765 I 12828 12828] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:05:04,767 I 12828 12828] (raylet) worker_pool.cc:494: Started worker process with pid 13724, the token is 16 -[2024-05-15 09:05:06,985 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:11,980 W 12828 12853] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:62660: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 09:05:11,989 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:16,992 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:21,993 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:26,996 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:31,998 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:37,000 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:42,002 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:47,004 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:52,006 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:05:57,008 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:06:01,973 I 12828 12859] (raylet) store.cc:564: ========== Plasma store: ================= -Current usage: 0 / 18.2754 GB -- num bytes created total: 144 -0 pending objects of total size 0MB -- objects spillable: 0 -- bytes spillable: 0 -- objects unsealed: 0 -- bytes unsealed: 0 -- objects in use: 0 -- bytes in use: 0 -- objects evictable: 0 -- bytes evictable: 0 - -- objects created by worker: 0 -- bytes created by worker: 0 -- objects restored: 0 -- bytes restored: 0 -- objects received: 0 -- bytes received: 0 -- objects errored: 0 -- bytes errored: 0 - -[2024-05-15 09:06:01,986 I 12828 12828] (raylet) node_manager.cc:521: [state-dump] NodeManager: -[state-dump] Node ID: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[state-dump] Node name: 192.168.1.56 -[state-dump] InitialConfigResources: {CPU: 120000, memory: 365508034560000, node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 182754017280000} -[state-dump] ClusterTaskManager: -[state-dump] ========== Node: 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 ================= -[state-dump] Infeasible queue length: 0 -[state-dump] Schedule queue length: 0 -[state-dump] Dispatch queue length: 0 -[state-dump] num_waiting_for_resource: 0 -[state-dump] num_waiting_for_plasma_memory: 0 -[state-dump] num_waiting_for_remote_node_resources: 0 -[state-dump] num_worker_not_started_by_job_config_not_exist: 0 -[state-dump] num_worker_not_started_by_registration_timeout: 0 -[state-dump] num_tasks_waiting_for_workers: 0 -[state-dump] num_cancelled_tasks: 0 -[state-dump] cluster_resource_scheduler state: -[state-dump] Local id: 4706454582974575632 Local resources: {"total":{object_store_memory: [182754017280000], node:192.168.1.56: [10000], CPU_group_1_656f9c54c57685dae4b6a63a81d401000000: [10000], bundle_group_1_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU_group_3_656f9c54c57685dae4b6a63a81d401000000: [10000], bundle_group_3_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU_group_656f9c54c57685dae4b6a63a81d401000000: [50000], bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: [10000000], bundle_group_2_656f9c54c57685dae4b6a63a81d401000000: [10000000], bundle_group_4_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU: [120000], bundle_group_656f9c54c57685dae4b6a63a81d401000000: [50000000], CPU_group_2_656f9c54c57685dae4b6a63a81d401000000: [10000], memory: [365508034560000], node:__internal_head__: [10000], CPU_group_0_656f9c54c57685dae4b6a63a81d401000000: [10000], CPU_group_4_656f9c54c57685dae4b6a63a81d401000000: [10000]}}, "available": {object_store_memory: [182754017280000], node:192.168.1.56: [10000], CPU_group_1_656f9c54c57685dae4b6a63a81d401000000: [10000], bundle_group_1_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU_group_3_656f9c54c57685dae4b6a63a81d401000000: [10000], bundle_group_3_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU_group_656f9c54c57685dae4b6a63a81d401000000: [0], bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: [9999990], bundle_group_2_656f9c54c57685dae4b6a63a81d401000000: [10000000], bundle_group_4_656f9c54c57685dae4b6a63a81d401000000: [10000000], CPU: [70000], bundle_group_656f9c54c57685dae4b6a63a81d401000000: [49999950], CPU_group_2_656f9c54c57685dae4b6a63a81d401000000: [10000], memory: [365508034560000], node:__internal_head__: [10000], CPU_group_0_656f9c54c57685dae4b6a63a81d401000000: [0], CPU_group_4_656f9c54c57685dae4b6a63a81d401000000: [10000]}}, "labels":{"ray.io/node_id":"2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691",} is_draining: 0 is_idle: 0 Cluster resources: node id: 4706454582974575632{"total":{CPU_group_656f9c54c57685dae4b6a63a81d401000000: 50000, bundle_group_2_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_4_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_3_656f9c54c57685dae4b6a63a81d401000000: 10000000, node:192.168.1.56: 10000, object_store_memory: 182754017280000, bundle_group_1_656f9c54c57685dae4b6a63a81d401000000: 10000000, CPU_group_3_656f9c54c57685dae4b6a63a81d401000000: 10000, node:__internal_head__: 10000, CPU_group_1_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_4_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_0_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 50000000, CPU: 120000, memory: 365508034560000, CPU_group_2_656f9c54c57685dae4b6a63a81d401000000: 10000}}, "available": {CPU_group_4_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_1_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_3_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_3_656f9c54c57685dae4b6a63a81d401000000: 10000000, node:192.168.1.56: 10000, object_store_memory: 182754017280000, node:__internal_head__: 10000, memory: 365508034560000, CPU_group_2_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 49999950, CPU: 70000, bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: 9999990, bundle_group_4_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_2_656f9c54c57685dae4b6a63a81d401000000: 10000000, bundle_group_1_656f9c54c57685dae4b6a63a81d401000000: 10000000}}, "labels":{"ray.io/node_id":"2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -[state-dump] Waiting tasks size: 0 -[state-dump] Number of executing tasks: 5 -[state-dump] Number of pinned task arguments: 0 -[state-dump] Number of total spilled tasks: 0 -[state-dump] Number of spilled waiting tasks: 0 -[state-dump] Number of spilled unschedulable tasks: 0 -[state-dump] Resource usage { -[state-dump] - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=13722): {CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10} -[state-dump] - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=13723): {CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10} -[state-dump] - (language=PYTHON actor_or_task=PPO.__init__ pid=13649): {bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10, CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, CPU_group_0_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_0_656f9c54c57685dae4b6a63a81d401000000: 10} -[state-dump] - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=13721): {CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10} -[state-dump] - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=13724): {CPU_group_656f9c54c57685dae4b6a63a81d401000000: 10000, bundle_group_656f9c54c57685dae4b6a63a81d401000000: 10} -[state-dump] } -[state-dump] Running tasks by scheduling class: -[state-dump] - {depth=1 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.algorithms.ppo.ppo, class_name=PPO, function_name=__init__, function_hash=1023ce2155184297951346509793ac63} scheduling_strategy=placement_group_scheduling_strategy { -[state-dump] placement_group_id: "eo\234T\305v\205\332\344\266\246:\201\324\001\000\000\000" -[state-dump] placement_group_capture_child_tasks: true -[state-dump] } -[state-dump] resource_set={bundle_group_0_656f9c54c57685dae4b6a63a81d401000000 : 0.001, CPU_group_656f9c54c57685dae4b6a63a81d401000000 : 1, CPU_group_0_656f9c54c57685dae4b6a63a81d401000000 : 1, bundle_group_656f9c54c57685dae4b6a63a81d401000000 : 0.001, }}: 1/18446744073709551615 -[state-dump] - {depth=2 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.evaluation.rollout_worker, class_name=RolloutWorker, function_name=__init__, function_hash=b8267bd017c84e5881fa6c6f47c3e4c0} scheduling_strategy=placement_group_scheduling_strategy { -[state-dump] placement_group_id: "eo\234T\305v\205\332\344\266\246:\201\324\001\000\000\000" -[state-dump] placement_group_bundle_index: -1 -[state-dump] placement_group_capture_child_tasks: true -[state-dump] } -[state-dump] resource_set={CPU_group_656f9c54c57685dae4b6a63a81d401000000 : 1, bundle_group_656f9c54c57685dae4b6a63a81d401000000 : 0.001, }}: 4/18446744073709551615 -[state-dump] ================================================== -[state-dump] -[state-dump] ClusterResources: -[state-dump] LocalObjectManager: -[state-dump] - num pinned objects: 0 -[state-dump] - pinned objects size: 0 -[state-dump] - num objects pending restore: 0 -[state-dump] - num objects pending spill: 0 -[state-dump] - num bytes pending spill: 0 -[state-dump] - num bytes currently spilled: 0 -[state-dump] - cumulative spill requests: 0 -[state-dump] - cumulative restore requests: 0 -[state-dump] - spilled objects pending delete: 0 -[state-dump] -[state-dump] ObjectManager: -[state-dump] - num local objects: 0 -[state-dump] - num unfulfilled push requests: 0 -[state-dump] - num object pull requests: 0 -[state-dump] - num chunks received total: 0 -[state-dump] - num chunks received failed (all): 0 -[state-dump] - num chunks received failed / cancelled: 0 -[state-dump] - num chunks received failed / plasma error: 0 -[state-dump] Event stats: -[state-dump] Global stats: 0 total (0 active) -[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] Execution time: mean = -nan s, total = 0.000 s -[state-dump] Event stats: -[state-dump] PushManager: -[state-dump] - num pushes in flight: 0 -[state-dump] - num chunks in flight: 0 -[state-dump] - num chunks remaining: 0 -[state-dump] - max chunks allowed: 409 -[state-dump] OwnershipBasedObjectDirectory: -[state-dump] - num listeners: 0 -[state-dump] - cumulative location updates: 0 -[state-dump] - num location updates per second: 0.000 -[state-dump] - num location lookups per second: 0.000 -[state-dump] - num locations added per second: 0.000 -[state-dump] - num locations removed per second: 0.000 -[state-dump] BufferPool: -[state-dump] - create buffer state map size: 0 -[state-dump] PullManager: -[state-dump] - num bytes available for pulled objects: 18275401728 -[state-dump] - num bytes being pulled (all): 0 -[state-dump] - num bytes being pulled / pinned: 0 -[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - first get request bundle: N/A -[state-dump] - first wait request bundle: N/A -[state-dump] - first task request bundle: N/A -[state-dump] - num objects queued: 0 -[state-dump] - num objects actively pulled (all): 0 -[state-dump] - num objects actively pulled / pinned: 0 -[state-dump] - num bundles being pulled: 0 -[state-dump] - num pull retries: 0 -[state-dump] - max timeout seconds: 0 -[state-dump] - max timeout request is already processed. No entry. -[state-dump] -[state-dump] WorkerPool: -[state-dump] - registered jobs: 1 -[state-dump] - process_failed_job_config_missing: 0 -[state-dump] - process_failed_rate_limited: 0 -[state-dump] - process_failed_pending_registration: 0 -[state-dump] - process_failed_runtime_env_setup_failed: 0 -[state-dump] - num PYTHON workers: 12 -[state-dump] - num PYTHON drivers: 1 -[state-dump] - num object spill callbacks queued: 0 -[state-dump] - num object restore queued: 0 -[state-dump] - num util functions queued: 0 -[state-dump] - num idle workers: 7 -[state-dump] TaskDependencyManager: -[state-dump] - task deps map size: 0 -[state-dump] - get req map size: 0 -[state-dump] - wait req map size: 0 -[state-dump] - local objects map size: 0 -[state-dump] WaitManager: -[state-dump] - num active wait requests: 0 -[state-dump] Subscriber: -[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_OBJECT_EVICTION -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_REF_REMOVED_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] num async plasma notifications: 0 -[state-dump] Remote node managers: -[state-dump] Event stats: -[state-dump] Global stats: 4609 total (27 active) -[state-dump] Queueing time: mean = 1.448 ms, max = 1.315 s, min = 70.000 ns, total = 6.673 s -[state-dump] Execution time: mean = 926.484 us, total = 4.270 s -[state-dump] Event stats: -[state-dump] NodeManagerService.grpc_server.ReportWorkerBacklog.HandleRequestImpl - 777 total (0 active), Execution time: mean = 9.890 us, total = 7.684 ms, Queueing time: mean = 11.345 us, max = 260.564 us, min = 5.370 us, total = 8.815 ms -[state-dump] NodeManagerService.grpc_server.ReportWorkerBacklog - 777 total (0 active), Execution time: mean = 95.817 us, total = 74.450 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.CheckGC - 600 total (1 active), Execution time: mean = 862.498 ns, total = 517.499 us, Queueing time: mean = 51.458 us, max = 20.877 ms, min = 6.798 us, total = 30.875 ms -[state-dump] RaySyncer.OnDemandBroadcasting - 600 total (1 active), Execution time: mean = 4.655 us, total = 2.793 ms, Queueing time: mean = 48.281 us, max = 20.869 ms, min = 7.100 us, total = 28.969 ms -[state-dump] ObjectManager.UpdateAvailableMemory - 600 total (0 active), Execution time: mean = 1.120 us, total = 671.870 us, Queueing time: mean = 8.347 us, max = 17.989 us, min = 4.399 us, total = 5.008 ms -[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 300 total (1 active), Execution time: mean = 9.071 us, total = 2.721 ms, Queueing time: mean = 24.985 us, max = 3.787 ms, min = 4.986 us, total = 7.495 ms -[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 240 total (1 active), Execution time: mean = 93.805 us, total = 22.513 ms, Queueing time: mean = 21.520 us, max = 1.959 ms, min = 4.529 us, total = 5.165 ms -[state-dump] ClientConnection.async_read.ProcessMessageHeader - 83 total (13 active), Execution time: mean = 2.637 us, total = 218.895 us, Queueing time: mean = 78.589 ms, max = 1.315 s, min = 10.660 us, total = 6.523 s -[state-dump] ClientConnection.async_read.ProcessMessage - 70 total (0 active), Execution time: mean = 556.123 us, total = 38.929 ms, Queueing time: mean = 20.546 us, max = 519.328 us, min = 1.760 us, total = 1.438 ms -[state-dump] NodeManager.ScheduleAndDispatchTasks - 61 total (1 active), Execution time: mean = 4.952 us, total = 302.072 us, Queueing time: mean = 10.238 us, max = 28.640 us, min = 5.442 us, total = 624.532 us -[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 60 total (1 active), Execution time: mean = 1.159 us, total = 69.518 us, Queueing time: mean = 124.476 us, max = 4.077 ms, min = 6.857 us, total = 7.469 ms -[state-dump] NodeManagerService.grpc_server.GetResourceLoad - 60 total (0 active), Execution time: mean = 156.631 us, total = 9.398 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.flush_free_objects - 60 total (1 active), Execution time: mean = 2.495 us, total = 149.680 us, Queueing time: mean = 123.346 us, max = 4.077 ms, min = 4.140 us, total = 7.401 ms -[state-dump] NodeManagerService.grpc_server.GetResourceLoad.HandleRequestImpl - 60 total (0 active), Execution time: mean = 53.331 us, total = 3.200 ms, Queueing time: mean = 13.716 us, max = 193.985 us, min = 6.280 us, total = 822.984 us -[state-dump] ClusterResourceManager.ResetRemoteNodeView - 21 total (1 active), Execution time: mean = 1.950 us, total = 40.949 us, Queueing time: mean = 11.393 us, max = 23.973 us, min = 7.953 us, total = 239.256 us -[state-dump] ClientConnection.async_write.DoAsyncWrites - 19 total (0 active), Execution time: mean = 595.789 ns, total = 11.320 us, Queueing time: mean = 12.061 us, max = 39.439 us, min = 6.350 us, total = 229.155 us -[state-dump] ObjectManager.ObjectAdded - 18 total (0 active), Execution time: mean = 6.807 us, total = 122.527 us, Queueing time: mean = 12.869 us, max = 27.039 us, min = 5.780 us, total = 231.644 us -[state-dump] NodeManagerService.grpc_server.GetSystemConfig.HandleRequestImpl - 18 total (0 active), Execution time: mean = 23.248 us, total = 418.471 us, Queueing time: mean = 10.128 us, max = 17.150 us, min = 6.550 us, total = 182.299 us -[state-dump] ObjectManager.ObjectDeleted - 18 total (0 active), Execution time: mean = 7.564 us, total = 136.158 us, Queueing time: mean = 29.838 us, max = 161.227 us, min = 11.850 us, total = 537.091 us -[state-dump] NodeManagerService.grpc_server.GetSystemConfig - 18 total (0 active), Execution time: mean = 114.145 us, total = 2.055 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] PeriodicalRunner.RunFnPeriodically - 13 total (0 active), Execution time: mean = 126.892 us, total = 1.650 ms, Queueing time: mean = 2.277 ms, max = 7.175 ms, min = 18.609 us, total = 29.601 ms -[state-dump] NodeManager.deadline_timer.record_metrics - 12 total (1 active), Execution time: mean = 236.378 us, total = 2.837 ms, Queueing time: mean = 376.783 us, max = 3.826 ms, min = 9.922 us, total = 4.521 ms -[state-dump] NodeInfoGcsService.grpc_client.CheckAlive - 12 total (0 active), Execution time: mean = 296.571 us, total = 3.559 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.GcsCheckAlive - 12 total (1 active), Execution time: mean = 96.280 us, total = 1.155 ms, Queueing time: mean = 504.905 us, max = 4.043 ms, min = 53.416 us, total = 6.059 ms -[state-dump] NodeInfoGcsService.grpc_client.CheckAlive.OnReplyReceived - 12 total (0 active), Execution time: mean = 13.821 us, total = 165.847 us, Queueing time: mean = 10.679 us, max = 16.709 us, min = 7.860 us, total = 128.146 us -[state-dump] NodeManagerService.grpc_server.RequestWorkerLease.HandleRequestImpl - 6 total (0 active), Execution time: mean = 129.099 us, total = 774.591 us, Queueing time: mean = 289.732 us, max = 937.888 us, min = 7.350 us, total = 1.738 ms -[state-dump] NodeManagerService.grpc_server.RequestWorkerLease - 6 total (0 active), Execution time: mean = 375.071 ms, total = 2.250 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.debug_state_dump - 6 total (1 active), Execution time: mean = 1.167 ms, total = 7.003 ms, Queueing time: mean = 8.847 us, max = 13.645 us, min = 8.547 us, total = 53.084 us -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 5 total (1 active), Execution time: mean = 361.638 ms, total = 1.808 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] CoreWorkerService.grpc_client.Exit - 5 total (0 active), Execution time: mean = 913.509 us, total = 4.568 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] CoreWorkerService.grpc_client.Exit.OnReplyReceived - 5 total (0 active), Execution time: mean = 11.781 us, total = 58.907 us, Queueing time: mean = 14.184 us, max = 38.179 us, min = 5.980 us, total = 70.919 us -[state-dump] - 5 total (0 active), Execution time: mean = 134.000 ns, total = 670.000 ns, Queueing time: mean = 8.842 us, max = 9.990 us, min = 8.019 us, total = 44.208 us -[state-dump] WorkerInfoGcsService.grpc_client.ReportWorkerFailure.OnReplyReceived - 5 total (0 active), Execution time: mean = 17.188 us, total = 85.938 us, Queueing time: mean = 42.755 us, max = 68.518 us, min = 5.860 us, total = 213.773 us -[state-dump] Subscriber.HandlePublishedMessage_GCS_WORKER_DELTA_CHANNEL - 5 total (0 active), Execution time: mean = 1.738 us, total = 8.689 us, Queueing time: mean = 93.382 us, max = 101.948 us, min = 69.249 us, total = 466.911 us -[state-dump] WorkerInfoGcsService.grpc_client.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 1.060 ms, total = 5.299 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] RaySyncer.BroadcastMessage - 5 total (0 active), Execution time: mean = 99.532 us, total = 497.658 us, Queueing time: mean = 352.000 ns, max = 1.100 us, min = 90.000 ns, total = 1.760 us -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll.OnReplyReceived - 4 total (0 active), Execution time: mean = 93.540 us, total = 374.162 us, Queueing time: mean = 9.925 us, max = 11.770 us, min = 6.350 us, total = 39.700 us -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 2 total (0 active), Execution time: mean = 70.573 us, total = 141.147 us, Queueing time: mean = 652.500 us, max = 1.244 ms, min = 60.769 us, total = 1.305 ms -[state-dump] RaySyncerRegister - 2 total (0 active), Execution time: mean = 800.000 ns, total = 1.600 us, Queueing time: mean = 130.000 ns, max = 190.000 ns, min = 70.000 ns, total = 260.000 ns -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 2 total (0 active), Execution time: mean = 241.449 us, total = 482.898 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.print_event_loop_stats - 1 total (1 active, 1 running), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 522.368 us, total = 522.368 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] WorkerPool.PopWorkerCallback - 1 total (0 active), Execution time: mean = 18.460 us, total = 18.460 us, Queueing time: mean = 6.200 us, max = 6.200 us, min = 6.200 us, total = 6.200 us -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 863.500 us, total = 863.500 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManagerService.grpc_server.CommitBundleResources - 1 total (0 active), Execution time: mean = 236.855 us, total = 236.855 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] Subscriber.HandlePublishedMessage_GCS_JOB_CHANNEL - 1 total (0 active), Execution time: mean = 25.699 us, total = 25.699 us, Queueing time: mean = 94.678 us, max = 94.678 us, min = 94.678 us, total = 94.678 us -[state-dump] JobInfoGcsService.grpc_client.GetAllJobInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 7.480 us, total = 7.480 us, Queueing time: mean = 10.389 us, max = 10.389 us, min = 10.389 us, total = 10.389 us -[state-dump] JobInfoGcsService.grpc_client.GetAllJobInfo - 1 total (0 active), Execution time: mean = 236.415 us, total = 236.415 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] JobInfoGcsService.grpc_client.AddJob - 1 total (0 active), Execution time: mean = 544.817 us, total = 544.817 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] JobInfoGcsService.grpc_client.AddJob.OnReplyReceived - 1 total (0 active), Execution time: mean = 14.470 us, total = 14.470 us, Queueing time: mean = 92.518 us, max = 92.518 us, min = 92.518 us, total = 92.518 us -[state-dump] NodeInfoGcsService.grpc_client.GetAllNodeInfo - 1 total (0 active), Execution time: mean = 257.524 us, total = 257.524 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 12.673 ms, total = 12.673 ms, Queueing time: mean = 5.610 us, max = 5.610 us, min = 5.610 us, total = 5.610 us -[state-dump] NodeManagerService.grpc_server.PrepareBundleResources - 1 total (0 active), Execution time: mean = 282.294 us, total = 282.294 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetAllNodeInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 80.008 us, total = 80.008 us, Queueing time: mean = 7.170 us, max = 7.170 us, min = 7.170 us, total = 7.170 us -[state-dump] NodeManagerService.grpc_server.ReturnWorker.HandleRequestImpl - 1 total (0 active), Execution time: mean = 47.639 us, total = 47.639 us, Queueing time: mean = 11.740 us, max = 11.740 us, min = 11.740 us, total = 11.740 us -[state-dump] NodeManagerService.grpc_server.ReturnWorker - 1 total (0 active), Execution time: mean = 148.266 us, total = 148.266 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManagerService.grpc_server.PrepareBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 197.545 us, total = 197.545 us, Queueing time: mean = 8.130 us, max = 8.130 us, min = 8.130 us, total = 8.130 us -[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 140.966 us, total = 140.966 us, Queueing time: mean = 6.160 us, max = 6.160 us, min = 6.160 us, total = 6.160 us -[state-dump] NodeManagerService.grpc_server.CommitBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 182.946 us, total = 182.946 us, Queueing time: mean = 5.770 us, max = 5.770 us, min = 5.770 us, total = 5.770 us -[state-dump] DebugString() time ms: 1 -[state-dump] -[state-dump] -[2024-05-15 09:06:02,011 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:06:06,928 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=2, has creation task exception = true -[2024-05-15 09:06:06,928 I 12828 12828] (raylet) node_manager.cc:1465: Formatted creation task exception: Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 1883, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1984, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1889, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1830, in ray._raylet.execute_task.function_executor - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/function_manager.py", line 691, in actor_method_executor - return method(__ray_actor, *args, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/util/tracing/tracing_helper.py", line 467, in _resume_span - return method(self, *_args, **_kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ - -TimeoutError: timed out - - -During handling of the above exception, another exception occurred: - - -Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 2281, in ray._raylet.task_execution_handler - - File "python/ray/_raylet.pyx", line 2177, in ray._raylet.execute_task_with_cancellation_handler - - File "python/ray/_raylet.pyx", line 1832, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1833, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 2071, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1098, in ray._raylet.store_task_errors - -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -, worker_id: 6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a -[2024-05-15 09:06:06,938 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=2, has creation task exception = true -[2024-05-15 09:06:06,938 I 12828 12828] (raylet) node_manager.cc:1465: Formatted creation task exception: Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 1883, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1984, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1889, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1830, in ray._raylet.execute_task.function_executor - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/function_manager.py", line 691, in actor_method_executor - return method(__ray_actor, *args, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/util/tracing/tracing_helper.py", line 467, in _resume_span - return method(self, *_args, **_kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ - -TimeoutError: timed out - - -During handling of the above exception, another exception occurred: - - -Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 2281, in ray._raylet.task_execution_handler - - File "python/ray/_raylet.pyx", line 2177, in ray._raylet.execute_task_with_cancellation_handler - - File "python/ray/_raylet.pyx", line 1832, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1833, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 2071, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1098, in ray._raylet.store_task_errors - -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13723, ip=192.168.1.56, actor_id=6c1e2f8e5911cf8ba4dcac5b01000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -, worker_id: cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91 -[2024-05-15 09:06:06,972 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=2, has creation task exception = true -[2024-05-15 09:06:06,972 I 12828 12828] (raylet) node_manager.cc:1465: Formatted creation task exception: Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 1883, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1984, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1889, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1830, in ray._raylet.execute_task.function_executor - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/function_manager.py", line 691, in actor_method_executor - return method(__ray_actor, *args, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/util/tracing/tracing_helper.py", line 467, in _resume_span - return method(self, *_args, **_kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ - -TimeoutError: timed out - - -During handling of the above exception, another exception occurred: - - -Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 2281, in ray._raylet.task_execution_handler - - File "python/ray/_raylet.pyx", line 2177, in ray._raylet.execute_task_with_cancellation_handler - - File "python/ray/_raylet.pyx", line 1832, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1833, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 2071, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1098, in ray._raylet.store_task_errors - -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13724, ip=192.168.1.56, actor_id=79760d3ce8052092d662e89401000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -, worker_id: 5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38 -[2024-05-15 09:06:07,013 W 12828 12828] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:06:07,044 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=2, has creation task exception = true -[2024-05-15 09:06:07,044 I 12828 12828] (raylet) node_manager.cc:1465: Formatted creation task exception: Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 1883, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1984, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1889, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1830, in ray._raylet.execute_task.function_executor - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/function_manager.py", line 691, in actor_method_executor - return method(__ray_actor, *args, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/util/tracing/tracing_helper.py", line 467, in _resume_span - return method(self, *_args, **_kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ - -TimeoutError: timed out - - -During handling of the above exception, another exception occurred: - - -Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 2281, in ray._raylet.task_execution_handler - - File "python/ray/_raylet.pyx", line 2177, in ray._raylet.execute_task_with_cancellation_handler - - File "python/ray/_raylet.pyx", line 1832, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1833, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 2071, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1098, in ray._raylet.store_task_errors - -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13722, ip=192.168.1.56, actor_id=64110d8601f56bf14e9797c501000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -, worker_id: 7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed -[2024-05-15 09:06:07,188 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=2, has creation task exception = true -[2024-05-15 09:06:07,188 I 12828 12828] (raylet) node_manager.cc:1465: Formatted creation task exception: Traceback (most recent call last): - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 169, in __init__ - self._setup( - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 239, in _setup - self.add_workers( - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 754, in add_workers - raise result.get() - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/utils/actor_manager.py", line 497, in _fetch_result - result = ray.get(r) - ^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper - return fn(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper - return func(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/worker.py", line 2623, in get - values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/worker.py", line 863, in get_objects - raise value - -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - - -During handling of the above exception, another exception occurred: - - -Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 1883, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1984, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1889, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1830, in ray._raylet.execute_task.function_executor - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/function_manager.py", line 691, in actor_method_executor - return method(__ray_actor, *args, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/util/tracing/tracing_helper.py", line 467, in _resume_span - return method(self, *_args, **_kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 554, in __init__ - super().__init__( - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__ - self.setup(copy.deepcopy(self.config)) - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/util/tracing/tracing_helper.py", line 467, in _resume_span - return method(self, *_args, **_kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup - self.workers = EnvRunnerGroup( - ^^^^^^^^^^^^^^^ - - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__ - raise e.args[0].args[2] - -TimeoutError: timed out - - -During handling of the above exception, another exception occurred: - - -Traceback (most recent call last): - - File "python/ray/_raylet.pyx", line 2281, in ray._raylet.task_execution_handler - - File "python/ray/_raylet.pyx", line 2177, in ray._raylet.execute_task_with_cancellation_handler - - File "python/ray/_raylet.pyx", line 1832, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1833, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 2071, in ray._raylet.execute_task - - File "python/ray/_raylet.pyx", line 1098, in ray._raylet.store_task_errors - -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 239, in _setup - self.add_workers( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 754, in add_workers - raise result.get() - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/utils/actor_manager.py", line 497, in _fetch_result - result = ray.get(r) - ^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -During handling of the above exception, another exception occurred: - -ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 554, in __init__ - super().__init__( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__ - self.setup(copy.deepcopy(self.config)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup - self.workers = EnvRunnerGroup( - ^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__ - raise e.args[0].args[2] -TimeoutError: timed out -, worker_id: 88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76 -[2024-05-15 09:06:07,704 I 12828 12828] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=3, has creation task exception = false -[2024-05-15 09:06:07,704 I 12828 12828] (raylet) node_manager.cc:1541: Driver (pid=12647) is disconnected. job_id: 01000000 -[2024-05-15 09:06:07,705 I 12828 12828] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 12647 is dead: 1 driver address: 192.168.1.56 -[2024-05-15 09:06:07,705 I 12828 12828] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 09:06:07,714 I 12828 12828] (raylet) main.cc:413: Raylet received SIGTERM, shutting down... -[2024-05-15 09:06:07,714 I 12828 12828] (raylet) accessor.cc:451: Unregistering node info, node id = 2808763200e5fb675aeed4f2c3a8a8a50b290dfd4d83a5db793b0691 -[2024-05-15 09:06:07,715 I 12828 12828] (raylet) agent_manager.cc:109: Killing agent dashboard_agent/424238335, pid 12889. -[2024-05-15 09:06:07,720 I 12828 12890] (raylet) agent_manager.cc:80: Agent process with name dashboard_agent/424238335 exited, exit code 0. -[2024-05-15 09:06:07,720 I 12828 12828] (raylet) agent_manager.cc:109: Killing agent runtime_env_agent, pid 12891. -[2024-05-15 09:06:07,724 I 12828 12892] (raylet) agent_manager.cc:80: Agent process with name runtime_env_agent exited, exit code 0. -[2024-05-15 09:06:07,724 I 12828 12828] (raylet) io_service_pool.cc:47: IOServicePool is stopped. -[2024-05-15 09:06:07,815 I 12828 12828] (raylet) stats.h:120: Stats module has shutdown. -[2024-05-15 09:06:07,815 I 12828 12894] (raylet) experimental_mutable_object_provider.cc:200: Core worker channel io service stopped. diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/runtime_env_agent.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/runtime_env_agent.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/runtime_env_agent.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/runtime_env_agent.out deleted file mode 100644 index 1b3daab..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/runtime_env_agent.out +++ /dev/null @@ -1,2 +0,0 @@ -======== Running on http://0.0.0.0:63152 ======== -(Press CTRL+C to quit) diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-2bf7d3c67af7b22c604647504435f7d7f2f0cab1786041e19bbd4d86-ffffffff-12915.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-2bf7d3c67af7b22c604647504435f7d7f2f0cab1786041e19bbd4d86-ffffffff-12915.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-2bf7d3c67af7b22c604647504435f7d7f2f0cab1786041e19bbd4d86-ffffffff-12915.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-2bf7d3c67af7b22c604647504435f7d7f2f0cab1786041e19bbd4d86-ffffffff-12915.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-54e4ae81bb48bc53c549cf82365f0631056670db569b639aa1fc6dcc-ffffffff-12912.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-54e4ae81bb48bc53c549cf82365f0631056670db569b639aa1fc6dcc-ffffffff-12912.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-54e4ae81bb48bc53c549cf82365f0631056670db569b639aa1fc6dcc-ffffffff-12912.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-54e4ae81bb48bc53c549cf82365f0631056670db569b639aa1fc6dcc-ffffffff-12912.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38-01000000-13724.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38-01000000-13724.err deleted file mode 100644 index e0b0be2..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38-01000000-13724.err +++ /dev/null @@ -1,58 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) -Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13724, ip=192.168.1.56, actor_id=79760d3ce8052092d662e89401000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38-01000000-13724.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38-01000000-13724.out deleted file mode 100644 index 9587b73..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-5f80103a7fc6da866dac6d9a1f8048732e7c48f9a9b01d7546969c38-01000000-13724.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11012 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a-01000000-13721.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a-01000000-13721.err deleted file mode 100644 index a50c39d..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a-01000000-13721.err +++ /dev/null @@ -1,58 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) -Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a-01000000-13721.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a-01000000-13721.out deleted file mode 100644 index 6628553..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-6edeb2bc45590aa290c046f72cb5570a133031f0fa08412f23f2528a-01000000-13721.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11009 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-702aaa7a78ea7761ed0f00dee65aa7b0e9ecb6efc4647117290caba6-ffffffff-12914.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-702aaa7a78ea7761ed0f00dee65aa7b0e9ecb6efc4647117290caba6-ffffffff-12914.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-702aaa7a78ea7761ed0f00dee65aa7b0e9ecb6efc4647117290caba6-ffffffff-12914.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-702aaa7a78ea7761ed0f00dee65aa7b0e9ecb6efc4647117290caba6-ffffffff-12914.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7619db422bbf3ef2e4c09d4e12cdfcf569298846ba3c19f3b9018c77-ffffffff-12913.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7619db422bbf3ef2e4c09d4e12cdfcf569298846ba3c19f3b9018c77-ffffffff-12913.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7619db422bbf3ef2e4c09d4e12cdfcf569298846ba3c19f3b9018c77-ffffffff-12913.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7619db422bbf3ef2e4c09d4e12cdfcf569298846ba3c19f3b9018c77-ffffffff-12913.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7936e9b9660ad0fb2b0031075ccfb4068e470476040b23804ae2ba99-ffffffff-12917.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7936e9b9660ad0fb2b0031075ccfb4068e470476040b23804ae2ba99-ffffffff-12917.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7936e9b9660ad0fb2b0031075ccfb4068e470476040b23804ae2ba99-ffffffff-12917.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7936e9b9660ad0fb2b0031075ccfb4068e470476040b23804ae2ba99-ffffffff-12917.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed-01000000-13722.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed-01000000-13722.err deleted file mode 100644 index e8bbb83..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed-01000000-13722.err +++ /dev/null @@ -1,58 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) -Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13722, ip=192.168.1.56, actor_id=64110d8601f56bf14e9797c501000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed-01000000-13722.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed-01000000-13722.out deleted file mode 100644 index 80b5e41..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-7abd3065dca4325130aae9b5fed632b86b1f3673140c122382f8f4ed-01000000-13722.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11010 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76-01000000-13649.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76-01000000-13649.err deleted file mode 100644 index 7107b60..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76-01000000-13649.err +++ /dev/null @@ -1,143 +0,0 @@ -:job_id:01000000 -:actor_name:PPO -2024-05-15 09:05:04,706 WARNING deprecation.py:50 -- DeprecationWarning: `AlgorithmConfig.num_envs_per_worker` has been deprecated. Use `AlgorithmConfig.num_envs_per_env_runner` instead. This will raise an error in the future! -2024-05-15 09:05:04,711 WARNING deprecation.py:50 -- DeprecationWarning: `WorkerSet(num_workers=... OR local_worker=...)` has been deprecated. Use `EnvRunnerGroup(num_env_runners=... AND local_env_runner=...)` instead. This will raise an error in the future! -2024-05-15 09:05:04,711 WARNING deprecation.py:50 -- DeprecationWarning: `max_num_worker_restarts` has been deprecated. Use `AlgorithmConfig.max_num_env_runner_restarts` instead. This will raise an error in the future! -2024-05-15 09:06:07,046 ERROR actor_manager.py:519 -- Ray error, taking actor 1 out of service. The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -2024-05-15 09:06:07,046 ERROR actor_manager.py:519 -- Ray error, taking actor 2 out of service. The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13722, ip=192.168.1.56, actor_id=64110d8601f56bf14e9797c501000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -2024-05-15 09:06:07,046 ERROR actor_manager.py:519 -- Ray error, taking actor 3 out of service. The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13723, ip=192.168.1.56, actor_id=6c1e2f8e5911cf8ba4dcac5b01000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -2024-05-15 09:06:07,046 ERROR actor_manager.py:519 -- Ray error, taking actor 4 out of service. The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13724, ip=192.168.1.56, actor_id=79760d3ce8052092d662e89401000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out -Exception raised in creation task: The actor died because of an error raised in its creation task, ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 239, in _setup - self.add_workers( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 754, in add_workers - raise result.get() - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/utils/actor_manager.py", line 497, in _fetch_result - result = ray.get(r) - ^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out - -During handling of the above exception, another exception occurred: - -ray::PPO.__init__() (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 554, in __init__ - super().__init__( - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__ - self.setup(copy.deepcopy(self.config)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup - self.workers = EnvRunnerGroup( - ^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__ - raise e.args[0].args[2] -TimeoutError: timed out diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76-01000000-13649.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76-01000000-13649.out deleted file mode 100644 index 1124c85..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88aa07750bd7c12dafbb7815838b15c7992641795664ee3b029e7c76-01000000-13649.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:actor_name:PPO diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88f5a13d82fe9091f929d1898eb0cc021e1078ca6094cd81a0b6f175-ffffffff-12916.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88f5a13d82fe9091f929d1898eb0cc021e1078ca6094cd81a0b6f175-ffffffff-12916.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88f5a13d82fe9091f929d1898eb0cc021e1078ca6094cd81a0b6f175-ffffffff-12916.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-88f5a13d82fe9091f929d1898eb0cc021e1078ca6094cd81a0b6f175-ffffffff-12916.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-9256fdcbbb57174994408d09f738067d9447c39639610bdf5592e558-ffffffff-12918.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-9256fdcbbb57174994408d09f738067d9447c39639610bdf5592e558-ffffffff-12918.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-9256fdcbbb57174994408d09f738067d9447c39639610bdf5592e558-ffffffff-12918.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-9256fdcbbb57174994408d09f738067d9447c39639610bdf5592e558-ffffffff-12918.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-aa626922a42a0d5a5bfe5bb859c4c452207dd77975f2c5e128771274-ffffffff-12911.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-aa626922a42a0d5a5bfe5bb859c4c452207dd77975f2c5e128771274-ffffffff-12911.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-aa626922a42a0d5a5bfe5bb859c4c452207dd77975f2c5e128771274-ffffffff-12911.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-aa626922a42a0d5a5bfe5bb859c4c452207dd77975f2c5e128771274-ffffffff-12911.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-ac25b7a37923dfbae865a40e6ea4f9c1e60a2047eaa2bcfb22b7b6aa-ffffffff-12921.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-ac25b7a37923dfbae865a40e6ea4f9c1e60a2047eaa2bcfb22b7b6aa-ffffffff-12921.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-ac25b7a37923dfbae865a40e6ea4f9c1e60a2047eaa2bcfb22b7b6aa-ffffffff-12921.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-ac25b7a37923dfbae865a40e6ea4f9c1e60a2047eaa2bcfb22b7b6aa-ffffffff-12921.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91-01000000-13723.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91-01000000-13723.err deleted file mode 100644 index 5376ea8..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91-01000000-13723.err +++ /dev/null @@ -1,58 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) -Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=13723, ip=192.168.1.56, actor_id=6c1e2f8e5911cf8ba4dcac5b01000000, repr=) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__ - self.env = env_creator(copy.deepcopy(self.env_context)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator - return RayVectorGodotEnv(config=env_config, port=port, seed=seed) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__ - self._env = GodotEnv( - ^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__ - self.connection = self._start_server() - ^^^^^^^^^^^^^^^^^^^^ - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server - connection, client_address = sock.accept() - ^^^^^^^^^^^^^ - File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept - fd, addr = self._accept() - ^^^^^^^^^^^^^^ -TimeoutError: timed out diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91-01000000-13723.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91-01000000-13723.out deleted file mode 100644 index 01371fe..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-cd64a88e73d60e90bea69d93f771cc429cb8accd2441e342cb915a91-01000000-13723.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11011 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-dcb3105e2724639b0e5a926650939c66da4809bd9e7fe0bb553d821b-ffffffff-12920.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-dcb3105e2724639b0e5a926650939c66da4809bd9e7fe0bb553d821b-ffffffff-12920.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-dcb3105e2724639b0e5a926650939c66da4809bd9e7fe0bb553d821b-ffffffff-12920.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-dcb3105e2724639b0e5a926650939c66da4809bd9e7fe0bb553d821b-ffffffff-12920.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-e0030e1be26dd66edc155d667282f827109105dfc2c188e43b0902ad-ffffffff-12910.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-e0030e1be26dd66edc155d667282f827109105dfc2c188e43b0902ad-ffffffff-12910.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-e0030e1be26dd66edc155d667282f827109105dfc2c188e43b0902ad-ffffffff-12910.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-e0030e1be26dd66edc155d667282f827109105dfc2c188e43b0902ad-ffffffff-12910.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-f1a712308a565bcb4f2dbb26cbb1789ec0da8e87f36005a14791e6e1-ffffffff-12919.err b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-f1a712308a565bcb4f2dbb26cbb1789ec0da8e87f36005a14791e6e1-ffffffff-12919.err deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-f1a712308a565bcb4f2dbb26cbb1789ec0da8e87f36005a14791e6e1-ffffffff-12919.err +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-f1a712308a565bcb4f2dbb26cbb1789ec0da8e87f36005a14791e6e1-ffffffff-12919.out b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-f1a712308a565bcb4f2dbb26cbb1789ec0da8e87f36005a14791e6e1-ffffffff-12919.out deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/logs/worker-f1a712308a565bcb4f2dbb26cbb1789ec0da8e87f36005a14791e6e1-ffffffff-12919.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/node_ip_address.json b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/node_ip_address.json deleted file mode 100644 index 8d998d8..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/node_ip_address.json +++ /dev/null @@ -1 +0,0 @@ -{"node_ip_address": "192.168.1.56"} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/node_ip_address.json.lock b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/node_ip_address.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/ports_by_node.json b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/ports_by_node.json deleted file mode 100644 index 03c5a8f..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/ports_by_node.json +++ /dev/null @@ -1 +0,0 @@ -{"192.168.1.56:/home/valapeos/Projects/pneumarl/logs/rllib/session_2024-05-15_09-05-00_541719_12647/sockets/plasma_store": {"metrics_agent_port": 62660, "metrics_export_port": 65512, "dashboard_agent_listen_port": 52365, "runtime_env_agent_port": 63152, "gcs_server_port": 51730}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/ports_by_node.json.lock b/Godot/logs/rllib/session_2024-05-15_09-05-00_541719_12647/ports_by_node.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/artifacts/2024-05-15_09-10-12/PPO_2024-05-15_09-10-12/driver_artifacts/basic-variant-state-2024-05-15_09-10-12.json b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/artifacts/2024-05-15_09-10-12/PPO_2024-05-15_09-10-12/driver_artifacts/basic-variant-state-2024-05-15_09-10-12.json deleted file mode 100644 index e0081cb..0000000 Binary files a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/artifacts/2024-05-15_09-10-12/PPO_2024-05-15_09-10-12/driver_artifacts/basic-variant-state-2024-05-15_09-10-12.json and /dev/null differ diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/artifacts/2024-05-15_09-10-12/PPO_2024-05-15_09-10-12/driver_artifacts/experiment_state-2024-05-15_09-10-12.json b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/artifacts/2024-05-15_09-10-12/PPO_2024-05-15_09-10-12/driver_artifacts/experiment_state-2024-05-15_09-10-12.json deleted file mode 100644 index c56f8f6..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/artifacts/2024-05-15_09-10-12/PPO_2024-05-15_09-10-12/driver_artifacts/experiment_state-2024-05-15_09-10-12.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"2c223_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30392d31302d3132948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f32633232335f30303030305f305f323032342d30352d31355f30392d31302d3132948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30392d31302d31329475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_2c223_00000_0_2024-05-15_09-10-12\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 107, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715757012.708886, "_session_str": "2024-05-15_09-10-12", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715757012.708886}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/agent-424238335.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/agent-424238335.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/agent-424238335.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/agent-424238335.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/dashboard.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/dashboard.err deleted file mode 100644 index c88b3c0..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/dashboard.err +++ /dev/null @@ -1,2 +0,0 @@ -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/dashboard/dashboard.py:236: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead - logger.warn("Exiting with SIGTERM immediately...") diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/debug_state.txt b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/debug_state.txt deleted file mode 100644 index da1b404..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/debug_state.txt +++ /dev/null @@ -1,208 +0,0 @@ -NodeManager: -Node ID: 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -Node name: 192.168.1.56 -InitialConfigResources: {memory: 368901046280000, node:__internal_head__: 10000, node:192.168.1.56: 10000, CPU: 120000, object_store_memory: 184450523130000} -ClusterTaskManager: -========== Node: 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 ================= -Infeasible queue length: 0 -Schedule queue length: 0 -Dispatch queue length: 0 -num_waiting_for_resource: 0 -num_waiting_for_plasma_memory: 0 -num_waiting_for_remote_node_resources: 0 -num_worker_not_started_by_job_config_not_exist: 0 -num_worker_not_started_by_registration_timeout: 0 -num_tasks_waiting_for_workers: 0 -num_cancelled_tasks: 0 -cluster_resource_scheduler state: -Local id: -3088399737359427104 Local resources: {"total":{bundle_group_4_64b1cbad44c9258a4011f30ebaf501000000: [10000000], CPU_group_4_64b1cbad44c9258a4011f30ebaf501000000: [10000], object_store_memory: [184450523130000], node:__internal_head__: [10000], bundle_group_64b1cbad44c9258a4011f30ebaf501000000: [50000000], CPU_group_2_64b1cbad44c9258a4011f30ebaf501000000: [10000], CPU_group_0_64b1cbad44c9258a4011f30ebaf501000000: [10000], bundle_group_0_64b1cbad44c9258a4011f30ebaf501000000: [10000000], bundle_group_3_64b1cbad44c9258a4011f30ebaf501000000: [10000000], bundle_group_2_64b1cbad44c9258a4011f30ebaf501000000: [10000000], CPU_group_1_64b1cbad44c9258a4011f30ebaf501000000: [10000], bundle_group_1_64b1cbad44c9258a4011f30ebaf501000000: [10000000], memory: [368901046280000], CPU: [120000], CPU_group_3_64b1cbad44c9258a4011f30ebaf501000000: [10000], CPU_group_64b1cbad44c9258a4011f30ebaf501000000: [50000], node:192.168.1.56: [10000]}}, "available": {bundle_group_4_64b1cbad44c9258a4011f30ebaf501000000: [10000000], CPU_group_4_64b1cbad44c9258a4011f30ebaf501000000: [10000], object_store_memory: [184450523130000], node:__internal_head__: [10000], bundle_group_64b1cbad44c9258a4011f30ebaf501000000: [49999950], CPU_group_2_64b1cbad44c9258a4011f30ebaf501000000: [10000], CPU_group_0_64b1cbad44c9258a4011f30ebaf501000000: [0], bundle_group_0_64b1cbad44c9258a4011f30ebaf501000000: [9999990], bundle_group_3_64b1cbad44c9258a4011f30ebaf501000000: [10000000], bundle_group_2_64b1cbad44c9258a4011f30ebaf501000000: [10000000], CPU_group_1_64b1cbad44c9258a4011f30ebaf501000000: [10000], bundle_group_1_64b1cbad44c9258a4011f30ebaf501000000: [10000000], memory: [368901046280000], CPU: [70000], CPU_group_3_64b1cbad44c9258a4011f30ebaf501000000: [10000], CPU_group_64b1cbad44c9258a4011f30ebaf501000000: [0], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50",} is_draining: 0 is_idle: 0 Cluster resources: node id: -3088399737359427104{"total":{bundle_group_64b1cbad44c9258a4011f30ebaf501000000: 50000000, node:__internal_head__: 10000, object_store_memory: 184450523130000, CPU_group_4_64b1cbad44c9258a4011f30ebaf501000000: 10000, bundle_group_4_64b1cbad44c9258a4011f30ebaf501000000: 10000000, bundle_group_2_64b1cbad44c9258a4011f30ebaf501000000: 10000000, bundle_group_3_64b1cbad44c9258a4011f30ebaf501000000: 10000000, bundle_group_0_64b1cbad44c9258a4011f30ebaf501000000: 10000000, CPU_group_2_64b1cbad44c9258a4011f30ebaf501000000: 10000, CPU_group_0_64b1cbad44c9258a4011f30ebaf501000000: 10000, CPU_group_3_64b1cbad44c9258a4011f30ebaf501000000: 10000, CPU: 120000, bundle_group_1_64b1cbad44c9258a4011f30ebaf501000000: 10000000, memory: 368901046280000, CPU_group_1_64b1cbad44c9258a4011f30ebaf501000000: 10000, node:192.168.1.56: 10000, CPU_group_64b1cbad44c9258a4011f30ebaf501000000: 50000}}, "available": {bundle_group_3_64b1cbad44c9258a4011f30ebaf501000000: 10000000, bundle_group_1_64b1cbad44c9258a4011f30ebaf501000000: 10000000, CPU_group_1_64b1cbad44c9258a4011f30ebaf501000000: 10000, memory: 368901046280000, CPU: 70000, CPU_group_3_64b1cbad44c9258a4011f30ebaf501000000: 10000, node:192.168.1.56: 10000, bundle_group_4_64b1cbad44c9258a4011f30ebaf501000000: 10000000, CPU_group_4_64b1cbad44c9258a4011f30ebaf501000000: 10000, node:__internal_head__: 10000, object_store_memory: 184450523130000, bundle_group_64b1cbad44c9258a4011f30ebaf501000000: 49999950, CPU_group_2_64b1cbad44c9258a4011f30ebaf501000000: 10000, bundle_group_0_64b1cbad44c9258a4011f30ebaf501000000: 9999990, bundle_group_2_64b1cbad44c9258a4011f30ebaf501000000: 10000000}}, "labels":{"ray.io/node_id":"08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -Waiting tasks size: 0 -Number of executing tasks: 5 -Number of pinned task arguments: 0 -Number of total spilled tasks: 0 -Number of spilled waiting tasks: 0 -Number of spilled unschedulable tasks: 0 -Resource usage { - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=17260): {bundle_group_64b1cbad44c9258a4011f30ebaf501000000: 10, CPU_group_64b1cbad44c9258a4011f30ebaf501000000: 10000} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=17261): {CPU_group_64b1cbad44c9258a4011f30ebaf501000000: 10000, bundle_group_64b1cbad44c9258a4011f30ebaf501000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=17259): {CPU_group_64b1cbad44c9258a4011f30ebaf501000000: 10000, bundle_group_64b1cbad44c9258a4011f30ebaf501000000: 10} - - (language=PYTHON actor_or_task=PPO.__init__ pid=17186): {bundle_group_64b1cbad44c9258a4011f30ebaf501000000: 10, CPU_group_64b1cbad44c9258a4011f30ebaf501000000: 10000, CPU_group_0_64b1cbad44c9258a4011f30ebaf501000000: 10000, bundle_group_0_64b1cbad44c9258a4011f30ebaf501000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=17258): {bundle_group_64b1cbad44c9258a4011f30ebaf501000000: 10, CPU_group_64b1cbad44c9258a4011f30ebaf501000000: 10000} -} -Running tasks by scheduling class: - - {depth=1 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.algorithms.ppo.ppo, class_name=PPO, function_name=__init__, function_hash=c65ff468889f4ee894fd49da99100c5a} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "d\261\313\255D\311%\212@\021\363\016\272\365\001\000\000\000" - placement_group_capture_child_tasks: true -} - resource_set={bundle_group_64b1cbad44c9258a4011f30ebaf501000000 : 0.001, bundle_group_0_64b1cbad44c9258a4011f30ebaf501000000 : 0.001, CPU_group_64b1cbad44c9258a4011f30ebaf501000000 : 1, CPU_group_0_64b1cbad44c9258a4011f30ebaf501000000 : 1, }}: 1/18446744073709551615 - - {depth=2 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.evaluation.rollout_worker, class_name=RolloutWorker, function_name=__init__, function_hash=c84d22087f674cad861c4e5518d46277} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "d\261\313\255D\311%\212@\021\363\016\272\365\001\000\000\000" - placement_group_bundle_index: -1 - placement_group_capture_child_tasks: true -} - resource_set={CPU_group_64b1cbad44c9258a4011f30ebaf501000000 : 1, bundle_group_64b1cbad44c9258a4011f30ebaf501000000 : 0.001, }}: 4/18446744073709551615 -================================================== - -ClusterResources: -LocalObjectManager: -- num pinned objects: 0 -- pinned objects size: 0 -- num objects pending restore: 0 -- num objects pending spill: 0 -- num bytes pending spill: 0 -- num bytes currently spilled: 0 -- cumulative spill requests: 0 -- cumulative restore requests: 0 -- spilled objects pending delete: 0 - -ObjectManager: -- num local objects: 0 -- num unfulfilled push requests: 0 -- num object pull requests: 0 -- num chunks received total: 0 -- num chunks received failed (all): 0 -- num chunks received failed / cancelled: 0 -- num chunks received failed / plasma error: 0 -Event stats: -Global stats: 0 total (0 active) -Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -Execution time: mean = -nan s, total = 0.000 s -Event stats: -PushManager: -- num pushes in flight: 0 -- num chunks in flight: 0 -- num chunks remaining: 0 -- max chunks allowed: 409 -OwnershipBasedObjectDirectory: -- num listeners: 0 -- cumulative location updates: 0 -- num location updates per second: 0.000 -- num location lookups per second: 0.000 -- num locations added per second: 0.000 -- num locations removed per second: 0.000 -BufferPool: -- create buffer state map size: 0 -PullManager: -- num bytes available for pulled objects: 18445052313 -- num bytes being pulled (all): 0 -- num bytes being pulled / pinned: 0 -- get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- first get request bundle: N/A -- first wait request bundle: N/A -- first task request bundle: N/A -- num objects queued: 0 -- num objects actively pulled (all): 0 -- num objects actively pulled / pinned: 0 -- num bundles being pulled: 0 -- num pull retries: 0 -- max timeout seconds: 0 -- max timeout request is already processed. No entry. - -WorkerPool: -- registered jobs: 1 -- process_failed_job_config_missing: 0 -- process_failed_rate_limited: 0 -- process_failed_pending_registration: 0 -- process_failed_runtime_env_setup_failed: 0 -- num PYTHON workers: 12 -- num PYTHON drivers: 1 -- num object spill callbacks queued: 0 -- num object restore queued: 0 -- num util functions queued: 0 -- num idle workers: 7 -TaskDependencyManager: -- task deps map size: 0 -- get req map size: 0 -- wait req map size: 0 -- local objects map size: 0 -WaitManager: -- num active wait requests: 0 -Subscriber: -Channel WORKER_OBJECT_LOCATIONS_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_OBJECT_EVICTION -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_REF_REMOVED_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -num async plasma notifications: 0 -Remote node managers: -Event stats: -Global stats: 1763 total (27 active) -Queueing time: mean = 3.926 ms, max = 1.320 s, min = 70.000 ns, total = 6.922 s -Execution time: mean = 2.318 ms, total = 4.087 s -Event stats: - NodeManagerService.grpc_server.ReportWorkerBacklog.HandleRequestImpl - 257 total (0 active), Execution time: mean = 10.183 us, total = 2.617 ms, Queueing time: mean = 11.510 us, max = 160.989 us, min = 5.100 us, total = 2.958 ms - NodeManagerService.grpc_server.ReportWorkerBacklog - 257 total (0 active), Execution time: mean = 98.570 us, total = 25.333 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ObjectManager.UpdateAvailableMemory - 200 total (0 active), Execution time: mean = 1.285 us, total = 257.087 us, Queueing time: mean = 8.026 us, max = 22.490 us, min = 4.370 us, total = 1.605 ms - NodeManager.CheckGC - 200 total (1 active), Execution time: mean = 962.295 ns, total = 192.459 us, Queueing time: mean = 66.225 us, max = 9.092 ms, min = 5.289 us, total = 13.245 ms - RaySyncer.OnDemandBroadcasting - 200 total (1 active), Execution time: mean = 7.627 us, total = 1.525 ms, Queueing time: mean = 60.136 us, max = 9.091 ms, min = 5.209 us, total = 12.027 ms - RayletWorkerPool.deadline_timer.kill_idle_workers - 100 total (1 active), Execution time: mean = 15.751 us, total = 1.575 ms, Queueing time: mean = 14.490 us, max = 268.628 us, min = 5.353 us, total = 1.449 ms - ClientConnection.async_read.ProcessMessageHeader - 83 total (13 active), Execution time: mean = 2.617 us, total = 217.186 us, Queueing time: mean = 82.530 ms, max = 1.320 s, min = 11.489 us, total = 6.850 s - MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 80 total (1 active), Execution time: mean = 203.673 us, total = 16.294 ms, Queueing time: mean = 37.757 us, max = 2.078 ms, min = 5.685 us, total = 3.021 ms - ClientConnection.async_read.ProcessMessage - 70 total (0 active), Execution time: mean = 531.214 us, total = 37.185 ms, Queueing time: mean = 15.486 us, max = 458.037 us, min = 1.780 us, total = 1.084 ms - NodeManager.ScheduleAndDispatchTasks - 21 total (1 active), Execution time: mean = 5.506 us, total = 115.628 us, Queueing time: mean = 11.051 us, max = 21.937 us, min = 5.951 us, total = 232.064 us - NodeManager.deadline_timer.spill_objects_when_over_threshold - 20 total (1 active), Execution time: mean = 1.119 us, total = 22.380 us, Queueing time: mean = 46.197 us, max = 404.799 us, min = 7.047 us, total = 923.950 us - NodeManagerService.grpc_server.GetResourceLoad - 20 total (0 active), Execution time: mean = 153.463 us, total = 3.069 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.deadline_timer.flush_free_objects - 20 total (1 active), Execution time: mean = 2.137 us, total = 42.740 us, Queueing time: mean = 45.274 us, max = 405.659 us, min = 6.907 us, total = 905.480 us - NodeManagerService.grpc_server.GetResourceLoad.HandleRequestImpl - 20 total (0 active), Execution time: mean = 55.124 us, total = 1.102 ms, Queueing time: mean = 11.796 us, max = 21.450 us, min = 7.180 us, total = 235.917 us - ClientConnection.async_write.DoAsyncWrites - 19 total (0 active), Execution time: mean = 504.737 ns, total = 9.590 us, Queueing time: mean = 13.737 us, max = 80.449 us, min = 6.300 us, total = 261.009 us - ObjectManager.ObjectDeleted - 18 total (0 active), Execution time: mean = 7.689 us, total = 138.398 us, Queueing time: mean = 31.822 us, max = 209.039 us, min = 13.380 us, total = 572.797 us - NodeManagerService.grpc_server.GetSystemConfig - 18 total (0 active), Execution time: mean = 213.778 us, total = 3.848 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.GetSystemConfig.HandleRequestImpl - 18 total (0 active), Execution time: mean = 29.682 us, total = 534.268 us, Queueing time: mean = 10.425 us, max = 18.930 us, min = 6.740 us, total = 187.650 us - ObjectManager.ObjectAdded - 18 total (0 active), Execution time: mean = 6.679 us, total = 120.230 us, Queueing time: mean = 13.392 us, max = 49.759 us, min = 5.700 us, total = 241.048 us - PeriodicalRunner.RunFnPeriodically - 13 total (0 active), Execution time: mean = 102.087 us, total = 1.327 ms, Queueing time: mean = 2.210 ms, max = 7.543 ms, min = 22.120 us, total = 28.735 ms - ClusterResourceManager.ResetRemoteNodeView - 7 total (1 active), Execution time: mean = 1.636 us, total = 11.450 us, Queueing time: mean = 10.837 us, max = 19.101 us, min = 8.953 us, total = 75.859 us - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 6 total (1 active), Execution time: mean = 301.467 ms, total = 1.809 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RaySyncer.BroadcastMessage - 6 total (0 active), Execution time: mean = 93.769 us, total = 562.616 us, Queueing time: mean = 338.333 ns, max = 960.000 ns, min = 100.000 ns, total = 2.030 us - - 6 total (0 active), Execution time: mean = 130.000 ns, total = 780.000 ns, Queueing time: mean = 9.272 us, max = 17.180 us, min = 6.720 us, total = 55.630 us - NodeManagerService.grpc_server.RequestWorkerLease.HandleRequestImpl - 6 total (0 active), Execution time: mean = 119.743 us, total = 718.455 us, Queueing time: mean = 295.510 us, max = 941.484 us, min = 6.540 us, total = 1.773 ms - NodeManagerService.grpc_server.RequestWorkerLease - 6 total (0 active), Execution time: mean = 359.073 ms, total = 2.154 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_client.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 910.156 us, total = 4.551 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - Subscriber.HandlePublishedMessage_GCS_WORKER_DELTA_CHANNEL - 5 total (0 active), Execution time: mean = 3.820 us, total = 19.100 us, Queueing time: mean = 94.944 us, max = 103.149 us, min = 80.180 us, total = 474.718 us - WorkerInfoGcsService.grpc_client.ReportWorkerFailure.OnReplyReceived - 5 total (0 active), Execution time: mean = 10.066 us, total = 50.330 us, Queueing time: mean = 43.582 us, max = 98.450 us, min = 7.420 us, total = 217.909 us - CoreWorkerService.grpc_client.Exit.OnReplyReceived - 5 total (0 active), Execution time: mean = 9.134 us, total = 45.670 us, Queueing time: mean = 16.626 us, max = 39.329 us, min = 5.120 us, total = 83.129 us - CoreWorkerService.grpc_client.Exit - 5 total (0 active), Execution time: mean = 760.827 us, total = 3.804 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll.OnReplyReceived - 5 total (0 active), Execution time: mean = 89.015 us, total = 445.077 us, Queueing time: mean = 9.492 us, max = 16.010 us, min = 4.260 us, total = 47.460 us - NodeInfoGcsService.grpc_client.CheckAlive.OnReplyReceived - 4 total (0 active), Execution time: mean = 12.790 us, total = 51.160 us, Queueing time: mean = 8.490 us, max = 9.890 us, min = 6.780 us, total = 33.960 us - NodeInfoGcsService.grpc_client.CheckAlive - 4 total (0 active), Execution time: mean = 302.955 us, total = 1.212 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.GcsCheckAlive - 4 total (1 active), Execution time: mean = 73.722 us, total = 294.887 us, Queueing time: mean = 108.355 us, max = 206.000 us, min = 100.884 us, total = 433.420 us - NodeManager.deadline_timer.record_metrics - 4 total (1 active), Execution time: mean = 212.776 us, total = 851.106 us, Queueing time: mean = 6.867 us, max = 13.210 us, min = 6.756 us, total = 27.470 us - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 2 total (0 active), Execution time: mean = 74.570 us, total = 149.140 us, Queueing time: mean = 560.916 us, max = 1.047 ms, min = 74.709 us, total = 1.122 ms - NodeManager.deadline_timer.debug_state_dump - 2 total (1 active, 1 running), Execution time: mean = 247.293 us, total = 494.587 us, Queueing time: mean = 3.348 us, max = 6.695 us, min = 6.695 us, total = 6.695 us - RaySyncerRegister - 2 total (0 active), Execution time: mean = 815.000 ns, total = 1.630 us, Queueing time: mean = 190.000 ns, max = 310.000 ns, min = 70.000 ns, total = 380.000 ns - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 2 total (0 active), Execution time: mean = 259.128 us, total = 518.257 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.GetAllJobInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 5.750 us, total = 5.750 us, Queueing time: mean = 7.730 us, max = 7.730 us, min = 7.730 us, total = 7.730 us - NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 11.326 ms, total = 11.326 ms, Queueing time: mean = 7.020 us, max = 7.020 us, min = 7.020 us, total = 7.020 us - NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 542.017 us, total = 542.017 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerPool.PopWorkerCallback - 1 total (0 active), Execution time: mean = 12.110 us, total = 12.110 us, Queueing time: mean = 5.170 us, max = 5.170 us, min = 5.170 us, total = 5.170 us - NodeManagerService.grpc_server.PrepareBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 228.679 us, total = 228.679 us, Queueing time: mean = 12.400 us, max = 12.400 us, min = 12.400 us, total = 12.400 us - NodeManagerService.grpc_server.ReturnWorker - 1 total (0 active), Execution time: mean = 124.159 us, total = 124.159 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetAllNodeInfo - 1 total (0 active), Execution time: mean = 273.689 us, total = 273.689 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.AddJob.OnReplyReceived - 1 total (0 active), Execution time: mean = 22.290 us, total = 22.290 us, Queueing time: mean = 5.330 us, max = 5.330 us, min = 5.330 us, total = 5.330 us - Subscriber.HandlePublishedMessage_GCS_JOB_CHANNEL - 1 total (0 active), Execution time: mean = 22.710 us, total = 22.710 us, Queueing time: mean = 49.919 us, max = 49.919 us, min = 49.919 us, total = 49.919 us - NodeManagerService.grpc_server.CommitBundleResources - 1 total (0 active), Execution time: mean = 255.428 us, total = 255.428 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetAllNodeInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 73.879 us, total = 73.879 us, Queueing time: mean = 1.840 us, max = 1.840 us, min = 1.840 us, total = 1.840 us - NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 401.647 us, total = 401.647 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 175.629 us, total = 175.629 us, Queueing time: mean = 7.760 us, max = 7.760 us, min = 7.760 us, total = 7.760 us - NodeManagerService.grpc_server.PrepareBundleResources - 1 total (0 active), Execution time: mean = 311.688 us, total = 311.688 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.AddJob - 1 total (0 active), Execution time: mean = 336.148 us, total = 336.148 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.CommitBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 188.319 us, total = 188.319 us, Queueing time: mean = 8.260 us, max = 8.260 us, min = 8.260 us, total = 8.260 us - NodeManager.deadline_timer.print_event_loop_stats - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.ReturnWorker.HandleRequestImpl - 1 total (0 active), Execution time: mean = 40.549 us, total = 40.549 us, Queueing time: mean = 10.450 us, max = 10.450 us, min = 10.450 us, total = 10.450 us - NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.GetAllJobInfo - 1 total (0 active), Execution time: mean = 238.329 us, total = 238.329 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -DebugString() time ms: 0 \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/debug_state_gcs.txt b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/debug_state_gcs.txt deleted file mode 100644 index cfb7cd4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/debug_state_gcs.txt +++ /dev/null @@ -1,150 +0,0 @@ -GcsNodeManager: -- RegisterNode request count: 1 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 27 -- GetInternalConfig request count: 1 - -GcsActorManager: -- RegisterActor request count: 5 -- CreateActor request count: 5 -- GetActorInfo request count: 5 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 5 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 1 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 5 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 4 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 1 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 5 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 1 -- Registered placement groups count: 1 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: -GCS_JOB_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 279 -- current buffered bytes: 279 -GCS_ACTOR_CHANNEL -- cumulative published messages: 10 -- cumulative published bytes: 5864 -- current buffered bytes: 1328 -GCS_NODE_INFO_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 557 -- current buffered bytes: 0 -RAY_LOG_CHANNEL -- cumulative published messages: 17 -- cumulative published bytes: 16702 -- current buffered bytes: 853 -GCS_WORKER_DELTA_CHANNEL -- cumulative published messages: 5 -- cumulative published bytes: 485 -- current buffered bytes: 291 - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 25 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 13 --Total num of actor creation tasks: 5 --Total num of actor tasks: 6 --Total num of normal tasks: 1 --Total num of driver tasks: 1 - - - - -Global stats: 866 total (19 active) -Queueing time: mean = 112.663 us, max = 25.782 ms, min = 570.000 ns, total = 97.566 ms -Execution time: mean = 2.625 ms, total = 2.273 s -Event stats: - GcsInMemoryStore.Put - 127 total (0 active), Execution time: mean = 226.563 us, total = 28.774 ms, Queueing time: mean = 214.108 us, max = 25.603 ms, min = 570.000 ns, total = 27.192 ms - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 81 total (0 active), Execution time: mean = 7.723 us, total = 625.567 us, Queueing time: mean = 37.898 us, max = 237.409 us, min = 2.170 us, total = 3.070 ms - InternalKVGcsService.grpc_server.InternalKVPut - 81 total (0 active), Execution time: mean = 140.734 us, total = 11.399 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Get - 77 total (0 active), Execution time: mean = 26.979 us, total = 2.077 ms, Queueing time: mean = 10.113 us, max = 231.118 us, min = 580.000 ns, total = 778.704 us - InternalKVGcsService.grpc_server.InternalKVGet - 70 total (0 active), Execution time: mean = 110.218 us, total = 7.715 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 70 total (0 active), Execution time: mean = 11.026 us, total = 771.806 us, Queueing time: mean = 17.028 us, max = 191.839 us, min = 4.160 us, total = 1.192 ms - NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 27 total (0 active), Execution time: mean = 11.678 us, total = 315.309 us, Queueing time: mean = 27.855 us, max = 349.738 us, min = 4.700 us, total = 752.075 us - NodeInfoGcsService.grpc_server.GetAllNodeInfo - 27 total (0 active), Execution time: mean = 125.746 us, total = 3.395 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut - 24 total (0 active), Execution time: mean = 522.262 us, total = 12.534 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 24 total (0 active), Execution time: mean = 4.945 us, total = 118.669 us, Queueing time: mean = 15.249 us, max = 31.919 us, min = 4.560 us, total = 365.976 us - RayletLoadPulled - 20 total (1 active), Execution time: mean = 82.377 us, total = 1.648 ms, Queueing time: mean = 14.873 us, max = 107.259 us, min = 6.300 us, total = 297.460 us - NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 18 total (0 active), Execution time: mean = 31.139 us, total = 560.498 us, Queueing time: mean = 10.374 us, max = 22.150 us, min = 5.750 us, total = 186.730 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 113.633 us, total = 2.045 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.GetResourceLoad - 18 total (0 active), Execution time: mean = 398.326 us, total = 7.170 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 15.332 us, total = 275.977 us, Queueing time: mean = 11.693 us, max = 36.500 us, min = 5.610 us, total = 210.479 us - NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 8 total (0 active), Execution time: mean = 8.315 us, total = 66.520 us, Queueing time: mean = 10.535 us, max = 15.430 us, min = 7.370 us, total = 84.278 us - NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 10.729 us, total = 85.828 us, Queueing time: mean = 1.223 ms, max = 3.736 ms, min = 8.480 us, total = 9.784 ms - NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 2.907 ms, total = 23.254 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.CheckAlive - 8 total (0 active), Execution time: mean = 104.724 us, total = 837.792 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ClusterResourceManager.ResetRemoteNodeView - 7 total (1 active), Execution time: mean = 2.651 us, total = 18.560 us, Queueing time: mean = 7.507 us, max = 11.596 us, min = 7.193 us, total = 52.547 us - GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 5.136 us, total = 35.950 us, Queueing time: mean = 11.218 us, max = 16.129 us, min = 2.280 us, total = 78.525 us - GcsResourceManager::Update - 6 total (0 active), Execution time: mean = 42.128 us, total = 252.769 us, Queueing time: mean = 9.148 us, max = 11.490 us, min = 7.530 us, total = 54.890 us - ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 330.032 us, total = 1.650 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - HealthCheck - 5 total (0 active), Execution time: mean = 2.516 us, total = 12.580 us, Queueing time: mean = 9.286 us, max = 11.020 us, min = 7.240 us, total = 46.429 us - WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 25.818 us, total = 129.088 us, Queueing time: mean = 118.913 us, max = 456.008 us, min = 14.030 us, total = 594.567 us - ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 523.595 us, total = 2.618 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 103.343 us, total = 516.715 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 133.637 us, total = 668.185 us, Queueing time: mean = 6.798 us, max = 7.470 us, min = 5.690 us, total = 33.990 us - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 11.144 us, total = 55.720 us, Queueing time: mean = 99.483 us, max = 280.898 us, min = 5.730 us, total = 497.416 us - ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 217.807 us, total = 1.089 ms, Queueing time: mean = 6.520 us, max = 7.370 us, min = 5.540 us, total = 32.600 us - ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 6.446 us, total = 32.230 us, Queueing time: mean = 34.118 us, max = 82.100 us, min = 4.480 us, total = 170.589 us - CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 134.183 us, total = 670.916 us, Queueing time: mean = 41.427 us, max = 115.259 us, min = 6.560 us, total = 207.137 us - NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 431.412 ms, total = 2.157 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 165.029 us, total = 825.144 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 57.844 us, total = 231.378 us, Queueing time: mean = 12.918 ms, max = 25.782 ms, min = 11.350 us, total = 51.672 ms - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 4 total (0 active), Execution time: mean = 144.512 us, total = 578.046 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 4 total (0 active), Execution time: mean = 30.884 us, total = 123.538 us, Queueing time: mean = 12.352 us, max = 15.450 us, min = 11.050 us, total = 49.410 us - GCSServer.deadline_timer.debug_state_dump - 2 total (1 active, 1 running), Execution time: mean = 233.453 us, total = 466.907 us, Queueing time: mean = 3.068 us, max = 6.136 us, min = 6.136 us, total = 6.136 us - JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 38.170 us, total = 38.170 us, Queueing time: mean = 6.830 us, max = 6.830 us, min = 6.830 us, total = 6.830 us - InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 10.860 us, total = 10.860 us, Queueing time: mean = 5.340 us, max = 5.340 us, min = 5.340 us, total = 5.340 us - GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 60.000 ns, total = 60.000 ns, Queueing time: mean = 79.250 us, max = 79.250 us, min = 79.250 us, total = 79.250 us - NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 278.498 us, total = 278.498 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 38.280 us, total = 38.280 us, Queueing time: mean = 7.630 us, max = 7.630 us, min = 7.630 us, total = 7.630 us - NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 56.229 us, total = 56.229 us, Queueing time: mean = 6.130 us, max = 6.130 us, min = 6.130 us, total = 6.130 us - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 96.770 us, total = 96.770 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 5.800 us, total = 5.800 us, Queueing time: mean = 890.000 ns, max = 890.000 ns, min = 890.000 ns, total = 890.000 ns - NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 42.089 us, total = 42.089 us, Queueing time: mean = 7.340 us, max = 7.340 us, min = 7.340 us, total = 7.340 us - NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 603.597 us, total = 603.597 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 64.909 us, total = 64.909 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 8.270 us, total = 8.270 us, Queueing time: mean = 9.830 us, max = 9.830 us, min = 9.830 us, total = 9.830 us - JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 4.480 us, total = 4.480 us, Queueing time: mean = 8.490 us, max = 8.490 us, min = 8.490 us, total = 8.490 us - InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 67.130 us, total = 67.130 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 56.680 us, total = 56.680 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 91.430 us, total = 91.430 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 7.810 us, total = 7.810 us, Queueing time: mean = 9.210 us, max = 9.210 us, min = 9.210 us, total = 9.210 us - NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 111.889 us, total = 111.889 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 390.218 us, total = 390.218 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 60.909 us, total = 60.909 us, Queueing time: mean = 7.450 us, max = 7.450 us, min = 7.450 us, total = 7.450 us - GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 2.310 us, total = 2.310 us, Queueing time: mean = 1.110 us, max = 1.110 us, min = 1.110 us, total = 1.110 us - JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 121.499 us, total = 121.499 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 2.580 us, total = 2.580 us, Queueing time: mean = 1.480 us, max = 1.480 us, min = 1.480 us, total = 1.480 us - InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 13.430 us, total = 13.430 us, Queueing time: mean = 5.880 us, max = 5.880 us, min = 5.880 us, total = 5.880 us \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/gcs_server.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/gcs_server.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/gcs_server.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/gcs_server.out deleted file mode 100644 index 3178612..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/gcs_server.out +++ /dev/null @@ -1,183 +0,0 @@ -[2024-05-15 09:10:10,637 I 16234 16234] (gcs_server) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) event.cc:234: Set ray event level to warning -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) event.cc:342: Ray Event initialized for GCS -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_server.cc:74: GCS storage type is StorageType::IN_MEMORY -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:42: Loading job table data. -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:54: Loading node table data. -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:80: Loading actor table data. -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:93: Loading actor task spec table data. -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:66: Loading placement group table data. -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:46: Finished loading job table data, size = 0 -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:58: Finished loading node table data, size = 0 -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:84: Finished loading actor table data, size = 0 -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:97: Finished loading actor task spec table data, size = 0 -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_init_data.cc:71: Finished loading placement group table data, size = 0 -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_server.cc:162: No existing server cluster ID found. Generating new ID: 4554569021e90e46163cb999ab6255921f2f493c80bfefaa5084d621 -[2024-05-15 09:10:10,638 I 16234 16234] (gcs_server) gcs_server.cc:648: Autoscaler V2 enabled: 0 -[2024-05-15 09:10:10,639 I 16234 16234] (gcs_server) grpc_server.cc:134: GcsServer server started, listening on port 59971. -[2024-05-15 09:10:10,664 I 16234 16234] (gcs_server) gcs_server.cc:250: GcsNodeManager: -- RegisterNode request count: 0 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 0 -- GetInternalConfig request count: 0 - -GcsActorManager: -- RegisterActor request count: 0 -- CreateActor request count: 0 -- GetActorInfo request count: 0 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 0 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 0 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 0 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 0 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 0 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 0 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 0 -- Registered placement groups count: 0 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 0 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 0 --Total num of actor creation tasks: 0 --Total num of actor tasks: 0 --Total num of normal tasks: 0 --Total num of driver tasks: 0 - - -[2024-05-15 09:10:10,664 I 16234 16234] (gcs_server) gcs_server.cc:844: Event stats: - - -Global stats: 22 total (11 active) -Queueing time: mean = 3.510 ms, max = 25.782 ms, min = 990.000 ns, total = 77.222 ms -Execution time: mean = 1.173 ms, total = 25.814 ms -Event stats: - InternalKVGcsService.grpc_client.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.GetAll - 5 total (0 active), Execution time: mean = 3.496 us, total = 17.480 us, Queueing time: mean = 14.751 us, max = 16.129 us, min = 13.199 us, total = 73.755 us - PeriodicalRunner.RunFnPeriodically - 4 total (2 active, 1 running), Execution time: mean = 615.000 ns, total = 2.460 us, Queueing time: mean = 12.885 ms, max = 25.782 ms, min = 25.759 ms, total = 51.540 ms - GcsInMemoryStore.Put - 3 total (0 active), Execution time: mean = 8.596 ms, total = 25.789 ms, Queueing time: mean = 8.535 ms, max = 25.603 ms, min = 990.000 ns, total = 25.606 ms - ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RayletLoadPulled - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Get - 1 total (0 active), Execution time: mean = 5.030 us, total = 5.030 us, Queueing time: mean = 1.410 us, max = 1.410 us, min = 1.410 us, total = 1.410 us - InternalKVGcsService.grpc_server.InternalKVPut - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 09:10:10,664 I 16234 16234] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats: - - -Global stats: 2 total (1 active) -Queueing time: mean = 35.635 us, max = 71.270 us, min = 71.270 us, total = 71.270 us -Execution time: mean = 11.695 us, total = 23.390 us -Event stats: - PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 23.390 us, total = 23.390 us, Queueing time: mean = 71.270 us, max = 71.270 us, min = 71.270 us, total = 71.270 us - GcsTaskManager.GcJobSummary - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 09:10:12,036 I 16234 16234] (gcs_server) gcs_node_manager.cc:55: Registering node info, node id = 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 09:10:12,036 I 16234 16234] (gcs_server) gcs_node_manager.cc:61: Finished registering node info, node id = 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 09:10:12,036 I 16234 16234] (gcs_server) gcs_placement_group_manager.cc:797: A new node: 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 registered, will try to reschedule all the infeasible placement groups. -[2024-05-15 09:10:12,684 I 16234 16234] (gcs_server) gcs_job_manager.cc:42: Adding job, job id = 01000000, driver pid = 16182 -[2024-05-15 09:10:12,684 I 16234 16234] (gcs_server) gcs_job_manager.cc:57: Finished adding job, job id = 01000000, driver pid = 16182 -[2024-05-15 09:10:12,716 I 16234 16234] (gcs_server) gcs_placement_group_manager.cc:432: Registering placement group, placement group id = 64b1cbad44c9258a4011f30ebaf501000000, name = , strategy = 0 -[2024-05-15 09:10:12,716 I 16234 16234] (gcs_server) gcs_placement_group_manager.cc:436: Finished registering placement group, placement group id = 64b1cbad44c9258a4011f30ebaf501000000, name = , strategy = 0 -[2024-05-15 09:10:12,716 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:185: Preparing resource from node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for bundles: {placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={0}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={1}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={2}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={3}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={4}}, -[2024-05-15 09:10:12,717 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:195: Finished leasing resource from 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for bundles: {placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={0}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={1}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={2}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={3}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={4}}, -[2024-05-15 09:10:12,717 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:213: Committing resource to a node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for bundles: {placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={1}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={2}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={0}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={3}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={4}}, -[2024-05-15 09:10:12,717 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:220: Finished committing resource to 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for bundles: {placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={1}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={2}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={0}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={3}},{placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={4}}, -[2024-05-15 09:10:12,717 I 16234 16234] (gcs_server) gcs_placement_group_manager.cc:321: Successfully created placement group , id: 64b1cbad44c9258a4011f30ebaf501000000 -[2024-05-15 09:10:12,900 I 16234 16234] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 76dc2a362a04e1f8850a8a8d01000000 -[2024-05-15 09:10:12,901 I 16234 16234] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 76dc2a362a04e1f8850a8a8d01000000 -[2024-05-15 09:10:12,902 I 16234 16234] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 76dc2a362a04e1f8850a8a8d01000000 -[2024-05-15 09:10:12,902 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor 76dc2a362a04e1f8850a8a8d01000000, job id = 01000000 -[2024-05-15 09:10:13,316 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor 76dc2a362a04e1f8850a8a8d01000000, job id = 01000000 -[2024-05-15 09:10:13,317 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 76dc2a362a04e1f8850a8a8d01000000 on worker 0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47 at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, job id = 01000000 -[2024-05-15 09:10:14,934 I 16234 16234] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 67e6f677752cb73b0ee3220c01000000 -[2024-05-15 09:10:14,935 I 16234 16234] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 67e6f677752cb73b0ee3220c01000000 -[2024-05-15 09:10:14,935 I 16234 16234] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 67e6f677752cb73b0ee3220c01000000 -[2024-05-15 09:10:14,935 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor 67e6f677752cb73b0ee3220c01000000, job id = 01000000 -[2024-05-15 09:10:14,936 I 16234 16234] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = fd700599cf86ea6f0edb007601000000 -[2024-05-15 09:10:14,936 I 16234 16234] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = fd700599cf86ea6f0edb007601000000 -[2024-05-15 09:10:14,936 I 16234 16234] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = fd700599cf86ea6f0edb007601000000 -[2024-05-15 09:10:14,936 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor fd700599cf86ea6f0edb007601000000, job id = 01000000 -[2024-05-15 09:10:14,937 I 16234 16234] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 9c3a7c392d8606e2a42d587b01000000 -[2024-05-15 09:10:14,937 I 16234 16234] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 9c3a7c392d8606e2a42d587b01000000 -[2024-05-15 09:10:14,937 I 16234 16234] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 9c3a7c392d8606e2a42d587b01000000 -[2024-05-15 09:10:14,937 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor 9c3a7c392d8606e2a42d587b01000000, job id = 01000000 -[2024-05-15 09:10:14,938 I 16234 16234] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = a4a5c26f92485fcacf43848901000000 -[2024-05-15 09:10:14,938 I 16234 16234] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = a4a5c26f92485fcacf43848901000000 -[2024-05-15 09:10:14,938 I 16234 16234] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = a4a5c26f92485fcacf43848901000000 -[2024-05-15 09:10:14,938 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor a4a5c26f92485fcacf43848901000000, job id = 01000000 -[2024-05-15 09:10:15,369 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor 67e6f677752cb73b0ee3220c01000000, job id = 01000000 -[2024-05-15 09:10:15,369 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 67e6f677752cb73b0ee3220c01000000 on worker 37b8d8ce9abd2b92e3cb7714083624ac186596bb5823a73b466595c6 at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, job id = 01000000 -[2024-05-15 09:10:15,370 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor fd700599cf86ea6f0edb007601000000, job id = 01000000 -[2024-05-15 09:10:15,370 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor fd700599cf86ea6f0edb007601000000 on worker 549f23b83d9019be85c71b8d27fda4f2212a81ee5a1202b5dbdea36a at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, job id = 01000000 -[2024-05-15 09:10:15,374 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor a4a5c26f92485fcacf43848901000000, job id = 01000000 -[2024-05-15 09:10:15,374 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor a4a5c26f92485fcacf43848901000000 on worker b057bb8fb21bee7fe002c4f0b754a20f76baaec5a6edf06ab1b16a58 at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, job id = 01000000 -[2024-05-15 09:10:15,377 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor 9c3a7c392d8606e2a42d587b01000000, job id = 01000000 -[2024-05-15 09:10:15,377 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 9c3a7c392d8606e2a42d587b01000000 on worker 63e25554b24d91bc63225717ce4fc19fe1d038b7d8b010c111f16372 at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, job id = 01000000 -[2024-05-15 09:10:20,645 W 16234 16255] (gcs_server) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:58358: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 09:10:33,538 I 16234 16234] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 76dc2a362a04e1f8850a8a8d01000000, job id = 01000000 -[2024-05-15 09:10:33,538 I 16234 16234] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 76dc2a362a04e1f8850a8a8d01000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 09:10:33,538 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:607: Cancelling all committed bundles of a placement group, id is 64b1cbad44c9258a4011f30ebaf501000000 -[2024-05-15 09:10:33,538 I 16234 16234] (gcs_server) gcs_placement_group_manager.cc:458: Placement group of an id, 64b1cbad44c9258a4011f30ebaf501000000 is removed successfully. -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_manager.cc:1167: Actor 67e6f677752cb73b0ee3220c01000000 is failed on worker 37b8d8ce9abd2b92e3cb7714083624ac186596bb5823a73b466595c6 at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000 -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 for actor 67e6f677752cb73b0ee3220c01000000, job id = 01000000 -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 67e6f677752cb73b0ee3220c01000000, job id = 01000000 -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 67e6f677752cb73b0ee3220c01000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = a4a5c26f92485fcacf43848901000000, job id = 01000000 -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = a4a5c26f92485fcacf43848901000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 9c3a7c392d8606e2a42d587b01000000, job id = 01000000 -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 9c3a7c392d8606e2a42d587b01000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 09:10:33,539 I 16234 16234] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = fd700599cf86ea6f0edb007601000000, job id = 01000000 -[2024-05-15 09:10:33,540 I 16234 16234] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = fd700599cf86ea6f0edb007601000000, status = SchedulingCancelled: Actor creation cancelled. -[2024-05-15 09:10:33,540 I 16234 16234] (gcs_server) gcs_actor_manager.cc:798: Actor 76dc2a362a04e1f8850a8a8d01000000 is out of scope, destroying actor, job id = 01000000 -[2024-05-15 09:10:33,540 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={1} at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:33,540 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={0} at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:33,540 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={2} at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:33,540 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={4} at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:33,540 I 16234 16234] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={64b1cbad44c9258a4011f30ebaf501000000}, bundle index={3} at node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:33,582 I 16234 16234] (gcs_server) gcs_actor_manager.cc:794: Worker 0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47 failed, destroying actor child, job id = 01000000 -[2024-05-15 09:10:33,582 I 16234 16234] (gcs_server) gcs_actor_manager.cc:794: Worker 0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47 failed, destroying actor child, job id = 01000000 -[2024-05-15 09:10:33,582 I 16234 16234] (gcs_server) gcs_actor_manager.cc:794: Worker 0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47 failed, destroying actor child, job id = 01000000 -[2024-05-15 09:10:33,582 I 16234 16234] (gcs_server) gcs_actor_manager.cc:794: Worker 0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47 failed, destroying actor child, job id = 01000000 -[2024-05-15 09:10:34,047 I 16234 16234] (gcs_server) gcs_job_manager.cc:87: Finished marking job state, job id = 01000000 -[2024-05-15 09:10:34,059 I 16234 16234] (gcs_server) gcs_node_manager.cc:128: Shutting down raylet, node id = 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:34,059 I 16234 16234] (gcs_server) gcs_node_manager.cc:252: Removing node, node id = 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, node name = 192.168.1.56 -[2024-05-15 09:10:34,059 I 16234 16234] (gcs_server) gcs_placement_group_manager.cc:767: Node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 failed, rescheduling the placement groups on the dead node. -[2024-05-15 09:10:34,059 I 16234 16234] (gcs_server) gcs_actor_manager.cc:1048: Node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 failed, reconstructing actors. -[2024-05-15 09:10:34,059 I 16234 16234] (gcs_server) gcs_job_manager.cc:302: Node 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 failed, mark all jobs from this node as finished -[2024-05-15 09:10:34,146 I 16234 16234] (gcs_server) gcs_node_manager.cc:170: Raylet 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 is drained. Status GrpcUnavailable: RPC Error message: Cancelling all calls; RPC Error details: . The information will be published to the cluster. -[2024-05-15 09:10:34,158 I 16234 16282] (gcs_server) ray_syncer-inl.h:318: Failed to read the message from: 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:34,172 I 16234 16234] (gcs_server) gcs_server_main.cc:111: GCS server received SIGTERM, shutting down... -[2024-05-15 09:10:34,172 I 16234 16234] (gcs_server) gcs_server.cc:272: Stopping GCS server. -[2024-05-15 09:10:34,177 I 16234 16234] (gcs_server) gcs_server.cc:292: GCS server stopped. -[2024-05-15 09:10:34,177 I 16234 16234] (gcs_server) io_service_pool.cc:47: IOServicePool is stopped. -[2024-05-15 09:10:34,253 I 16234 16234] (gcs_server) stats.h:120: Stats module has shutdown. diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/log_monitor.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/log_monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/monitor.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/monitor.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/monitor.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/raylet.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/raylet.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/raylet.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/raylet.out deleted file mode 100644 index 0ac5aa4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/raylet.out +++ /dev/null @@ -1,255 +0,0 @@ -[2024-05-15 09:10:12,023 I 16364 16364] (raylet) main.cc:179: Setting cluster ID to: 4554569021e90e46163cb999ab6255921f2f493c80bfefaa5084d621 -[2024-05-15 09:10:12,025 I 16364 16364] (raylet) main.cc:242: Raylet is not set to kill unknown children. -[2024-05-15 09:10:12,025 I 16364 16364] (raylet) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 09:10:12,025 I 16364 16364] (raylet) main.cc:371: Setting node ID to: 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:12,025 I 16364 16364] (raylet) store_runner.cc:32: Allowing the Plasma store to use up to 18.4451GB of memory. -[2024-05-15 09:10:12,025 I 16364 16364] (raylet) store_runner.cc:48: Starting object store with directory /dev/shm, fallback /home/valapeos/Projects/pneumarl/logs/rllib, and huge page support disabled -[2024-05-15 09:10:12,025 I 16364 16392] (raylet) dlmalloc.cc:154: create_and_mmap_buffer(18445107208, /dev/shm/plasmaXXXXXX) -[2024-05-15 09:10:12,026 I 16364 16392] (raylet) store.cc:564: ========== Plasma store: ================= -Current usage: 0 / 18.4451 GB -- num bytes created total: 0 -0 pending objects of total size 0MB -- objects spillable: 0 -- bytes spillable: 0 -- objects unsealed: 0 -- bytes unsealed: 0 -- objects in use: 0 -- bytes in use: 0 -- objects evictable: 0 -- bytes evictable: 0 - -- objects created by worker: 0 -- bytes created by worker: 0 -- objects restored: 0 -- bytes restored: 0 -- objects received: 0 -- bytes received: 0 -- objects errored: 0 -- bytes errored: 0 - -[2024-05-15 09:10:12,027 I 16364 16364] (raylet) grpc_server.cc:134: ObjectManager server started, listening on port 38991. -[2024-05-15 09:10:12,029 I 16364 16364] (raylet) worker_killing_policy.cc:101: Running GroupByOwner policy. -[2024-05-15 09:10:12,029 W 16364 16364] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:10:12,029 I 16364 16364] (raylet) memory_monitor.cc:47: MemoryMonitor initialized with usage threshold at 63982133248 bytes (0.95 system memory), total system memory bytes: 67349614592 -[2024-05-15 09:10:12,029 I 16364 16364] (raylet) node_manager.cc:285: Initializing NodeManager with ID 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:12,029 I 16364 16364] (raylet) grpc_server.cc:134: NodeManager server started, listening on port 36141. -[2024-05-15 09:10:12,034 I 16364 16426] (raylet) agent_manager.cc:78: Monitor agent process with name dashboard_agent/424238335 -[2024-05-15 09:10:12,036 I 16364 16428] (raylet) agent_manager.cc:78: Monitor agent process with name runtime_env_agent -[2024-05-15 09:10:12,036 I 16364 16364] (raylet) event.cc:234: Set ray event level to warning -[2024-05-15 09:10:12,036 I 16364 16364] (raylet) event.cc:342: Ray Event initialized for RAYLET -[2024-05-15 09:10:12,037 I 16364 16364] (raylet) raylet.cc:128: Raylet of id, 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 started. Raylet consists of node_manager and object_manager. node_manager address: 192.168.1.56:36141 object_manager address: 192.168.1.56:38991 hostname: valanixos -[2024-05-15 09:10:12,038 I 16364 16364] (raylet) node_manager.cc:521: [state-dump] NodeManager: -[state-dump] Node ID: 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[state-dump] Node name: 192.168.1.56 -[state-dump] InitialConfigResources: {memory: 368901046280000, node:__internal_head__: 10000, node:192.168.1.56: 10000, CPU: 120000, object_store_memory: 184450523130000} -[state-dump] ClusterTaskManager: -[state-dump] ========== Node: 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 ================= -[state-dump] Infeasible queue length: 0 -[state-dump] Schedule queue length: 0 -[state-dump] Dispatch queue length: 0 -[state-dump] num_waiting_for_resource: 0 -[state-dump] num_waiting_for_plasma_memory: 0 -[state-dump] num_waiting_for_remote_node_resources: 0 -[state-dump] num_worker_not_started_by_job_config_not_exist: 0 -[state-dump] num_worker_not_started_by_registration_timeout: 0 -[state-dump] num_tasks_waiting_for_workers: 0 -[state-dump] num_cancelled_tasks: 0 -[state-dump] cluster_resource_scheduler state: -[state-dump] Local id: -3088399737359427104 Local resources: {"total":{node:192.168.1.56: [10000], node:__internal_head__: [10000], CPU: [120000], memory: [368901046280000], object_store_memory: [184450523130000]}}, "available": {node:192.168.1.56: [10000], node:__internal_head__: [10000], CPU: [120000], memory: [368901046280000], object_store_memory: [184450523130000]}}, "labels":{"ray.io/node_id":"08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50",} is_draining: 0 is_idle: 1 Cluster resources: node id: -3088399737359427104{"total":{CPU: 120000, memory: 368901046280000, node:192.168.1.56: 10000, node:__internal_head__: 10000, object_store_memory: 184450523130000}}, "available": {CPU: 120000, memory: 368901046280000, node:192.168.1.56: 10000, node:__internal_head__: 10000, object_store_memory: 184450523130000}}, "labels":{"ray.io/node_id":"08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -[state-dump] Waiting tasks size: 0 -[state-dump] Number of executing tasks: 0 -[state-dump] Number of pinned task arguments: 0 -[state-dump] Number of total spilled tasks: 0 -[state-dump] Number of spilled waiting tasks: 0 -[state-dump] Number of spilled unschedulable tasks: 0 -[state-dump] Resource usage { -[state-dump] } -[state-dump] Running tasks by scheduling class: -[state-dump] ================================================== -[state-dump] -[state-dump] ClusterResources: -[state-dump] LocalObjectManager: -[state-dump] - num pinned objects: 0 -[state-dump] - pinned objects size: 0 -[state-dump] - num objects pending restore: 0 -[state-dump] - num objects pending spill: 0 -[state-dump] - num bytes pending spill: 0 -[state-dump] - num bytes currently spilled: 0 -[state-dump] - cumulative spill requests: 0 -[state-dump] - cumulative restore requests: 0 -[state-dump] - spilled objects pending delete: 0 -[state-dump] -[state-dump] ObjectManager: -[state-dump] - num local objects: 0 -[state-dump] - num unfulfilled push requests: 0 -[state-dump] - num object pull requests: 0 -[state-dump] - num chunks received total: 0 -[state-dump] - num chunks received failed (all): 0 -[state-dump] - num chunks received failed / cancelled: 0 -[state-dump] - num chunks received failed / plasma error: 0 -[state-dump] Event stats: -[state-dump] Global stats: 0 total (0 active) -[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] Execution time: mean = -nan s, total = 0.000 s -[state-dump] Event stats: -[state-dump] PushManager: -[state-dump] - num pushes in flight: 0 -[state-dump] - num chunks in flight: 0 -[state-dump] - num chunks remaining: 0 -[state-dump] - max chunks allowed: 409 -[state-dump] OwnershipBasedObjectDirectory: -[state-dump] - num listeners: 0 -[state-dump] - cumulative location updates: 0 -[state-dump] - num location updates per second: 0.000 -[state-dump] - num location lookups per second: 0.000 -[state-dump] - num locations added per second: 0.000 -[state-dump] - num locations removed per second: 0.000 -[state-dump] BufferPool: -[state-dump] - create buffer state map size: 0 -[state-dump] PullManager: -[state-dump] - num bytes available for pulled objects: 18445052313 -[state-dump] - num bytes being pulled (all): 0 -[state-dump] - num bytes being pulled / pinned: 0 -[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - first get request bundle: N/A -[state-dump] - first wait request bundle: N/A -[state-dump] - first task request bundle: N/A -[state-dump] - num objects queued: 0 -[state-dump] - num objects actively pulled (all): 0 -[state-dump] - num objects actively pulled / pinned: 0 -[state-dump] - num bundles being pulled: 0 -[state-dump] - num pull retries: 0 -[state-dump] - max timeout seconds: 0 -[state-dump] - max timeout request is already processed. No entry. -[state-dump] -[state-dump] WorkerPool: -[state-dump] - registered jobs: 0 -[state-dump] - process_failed_job_config_missing: 0 -[state-dump] - process_failed_rate_limited: 0 -[state-dump] - process_failed_pending_registration: 0 -[state-dump] - process_failed_runtime_env_setup_failed: 0 -[state-dump] - num PYTHON workers: 0 -[state-dump] - num PYTHON drivers: 0 -[state-dump] - num object spill callbacks queued: 0 -[state-dump] - num object restore queued: 0 -[state-dump] - num util functions queued: 0 -[state-dump] - num idle workers: 0 -[state-dump] TaskDependencyManager: -[state-dump] - task deps map size: 0 -[state-dump] - get req map size: 0 -[state-dump] - wait req map size: 0 -[state-dump] - local objects map size: 0 -[state-dump] WaitManager: -[state-dump] - num active wait requests: 0 -[state-dump] Subscriber: -[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_OBJECT_EVICTION -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_REF_REMOVED_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] num async plasma notifications: 0 -[state-dump] Remote node managers: -[state-dump] Event stats: -[state-dump] Global stats: 27 total (13 active) -[state-dump] Queueing time: mean = 978.578 us, max = 7.543 ms, min = 7.020 us, total = 26.422 ms -[state-dump] Execution time: mean = 509.651 us, total = 13.761 ms -[state-dump] Event stats: -[state-dump] PeriodicalRunner.RunFnPeriodically - 11 total (2 active, 1 running), Execution time: mean = 92.548 us, total = 1.018 ms, Queueing time: mean = 2.401 ms, max = 7.543 ms, min = 22.120 us, total = 26.407 ms -[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 401.647 us, total = 401.647 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 175.629 us, total = 175.629 us, Queueing time: mean = 7.760 us, max = 7.760 us, min = 7.760 us, total = 7.760 us -[state-dump] NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 542.017 us, total = 542.017 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 11.326 ms, total = 11.326 ms, Queueing time: mean = 7.020 us, max = 7.020 us, min = 7.020 us, total = 7.020 us -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 297.038 us, total = 297.038 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.debug_state_dump - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.record_metrics - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.flush_free_objects - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] DebugString() time ms: 0 -[state-dump] -[state-dump] -[2024-05-15 09:10:12,038 I 16364 16364] (raylet) accessor.cc:627: Received notification for node id = 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50, IsAlive = 1 -[2024-05-15 09:10:12,143 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16446, the token is 0 -[2024-05-15 09:10:12,145 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16447, the token is 1 -[2024-05-15 09:10:12,147 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16448, the token is 2 -[2024-05-15 09:10:12,149 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16449, the token is 3 -[2024-05-15 09:10:12,151 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16450, the token is 4 -[2024-05-15 09:10:12,153 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16451, the token is 5 -[2024-05-15 09:10:12,155 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16452, the token is 6 -[2024-05-15 09:10:12,157 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16453, the token is 7 -[2024-05-15 09:10:12,159 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16454, the token is 8 -[2024-05-15 09:10:12,164 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16455, the token is 9 -[2024-05-15 09:10:12,167 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16456, the token is 10 -[2024-05-15 09:10:12,173 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 16457, the token is 11 -[2024-05-15 09:10:12,507 I 16364 16392] (raylet) object_store.cc:35: Object store current usage 8e-09 / 18.4451 GB. -[2024-05-15 09:10:12,684 I 16364 16364] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 16182 is dead: 0 driver address: 192.168.1.56 -[2024-05-15 09:10:12,684 I 16364 16364] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 09:10:12,904 I 16364 16364] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:10:12,906 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 17186, the token is 12 -[2024-05-15 09:10:13,643 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:13,644 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:13,844 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:13,845 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:13,845 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:14,937 I 16364 16364] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:10:14,939 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 17258, the token is 13 -[2024-05-15 09:10:14,940 I 16364 16364] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:10:14,942 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 17259, the token is 14 -[2024-05-15 09:10:14,942 I 16364 16364] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:10:14,944 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 17260, the token is 15 -[2024-05-15 09:10:14,944 I 16364 16364] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:10:14,947 I 16364 16364] (raylet) worker_pool.cc:494: Started worker process with pid 17261, the token is 16 -[2024-05-15 09:10:17,049 W 16364 16364] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:10:22,036 W 16364 16386] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:58358: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 09:10:22,051 W 16364 16364] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:10:27,053 W 16364 16364] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:10:32,055 W 16364 16364] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:10:33,538 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:33,539 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:33,539 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:33,539 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:33,539 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:10:33,539 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false -[2024-05-15 09:10:33,540 I 16364 16364] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected. -[2024-05-15 09:10:34,047 I 16364 16364] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=3, has creation task exception = false -[2024-05-15 09:10:34,047 I 16364 16364] (raylet) node_manager.cc:1541: Driver (pid=16182) is disconnected. job_id: 01000000 -[2024-05-15 09:10:34,048 I 16364 16364] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 16182 is dead: 1 driver address: 192.168.1.56 -[2024-05-15 09:10:34,048 I 16364 16364] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 09:10:34,059 I 16364 16364] (raylet) main.cc:413: Raylet received SIGTERM, shutting down... -[2024-05-15 09:10:34,059 I 16364 16364] (raylet) accessor.cc:451: Unregistering node info, node id = 08a523a8e3ba0d76e6f2575ecdc29fe986478ad86693866a248f5e50 -[2024-05-15 09:10:34,059 I 16364 16364] (raylet) agent_manager.cc:109: Killing agent dashboard_agent/424238335, pid 16423. -[2024-05-15 09:10:34,065 I 16364 16426] (raylet) agent_manager.cc:80: Agent process with name dashboard_agent/424238335 exited, exit code 0. -[2024-05-15 09:10:34,065 I 16364 16364] (raylet) agent_manager.cc:109: Killing agent runtime_env_agent, pid 16427. -[2024-05-15 09:10:34,070 I 16364 16428] (raylet) agent_manager.cc:80: Agent process with name runtime_env_agent exited, exit code 0. -[2024-05-15 09:10:34,070 I 16364 16364] (raylet) io_service_pool.cc:47: IOServicePool is stopped. -[2024-05-15 09:10:34,143 I 16364 16364] (raylet) stats.h:120: Stats module has shutdown. -[2024-05-15 09:10:34,143 I 16364 16430] (raylet) experimental_mutable_object_provider.cc:200: Core worker channel io service stopped. diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/runtime_env_agent.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/runtime_env_agent.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/runtime_env_agent.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/runtime_env_agent.out deleted file mode 100644 index 676b7de..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/runtime_env_agent.out +++ /dev/null @@ -1,2 +0,0 @@ -======== Running on http://0.0.0.0:58554 ======== -(Press CTRL+C to quit) diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47-01000000-17186.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47-01000000-17186.err deleted file mode 100644 index a8ac8cb..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47-01000000-17186.err +++ /dev/null @@ -1,5 +0,0 @@ -:job_id:01000000 -:actor_name:PPO -2024-05-15 09:10:14,883 WARNING deprecation.py:50 -- DeprecationWarning: `AlgorithmConfig.num_envs_per_worker` has been deprecated. Use `AlgorithmConfig.num_envs_per_env_runner` instead. This will raise an error in the future! -2024-05-15 09:10:14,889 WARNING deprecation.py:50 -- DeprecationWarning: `WorkerSet(num_workers=... OR local_worker=...)` has been deprecated. Use `EnvRunnerGroup(num_env_runners=... AND local_env_runner=...)` instead. This will raise an error in the future! -2024-05-15 09:10:14,889 WARNING deprecation.py:50 -- DeprecationWarning: `max_num_worker_restarts` has been deprecated. Use `AlgorithmConfig.max_num_env_runner_restarts` instead. This will raise an error in the future! diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47-01000000-17186.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47-01000000-17186.out deleted file mode 100644 index 1124c85..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-0dde64b2aab2ddbc119ec17794a2c3e87160c7e81cc046bb6809fa47-01000000-17186.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:actor_name:PPO diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-18030d82e7dcf57332ad8b982f33ea2be3456e927ff3b646282302e1-ffffffff-16453.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-18030d82e7dcf57332ad8b982f33ea2be3456e927ff3b646282302e1-ffffffff-16453.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-18030d82e7dcf57332ad8b982f33ea2be3456e927ff3b646282302e1-ffffffff-16453.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-18030d82e7dcf57332ad8b982f33ea2be3456e927ff3b646282302e1-ffffffff-16453.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-1c34556010e5dc86b36b2a8adb29d2fbce107f63a32e13234dd8d31b-ffffffff-16446.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-1c34556010e5dc86b36b2a8adb29d2fbce107f63a32e13234dd8d31b-ffffffff-16446.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-1c34556010e5dc86b36b2a8adb29d2fbce107f63a32e13234dd8d31b-ffffffff-16446.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-1c34556010e5dc86b36b2a8adb29d2fbce107f63a32e13234dd8d31b-ffffffff-16446.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-2848e978174064f8adaa82eb878ee371185a397a835d20c7e7e65dca-ffffffff-16449.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-2848e978174064f8adaa82eb878ee371185a397a835d20c7e7e65dca-ffffffff-16449.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-2848e978174064f8adaa82eb878ee371185a397a835d20c7e7e65dca-ffffffff-16449.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-2848e978174064f8adaa82eb878ee371185a397a835d20c7e7e65dca-ffffffff-16449.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-315337587e61b4335c8e853c14c09ff28051fd2ac34a9402fe9d1a2f-ffffffff-16457.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-315337587e61b4335c8e853c14c09ff28051fd2ac34a9402fe9d1a2f-ffffffff-16457.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-315337587e61b4335c8e853c14c09ff28051fd2ac34a9402fe9d1a2f-ffffffff-16457.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-315337587e61b4335c8e853c14c09ff28051fd2ac34a9402fe9d1a2f-ffffffff-16457.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-37b8d8ce9abd2b92e3cb7714083624ac186596bb5823a73b466595c6-01000000-17258.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-37b8d8ce9abd2b92e3cb7714083624ac186596bb5823a73b466595c6-01000000-17258.err deleted file mode 100644 index 7a6ed25..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-37b8d8ce9abd2b92e3cb7714083624ac186596bb5823a73b466595c6-01000000-17258.err +++ /dev/null @@ -1,68 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) -[2024-05-15 09:10:33,590 C 17258 17258] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe -*** StackTrace Information *** -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7fd2f5e0dbba] ray::operator<<() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7fd2f5e0f477] ray::SpdLogMessage::Flush() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7fd2f5e0f917] ray::RayLog::~RayLog() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7fd2f55d2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7fd2f55d319a] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7fd2f55ea0de] ray::core::InboundRequest::Accept() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7fd2f55bc440] ray::core::NormalSchedulingQueue::ScheduleRequests() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7fd2f584e71e] EventTracker::RecordExecution() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7fd2f5847b0e] std::_Function_handler<>::_M_invoke() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7fd2f5847f86] boost::asio::detail::completion_handler<>::do_complete() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7fd2f5ef78eb] boost::asio::detail::scheduler::do_run_one() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7fd2f5ef9269] boost::asio::detail::scheduler::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7fd2f5ef9972] boost::asio::io_context::run() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7fd2f55713fd] ray::core::CoreWorker::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7fd2f55b5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7fd2f55b60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop() -/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7fd2f53bd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop() -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7fd2f72413c2] method_vectorcall_NOARGS -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7fd2f725ad93] PyObject_Vectorcall -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7fd2f70f7c5c] _PyEval_EvalFrameDefault -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7fd2f732bc9c] _PyEval_Vector.constprop.0 -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7fd2f732be3a] PyEval_EvalCode -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7fd2f7356a10] run_mod -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7fd2f7376752] _PyRun_SimpleFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7fd2f7376ff1] _PyRun_AnyFileObject -/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7fd2f737990f] Py_RunMain -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7fd2f6e3d10e] __libc_start_call_main -/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7fd2f6e3d1c9] __libc_start_main@GLIBC_2.2.5 -ray::RolloutWorker(_start+0x25) [0x401075] _start - diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-37b8d8ce9abd2b92e3cb7714083624ac186596bb5823a73b466595c6-01000000-17258.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-37b8d8ce9abd2b92e3cb7714083624ac186596bb5823a73b466595c6-01000000-17258.out deleted file mode 100644 index 6628553..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-37b8d8ce9abd2b92e3cb7714083624ac186596bb5823a73b466595c6-01000000-17258.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11009 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-48466548ee6f27e97ecee263a02ad52900d76875015094becf81d779-ffffffff-16447.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-48466548ee6f27e97ecee263a02ad52900d76875015094becf81d779-ffffffff-16447.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-48466548ee6f27e97ecee263a02ad52900d76875015094becf81d779-ffffffff-16447.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-48466548ee6f27e97ecee263a02ad52900d76875015094becf81d779-ffffffff-16447.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-549f23b83d9019be85c71b8d27fda4f2212a81ee5a1202b5dbdea36a-01000000-17259.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-549f23b83d9019be85c71b8d27fda4f2212a81ee5a1202b5dbdea36a-01000000-17259.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-549f23b83d9019be85c71b8d27fda4f2212a81ee5a1202b5dbdea36a-01000000-17259.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-549f23b83d9019be85c71b8d27fda4f2212a81ee5a1202b5dbdea36a-01000000-17259.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-549f23b83d9019be85c71b8d27fda4f2212a81ee5a1202b5dbdea36a-01000000-17259.out deleted file mode 100644 index 80b5e41..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-549f23b83d9019be85c71b8d27fda4f2212a81ee5a1202b5dbdea36a-01000000-17259.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11010 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-616cf4ccca96bffec0abccbb05d160f6e73f9be3a3d48738e78dd924-ffffffff-16455.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-616cf4ccca96bffec0abccbb05d160f6e73f9be3a3d48738e78dd924-ffffffff-16455.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-616cf4ccca96bffec0abccbb05d160f6e73f9be3a3d48738e78dd924-ffffffff-16455.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-616cf4ccca96bffec0abccbb05d160f6e73f9be3a3d48738e78dd924-ffffffff-16455.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-63e25554b24d91bc63225717ce4fc19fe1d038b7d8b010c111f16372-01000000-17260.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-63e25554b24d91bc63225717ce4fc19fe1d038b7d8b010c111f16372-01000000-17260.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-63e25554b24d91bc63225717ce4fc19fe1d038b7d8b010c111f16372-01000000-17260.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-63e25554b24d91bc63225717ce4fc19fe1d038b7d8b010c111f16372-01000000-17260.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-63e25554b24d91bc63225717ce4fc19fe1d038b7d8b010c111f16372-01000000-17260.out deleted file mode 100644 index 01371fe..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-63e25554b24d91bc63225717ce4fc19fe1d038b7d8b010c111f16372-01000000-17260.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11011 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-7e837560f39eef4e5b4e2f169edf869152f6a81cfbde0c5341c5901f-ffffffff-16451.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-7e837560f39eef4e5b4e2f169edf869152f6a81cfbde0c5341c5901f-ffffffff-16451.err deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-7e837560f39eef4e5b4e2f169edf869152f6a81cfbde0c5341c5901f-ffffffff-16451.err +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-7e837560f39eef4e5b4e2f169edf869152f6a81cfbde0c5341c5901f-ffffffff-16451.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-7e837560f39eef4e5b4e2f169edf869152f6a81cfbde0c5341c5901f-ffffffff-16451.out deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-7e837560f39eef4e5b4e2f169edf869152f6a81cfbde0c5341c5901f-ffffffff-16451.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-85f07180421c47d0d52b2f00126c071493d95e7446a9f7d173344be0-ffffffff-16452.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-85f07180421c47d0d52b2f00126c071493d95e7446a9f7d173344be0-ffffffff-16452.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-85f07180421c47d0d52b2f00126c071493d95e7446a9f7d173344be0-ffffffff-16452.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-85f07180421c47d0d52b2f00126c071493d95e7446a9f7d173344be0-ffffffff-16452.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-9f60c1bf79716be47586824888aa6e6ed798dc16d8612b9822c0327f-ffffffff-16450.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-9f60c1bf79716be47586824888aa6e6ed798dc16d8612b9822c0327f-ffffffff-16450.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-9f60c1bf79716be47586824888aa6e6ed798dc16d8612b9822c0327f-ffffffff-16450.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-9f60c1bf79716be47586824888aa6e6ed798dc16d8612b9822c0327f-ffffffff-16450.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-ac37600498ac6c14520e9c642e0573de3f60fdae7f90398d10866019-ffffffff-16456.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-ac37600498ac6c14520e9c642e0573de3f60fdae7f90398d10866019-ffffffff-16456.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-ac37600498ac6c14520e9c642e0573de3f60fdae7f90398d10866019-ffffffff-16456.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-ac37600498ac6c14520e9c642e0573de3f60fdae7f90398d10866019-ffffffff-16456.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-adde8d577bf582bc4b024ca5f9682984d2a9ad5f29e206e71a240cb0-ffffffff-16448.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-adde8d577bf582bc4b024ca5f9682984d2a9ad5f29e206e71a240cb0-ffffffff-16448.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-adde8d577bf582bc4b024ca5f9682984d2a9ad5f29e206e71a240cb0-ffffffff-16448.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-adde8d577bf582bc4b024ca5f9682984d2a9ad5f29e206e71a240cb0-ffffffff-16448.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-b057bb8fb21bee7fe002c4f0b754a20f76baaec5a6edf06ab1b16a58-01000000-17261.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-b057bb8fb21bee7fe002c4f0b754a20f76baaec5a6edf06ab1b16a58-01000000-17261.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-b057bb8fb21bee7fe002c4f0b754a20f76baaec5a6edf06ab1b16a58-01000000-17261.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-b057bb8fb21bee7fe002c4f0b754a20f76baaec5a6edf06ab1b16a58-01000000-17261.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-b057bb8fb21bee7fe002c4f0b754a20f76baaec5a6edf06ab1b16a58-01000000-17261.out deleted file mode 100644 index 9587b73..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-b057bb8fb21bee7fe002c4f0b754a20f76baaec5a6edf06ab1b16a58-01000000-17261.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11012 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-ef36a4325f3dd9afb1b0385af8324dc907f066b34f065ab5fcf72aaa-ffffffff-16454.err b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-ef36a4325f3dd9afb1b0385af8324dc907f066b34f065ab5fcf72aaa-ffffffff-16454.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-ef36a4325f3dd9afb1b0385af8324dc907f066b34f065ab5fcf72aaa-ffffffff-16454.out b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/logs/worker-ef36a4325f3dd9afb1b0385af8324dc907f066b34f065ab5fcf72aaa-ffffffff-16454.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/node_ip_address.json b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/node_ip_address.json deleted file mode 100644 index 8d998d8..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/node_ip_address.json +++ /dev/null @@ -1 +0,0 @@ -{"node_ip_address": "192.168.1.56"} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/node_ip_address.json.lock b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/node_ip_address.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/ports_by_node.json b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/ports_by_node.json deleted file mode 100644 index 4838a99..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/ports_by_node.json +++ /dev/null @@ -1 +0,0 @@ -{"192.168.1.56:/home/valapeos/Projects/pneumarl/logs/rllib/session_2024-05-15_09-10-10_617865_16182/sockets/plasma_store": {"metrics_agent_port": 58358, "metrics_export_port": 55563, "dashboard_agent_listen_port": 52365, "runtime_env_agent_port": 58554, "gcs_server_port": 59971}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/ports_by_node.json.lock b/Godot/logs/rllib/session_2024-05-15_09-10-10_617865_16182/ports_by_node.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/artifacts/2024-05-15_09-19-31/PPO_2024-05-15_09-19-31/driver_artifacts/basic-variant-state-2024-05-15_09-19-31.json b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/artifacts/2024-05-15_09-19-31/PPO_2024-05-15_09-19-31/driver_artifacts/basic-variant-state-2024-05-15_09-19-31.json deleted file mode 100644 index 536e3bb..0000000 Binary files a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/artifacts/2024-05-15_09-19-31/PPO_2024-05-15_09-19-31/driver_artifacts/basic-variant-state-2024-05-15_09-19-31.json and /dev/null differ diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/artifacts/2024-05-15_09-19-31/PPO_2024-05-15_09-19-31/driver_artifacts/experiment_state-2024-05-15_09-19-31.json b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/artifacts/2024-05-15_09-19-31/PPO_2024-05-15_09-19-31/driver_artifacts/experiment_state-2024-05-15_09-19-31.json deleted file mode 100644 index 4bda095..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/artifacts/2024-05-15_09-19-31/PPO_2024-05-15_09-19-31/driver_artifacts/experiment_state-2024-05-15_09-19-31.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"79228_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30392d31392d3331948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f37393232385f30303030305f305f323032342d30352d31355f30392d31392d3331948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30392d31392d33319475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_79228_00000_0_2024-05-15_09-19-31\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 256, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715757571.3920052, "_session_str": "2024-05-15_09-19-31", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715757571.3920052}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/agent-424238335.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/agent-424238335.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/agent-424238335.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/agent-424238335.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/dashboard.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/dashboard.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/debug_state.txt b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/debug_state.txt deleted file mode 100644 index 2e0bbc4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/debug_state.txt +++ /dev/null @@ -1,208 +0,0 @@ -NodeManager: -Node ID: 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b -Node name: 192.168.1.56 -InitialConfigResources: {object_store_memory: 183210811390000, node:192.168.1.56: 10000, memory: 366421622790000, node:__internal_head__: 10000, CPU: 120000} -ClusterTaskManager: -========== Node: 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b ================= -Infeasible queue length: 0 -Schedule queue length: 0 -Dispatch queue length: 0 -num_waiting_for_resource: 0 -num_waiting_for_plasma_memory: 0 -num_waiting_for_remote_node_resources: 0 -num_worker_not_started_by_job_config_not_exist: 0 -num_worker_not_started_by_registration_timeout: 0 -num_tasks_waiting_for_workers: 0 -num_cancelled_tasks: 0 -cluster_resource_scheduler state: -Local id: 6857835085091698370 Local resources: {"total":{node:__internal_head__: [10000], bundle_group_4_486f798c40a870f1d3f75646414a01000000: [10000000], CPU_group_4_486f798c40a870f1d3f75646414a01000000: [10000], bundle_group_486f798c40a870f1d3f75646414a01000000: [50000000], CPU: [120000], CPU_group_1_486f798c40a870f1d3f75646414a01000000: [10000], memory: [366421622790000], bundle_group_0_486f798c40a870f1d3f75646414a01000000: [10000000], CPU_group_0_486f798c40a870f1d3f75646414a01000000: [10000], CPU_group_486f798c40a870f1d3f75646414a01000000: [50000], bundle_group_1_486f798c40a870f1d3f75646414a01000000: [10000000], CPU_group_2_486f798c40a870f1d3f75646414a01000000: [10000], bundle_group_2_486f798c40a870f1d3f75646414a01000000: [10000000], object_store_memory: [183210811390000], node:192.168.1.56: [10000], bundle_group_3_486f798c40a870f1d3f75646414a01000000: [10000000], CPU_group_3_486f798c40a870f1d3f75646414a01000000: [10000]}}, "available": {node:__internal_head__: [10000], bundle_group_4_486f798c40a870f1d3f75646414a01000000: [10000000], CPU_group_4_486f798c40a870f1d3f75646414a01000000: [10000], bundle_group_486f798c40a870f1d3f75646414a01000000: [49999950], CPU: [70000], CPU_group_1_486f798c40a870f1d3f75646414a01000000: [10000], memory: [366421622790000], bundle_group_0_486f798c40a870f1d3f75646414a01000000: [9999990], CPU_group_0_486f798c40a870f1d3f75646414a01000000: [0], CPU_group_486f798c40a870f1d3f75646414a01000000: [0], bundle_group_1_486f798c40a870f1d3f75646414a01000000: [10000000], CPU_group_2_486f798c40a870f1d3f75646414a01000000: [10000], bundle_group_2_486f798c40a870f1d3f75646414a01000000: [10000000], object_store_memory: [183210811390000], node:192.168.1.56: [10000], bundle_group_3_486f798c40a870f1d3f75646414a01000000: [10000000], CPU_group_3_486f798c40a870f1d3f75646414a01000000: [10000]}}, "labels":{"ray.io/node_id":"322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b",} is_draining: 0 is_idle: 0 Cluster resources: node id: 6857835085091698370{"total":{bundle_group_2_486f798c40a870f1d3f75646414a01000000: 10000000, node:192.168.1.56: 10000, object_store_memory: 183210811390000, bundle_group_3_486f798c40a870f1d3f75646414a01000000: 10000000, CPU_group_3_486f798c40a870f1d3f75646414a01000000: 10000, bundle_group_0_486f798c40a870f1d3f75646414a01000000: 10000000, CPU_group_0_486f798c40a870f1d3f75646414a01000000: 10000, CPU_group_486f798c40a870f1d3f75646414a01000000: 50000, bundle_group_1_486f798c40a870f1d3f75646414a01000000: 10000000, CPU_group_2_486f798c40a870f1d3f75646414a01000000: 10000, CPU: 120000, memory: 366421622790000, CPU_group_1_486f798c40a870f1d3f75646414a01000000: 10000, node:__internal_head__: 10000, bundle_group_4_486f798c40a870f1d3f75646414a01000000: 10000000, CPU_group_4_486f798c40a870f1d3f75646414a01000000: 10000, bundle_group_486f798c40a870f1d3f75646414a01000000: 50000000}}, "available": {bundle_group_2_486f798c40a870f1d3f75646414a01000000: 10000000, bundle_group_486f798c40a870f1d3f75646414a01000000: 49999950, CPU_group_4_486f798c40a870f1d3f75646414a01000000: 10000, bundle_group_4_486f798c40a870f1d3f75646414a01000000: 10000000, node:__internal_head__: 10000, memory: 366421622790000, CPU_group_1_486f798c40a870f1d3f75646414a01000000: 10000, CPU: 70000, bundle_group_1_486f798c40a870f1d3f75646414a01000000: 10000000, CPU_group_2_486f798c40a870f1d3f75646414a01000000: 10000, bundle_group_0_486f798c40a870f1d3f75646414a01000000: 9999990, CPU_group_3_486f798c40a870f1d3f75646414a01000000: 10000, object_store_memory: 183210811390000, bundle_group_3_486f798c40a870f1d3f75646414a01000000: 10000000, node:192.168.1.56: 10000}}, "labels":{"ray.io/node_id":"322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -Waiting tasks size: 0 -Number of executing tasks: 5 -Number of pinned task arguments: 0 -Number of total spilled tasks: 0 -Number of spilled waiting tasks: 0 -Number of spilled unschedulable tasks: 0 -Resource usage { - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=25007): {bundle_group_486f798c40a870f1d3f75646414a01000000: 10, CPU_group_486f798c40a870f1d3f75646414a01000000: 10000} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=25005): {CPU_group_486f798c40a870f1d3f75646414a01000000: 10000, bundle_group_486f798c40a870f1d3f75646414a01000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=25006): {CPU_group_486f798c40a870f1d3f75646414a01000000: 10000, bundle_group_486f798c40a870f1d3f75646414a01000000: 10} - - (language=PYTHON actor_or_task=PPO.__init__ pid=24919): {bundle_group_0_486f798c40a870f1d3f75646414a01000000: 10, CPU_group_0_486f798c40a870f1d3f75646414a01000000: 10000, CPU_group_486f798c40a870f1d3f75646414a01000000: 10000, bundle_group_486f798c40a870f1d3f75646414a01000000: 10} - - (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=25008): {bundle_group_486f798c40a870f1d3f75646414a01000000: 10, CPU_group_486f798c40a870f1d3f75646414a01000000: 10000} -} -Running tasks by scheduling class: - - {depth=2 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.evaluation.rollout_worker, class_name=RolloutWorker, function_name=__init__, function_hash=b812dba7dd5f4e829bf85572baaaa9ff} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "Hoy\214@\250p\361\323\367VFAJ\001\000\000\000" - placement_group_bundle_index: -1 - placement_group_capture_child_tasks: true -} - resource_set={bundle_group_486f798c40a870f1d3f75646414a01000000 : 0.001, CPU_group_486f798c40a870f1d3f75646414a01000000 : 1, }}: 4/18446744073709551615 - - {depth=1 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.algorithms.ppo.ppo, class_name=PPO, function_name=__init__, function_hash=775f3a1c224747a680a5cbe7774445f3} scheduling_strategy=placement_group_scheduling_strategy { - placement_group_id: "Hoy\214@\250p\361\323\367VFAJ\001\000\000\000" - placement_group_capture_child_tasks: true -} - resource_set={CPU_group_486f798c40a870f1d3f75646414a01000000 : 1, CPU_group_0_486f798c40a870f1d3f75646414a01000000 : 1, bundle_group_0_486f798c40a870f1d3f75646414a01000000 : 0.001, bundle_group_486f798c40a870f1d3f75646414a01000000 : 0.001, }}: 1/18446744073709551615 -================================================== - -ClusterResources: -LocalObjectManager: -- num pinned objects: 0 -- pinned objects size: 0 -- num objects pending restore: 0 -- num objects pending spill: 0 -- num bytes pending spill: 0 -- num bytes currently spilled: 0 -- cumulative spill requests: 0 -- cumulative restore requests: 0 -- spilled objects pending delete: 0 - -ObjectManager: -- num local objects: 0 -- num unfulfilled push requests: 0 -- num object pull requests: 0 -- num chunks received total: 0 -- num chunks received failed (all): 0 -- num chunks received failed / cancelled: 0 -- num chunks received failed / plasma error: 0 -Event stats: -Global stats: 0 total (0 active) -Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -Execution time: mean = -nan s, total = 0.000 s -Event stats: -PushManager: -- num pushes in flight: 0 -- num chunks in flight: 0 -- num chunks remaining: 0 -- max chunks allowed: 409 -OwnershipBasedObjectDirectory: -- num listeners: 0 -- cumulative location updates: 0 -- num location updates per second: 0.000 -- num location lookups per second: 0.000 -- num locations added per second: 0.000 -- num locations removed per second: 0.000 -BufferPool: -- create buffer state map size: 0 -PullManager: -- num bytes available for pulled objects: 18321081139 -- num bytes being pulled (all): 0 -- num bytes being pulled / pinned: 0 -- get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- first get request bundle: N/A -- first wait request bundle: N/A -- first task request bundle: N/A -- num objects queued: 0 -- num objects actively pulled (all): 0 -- num objects actively pulled / pinned: 0 -- num bundles being pulled: 0 -- num pull retries: 0 -- max timeout seconds: 0 -- max timeout request is already processed. No entry. - -WorkerPool: -- registered jobs: 1 -- process_failed_job_config_missing: 0 -- process_failed_rate_limited: 0 -- process_failed_pending_registration: 0 -- process_failed_runtime_env_setup_failed: 0 -- num PYTHON workers: 12 -- num PYTHON drivers: 1 -- num object spill callbacks queued: 0 -- num object restore queued: 0 -- num util functions queued: 0 -- num idle workers: 7 -TaskDependencyManager: -- task deps map size: 0 -- get req map size: 0 -- wait req map size: 0 -- local objects map size: 0 -WaitManager: -- num active wait requests: 0 -Subscriber: -Channel WORKER_OBJECT_EVICTION -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_REF_REMOVED_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_OBJECT_LOCATIONS_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -num async plasma notifications: 0 -Remote node managers: -Event stats: -Global stats: 1759 total (27 active) -Queueing time: mean = 3.722 ms, max = 1.304 s, min = 60.000 ns, total = 6.548 s -Execution time: mean = 2.446 ms, total = 4.302 s -Event stats: - NodeManagerService.grpc_server.ReportWorkerBacklog - 257 total (0 active), Execution time: mean = 105.462 us, total = 27.104 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.ReportWorkerBacklog.HandleRequestImpl - 257 total (0 active), Execution time: mean = 11.392 us, total = 2.928 ms, Queueing time: mean = 13.103 us, max = 43.590 us, min = 5.220 us, total = 3.367 ms - NodeManager.CheckGC - 200 total (1 active), Execution time: mean = 953.750 ns, total = 190.750 us, Queueing time: mean = 22.266 us, max = 259.501 us, min = 5.115 us, total = 4.453 ms - RaySyncer.OnDemandBroadcasting - 200 total (1 active), Execution time: mean = 6.568 us, total = 1.314 ms, Queueing time: mean = 16.998 us, max = 256.821 us, min = 5.975 us, total = 3.400 ms - ObjectManager.UpdateAvailableMemory - 200 total (0 active), Execution time: mean = 1.388 us, total = 277.570 us, Queueing time: mean = 12.168 us, max = 90.500 us, min = 5.480 us, total = 2.434 ms - RayletWorkerPool.deadline_timer.kill_idle_workers - 100 total (1 active), Execution time: mean = 14.638 us, total = 1.464 ms, Queueing time: mean = 18.699 us, max = 409.514 us, min = 4.904 us, total = 1.870 ms - ClientConnection.async_read.ProcessMessageHeader - 83 total (13 active), Execution time: mean = 2.870 us, total = 238.190 us, Queueing time: mean = 78.263 ms, max = 1.304 s, min = 11.370 us, total = 6.496 s - MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 80 total (1 active), Execution time: mean = 98.623 us, total = 7.890 ms, Queueing time: mean = 17.748 us, max = 223.360 us, min = 3.698 us, total = 1.420 ms - ClientConnection.async_read.ProcessMessage - 70 total (0 active), Execution time: mean = 363.087 us, total = 25.416 ms, Queueing time: mean = 9.435 us, max = 338.320 us, min = 1.950 us, total = 660.460 us - NodeManager.ScheduleAndDispatchTasks - 21 total (1 active), Execution time: mean = 4.883 us, total = 102.548 us, Queueing time: mean = 14.413 us, max = 38.182 us, min = 6.274 us, total = 302.674 us - NodeManager.deadline_timer.spill_objects_when_over_threshold - 20 total (1 active), Execution time: mean = 1.093 us, total = 21.860 us, Queueing time: mean = 59.319 us, max = 466.314 us, min = 5.591 us, total = 1.186 ms - NodeManagerService.grpc_server.GetResourceLoad.HandleRequestImpl - 20 total (0 active), Execution time: mean = 56.994 us, total = 1.140 ms, Queueing time: mean = 16.232 us, max = 50.800 us, min = 9.360 us, total = 324.640 us - NodeManager.deadline_timer.flush_free_objects - 20 total (1 active), Execution time: mean = 2.309 us, total = 46.190 us, Queueing time: mean = 58.293 us, max = 467.674 us, min = 5.191 us, total = 1.166 ms - NodeManagerService.grpc_server.GetResourceLoad - 20 total (0 active), Execution time: mean = 166.756 us, total = 3.335 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ClientConnection.async_write.DoAsyncWrites - 19 total (0 active), Execution time: mean = 594.737 ns, total = 11.300 us, Queueing time: mean = 11.335 us, max = 39.560 us, min = 6.460 us, total = 215.370 us - ObjectManager.ObjectAdded - 18 total (0 active), Execution time: mean = 7.137 us, total = 128.460 us, Queueing time: mean = 25.431 us, max = 265.729 us, min = 5.590 us, total = 457.759 us - NodeManagerService.grpc_server.GetSystemConfig.HandleRequestImpl - 18 total (0 active), Execution time: mean = 23.596 us, total = 424.720 us, Queueing time: mean = 12.937 us, max = 38.620 us, min = 7.610 us, total = 232.870 us - ObjectManager.ObjectDeleted - 18 total (0 active), Execution time: mean = 7.710 us, total = 138.780 us, Queueing time: mean = 27.275 us, max = 133.720 us, min = 13.230 us, total = 490.950 us - NodeManagerService.grpc_server.GetSystemConfig - 18 total (0 active), Execution time: mean = 283.507 us, total = 5.103 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 13 total (0 active), Execution time: mean = 90.615 us, total = 1.178 ms, Queueing time: mean = 1.936 ms, max = 6.594 ms, min = 20.180 us, total = 25.162 ms - ClusterResourceManager.ResetRemoteNodeView - 7 total (1 active), Execution time: mean = 1.929 us, total = 13.500 us, Queueing time: mean = 20.530 us, max = 37.382 us, min = 9.586 us, total = 143.710 us - NodeManagerService.grpc_server.RequestWorkerLease.HandleRequestImpl - 6 total (0 active), Execution time: mean = 117.636 us, total = 705.819 us, Queueing time: mean = 241.475 us, max = 772.869 us, min = 7.060 us, total = 1.449 ms - NodeManagerService.grpc_server.RequestWorkerLease - 6 total (0 active), Execution time: mean = 397.816 ms, total = 2.387 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_client.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 1.225 ms, total = 6.123 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - Subscriber.HandlePublishedMessage_GCS_WORKER_DELTA_CHANNEL - 5 total (0 active), Execution time: mean = 1.968 us, total = 9.840 us, Queueing time: mean = 64.544 us, max = 90.480 us, min = 55.670 us, total = 322.720 us - CoreWorkerService.grpc_client.Exit - 5 total (0 active), Execution time: mean = 770.345 us, total = 3.852 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RaySyncer.BroadcastMessage - 5 total (0 active), Execution time: mean = 100.592 us, total = 502.960 us, Queueing time: mean = 170.000 ns, max = 200.000 ns, min = 130.000 ns, total = 850.000 ns - WorkerInfoGcsService.grpc_client.ReportWorkerFailure.OnReplyReceived - 5 total (0 active), Execution time: mean = 11.874 us, total = 59.370 us, Queueing time: mean = 77.160 us, max = 124.110 us, min = 16.800 us, total = 385.800 us - - 5 total (0 active), Execution time: mean = 122.000 ns, total = 610.000 ns, Queueing time: mean = 8.958 us, max = 11.100 us, min = 7.330 us, total = 44.790 us - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 5 total (1 active), Execution time: mean = 361.555 ms, total = 1.808 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.Exit.OnReplyReceived - 5 total (0 active), Execution time: mean = 8.698 us, total = 43.490 us, Queueing time: mean = 12.362 us, max = 31.070 us, min = 5.690 us, total = 61.810 us - NodeInfoGcsService.grpc_client.CheckAlive.OnReplyReceived - 4 total (0 active), Execution time: mean = 16.128 us, total = 64.510 us, Queueing time: mean = 32.270 us, max = 94.140 us, min = 4.490 us, total = 129.080 us - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll.OnReplyReceived - 4 total (0 active), Execution time: mean = 70.730 us, total = 282.920 us, Queueing time: mean = 12.440 us, max = 29.600 us, min = 5.460 us, total = 49.760 us - NodeInfoGcsService.grpc_client.CheckAlive - 4 total (0 active), Execution time: mean = 341.707 us, total = 1.367 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.GcsCheckAlive - 4 total (1 active), Execution time: mean = 95.910 us, total = 383.640 us, Queueing time: mean = 155.923 us, max = 305.146 us, min = 43.668 us, total = 623.694 us - NodeManager.deadline_timer.record_metrics - 4 total (1 active), Execution time: mean = 269.715 us, total = 1.079 ms, Queueing time: mean = 10.479 us, max = 17.690 us, min = 10.608 us, total = 41.915 us - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 2 total (0 active), Execution time: mean = 66.680 us, total = 133.360 us, Queueing time: mean = 506.425 us, max = 942.269 us, min = 70.580 us, total = 1.013 ms - RaySyncerRegister - 2 total (0 active), Execution time: mean = 720.000 ns, total = 1.440 us, Queueing time: mean = 140.000 ns, max = 220.000 ns, min = 60.000 ns, total = 280.000 ns - NodeManager.deadline_timer.debug_state_dump - 2 total (1 active, 1 running), Execution time: mean = 382.860 us, total = 765.720 us, Queueing time: mean = 5.233 us, max = 10.466 us, min = 10.466 us, total = 10.466 us - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 2 total (0 active), Execution time: mean = 216.200 us, total = 432.400 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_client.AddJob.OnReplyReceived - 1 total (0 active), Execution time: mean = 12.650 us, total = 12.650 us, Queueing time: mean = 80.690 us, max = 80.690 us, min = 80.690 us, total = 80.690 us - JobInfoGcsService.grpc_client.GetAllJobInfo - 1 total (0 active), Execution time: mean = 246.559 us, total = 246.559 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 10.163 ms, total = 10.163 ms, Queueing time: mean = 7.490 us, max = 7.490 us, min = 7.490 us, total = 7.490 us - NodeInfoGcsService.grpc_client.GetAllNodeInfo - 1 total (0 active), Execution time: mean = 224.380 us, total = 224.380 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 356.750 us, total = 356.750 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.CommitBundleResources - 1 total (0 active), Execution time: mean = 274.050 us, total = 274.050 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.ReturnWorker - 1 total (0 active), Execution time: mean = 123.909 us, total = 123.909 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - Subscriber.HandlePublishedMessage_GCS_JOB_CHANNEL - 1 total (0 active), Execution time: mean = 27.210 us, total = 27.210 us, Queueing time: mean = 79.790 us, max = 79.790 us, min = 79.790 us, total = 79.790 us - NodeManagerService.grpc_server.ReturnWorker.HandleRequestImpl - 1 total (0 active), Execution time: mean = 42.720 us, total = 42.720 us, Queueing time: mean = 11.020 us, max = 11.020 us, min = 11.020 us, total = 11.020 us - JobInfoGcsService.grpc_client.AddJob - 1 total (0 active), Execution time: mean = 337.319 us, total = 337.319 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 144.900 us, total = 144.900 us, Queueing time: mean = 4.900 us, max = 4.900 us, min = 4.900 us, total = 4.900 us - NodeManagerService.grpc_server.PrepareBundleResources - 1 total (0 active), Execution time: mean = 284.180 us, total = 284.180 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerPool.PopWorkerCallback - 1 total (0 active), Execution time: mean = 12.250 us, total = 12.250 us, Queueing time: mean = 33.810 us, max = 33.810 us, min = 33.810 us, total = 33.810 us - NodeInfoGcsService.grpc_client.GetAllNodeInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 68.730 us, total = 68.730 us, Queueing time: mean = 3.450 us, max = 3.450 us, min = 3.450 us, total = 3.450 us - NodeManagerService.grpc_server.CommitBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 225.290 us, total = 225.290 us, Queueing time: mean = 6.500 us, max = 6.500 us, min = 6.500 us, total = 6.500 us - JobInfoGcsService.grpc_client.GetAllJobInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 6.340 us, total = 6.340 us, Queueing time: mean = 4.740 us, max = 4.740 us, min = 4.740 us, total = 4.740 us - NodeManager.deadline_timer.print_event_loop_stats - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_server.PrepareBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 220.200 us, total = 220.200 us, Queueing time: mean = 8.490 us, max = 8.490 us, min = 8.490 us, total = 8.490 us - NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 498.049 us, total = 498.049 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -DebugString() time ms: 0 \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/debug_state_gcs.txt b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/debug_state_gcs.txt deleted file mode 100644 index 0a7aec7..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/debug_state_gcs.txt +++ /dev/null @@ -1,150 +0,0 @@ -GcsNodeManager: -- RegisterNode request count: 1 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 27 -- GetInternalConfig request count: 1 - -GcsActorManager: -- RegisterActor request count: 5 -- CreateActor request count: 5 -- GetActorInfo request count: 5 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 5 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 1 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 5 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 4 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 1 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 5 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 1 -- Registered placement groups count: 1 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: -GCS_JOB_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 280 -- current buffered bytes: 280 -GCS_ACTOR_CHANNEL -- cumulative published messages: 10 -- cumulative published bytes: 5864 -- current buffered bytes: 665 -GCS_NODE_INFO_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 557 -- current buffered bytes: 0 -RAY_LOG_CHANNEL -- cumulative published messages: 13 -- cumulative published bytes: 16445 -- current buffered bytes: 853 -GCS_WORKER_DELTA_CHANNEL -- cumulative published messages: 5 -- cumulative published bytes: 485 -- current buffered bytes: 97 - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 23 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 12 --Total num of actor creation tasks: 5 --Total num of actor tasks: 5 --Total num of normal tasks: 1 --Total num of driver tasks: 1 - - - - -Global stats: 865 total (19 active) -Queueing time: mean = 101.782 us, max = 25.578 ms, min = 180.000 ns, total = 88.041 ms -Execution time: mean = 2.878 ms, total = 2.490 s -Event stats: - GcsInMemoryStore.Put - 127 total (0 active), Execution time: mean = 223.503 us, total = 28.385 ms, Queueing time: mean = 209.819 us, max = 25.414 ms, min = 660.000 ns, total = 26.647 ms - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 81 total (0 active), Execution time: mean = 6.953 us, total = 563.160 us, Queueing time: mean = 37.330 us, max = 226.460 us, min = 3.730 us, total = 3.024 ms - InternalKVGcsService.grpc_server.InternalKVPut - 81 total (0 active), Execution time: mean = 134.171 us, total = 10.868 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Get - 77 total (0 active), Execution time: mean = 20.107 us, total = 1.548 ms, Queueing time: mean = 13.570 us, max = 294.989 us, min = 710.000 ns, total = 1.045 ms - InternalKVGcsService.grpc_server.InternalKVGet - 70 total (0 active), Execution time: mean = 94.341 us, total = 6.604 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 70 total (0 active), Execution time: mean = 10.107 us, total = 707.519 us, Queueing time: mean = 11.047 us, max = 135.410 us, min = 4.960 us, total = 773.290 us - NodeInfoGcsService.grpc_server.GetAllNodeInfo - 27 total (0 active), Execution time: mean = 88.741 us, total = 2.396 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 27 total (0 active), Execution time: mean = 12.016 us, total = 324.420 us, Queueing time: mean = 9.041 us, max = 26.670 us, min = 5.000 us, total = 244.120 us - InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 24 total (0 active), Execution time: mean = 4.120 us, total = 98.890 us, Queueing time: mean = 14.020 us, max = 27.410 us, min = 2.380 us, total = 336.480 us - InternalKVGcsService.grpc_client.InternalKVPut - 24 total (0 active), Execution time: mean = 487.867 us, total = 11.709 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - RayletLoadPulled - 20 total (1 active), Execution time: mean = 75.652 us, total = 1.513 ms, Queueing time: mean = 42.562 us, max = 410.494 us, min = 4.888 us, total = 851.250 us - NodeManagerService.grpc_client.GetResourceLoad - 18 total (0 active), Execution time: mean = 399.040 us, total = 7.183 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 18 total (0 active), Execution time: mean = 26.549 us, total = 477.888 us, Queueing time: mean = 10.987 us, max = 16.360 us, min = 6.770 us, total = 197.770 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 154.495 us, total = 2.781 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 15.402 us, total = 277.230 us, Queueing time: mean = 17.226 us, max = 149.360 us, min = 5.530 us, total = 310.060 us - NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 8.620 us, total = 68.960 us, Queueing time: mean = 18.786 us, max = 55.660 us, min = 6.280 us, total = 150.290 us - NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 1.099 ms, total = 8.794 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.CheckAlive - 8 total (0 active), Execution time: mean = 106.093 us, total = 848.740 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 8 total (0 active), Execution time: mean = 6.588 us, total = 52.700 us, Queueing time: mean = 11.845 us, max = 17.500 us, min = 4.400 us, total = 94.760 us - GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 5.030 us, total = 35.210 us, Queueing time: mean = 12.010 us, max = 17.640 us, min = 1.470 us, total = 84.070 us - ClusterResourceManager.ResetRemoteNodeView - 7 total (1 active), Execution time: mean = 2.347 us, total = 16.430 us, Queueing time: mean = 8.278 us, max = 12.074 us, min = 7.520 us, total = 57.943 us - ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsResourceManager::Update - 5 total (0 active), Execution time: mean = 48.128 us, total = 240.640 us, Queueing time: mean = 11.748 us, max = 14.640 us, min = 6.680 us, total = 58.740 us - HealthCheck - 5 total (0 active), Execution time: mean = 2.732 us, total = 13.660 us, Queueing time: mean = 16.474 us, max = 36.620 us, min = 8.640 us, total = 82.370 us - ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 105.628 us, total = 528.139 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 10.412 us, total = 52.060 us, Queueing time: mean = 99.082 us, max = 280.700 us, min = 5.070 us, total = 495.409 us - WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 15.830 us, total = 79.150 us, Queueing time: mean = 351.328 us, max = 723.950 us, min = 9.420 us, total = 1.757 ms - ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 6.240 us, total = 31.200 us, Queueing time: mean = 41.312 us, max = 107.930 us, min = 5.020 us, total = 206.560 us - ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 216.202 us, total = 1.081 ms, Queueing time: mean = 6.288 us, max = 6.820 us, min = 5.850 us, total = 31.440 us - NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 477.971 ms, total = 2.390 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 176.994 us, total = 884.970 us, Queueing time: mean = 12.884 us, max = 33.030 us, min = 5.519 us, total = 64.419 us - CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 318.232 us, total = 1.591 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 163.046 us, total = 815.228 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 134.546 us, total = 672.730 us, Queueing time: mean = 7.994 us, max = 9.410 us, min = 6.570 us, total = 39.970 us - WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 947.435 us, total = 4.737 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 4 total (0 active), Execution time: mean = 132.400 us, total = 529.600 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 55.992 us, total = 223.970 us, Queueing time: mean = 12.819 ms, max = 25.578 ms, min = 12.010 us, total = 51.275 ms - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 4 total (0 active), Execution time: mean = 25.195 us, total = 100.780 us, Queueing time: mean = 11.495 us, max = 13.250 us, min = 7.370 us, total = 45.980 us - GCSServer.deadline_timer.debug_state_dump - 2 total (1 active, 1 running), Execution time: mean = 377.779 us, total = 755.559 us, Queueing time: mean = 5.543 us, max = 11.085 us, min = 11.085 us, total = 11.085 us - JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 132.460 us, total = 132.460 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 34.060 us, total = 34.060 us, Queueing time: mean = 8.220 us, max = 8.220 us, min = 8.220 us, total = 8.220 us - NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 92.910 us, total = 92.910 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 816.319 us, total = 816.319 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 407.479 us, total = 407.479 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 4.100 us, total = 4.100 us, Queueing time: mean = 7.860 us, max = 7.860 us, min = 7.860 us, total = 7.860 us - InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.950 us, total = 9.950 us, Queueing time: mean = 5.500 us, max = 5.500 us, min = 5.500 us, total = 5.500 us - NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 42.260 us, total = 42.260 us, Queueing time: mean = 7.730 us, max = 7.730 us, min = 7.730 us, total = 7.730 us - JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 7.180 us, total = 7.180 us, Queueing time: mean = 5.460 us, max = 5.460 us, min = 5.460 us, total = 5.460 us - NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 239.230 us, total = 239.230 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 53.819 us, total = 53.819 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 1.850 us, total = 1.850 us, Queueing time: mean = 1.100 us, max = 1.100 us, min = 1.100 us, total = 1.100 us - NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 51.730 us, total = 51.730 us, Queueing time: mean = 6.120 us, max = 6.120 us, min = 6.120 us, total = 6.120 us - GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 2.440 us, total = 2.440 us, Queueing time: mean = 939.000 ns, max = 939.000 ns, min = 939.000 ns, total = 939.000 ns - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 95.290 us, total = 95.290 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 50.649 us, total = 50.649 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 7.410 us, total = 7.410 us, Queueing time: mean = 13.080 us, max = 13.080 us, min = 13.080 us, total = 13.080 us - JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 61.030 us, total = 61.030 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.839 us, total = 9.839 us, Queueing time: mean = 4.880 us, max = 4.880 us, min = 4.880 us, total = 4.880 us - NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 48.830 us, total = 48.830 us, Queueing time: mean = 10.230 us, max = 10.230 us, min = 10.230 us, total = 10.230 us - GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 5.500 us, total = 5.500 us, Queueing time: mean = 180.000 ns, max = 180.000 ns, min = 180.000 ns, total = 180.000 ns - JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 111.640 us, total = 111.640 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 70.000 ns, total = 70.000 ns, Queueing time: mean = 78.380 us, max = 78.380 us, min = 78.380 us, total = 78.380 us - JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 45.100 us, total = 45.100 us, Queueing time: mean = 8.550 us, max = 8.550 us, min = 8.550 us, total = 8.550 us \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/gcs_server.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/gcs_server.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/gcs_server.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/gcs_server.out deleted file mode 100644 index bbfde82..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/gcs_server.out +++ /dev/null @@ -1,147 +0,0 @@ -[2024-05-15 09:19:29,376 I 23974 23974] (gcs_server) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 09:19:29,377 I 23974 23974] (gcs_server) event.cc:234: Set ray event level to warning -[2024-05-15 09:19:29,377 I 23974 23974] (gcs_server) event.cc:342: Ray Event initialized for GCS -[2024-05-15 09:19:29,377 I 23974 23974] (gcs_server) gcs_server.cc:74: GCS storage type is StorageType::IN_MEMORY -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:42: Loading job table data. -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:54: Loading node table data. -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:80: Loading actor table data. -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:93: Loading actor task spec table data. -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:66: Loading placement group table data. -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:46: Finished loading job table data, size = 0 -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:58: Finished loading node table data, size = 0 -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:84: Finished loading actor table data, size = 0 -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:97: Finished loading actor task spec table data, size = 0 -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_init_data.cc:71: Finished loading placement group table data, size = 0 -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_server.cc:162: No existing server cluster ID found. Generating new ID: 25b549eeebc5d14c188b2758b99248779753f165a1262902a90adf06 -[2024-05-15 09:19:29,378 I 23974 23974] (gcs_server) gcs_server.cc:648: Autoscaler V2 enabled: 0 -[2024-05-15 09:19:29,379 I 23974 23974] (gcs_server) grpc_server.cc:134: GcsServer server started, listening on port 62418. -[2024-05-15 09:19:29,403 I 23974 23974] (gcs_server) gcs_server.cc:250: GcsNodeManager: -- RegisterNode request count: 0 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 0 -- GetInternalConfig request count: 0 - -GcsActorManager: -- RegisterActor request count: 0 -- CreateActor request count: 0 -- GetActorInfo request count: 0 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 0 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 0 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 0 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 0 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 0 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 0 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 0 -- Registered placement groups count: 0 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 0 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 0 --Total num of actor creation tasks: 0 --Total num of actor tasks: 0 --Total num of normal tasks: 0 --Total num of driver tasks: 0 - - -[2024-05-15 09:19:29,404 I 23974 23974] (gcs_server) gcs_server.cc:844: Event stats: - - -Global stats: 33 total (22 active) -Queueing time: mean = 2.322 ms, max = 25.578 ms, min = 980.000 ns, total = 76.628 ms -Execution time: mean = 776.303 us, total = 25.618 ms -Event stats: - InternalKVGcsService.grpc_client.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.GetAll - 5 total (0 active), Execution time: mean = 4.218 us, total = 21.090 us, Queueing time: mean = 16.136 us, max = 17.640 us, min = 14.620 us, total = 80.680 us - PeriodicalRunner.RunFnPeriodically - 4 total (2 active, 1 running), Execution time: mean = 610.000 ns, total = 2.440 us, Queueing time: mean = 12.782 ms, max = 25.578 ms, min = 25.552 ms, total = 51.130 ms - GcsInMemoryStore.Put - 3 total (0 active), Execution time: mean = 8.530 ms, total = 25.589 ms, Queueing time: mean = 8.472 ms, max = 25.414 ms, min = 980.000 ns, total = 25.416 ms - RayletLoadPulled - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Get - 1 total (0 active), Execution time: mean = 5.020 us, total = 5.020 us, Queueing time: mean = 1.280 us, max = 1.280 us, min = 1.280 us, total = 1.280 us - - -[2024-05-15 09:19:29,404 I 23974 23974] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats: - - -Global stats: 2 total (1 active) -Queueing time: mean = 46.570 us, max = 93.140 us, min = 93.140 us, total = 93.140 us -Execution time: mean = 12.700 us, total = 25.400 us -Event stats: - PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 25.400 us, total = 25.400 us, Queueing time: mean = 93.140 us, max = 93.140 us, min = 93.140 us, total = 93.140 us - GcsTaskManager.GcJobSummary - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 09:19:30,775 I 23974 23974] (gcs_server) gcs_node_manager.cc:55: Registering node info, node id = 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 09:19:30,775 I 23974 23974] (gcs_server) gcs_node_manager.cc:61: Finished registering node info, node id = 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 09:19:30,776 I 23974 23974] (gcs_server) gcs_placement_group_manager.cc:797: A new node: 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b registered, will try to reschedule all the infeasible placement groups. -[2024-05-15 09:19:31,367 I 23974 23974] (gcs_server) gcs_job_manager.cc:42: Adding job, job id = 01000000, driver pid = 23913 -[2024-05-15 09:19:31,367 I 23974 23974] (gcs_server) gcs_job_manager.cc:57: Finished adding job, job id = 01000000, driver pid = 23913 -[2024-05-15 09:19:31,399 I 23974 23974] (gcs_server) gcs_placement_group_manager.cc:432: Registering placement group, placement group id = 486f798c40a870f1d3f75646414a01000000, name = , strategy = 0 -[2024-05-15 09:19:31,399 I 23974 23974] (gcs_server) gcs_placement_group_manager.cc:436: Finished registering placement group, placement group id = 486f798c40a870f1d3f75646414a01000000, name = , strategy = 0 -[2024-05-15 09:19:31,399 I 23974 23974] (gcs_server) gcs_placement_group_scheduler.cc:185: Preparing resource from node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for bundles: {placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={0}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={1}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={2}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={3}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={4}}, -[2024-05-15 09:19:31,400 I 23974 23974] (gcs_server) gcs_placement_group_scheduler.cc:195: Finished leasing resource from 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for bundles: {placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={0}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={1}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={2}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={3}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={4}}, -[2024-05-15 09:19:31,400 I 23974 23974] (gcs_server) gcs_placement_group_scheduler.cc:213: Committing resource to a node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for bundles: {placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={0}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={1}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={2}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={3}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={4}}, -[2024-05-15 09:19:31,400 I 23974 23974] (gcs_server) gcs_placement_group_scheduler.cc:220: Finished committing resource to 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for bundles: {placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={0}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={1}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={2}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={3}},{placement group id={486f798c40a870f1d3f75646414a01000000}, bundle index={4}}, -[2024-05-15 09:19:31,401 I 23974 23974] (gcs_server) gcs_placement_group_manager.cc:321: Successfully created placement group , id: 486f798c40a870f1d3f75646414a01000000 -[2024-05-15 09:19:31,581 I 23974 23974] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 0719ba00ba0b5675585de11d01000000 -[2024-05-15 09:19:31,581 I 23974 23974] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 0719ba00ba0b5675585de11d01000000 -[2024-05-15 09:19:31,582 I 23974 23974] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 0719ba00ba0b5675585de11d01000000 -[2024-05-15 09:19:31,583 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor 0719ba00ba0b5675585de11d01000000, job id = 01000000 -[2024-05-15 09:19:31,987 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor 0719ba00ba0b5675585de11d01000000, job id = 01000000 -[2024-05-15 09:19:31,987 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 0719ba00ba0b5675585de11d01000000 on worker 09a021e1580a8a10e3f9e66f2eed1bc46e2049f3a43caeb389090ae2 at node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b, job id = 01000000 -[2024-05-15 09:19:33,538 I 23974 23974] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 5d51a5db74f9d1a22bb0eef401000000 -[2024-05-15 09:19:33,539 I 23974 23974] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 5d51a5db74f9d1a22bb0eef401000000 -[2024-05-15 09:19:33,539 I 23974 23974] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 5d51a5db74f9d1a22bb0eef401000000 -[2024-05-15 09:19:33,539 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor 5d51a5db74f9d1a22bb0eef401000000, job id = 01000000 -[2024-05-15 09:19:33,539 I 23974 23974] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = cccd744a25e73a453f97355401000000 -[2024-05-15 09:19:33,540 I 23974 23974] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = cccd744a25e73a453f97355401000000 -[2024-05-15 09:19:33,540 I 23974 23974] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = cccd744a25e73a453f97355401000000 -[2024-05-15 09:19:33,540 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor cccd744a25e73a453f97355401000000, job id = 01000000 -[2024-05-15 09:19:33,540 I 23974 23974] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 835b092645f7e78f977631cd01000000 -[2024-05-15 09:19:33,540 I 23974 23974] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 835b092645f7e78f977631cd01000000 -[2024-05-15 09:19:33,541 I 23974 23974] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 835b092645f7e78f977631cd01000000 -[2024-05-15 09:19:33,541 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor 835b092645f7e78f977631cd01000000, job id = 01000000 -[2024-05-15 09:19:33,541 I 23974 23974] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = d8cbd19b2d257ee31de31a8901000000 -[2024-05-15 09:19:33,542 I 23974 23974] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = d8cbd19b2d257ee31de31a8901000000 -[2024-05-15 09:19:33,542 I 23974 23974] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = d8cbd19b2d257ee31de31a8901000000 -[2024-05-15 09:19:33,542 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor d8cbd19b2d257ee31de31a8901000000, job id = 01000000 -[2024-05-15 09:19:33,961 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor cccd744a25e73a453f97355401000000, job id = 01000000 -[2024-05-15 09:19:33,961 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor cccd744a25e73a453f97355401000000 on worker cf13e1dd7b63e78f5580dc3e7561fc56819d9d92ccdc134e3d810783 at node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b, job id = 01000000 -[2024-05-15 09:19:34,011 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor d8cbd19b2d257ee31de31a8901000000, job id = 01000000 -[2024-05-15 09:19:34,011 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor d8cbd19b2d257ee31de31a8901000000 on worker e76f309471a55a309afd864f3af9ce8abe6bb8b35beadcfa46679e2d at node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b, job id = 01000000 -[2024-05-15 09:19:34,061 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor 5d51a5db74f9d1a22bb0eef401000000, job id = 01000000 -[2024-05-15 09:19:34,062 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 5d51a5db74f9d1a22bb0eef401000000 on worker fb9598648ce04044c1d12aea113547920f6478f03059d34c2f39552e at node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b, job id = 01000000 -[2024-05-15 09:19:34,114 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b for actor 835b092645f7e78f977631cd01000000, job id = 01000000 -[2024-05-15 09:19:34,115 I 23974 23974] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 835b092645f7e78f977631cd01000000 on worker ca15d91423a5ad101b3a5cea1bab7be41c8978e2eb6850584981223a at node 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b, job id = 01000000 -[2024-05-15 09:19:39,390 W 23974 23995] (gcs_server) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:60081: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/log_monitor.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/log_monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/monitor.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/monitor.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/monitor.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/raylet.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/raylet.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/raylet.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/raylet.out deleted file mode 100644 index 947e094..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/raylet.out +++ /dev/null @@ -1,228 +0,0 @@ -[2024-05-15 09:19:30,764 I 24097 24097] (raylet) main.cc:179: Setting cluster ID to: 25b549eeebc5d14c188b2758b99248779753f165a1262902a90adf06 -[2024-05-15 09:19:30,765 I 24097 24097] (raylet) main.cc:242: Raylet is not set to kill unknown children. -[2024-05-15 09:19:30,765 I 24097 24097] (raylet) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 09:19:30,766 I 24097 24097] (raylet) main.cc:371: Setting node ID to: 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b -[2024-05-15 09:19:30,766 I 24097 24097] (raylet) store_runner.cc:32: Allowing the Plasma store to use up to 18.3211GB of memory. -[2024-05-15 09:19:30,766 I 24097 24097] (raylet) store_runner.cc:48: Starting object store with directory /dev/shm, fallback /home/valapeos/Projects/pneumarl/logs/rllib, and huge page support disabled -[2024-05-15 09:19:30,766 I 24097 24125] (raylet) dlmalloc.cc:154: create_and_mmap_buffer(18321113096, /dev/shm/plasmaXXXXXX) -[2024-05-15 09:19:30,766 I 24097 24125] (raylet) store.cc:564: ========== Plasma store: ================= -Current usage: 0 / 18.3211 GB -- num bytes created total: 0 -0 pending objects of total size 0MB -- objects spillable: 0 -- bytes spillable: 0 -- objects unsealed: 0 -- bytes unsealed: 0 -- objects in use: 0 -- bytes in use: 0 -- objects evictable: 0 -- bytes evictable: 0 - -- objects created by worker: 0 -- bytes created by worker: 0 -- objects restored: 0 -- bytes restored: 0 -- objects received: 0 -- bytes received: 0 -- objects errored: 0 -- bytes errored: 0 - -[2024-05-15 09:19:30,767 I 24097 24097] (raylet) grpc_server.cc:134: ObjectManager server started, listening on port 38613. -[2024-05-15 09:19:30,769 I 24097 24097] (raylet) worker_killing_policy.cc:101: Running GroupByOwner policy. -[2024-05-15 09:19:30,769 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:19:30,769 I 24097 24097] (raylet) memory_monitor.cc:47: MemoryMonitor initialized with usage threshold at 63982133248 bytes (0.95 system memory), total system memory bytes: 67349614592 -[2024-05-15 09:19:30,769 I 24097 24097] (raylet) node_manager.cc:285: Initializing NodeManager with ID 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b -[2024-05-15 09:19:30,769 I 24097 24097] (raylet) grpc_server.cc:134: NodeManager server started, listening on port 40935. -[2024-05-15 09:19:30,774 I 24097 24159] (raylet) agent_manager.cc:78: Monitor agent process with name dashboard_agent/424238335 -[2024-05-15 09:19:30,775 I 24097 24161] (raylet) agent_manager.cc:78: Monitor agent process with name runtime_env_agent -[2024-05-15 09:19:30,775 I 24097 24097] (raylet) event.cc:234: Set ray event level to warning -[2024-05-15 09:19:30,775 I 24097 24097] (raylet) event.cc:342: Ray Event initialized for RAYLET -[2024-05-15 09:19:30,776 I 24097 24097] (raylet) raylet.cc:128: Raylet of id, 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b started. Raylet consists of node_manager and object_manager. node_manager address: 192.168.1.56:40935 object_manager address: 192.168.1.56:38613 hostname: valanixos -[2024-05-15 09:19:30,777 I 24097 24097] (raylet) node_manager.cc:521: [state-dump] NodeManager: -[state-dump] Node ID: 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b -[state-dump] Node name: 192.168.1.56 -[state-dump] InitialConfigResources: {object_store_memory: 183210811390000, node:192.168.1.56: 10000, memory: 366421622790000, node:__internal_head__: 10000, CPU: 120000} -[state-dump] ClusterTaskManager: -[state-dump] ========== Node: 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b ================= -[state-dump] Infeasible queue length: 0 -[state-dump] Schedule queue length: 0 -[state-dump] Dispatch queue length: 0 -[state-dump] num_waiting_for_resource: 0 -[state-dump] num_waiting_for_plasma_memory: 0 -[state-dump] num_waiting_for_remote_node_resources: 0 -[state-dump] num_worker_not_started_by_job_config_not_exist: 0 -[state-dump] num_worker_not_started_by_registration_timeout: 0 -[state-dump] num_tasks_waiting_for_workers: 0 -[state-dump] num_cancelled_tasks: 0 -[state-dump] cluster_resource_scheduler state: -[state-dump] Local id: 6857835085091698370 Local resources: {"total":{node:__internal_head__: [10000], memory: [366421622790000], object_store_memory: [183210811390000], CPU: [120000], node:192.168.1.56: [10000]}}, "available": {node:__internal_head__: [10000], memory: [366421622790000], object_store_memory: [183210811390000], CPU: [120000], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b",} is_draining: 0 is_idle: 1 Cluster resources: node id: 6857835085091698370{"total":{object_store_memory: 183210811390000, CPU: 120000, node:__internal_head__: 10000, node:192.168.1.56: 10000, memory: 366421622790000}}, "available": {object_store_memory: 183210811390000, CPU: 120000, node:__internal_head__: 10000, node:192.168.1.56: 10000, memory: 366421622790000}}, "labels":{"ray.io/node_id":"322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -[state-dump] Waiting tasks size: 0 -[state-dump] Number of executing tasks: 0 -[state-dump] Number of pinned task arguments: 0 -[state-dump] Number of total spilled tasks: 0 -[state-dump] Number of spilled waiting tasks: 0 -[state-dump] Number of spilled unschedulable tasks: 0 -[state-dump] Resource usage { -[state-dump] } -[state-dump] Running tasks by scheduling class: -[state-dump] ================================================== -[state-dump] -[state-dump] ClusterResources: -[state-dump] LocalObjectManager: -[state-dump] - num pinned objects: 0 -[state-dump] - pinned objects size: 0 -[state-dump] - num objects pending restore: 0 -[state-dump] - num objects pending spill: 0 -[state-dump] - num bytes pending spill: 0 -[state-dump] - num bytes currently spilled: 0 -[state-dump] - cumulative spill requests: 0 -[state-dump] - cumulative restore requests: 0 -[state-dump] - spilled objects pending delete: 0 -[state-dump] -[state-dump] ObjectManager: -[state-dump] - num local objects: 0 -[state-dump] - num unfulfilled push requests: 0 -[state-dump] - num object pull requests: 0 -[state-dump] - num chunks received total: 0 -[state-dump] - num chunks received failed (all): 0 -[state-dump] - num chunks received failed / cancelled: 0 -[state-dump] - num chunks received failed / plasma error: 0 -[state-dump] Event stats: -[state-dump] Global stats: 0 total (0 active) -[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] Execution time: mean = -nan s, total = 0.000 s -[state-dump] Event stats: -[state-dump] PushManager: -[state-dump] - num pushes in flight: 0 -[state-dump] - num chunks in flight: 0 -[state-dump] - num chunks remaining: 0 -[state-dump] - max chunks allowed: 409 -[state-dump] OwnershipBasedObjectDirectory: -[state-dump] - num listeners: 0 -[state-dump] - cumulative location updates: 0 -[state-dump] - num location updates per second: 0.000 -[state-dump] - num location lookups per second: 0.000 -[state-dump] - num locations added per second: 0.000 -[state-dump] - num locations removed per second: 0.000 -[state-dump] BufferPool: -[state-dump] - create buffer state map size: 0 -[state-dump] PullManager: -[state-dump] - num bytes available for pulled objects: 18321081139 -[state-dump] - num bytes being pulled (all): 0 -[state-dump] - num bytes being pulled / pinned: 0 -[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - first get request bundle: N/A -[state-dump] - first wait request bundle: N/A -[state-dump] - first task request bundle: N/A -[state-dump] - num objects queued: 0 -[state-dump] - num objects actively pulled (all): 0 -[state-dump] - num objects actively pulled / pinned: 0 -[state-dump] - num bundles being pulled: 0 -[state-dump] - num pull retries: 0 -[state-dump] - max timeout seconds: 0 -[state-dump] - max timeout request is already processed. No entry. -[state-dump] -[state-dump] WorkerPool: -[state-dump] - registered jobs: 0 -[state-dump] - process_failed_job_config_missing: 0 -[state-dump] - process_failed_rate_limited: 0 -[state-dump] - process_failed_pending_registration: 0 -[state-dump] - process_failed_runtime_env_setup_failed: 0 -[state-dump] - num PYTHON workers: 0 -[state-dump] - num PYTHON drivers: 0 -[state-dump] - num object spill callbacks queued: 0 -[state-dump] - num object restore queued: 0 -[state-dump] - num util functions queued: 0 -[state-dump] - num idle workers: 0 -[state-dump] TaskDependencyManager: -[state-dump] - task deps map size: 0 -[state-dump] - get req map size: 0 -[state-dump] - wait req map size: 0 -[state-dump] - local objects map size: 0 -[state-dump] WaitManager: -[state-dump] - num active wait requests: 0 -[state-dump] Subscriber: -[state-dump] Channel WORKER_OBJECT_EVICTION -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_REF_REMOVED_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] num async plasma notifications: 0 -[state-dump] Remote node managers: -[state-dump] Event stats: -[state-dump] Global stats: 27 total (13 active) -[state-dump] Queueing time: mean = 855.480 us, max = 6.594 ms, min = 4.900 us, total = 23.098 ms -[state-dump] Execution time: mean = 455.663 us, total = 12.303 ms -[state-dump] Event stats: -[state-dump] PeriodicalRunner.RunFnPeriodically - 11 total (2 active, 1 running), Execution time: mean = 81.224 us, total = 893.469 us, Queueing time: mean = 2.099 ms, max = 6.594 ms, min = 20.180 us, total = 23.086 ms -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 10.163 ms, total = 10.163 ms, Queueing time: mean = 7.490 us, max = 7.490 us, min = 7.490 us, total = 7.490 us -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 247.170 us, total = 247.170 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 498.049 us, total = 498.049 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.flush_free_objects - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.record_metrics - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 144.900 us, total = 144.900 us, Queueing time: mean = 4.900 us, max = 4.900 us, min = 4.900 us, total = 4.900 us -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.debug_state_dump - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 356.750 us, total = 356.750 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] DebugString() time ms: 0 -[state-dump] -[state-dump] -[2024-05-15 09:19:30,777 I 24097 24097] (raylet) accessor.cc:627: Received notification for node id = 322143802a8bacf17c2cb297db90b6d597468f55c850b24b440b447b, IsAlive = 1 -[2024-05-15 09:19:30,881 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24179, the token is 0 -[2024-05-15 09:19:30,882 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24180, the token is 1 -[2024-05-15 09:19:30,883 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24181, the token is 2 -[2024-05-15 09:19:30,885 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24182, the token is 3 -[2024-05-15 09:19:30,887 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24183, the token is 4 -[2024-05-15 09:19:30,888 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24184, the token is 5 -[2024-05-15 09:19:30,890 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24185, the token is 6 -[2024-05-15 09:19:30,892 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24186, the token is 7 -[2024-05-15 09:19:30,894 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24187, the token is 8 -[2024-05-15 09:19:30,896 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24188, the token is 9 -[2024-05-15 09:19:30,897 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24189, the token is 10 -[2024-05-15 09:19:30,899 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24190, the token is 11 -[2024-05-15 09:19:31,232 I 24097 24125] (raylet) object_store.cc:35: Object store current usage 8e-09 / 18.3211 GB. -[2024-05-15 09:19:31,367 I 24097 24097] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 23913 is dead: 0 driver address: 192.168.1.56 -[2024-05-15 09:19:31,367 I 24097 24097] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 09:19:31,584 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:19:31,586 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24919, the token is 12 -[2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:19:32,583 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:19:33,540 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:19:33,543 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25005, the token is 13 -[2024-05-15 09:19:33,543 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:19:33,545 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25006, the token is 14 -[2024-05-15 09:19:33,546 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:19:33,548 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25007, the token is 15 -[2024-05-15 09:19:33,548 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:19:33,549 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25008, the token is 16 -[2024-05-15 09:19:35,778 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:19:40,778 W 24097 24119] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:60081: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. -[2024-05-15 09:19:40,780 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:19:45,782 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:19:50,785 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:19:55,787 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/runtime_env_agent.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/runtime_env_agent.err deleted file mode 100644 index 8ebe7f0..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/runtime_env_agent.err +++ /dev/null @@ -1,50 +0,0 @@ -Raylet is terminated. Termination is unexpected. Possible reasons include: (1) SIGKILL by the user or system OOM killer, (2) Invalid memory access from Raylet causing SIGSEGV or SIGBUS, (3) Other termination signals. Last 20 lines of the Raylet logs: - [2024-05-15 09:19:31,586 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24919, the token is 12 - [2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:32,583 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:33,540 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 - [2024-05-15 09:19:33,543 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25005, the token is 13 - [2024-05-15 09:19:33,543 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 - [2024-05-15 09:19:33,545 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25006, the token is 14 - [2024-05-15 09:19:33,546 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 - [2024-05-15 09:19:33,548 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25007, the token is 15 - [2024-05-15 09:19:33,548 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 - [2024-05-15 09:19:33,549 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25008, the token is 16 - [2024-05-15 09:19:35,778 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [2024-05-15 09:19:40,778 W 24097 24119] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:60081: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. - [2024-05-15 09:19:40,780 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [2024-05-15 09:19:45,782 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [2024-05-15 09:19:50,785 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [2024-05-15 09:19:55,787 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - -Failed to publish error: Raylet is terminated. Termination is unexpected. Possible reasons include: (1) SIGKILL by the user or system OOM killer, (2) Invalid memory access from Raylet causing SIGSEGV or SIGBUS, (3) Other termination signals. Last 20 lines of the Raylet logs: - [2024-05-15 09:19:31,586 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 24919, the token is 12 - [2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:32,384 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:32,583 I 24097 24097] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false - [2024-05-15 09:19:33,540 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 - [2024-05-15 09:19:33,543 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25005, the token is 13 - [2024-05-15 09:19:33,543 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 - [2024-05-15 09:19:33,545 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25006, the token is 14 - [2024-05-15 09:19:33,546 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 - [2024-05-15 09:19:33,548 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25007, the token is 15 - [2024-05-15 09:19:33,548 I 24097 24097] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 - [2024-05-15 09:19:33,549 I 24097 24097] (raylet) worker_pool.cc:494: Started worker process with pid 25008, the token is 16 - [2024-05-15 09:19:35,778 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [2024-05-15 09:19:40,778 W 24097 24119] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:60081: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. - [2024-05-15 09:19:40,780 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [2024-05-15 09:19:45,782 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [2024-05-15 09:19:50,785 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [2024-05-15 09:19:55,787 W 24097 24097] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero - [type raylet_died] -Traceback (most recent call last): - File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_private/utils.py", line 207, in publish_error_to_driver - gcs_publisher.publish_error( - File "python/ray/_raylet.pyx", line 3038, in ray._raylet.GcsPublisher.publish_error - File "python/ray/_raylet.pyx", line 573, in ray._raylet.check_status -ray.exceptions.GetTimeoutError: Failed to publish after retries: failed to connect to all addresses; last error: UNKNOWN: ipv4:192.168.1.56:62418: Failed to connect to remote host: Connection refused diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/runtime_env_agent.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/runtime_env_agent.out deleted file mode 100644 index 2961f93..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/runtime_env_agent.out +++ /dev/null @@ -1,2 +0,0 @@ -======== Running on http://0.0.0.0:45975 ======== -(Press CTRL+C to quit) diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-096f24373a18f043736f5567e99c93956e4e82cedd5fb3d33edc810f-ffffffff-24181.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-096f24373a18f043736f5567e99c93956e4e82cedd5fb3d33edc810f-ffffffff-24181.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-096f24373a18f043736f5567e99c93956e4e82cedd5fb3d33edc810f-ffffffff-24181.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-096f24373a18f043736f5567e99c93956e4e82cedd5fb3d33edc810f-ffffffff-24181.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-09a021e1580a8a10e3f9e66f2eed1bc46e2049f3a43caeb389090ae2-01000000-24919.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-09a021e1580a8a10e3f9e66f2eed1bc46e2049f3a43caeb389090ae2-01000000-24919.err deleted file mode 100644 index e4d1bd1..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-09a021e1580a8a10e3f9e66f2eed1bc46e2049f3a43caeb389090ae2-01000000-24919.err +++ /dev/null @@ -1,5 +0,0 @@ -:job_id:01000000 -:actor_name:PPO -2024-05-15 09:19:33,488 WARNING deprecation.py:50 -- DeprecationWarning: `AlgorithmConfig.num_envs_per_worker` has been deprecated. Use `AlgorithmConfig.num_envs_per_env_runner` instead. This will raise an error in the future! -2024-05-15 09:19:33,494 WARNING deprecation.py:50 -- DeprecationWarning: `WorkerSet(num_workers=... OR local_worker=...)` has been deprecated. Use `EnvRunnerGroup(num_env_runners=... AND local_env_runner=...)` instead. This will raise an error in the future! -2024-05-15 09:19:33,494 WARNING deprecation.py:50 -- DeprecationWarning: `max_num_worker_restarts` has been deprecated. Use `AlgorithmConfig.max_num_env_runner_restarts` instead. This will raise an error in the future! diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-09a021e1580a8a10e3f9e66f2eed1bc46e2049f3a43caeb389090ae2-01000000-24919.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-09a021e1580a8a10e3f9e66f2eed1bc46e2049f3a43caeb389090ae2-01000000-24919.out deleted file mode 100644 index 1124c85..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-09a021e1580a8a10e3f9e66f2eed1bc46e2049f3a43caeb389090ae2-01000000-24919.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:actor_name:PPO diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-1eddae27c55c81b32f99a60b22fbb30eef6451dfd0e2ed46086d5f7e-ffffffff-24180.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-1eddae27c55c81b32f99a60b22fbb30eef6451dfd0e2ed46086d5f7e-ffffffff-24180.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-1eddae27c55c81b32f99a60b22fbb30eef6451dfd0e2ed46086d5f7e-ffffffff-24180.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-1eddae27c55c81b32f99a60b22fbb30eef6451dfd0e2ed46086d5f7e-ffffffff-24180.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-20a442b1333f9b1c648302a8074efb1c176d60eb12acb688975efafb-ffffffff-24182.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-20a442b1333f9b1c648302a8074efb1c176d60eb12acb688975efafb-ffffffff-24182.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-20a442b1333f9b1c648302a8074efb1c176d60eb12acb688975efafb-ffffffff-24182.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-20a442b1333f9b1c648302a8074efb1c176d60eb12acb688975efafb-ffffffff-24182.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-2626b0fabba4a24286afcbe1a657c6922d921da51e5f3ca4addc9027-ffffffff-24185.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-2626b0fabba4a24286afcbe1a657c6922d921da51e5f3ca4addc9027-ffffffff-24185.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-2626b0fabba4a24286afcbe1a657c6922d921da51e5f3ca4addc9027-ffffffff-24185.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-2626b0fabba4a24286afcbe1a657c6922d921da51e5f3ca4addc9027-ffffffff-24185.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-3b8cfe41aec33b8854f7eca52ced3c6e25ec8eb1f0a67152aa6aadc3-ffffffff-24183.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-3b8cfe41aec33b8854f7eca52ced3c6e25ec8eb1f0a67152aa6aadc3-ffffffff-24183.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-3b8cfe41aec33b8854f7eca52ced3c6e25ec8eb1f0a67152aa6aadc3-ffffffff-24183.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-3b8cfe41aec33b8854f7eca52ced3c6e25ec8eb1f0a67152aa6aadc3-ffffffff-24183.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-639218e374c6f73af00bf768fcbd8d4c6940c4354a77161f763cfcf2-ffffffff-24190.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-639218e374c6f73af00bf768fcbd8d4c6940c4354a77161f763cfcf2-ffffffff-24190.err deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-639218e374c6f73af00bf768fcbd8d4c6940c4354a77161f763cfcf2-ffffffff-24190.err +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-639218e374c6f73af00bf768fcbd8d4c6940c4354a77161f763cfcf2-ffffffff-24190.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-639218e374c6f73af00bf768fcbd8d4c6940c4354a77161f763cfcf2-ffffffff-24190.out deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-639218e374c6f73af00bf768fcbd8d4c6940c4354a77161f763cfcf2-ffffffff-24190.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-7609c75ec49367879358051664321619632380f1fda88186fe1d26c2-ffffffff-24186.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-7609c75ec49367879358051664321619632380f1fda88186fe1d26c2-ffffffff-24186.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-7609c75ec49367879358051664321619632380f1fda88186fe1d26c2-ffffffff-24186.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-7609c75ec49367879358051664321619632380f1fda88186fe1d26c2-ffffffff-24186.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-8d3e7fc6b49613c46055d7269009837e037d1e463065bd676313dd30-ffffffff-24179.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-8d3e7fc6b49613c46055d7269009837e037d1e463065bd676313dd30-ffffffff-24179.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-8d3e7fc6b49613c46055d7269009837e037d1e463065bd676313dd30-ffffffff-24179.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-8d3e7fc6b49613c46055d7269009837e037d1e463065bd676313dd30-ffffffff-24179.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-b77445844ba2f87fee46b6c1b5579e1447b7bd06d3ebf8c32b49a2fc-ffffffff-24188.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-b77445844ba2f87fee46b6c1b5579e1447b7bd06d3ebf8c32b49a2fc-ffffffff-24188.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-b77445844ba2f87fee46b6c1b5579e1447b7bd06d3ebf8c32b49a2fc-ffffffff-24188.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-b77445844ba2f87fee46b6c1b5579e1447b7bd06d3ebf8c32b49a2fc-ffffffff-24188.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-c0d2f71f6769d732435bb3c570ed13652034a8472b6f7053961487a5-ffffffff-24184.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-c0d2f71f6769d732435bb3c570ed13652034a8472b6f7053961487a5-ffffffff-24184.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-c0d2f71f6769d732435bb3c570ed13652034a8472b6f7053961487a5-ffffffff-24184.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-c0d2f71f6769d732435bb3c570ed13652034a8472b6f7053961487a5-ffffffff-24184.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-ca15d91423a5ad101b3a5cea1bab7be41c8978e2eb6850584981223a-01000000-25007.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-ca15d91423a5ad101b3a5cea1bab7be41c8978e2eb6850584981223a-01000000-25007.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-ca15d91423a5ad101b3a5cea1bab7be41c8978e2eb6850584981223a-01000000-25007.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-ca15d91423a5ad101b3a5cea1bab7be41c8978e2eb6850584981223a-01000000-25007.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-ca15d91423a5ad101b3a5cea1bab7be41c8978e2eb6850584981223a-01000000-25007.out deleted file mode 100644 index 01371fe..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-ca15d91423a5ad101b3a5cea1bab7be41c8978e2eb6850584981223a-01000000-25007.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11011 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-cf13e1dd7b63e78f5580dc3e7561fc56819d9d92ccdc134e3d810783-01000000-25006.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-cf13e1dd7b63e78f5580dc3e7561fc56819d9d92ccdc134e3d810783-01000000-25006.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-cf13e1dd7b63e78f5580dc3e7561fc56819d9d92ccdc134e3d810783-01000000-25006.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-cf13e1dd7b63e78f5580dc3e7561fc56819d9d92ccdc134e3d810783-01000000-25006.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-cf13e1dd7b63e78f5580dc3e7561fc56819d9d92ccdc134e3d810783-01000000-25006.out deleted file mode 100644 index 80b5e41..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-cf13e1dd7b63e78f5580dc3e7561fc56819d9d92ccdc134e3d810783-01000000-25006.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11010 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-dc52f55b5e2b0083ebe19eac85015e74c939c1e1d930b2b0a382e7cb-ffffffff-24187.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-dc52f55b5e2b0083ebe19eac85015e74c939c1e1d930b2b0a382e7cb-ffffffff-24187.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-dc52f55b5e2b0083ebe19eac85015e74c939c1e1d930b2b0a382e7cb-ffffffff-24187.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-dc52f55b5e2b0083ebe19eac85015e74c939c1e1d930b2b0a382e7cb-ffffffff-24187.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e35832965c64fad7ca464b42dbce07d17ba28aaa234146add5a91e65-ffffffff-24189.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e35832965c64fad7ca464b42dbce07d17ba28aaa234146add5a91e65-ffffffff-24189.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e35832965c64fad7ca464b42dbce07d17ba28aaa234146add5a91e65-ffffffff-24189.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e35832965c64fad7ca464b42dbce07d17ba28aaa234146add5a91e65-ffffffff-24189.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e76f309471a55a309afd864f3af9ce8abe6bb8b35beadcfa46679e2d-01000000-25008.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e76f309471a55a309afd864f3af9ce8abe6bb8b35beadcfa46679e2d-01000000-25008.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e76f309471a55a309afd864f3af9ce8abe6bb8b35beadcfa46679e2d-01000000-25008.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e76f309471a55a309afd864f3af9ce8abe6bb8b35beadcfa46679e2d-01000000-25008.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e76f309471a55a309afd864f3af9ce8abe6bb8b35beadcfa46679e2d-01000000-25008.out deleted file mode 100644 index 9587b73..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-e76f309471a55a309afd864f3af9ce8abe6bb8b35beadcfa46679e2d-01000000-25008.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11012 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-fb9598648ce04044c1d12aea113547920f6478f03059d34c2f39552e-01000000-25005.err b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-fb9598648ce04044c1d12aea113547920f6478f03059d34c2f39552e-01000000-25005.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-fb9598648ce04044c1d12aea113547920f6478f03059d34c2f39552e-01000000-25005.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-fb9598648ce04044c1d12aea113547920f6478f03059d34c2f39552e-01000000-25005.out b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-fb9598648ce04044c1d12aea113547920f6478f03059d34c2f39552e-01000000-25005.out deleted file mode 100644 index 6628553..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/logs/worker-fb9598648ce04044c1d12aea113547920f6478f03059d34c2f39552e-01000000-25005.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11009 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/node_ip_address.json b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/node_ip_address.json deleted file mode 100644 index 8d998d8..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/node_ip_address.json +++ /dev/null @@ -1 +0,0 @@ -{"node_ip_address": "192.168.1.56"} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/node_ip_address.json.lock b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/node_ip_address.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/ports_by_node.json b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/ports_by_node.json deleted file mode 100644 index 1257bb6..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/ports_by_node.json +++ /dev/null @@ -1 +0,0 @@ -{"192.168.1.56:/home/valapeos/Projects/pneumarl/logs/rllib/session_2024-05-15_09-19-29_358196_23913/sockets/plasma_store": {"metrics_agent_port": 60081, "metrics_export_port": 64325, "dashboard_agent_listen_port": 52365, "runtime_env_agent_port": 45975, "gcs_server_port": 62418}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/ports_by_node.json.lock b/Godot/logs/rllib/session_2024-05-15_09-19-29_358196_23913/ports_by_node.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/artifacts/2024-05-15_09-23-16/PPO_2024-05-15_09-23-16/driver_artifacts/basic-variant-state-2024-05-15_09-23-16.json b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/artifacts/2024-05-15_09-23-16/PPO_2024-05-15_09-23-16/driver_artifacts/basic-variant-state-2024-05-15_09-23-16.json deleted file mode 100644 index 8dfa826..0000000 Binary files a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/artifacts/2024-05-15_09-23-16/PPO_2024-05-15_09-23-16/driver_artifacts/basic-variant-state-2024-05-15_09-23-16.json and /dev/null differ diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/artifacts/2024-05-15_09-23-16/PPO_2024-05-15_09-23-16/driver_artifacts/experiment_state-2024-05-15_09-23-16.json b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/artifacts/2024-05-15_09-23-16/PPO_2024-05-15_09-23-16/driver_artifacts/experiment_state-2024-05-15_09-23-16.json deleted file mode 100644 index 3d1dc6b..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/artifacts/2024-05-15_09-23-16/PPO_2024-05-15_09-23-16/driver_artifacts/experiment_state-2024-05-15_09-23-16.json +++ /dev/null @@ -1 +0,0 @@ -{"trial_data": [["{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"ff92f_00000\",\n \"storage\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"800595e8020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c1750504f5f323032342d30352d31355f30392d32332d3136948c0e747269616c5f6469725f6e616d65948c2b50504f5f676f646f745f66663932665f30303030305f305f323032342d30352d31355f30392d32332d3136948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e7494888c0a75706c6f61645f646972948c0a44455052454341544544948c0673796e6365729468168c1273796e635f6f6e5f636865636b706f696e7494681675628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1c4c6f63616c46696c6553797374656d2e5f7265636f6e7374727563749493947d948c087573655f6d6d6170948973859452948c0f73746f726167655f66735f70617468948c2b2f686f6d652f76616c6170656f732f50726f6a656374732f706e65756d61726c2f6c6f67732f726c6c696294681768008c115f46696c6573797374656d53796e6365729493942981947d94286819682068114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032342d30352d31355f30392d32332d31369475622e\"\n },\n \"config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"_Trial__unresolved_config\": {\n \"env\": \"godot\",\n \"env_config\": {\n \"env_path\": \"/home/valapeos/Projects/pneumarl/pneuma.x86_64\",\n \"action_repeat\": null,\n \"show_window\": true,\n \"speedup\": 30\n },\n \"framework\": \"torch\",\n \"lr\": 0.0003,\n \"lambda\": 0.95,\n \"gamma\": 0.99,\n \"vf_loss_coeff\": 0.5,\n \"vf_clip_param\": Infinity,\n \"entropy_coeff\": 0.0001,\n \"entropy_coeff_schedule\": null,\n \"normalize_actions\": false,\n \"clip_actions\": true,\n \"rollout_fragment_length\": 32,\n \"sgd_minibatch_size\": 128,\n \"num_workers\": 4,\n \"num_envs_per_worker\": 1,\n \"train_batch_size\": 2048,\n \"num_sgd_iter\": 4,\n \"batch_mode\": \"truncate_episodes\",\n \"num_gpus\": 0,\n \"model\": {\n \"vf_share_layers\": false,\n \"fcnet_hiddens\": [\n 64,\n 64\n ]\n }\n },\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"stopping_criterion\": {\n \"time_total_s\": 10000000\n },\n \"_setup_default_resource\": true,\n \"_default_placement_group_factory\": \"80054e2e\",\n \"placement_group_factory\": \"800595e3000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d94287d948c0343505594473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff0000000000000737d946808473ff000000000000073658c155f686561645f62756e646c655f69735f656d70747994898c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_default_result_or_future\": null,\n \"export_formats\": [],\n \"status\": \"PENDING\",\n \"relative_logdir\": \"PPO_godot_ff92f_00000_0_2024-05-15_09-23-16\",\n \"trial_name_creator\": null,\n \"trial_dirname_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"restore_path\": null,\n \"_restore_checkpoint_result\": null,\n \"_state_json\": null,\n \"results\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_resources\": \"80054e2e\"\n}", "{\n \"start_time\": null,\n \"num_failures\": 0,\n \"num_failures_after_restore\": 0,\n \"error_filename\": null,\n \"pickled_error_filename\": null,\n \"last_result\": {},\n \"last_result_time\": -Infinity,\n \"metric_analysis\": {},\n \"_n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 70, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1715757796.9438848, "_session_str": "2024-05-15_09-23-16", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b148c11636865636b706f696e745f61745f656e6494888c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1715757796.9438848}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/agent-424238335.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/agent-424238335.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/agent-424238335.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/agent-424238335.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/dashboard.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/dashboard.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/debug_state.txt b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/debug_state.txt deleted file mode 100644 index 7dc9d32..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/debug_state.txt +++ /dev/null @@ -1,145 +0,0 @@ -NodeManager: -Node ID: bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 -Node name: 192.168.1.56 -InitialConfigResources: {object_store_memory: 182564241400000, CPU: 120000, memory: 365128482830000, node:__internal_head__: 10000, node:192.168.1.56: 10000} -ClusterTaskManager: -========== Node: bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 ================= -Infeasible queue length: 0 -Schedule queue length: 0 -Dispatch queue length: 0 -num_waiting_for_resource: 0 -num_waiting_for_plasma_memory: 0 -num_waiting_for_remote_node_resources: 0 -num_worker_not_started_by_job_config_not_exist: 0 -num_worker_not_started_by_registration_timeout: 0 -num_tasks_waiting_for_workers: 0 -num_cancelled_tasks: 0 -cluster_resource_scheduler state: -Local id: -6954002209980175059 Local resources: {"total":{node:__internal_head__: [10000], memory: [365128482830000], object_store_memory: [182564241400000], CPU: [120000], node:192.168.1.56: [10000]}}, "available": {node:__internal_head__: [10000], memory: [365128482830000], object_store_memory: [182564241400000], CPU: [120000], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0",} is_draining: 0 is_idle: 1 Cluster resources: node id: -6954002209980175059{"total":{object_store_memory: 182564241400000, node:192.168.1.56: 10000, node:__internal_head__: 10000, CPU: 120000, memory: 365128482830000}}, "available": {object_store_memory: 182564241400000, node:192.168.1.56: 10000, node:__internal_head__: 10000, CPU: 120000, memory: 365128482830000}}, "labels":{"ray.io/node_id":"bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -Waiting tasks size: 0 -Number of executing tasks: 0 -Number of pinned task arguments: 0 -Number of total spilled tasks: 0 -Number of spilled waiting tasks: 0 -Number of spilled unschedulable tasks: 0 -Resource usage { -} -Running tasks by scheduling class: -================================================== - -ClusterResources: -LocalObjectManager: -- num pinned objects: 0 -- pinned objects size: 0 -- num objects pending restore: 0 -- num objects pending spill: 0 -- num bytes pending spill: 0 -- num bytes currently spilled: 0 -- cumulative spill requests: 0 -- cumulative restore requests: 0 -- spilled objects pending delete: 0 - -ObjectManager: -- num local objects: 0 -- num unfulfilled push requests: 0 -- num object pull requests: 0 -- num chunks received total: 0 -- num chunks received failed (all): 0 -- num chunks received failed / cancelled: 0 -- num chunks received failed / plasma error: 0 -Event stats: -Global stats: 0 total (0 active) -Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -Execution time: mean = -nan s, total = 0.000 s -Event stats: -PushManager: -- num pushes in flight: 0 -- num chunks in flight: 0 -- num chunks remaining: 0 -- max chunks allowed: 409 -OwnershipBasedObjectDirectory: -- num listeners: 0 -- cumulative location updates: 0 -- num location updates per second: 0.000 -- num location lookups per second: 0.000 -- num locations added per second: 0.000 -- num locations removed per second: 0.000 -BufferPool: -- create buffer state map size: 0 -PullManager: -- num bytes available for pulled objects: 18256424140 -- num bytes being pulled (all): 0 -- num bytes being pulled / pinned: 0 -- get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -- first get request bundle: N/A -- first wait request bundle: N/A -- first task request bundle: N/A -- num objects queued: 0 -- num objects actively pulled (all): 0 -- num objects actively pulled / pinned: 0 -- num bundles being pulled: 0 -- num pull retries: 0 -- max timeout seconds: 0 -- max timeout request is already processed. No entry. - -WorkerPool: -- registered jobs: 0 -- process_failed_job_config_missing: 0 -- process_failed_rate_limited: 0 -- process_failed_pending_registration: 0 -- process_failed_runtime_env_setup_failed: 0 -- num PYTHON workers: 0 -- num PYTHON drivers: 0 -- num object spill callbacks queued: 0 -- num object restore queued: 0 -- num util functions queued: 0 -- num idle workers: 0 -TaskDependencyManager: -- task deps map size: 0 -- get req map size: 0 -- wait req map size: 0 -- local objects map size: 0 -WaitManager: -- num active wait requests: 0 -Subscriber: -Channel WORKER_OBJECT_EVICTION -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_REF_REMOVED_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -Channel WORKER_OBJECT_LOCATIONS_CHANNEL -- cumulative subscribe requests: 0 -- cumulative unsubscribe requests: 0 -- active subscribed publishers: 0 -- cumulative published messages: 0 -- cumulative processed messages: 0 -num async plasma notifications: 0 -Remote node managers: -Event stats: -Global stats: 23 total (13 active) -Queueing time: mean = 910.431 us, max = 6.631 ms, min = 5.910 us, total = 20.940 ms -Execution time: mean = 497.348 us, total = 11.439 ms -Event stats: - PeriodicalRunner.RunFnPeriodically - 11 total (6 active, 1 running), Execution time: mean = 5.706 us, total = 62.770 us, Queueing time: mean = 1.903 ms, max = 6.631 ms, min = 546.500 us, total = 20.928 ms - NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 351.110 us, total = 351.110 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 130.460 us, total = 130.460 us, Queueing time: mean = 5.910 us, max = 5.910 us, min = 5.910 us, total = 5.910 us - NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 507.130 us, total = 507.130 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 249.870 us, total = 249.870 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 10.138 ms, total = 10.138 ms, Queueing time: mean = 6.040 us, max = 6.040 us, min = 6.040 us, total = 6.040 us - RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -DebugString() time ms: 1 \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/debug_state_gcs.txt b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/debug_state_gcs.txt deleted file mode 100644 index f3f7fd4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/debug_state_gcs.txt +++ /dev/null @@ -1,150 +0,0 @@ -GcsNodeManager: -- RegisterNode request count: 1 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 26 -- GetInternalConfig request count: 1 - -GcsActorManager: -- RegisterActor request count: 5 -- CreateActor request count: 5 -- GetActorInfo request count: 5 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 5 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 1 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 5 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 2 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 1 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 5 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 1 -- Registered placement groups count: 1 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: -RAY_LOG_CHANNEL -- cumulative published messages: 11 -- cumulative published bytes: 14739 -- current buffered bytes: 853 -GCS_JOB_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 280 -- current buffered bytes: 280 -GCS_ACTOR_CHANNEL -- cumulative published messages: 10 -- cumulative published bytes: 5864 -- current buffered bytes: 1328 -GCS_WORKER_DELTA_CHANNEL -- cumulative published messages: 5 -- cumulative published bytes: 485 -- current buffered bytes: 97 -GCS_NODE_INFO_CHANNEL -- cumulative published messages: 1 -- cumulative published bytes: 557 -- current buffered bytes: 0 - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 25 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 13 --Total num of actor creation tasks: 5 --Total num of actor tasks: 6 --Total num of normal tasks: 1 --Total num of driver tasks: 1 - - - - -Global stats: 736 total (19 active) -Queueing time: mean = 116.083 us, max = 25.149 ms, min = 170.000 ns, total = 85.437 ms -Execution time: mean = 3.024 ms, total = 2.226 s -Event stats: - GcsInMemoryStore.Put - 111 total (0 active), Execution time: mean = 251.458 us, total = 27.912 ms, Queueing time: mean = 243.327 us, max = 24.971 ms, min = 740.000 ns, total = 27.009 ms - GcsInMemoryStore.Get - 75 total (0 active), Execution time: mean = 18.990 us, total = 1.424 ms, Queueing time: mean = 10.724 us, max = 282.850 us, min = 830.000 ns, total = 804.280 us - InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 68 total (0 active), Execution time: mean = 9.513 us, total = 646.880 us, Queueing time: mean = 10.957 us, max = 103.820 us, min = 4.370 us, total = 745.099 us - InternalKVGcsService.grpc_server.InternalKVGet - 68 total (0 active), Execution time: mean = 106.234 us, total = 7.224 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 65 total (0 active), Execution time: mean = 7.201 us, total = 468.040 us, Queueing time: mean = 29.154 us, max = 208.990 us, min = 4.240 us, total = 1.895 ms - InternalKVGcsService.grpc_server.InternalKVPut - 65 total (0 active), Execution time: mean = 118.628 us, total = 7.711 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetAllNodeInfo - 26 total (0 active), Execution time: mean = 88.700 us, total = 2.306 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 26 total (0 active), Execution time: mean = 12.840 us, total = 333.829 us, Queueing time: mean = 9.878 us, max = 42.440 us, min = 5.040 us, total = 256.820 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 13.084 us, total = 235.520 us, Queueing time: mean = 17.323 us, max = 141.990 us, min = 5.220 us, total = 311.809 us - WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 130.728 us, total = 2.353 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut - 12 total (0 active), Execution time: mean = 591.939 us, total = 7.103 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 12 total (0 active), Execution time: mean = 4.240 us, total = 50.880 us, Queueing time: mean = 21.115 us, max = 30.370 us, min = 8.430 us, total = 253.380 us - RayletLoadPulled - 10 total (1 active), Execution time: mean = 59.404 us, total = 594.040 us, Queueing time: mean = 36.008 us, max = 286.699 us, min = 5.841 us, total = 360.083 us - NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 8 total (0 active), Execution time: mean = 24.582 us, total = 196.660 us, Queueing time: mean = 10.119 us, max = 19.840 us, min = 6.130 us, total = 80.950 us - NodeManagerService.grpc_client.GetResourceLoad - 8 total (0 active), Execution time: mean = 406.672 us, total = 3.253 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 8.041 us, total = 64.329 us, Queueing time: mean = 142.304 us, max = 1.048 ms, min = 4.290 us, total = 1.138 ms - NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 481.065 us, total = 3.849 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 4.499 us, total = 31.490 us, Queueing time: mean = 11.060 us, max = 16.150 us, min = 1.440 us, total = 77.420 us - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 9.070 us, total = 45.350 us, Queueing time: mean = 100.406 us, max = 292.930 us, min = 3.660 us, total = 502.030 us - NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 429.376 ms, total = 2.147 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 230.880 us, total = 1.154 ms, Queueing time: mean = 8.102 us, max = 9.480 us, min = 6.610 us, total = 40.510 us - GcsResourceManager::Update - 5 total (0 active), Execution time: mean = 41.324 us, total = 206.620 us, Queueing time: mean = 8.316 us, max = 12.450 us, min = 4.900 us, total = 41.580 us - ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 337.672 us, total = 1.688 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 796.420 us, total = 3.982 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 20.694 us, total = 103.470 us, Queueing time: mean = 140.718 us, max = 573.970 us, min = 11.020 us, total = 703.590 us - ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 122.230 us, total = 611.150 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 6.790 us, total = 33.950 us, Queueing time: mean = 49.494 us, max = 143.530 us, min = 4.940 us, total = 247.470 us - CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 175.296 us, total = 876.480 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 156.226 us, total = 781.129 us, Queueing time: mean = 8.382 us, max = 10.890 us, min = 5.820 us, total = 41.910 us - NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 141.290 us, total = 706.450 us, Queueing time: mean = 47.664 us, max = 166.380 us, min = 8.000 us, total = 238.320 us - PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 53.875 us, total = 215.500 us, Queueing time: mean = 12.598 ms, max = 25.149 ms, min = 9.169 us, total = 50.393 ms - ClusterResourceManager.ResetRemoteNodeView - 4 total (1 active), Execution time: mean = 2.058 us, total = 8.230 us, Queueing time: mean = 6.953 us, max = 10.174 us, min = 7.855 us, total = 27.813 us - NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 4 total (0 active), Execution time: mean = 5.452 us, total = 21.810 us, Queueing time: mean = 7.878 us, max = 11.000 us, min = 6.490 us, total = 31.510 us - NodeInfoGcsService.grpc_server.CheckAlive - 4 total (0 active), Execution time: mean = 90.147 us, total = 360.590 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 2 total (0 active), Execution time: mean = 96.865 us, total = 193.730 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - HealthCheck - 2 total (0 active), Execution time: mean = 2.645 us, total = 5.290 us, Queueing time: mean = 14.435 us, max = 16.010 us, min = 12.860 us, total = 28.870 us - NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 2 total (0 active), Execution time: mean = 17.270 us, total = 34.540 us, Queueing time: mean = 9.595 us, max = 11.840 us, min = 7.350 us, total = 19.190 us - NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 341.970 us, total = 341.970 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GCSServer.deadline_timer.debug_state_dump - 1 total (1 active, 1 running), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 768.630 us, total = 768.630 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 42.420 us, total = 42.420 us, Queueing time: mean = 7.060 us, max = 7.060 us, min = 7.060 us, total = 7.060 us - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 87.170 us, total = 87.170 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 273.010 us, total = 273.010 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 74.320 us, total = 74.320 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 1.890 us, total = 1.890 us, Queueing time: mean = 1.000 us, max = 1.000 us, min = 1.000 us, total = 1.000 us - PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 30.940 us, total = 30.940 us, Queueing time: mean = 7.280 us, max = 7.280 us, min = 7.280 us, total = 7.280 us - GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 12.470 us, total = 12.470 us, Queueing time: mean = 7.690 us, max = 7.690 us, min = 7.690 us, total = 7.690 us - NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 67.510 us, total = 67.510 us, Queueing time: mean = 8.780 us, max = 8.780 us, min = 8.780 us, total = 8.780 us - JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 63.970 us, total = 63.970 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 8.580 us, total = 8.580 us, Queueing time: mean = 7.040 us, max = 7.040 us, min = 7.040 us, total = 7.040 us - InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 61.600 us, total = 61.600 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 123.280 us, total = 123.280 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 105.260 us, total = 105.260 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 6.220 us, total = 6.220 us, Queueing time: mean = 170.000 ns, max = 170.000 ns, min = 170.000 ns, total = 170.000 ns - NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.030 us, total = 9.030 us, Queueing time: mean = 11.340 us, max = 11.340 us, min = 11.340 us, total = 11.340 us - JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 4.570 us, total = 4.570 us, Queueing time: mean = 6.950 us, max = 6.950 us, min = 6.950 us, total = 6.950 us - InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 52.970 us, total = 52.970 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 80.000 ns, total = 80.000 ns, Queueing time: mean = 107.590 us, max = 107.590 us, min = 107.590 us, total = 107.590 us - NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 47.970 us, total = 47.970 us, Queueing time: mean = 6.710 us, max = 6.710 us, min = 6.710 us, total = 6.710 us - InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 8.420 us, total = 8.420 us, Queueing time: mean = 5.030 us, max = 5.030 us, min = 5.030 us, total = 5.030 us - GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 2.480 us, total = 2.480 us, Queueing time: mean = 1.880 us, max = 1.880 us, min = 1.880 us, total = 1.880 us - NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 55.360 us, total = 55.360 us, Queueing time: mean = 10.000 us, max = 10.000 us, min = 10.000 us, total = 10.000 us \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/gcs_server.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/gcs_server.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/gcs_server.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/gcs_server.out deleted file mode 100644 index 722ba00..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/gcs_server.out +++ /dev/null @@ -1,145 +0,0 @@ -[2024-05-15 09:23:14,934 I 30311 30311] (gcs_server) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) event.cc:234: Set ray event level to warning -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) event.cc:342: Ray Event initialized for GCS -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_server.cc:74: GCS storage type is StorageType::IN_MEMORY -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:42: Loading job table data. -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:54: Loading node table data. -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:80: Loading actor table data. -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:93: Loading actor task spec table data. -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:66: Loading placement group table data. -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:46: Finished loading job table data, size = 0 -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:58: Finished loading node table data, size = 0 -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:84: Finished loading actor table data, size = 0 -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:97: Finished loading actor task spec table data, size = 0 -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_init_data.cc:71: Finished loading placement group table data, size = 0 -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_server.cc:162: No existing server cluster ID found. Generating new ID: cee699ec84e6b5d0073a5570221a4c3b30049386d7e4573749792f08 -[2024-05-15 09:23:14,935 I 30311 30311] (gcs_server) gcs_server.cc:648: Autoscaler V2 enabled: 0 -[2024-05-15 09:23:14,936 I 30311 30311] (gcs_server) grpc_server.cc:134: GcsServer server started, listening on port 64798. -[2024-05-15 09:23:14,960 I 30311 30311] (gcs_server) gcs_server.cc:250: GcsNodeManager: -- RegisterNode request count: 0 -- DrainNode request count: 0 -- GetAllNodeInfo request count: 0 -- GetInternalConfig request count: 0 - -GcsActorManager: -- RegisterActor request count: 0 -- CreateActor request count: 0 -- GetActorInfo request count: 0 -- GetNamedActorInfo request count: 0 -- GetAllActorInfo request count: 0 -- KillActor request count: 0 -- ListNamedActors request count: 0 -- Registered actors count: 0 -- Destroyed actors count: 0 -- Named actors count: 0 -- Unresolved actors count: 0 -- Pending actors count: 0 -- Created actors count: 0 -- owners_: 0 -- actor_to_register_callbacks_: 0 -- actor_to_create_callbacks_: 0 -- sorted_destroyed_actor_list_: 0 - -GcsResourceManager: -- GetAllAvailableResources request count0 -- GetAllResourceUsage request count: 0 - -GcsPlacementGroupManager: -- CreatePlacementGroup request count: 0 -- RemovePlacementGroup request count: 0 -- GetPlacementGroup request count: 0 -- GetAllPlacementGroup request count: 0 -- WaitPlacementGroupUntilReady request count: 0 -- GetNamedPlacementGroup request count: 0 -- Scheduling pending placement group count: 0 -- Registered placement groups count: 0 -- Named placement group count: 0 -- Pending placement groups count: 0 -- Infeasible placement groups count: 0 - -Publisher: - -[runtime env manager] ID to URIs table: -[runtime env manager] URIs reference table: - -GcsTaskManager: --Total num task events reported: 0 --Total num status task events dropped: 0 --Total num profile events dropped: 0 --Current num of task events stored: 0 --Total num of actor creation tasks: 0 --Total num of actor tasks: 0 --Total num of normal tasks: 0 --Total num of driver tasks: 0 - - -[2024-05-15 09:23:14,960 I 30311 30311] (gcs_server) gcs_server.cc:844: Event stats: - - -Global stats: 21 total (10 active) -Queueing time: mean = 3.587 ms, max = 25.149 ms, min = 1.030 us, total = 75.318 ms -Execution time: mean = 1.199 ms, total = 25.183 ms -Event stats: - InternalKVGcsService.grpc_client.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.GetAll - 5 total (0 active), Execution time: mean = 3.542 us, total = 17.710 us, Queueing time: mean = 14.838 us, max = 16.150 us, min = 13.350 us, total = 74.190 us - PeriodicalRunner.RunFnPeriodically - 4 total (2 active, 1 running), Execution time: mean = 545.000 ns, total = 2.180 us, Queueing time: mean = 12.567 ms, max = 25.149 ms, min = 25.120 ms, total = 50.269 ms - GcsInMemoryStore.Put - 3 total (0 active), Execution time: mean = 8.386 ms, total = 25.158 ms, Queueing time: mean = 8.325 ms, max = 24.971 ms, min = 1.030 us, total = 24.974 ms - RayletLoadPulled - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - GcsInMemoryStore.Get - 1 total (0 active), Execution time: mean = 5.390 us, total = 5.390 us, Queueing time: mean = 1.240 us, max = 1.240 us, min = 1.240 us, total = 1.240 us - ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - - -[2024-05-15 09:23:14,961 I 30311 30311] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats: - - -Global stats: 2 total (1 active) -Queueing time: mean = 30.590 us, max = 61.180 us, min = 61.180 us, total = 61.180 us -Execution time: mean = 9.495 us, total = 18.990 us -Event stats: - GcsTaskManager.GcJobSummary - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s - PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 18.990 us, total = 18.990 us, Queueing time: mean = 61.180 us, max = 61.180 us, min = 61.180 us, total = 61.180 us - - -[2024-05-15 09:23:16,337 I 30311 30311] (gcs_server) gcs_node_manager.cc:55: Registering node info, node id = bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 09:23:16,337 I 30311 30311] (gcs_server) gcs_node_manager.cc:61: Finished registering node info, node id = bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0, address = 192.168.1.56, node name = 192.168.1.56 -[2024-05-15 09:23:16,337 I 30311 30311] (gcs_server) gcs_placement_group_manager.cc:797: A new node: bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 registered, will try to reschedule all the infeasible placement groups. -[2024-05-15 09:23:16,919 I 30311 30311] (gcs_server) gcs_job_manager.cc:42: Adding job, job id = 01000000, driver pid = 30255 -[2024-05-15 09:23:16,919 I 30311 30311] (gcs_server) gcs_job_manager.cc:57: Finished adding job, job id = 01000000, driver pid = 30255 -[2024-05-15 09:23:16,951 I 30311 30311] (gcs_server) gcs_placement_group_manager.cc:432: Registering placement group, placement group id = 32d29e0f4ce311c5b9f22a81bbf501000000, name = , strategy = 0 -[2024-05-15 09:23:16,951 I 30311 30311] (gcs_server) gcs_placement_group_manager.cc:436: Finished registering placement group, placement group id = 32d29e0f4ce311c5b9f22a81bbf501000000, name = , strategy = 0 -[2024-05-15 09:23:16,951 I 30311 30311] (gcs_server) gcs_placement_group_scheduler.cc:185: Preparing resource from node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for bundles: {placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={0}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={1}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={2}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={3}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={4}}, -[2024-05-15 09:23:16,952 I 30311 30311] (gcs_server) gcs_placement_group_scheduler.cc:195: Finished leasing resource from bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for bundles: {placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={0}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={1}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={2}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={3}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={4}}, -[2024-05-15 09:23:16,952 I 30311 30311] (gcs_server) gcs_placement_group_scheduler.cc:213: Committing resource to a node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for bundles: {placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={2}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={0}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={3}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={4}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={1}}, -[2024-05-15 09:23:16,952 I 30311 30311] (gcs_server) gcs_placement_group_scheduler.cc:220: Finished committing resource to bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for bundles: {placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={2}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={0}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={3}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={4}},{placement group id={32d29e0f4ce311c5b9f22a81bbf501000000}, bundle index={1}}, -[2024-05-15 09:23:16,952 I 30311 30311] (gcs_server) gcs_placement_group_manager.cc:321: Successfully created placement group , id: 32d29e0f4ce311c5b9f22a81bbf501000000 -[2024-05-15 09:23:17,128 I 30311 30311] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 197fe642ac70308cd2acc9ce01000000 -[2024-05-15 09:23:17,129 I 30311 30311] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 197fe642ac70308cd2acc9ce01000000 -[2024-05-15 09:23:17,130 I 30311 30311] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 197fe642ac70308cd2acc9ce01000000 -[2024-05-15 09:23:17,130 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor 197fe642ac70308cd2acc9ce01000000, job id = 01000000 -[2024-05-15 09:23:17,536 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor 197fe642ac70308cd2acc9ce01000000, job id = 01000000 -[2024-05-15 09:23:17,536 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 197fe642ac70308cd2acc9ce01000000 on worker 637f3a2677e8ae8805c6719f23410b57166c3f363b57fdf8dc071841 at node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0, job id = 01000000 -[2024-05-15 09:23:19,044 I 30311 30311] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 0b79d814049abf6a43ebd5b601000000 -[2024-05-15 09:23:19,044 I 30311 30311] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 0b79d814049abf6a43ebd5b601000000 -[2024-05-15 09:23:19,045 I 30311 30311] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 0b79d814049abf6a43ebd5b601000000 -[2024-05-15 09:23:19,045 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor 0b79d814049abf6a43ebd5b601000000, job id = 01000000 -[2024-05-15 09:23:19,045 I 30311 30311] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 1f58d0d20a228e11f3006ade01000000 -[2024-05-15 09:23:19,045 I 30311 30311] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 1f58d0d20a228e11f3006ade01000000 -[2024-05-15 09:23:19,046 I 30311 30311] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 1f58d0d20a228e11f3006ade01000000 -[2024-05-15 09:23:19,046 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor 1f58d0d20a228e11f3006ade01000000, job id = 01000000 -[2024-05-15 09:23:19,046 I 30311 30311] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = b3d076f5c2c7d947cadec90701000000 -[2024-05-15 09:23:19,046 I 30311 30311] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = b3d076f5c2c7d947cadec90701000000 -[2024-05-15 09:23:19,047 I 30311 30311] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = b3d076f5c2c7d947cadec90701000000 -[2024-05-15 09:23:19,047 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor b3d076f5c2c7d947cadec90701000000, job id = 01000000 -[2024-05-15 09:23:19,047 I 30311 30311] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = bc53c5eb34f23c1c4a59815401000000 -[2024-05-15 09:23:19,047 I 30311 30311] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = bc53c5eb34f23c1c4a59815401000000 -[2024-05-15 09:23:19,048 I 30311 30311] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = bc53c5eb34f23c1c4a59815401000000 -[2024-05-15 09:23:19,048 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor bc53c5eb34f23c1c4a59815401000000, job id = 01000000 -[2024-05-15 09:23:19,468 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor 1f58d0d20a228e11f3006ade01000000, job id = 01000000 -[2024-05-15 09:23:19,468 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor 0b79d814049abf6a43ebd5b601000000, job id = 01000000 -[2024-05-15 09:23:19,468 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 1f58d0d20a228e11f3006ade01000000 on worker d34ed35583a5c164ec58fcbdfaf3fae61607238063df062abb1e62e3 at node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0, job id = 01000000 -[2024-05-15 09:23:19,468 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 0b79d814049abf6a43ebd5b601000000 on worker e56e6fb20e31fac4f33cbea6b3f727c3f794f6951e14f6fd53d22a24 at node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0, job id = 01000000 -[2024-05-15 09:23:19,469 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor b3d076f5c2c7d947cadec90701000000, job id = 01000000 -[2024-05-15 09:23:19,469 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor b3d076f5c2c7d947cadec90701000000 on worker 6c394590748634c65f828b8780152978379dcbb2dfa93fd2c8e9c177 at node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0, job id = 01000000 -[2024-05-15 09:23:19,522 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 for actor bc53c5eb34f23c1c4a59815401000000, job id = 01000000 -[2024-05-15 09:23:19,522 I 30311 30311] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor bc53c5eb34f23c1c4a59815401000000 on worker 470cc11134b5185b08ccb39eb35538c57bdff5da02ade1ac9a28dc2e at node bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0, job id = 01000000 -[2024-05-15 09:23:24,944 W 30311 30332] (gcs_server) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:65072: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster. diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/log_monitor.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/log_monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/monitor.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/monitor.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/monitor.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/monitor.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/raylet.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/raylet.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/raylet.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/raylet.out deleted file mode 100644 index 902557d..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/raylet.out +++ /dev/null @@ -1,223 +0,0 @@ -[2024-05-15 09:23:16,326 I 30434 30434] (raylet) main.cc:179: Setting cluster ID to: cee699ec84e6b5d0073a5570221a4c3b30049386d7e4573749792f08 -[2024-05-15 09:23:16,327 I 30434 30434] (raylet) main.cc:242: Raylet is not set to kill unknown children. -[2024-05-15 09:23:16,327 I 30434 30434] (raylet) io_service_pool.cc:35: IOServicePool is running with 1 io_service. -[2024-05-15 09:23:16,328 I 30434 30434] (raylet) main.cc:371: Setting node ID to: bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 -[2024-05-15 09:23:16,328 I 30434 30434] (raylet) store_runner.cc:32: Allowing the Plasma store to use up to 18.2564GB of memory. -[2024-05-15 09:23:16,328 I 30434 30434] (raylet) store_runner.cc:48: Starting object store with directory /dev/shm, fallback /home/valapeos/Projects/pneumarl/logs/rllib, and huge page support disabled -[2024-05-15 09:23:16,328 I 30434 30462] (raylet) dlmalloc.cc:154: create_and_mmap_buffer(18256494600, /dev/shm/plasmaXXXXXX) -[2024-05-15 09:23:16,328 I 30434 30462] (raylet) store.cc:564: ========== Plasma store: ================= -Current usage: 0 / 18.2564 GB -- num bytes created total: 0 -0 pending objects of total size 0MB -- objects spillable: 0 -- bytes spillable: 0 -- objects unsealed: 0 -- bytes unsealed: 0 -- objects in use: 0 -- bytes in use: 0 -- objects evictable: 0 -- bytes evictable: 0 - -- objects created by worker: 0 -- bytes created by worker: 0 -- objects restored: 0 -- bytes restored: 0 -- objects received: 0 -- bytes received: 0 -- objects errored: 0 -- bytes errored: 0 - -[2024-05-15 09:23:16,329 I 30434 30434] (raylet) grpc_server.cc:134: ObjectManager server started, listening on port 39761. -[2024-05-15 09:23:16,331 I 30434 30434] (raylet) worker_killing_policy.cc:101: Running GroupByOwner policy. -[2024-05-15 09:23:16,331 W 30434 30434] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero -[2024-05-15 09:23:16,331 I 30434 30434] (raylet) memory_monitor.cc:47: MemoryMonitor initialized with usage threshold at 63982133248 bytes (0.95 system memory), total system memory bytes: 67349614592 -[2024-05-15 09:23:16,331 I 30434 30434] (raylet) node_manager.cc:285: Initializing NodeManager with ID bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 -[2024-05-15 09:23:16,331 I 30434 30434] (raylet) grpc_server.cc:134: NodeManager server started, listening on port 40585. -[2024-05-15 09:23:16,337 I 30434 30496] (raylet) agent_manager.cc:78: Monitor agent process with name dashboard_agent/424238335 -[2024-05-15 09:23:16,337 I 30434 30498] (raylet) agent_manager.cc:78: Monitor agent process with name runtime_env_agent -[2024-05-15 09:23:16,337 I 30434 30434] (raylet) event.cc:234: Set ray event level to warning -[2024-05-15 09:23:16,337 I 30434 30434] (raylet) event.cc:342: Ray Event initialized for RAYLET -[2024-05-15 09:23:16,338 I 30434 30434] (raylet) raylet.cc:128: Raylet of id, bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 started. Raylet consists of node_manager and object_manager. node_manager address: 192.168.1.56:40585 object_manager address: 192.168.1.56:39761 hostname: valanixos -[2024-05-15 09:23:16,339 I 30434 30434] (raylet) node_manager.cc:521: [state-dump] NodeManager: -[state-dump] Node ID: bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 -[state-dump] Node name: 192.168.1.56 -[state-dump] InitialConfigResources: {object_store_memory: 182564241400000, CPU: 120000, memory: 365128482830000, node:__internal_head__: 10000, node:192.168.1.56: 10000} -[state-dump] ClusterTaskManager: -[state-dump] ========== Node: bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0 ================= -[state-dump] Infeasible queue length: 0 -[state-dump] Schedule queue length: 0 -[state-dump] Dispatch queue length: 0 -[state-dump] num_waiting_for_resource: 0 -[state-dump] num_waiting_for_plasma_memory: 0 -[state-dump] num_waiting_for_remote_node_resources: 0 -[state-dump] num_worker_not_started_by_job_config_not_exist: 0 -[state-dump] num_worker_not_started_by_registration_timeout: 0 -[state-dump] num_tasks_waiting_for_workers: 0 -[state-dump] num_cancelled_tasks: 0 -[state-dump] cluster_resource_scheduler state: -[state-dump] Local id: -6954002209980175059 Local resources: {"total":{node:__internal_head__: [10000], memory: [365128482830000], object_store_memory: [182564241400000], CPU: [120000], node:192.168.1.56: [10000]}}, "available": {node:__internal_head__: [10000], memory: [365128482830000], object_store_memory: [182564241400000], CPU: [120000], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0",} is_draining: 0 is_idle: 1 Cluster resources: node id: -6954002209980175059{"total":{object_store_memory: 182564241400000, node:192.168.1.56: 10000, node:__internal_head__: 10000, CPU: 120000, memory: 365128482830000}}, "available": {object_store_memory: 182564241400000, node:192.168.1.56: 10000, node:__internal_head__: 10000, CPU: 120000, memory: 365128482830000}}, "labels":{"ray.io/node_id":"bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []} -[state-dump] Waiting tasks size: 0 -[state-dump] Number of executing tasks: 0 -[state-dump] Number of pinned task arguments: 0 -[state-dump] Number of total spilled tasks: 0 -[state-dump] Number of spilled waiting tasks: 0 -[state-dump] Number of spilled unschedulable tasks: 0 -[state-dump] Resource usage { -[state-dump] } -[state-dump] Running tasks by scheduling class: -[state-dump] ================================================== -[state-dump] -[state-dump] ClusterResources: -[state-dump] LocalObjectManager: -[state-dump] - num pinned objects: 0 -[state-dump] - pinned objects size: 0 -[state-dump] - num objects pending restore: 0 -[state-dump] - num objects pending spill: 0 -[state-dump] - num bytes pending spill: 0 -[state-dump] - num bytes currently spilled: 0 -[state-dump] - cumulative spill requests: 0 -[state-dump] - cumulative restore requests: 0 -[state-dump] - spilled objects pending delete: 0 -[state-dump] -[state-dump] ObjectManager: -[state-dump] - num local objects: 0 -[state-dump] - num unfulfilled push requests: 0 -[state-dump] - num object pull requests: 0 -[state-dump] - num chunks received total: 0 -[state-dump] - num chunks received failed (all): 0 -[state-dump] - num chunks received failed / cancelled: 0 -[state-dump] - num chunks received failed / plasma error: 0 -[state-dump] Event stats: -[state-dump] Global stats: 0 total (0 active) -[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] Execution time: mean = -nan s, total = 0.000 s -[state-dump] Event stats: -[state-dump] PushManager: -[state-dump] - num pushes in flight: 0 -[state-dump] - num chunks in flight: 0 -[state-dump] - num chunks remaining: 0 -[state-dump] - max chunks allowed: 409 -[state-dump] OwnershipBasedObjectDirectory: -[state-dump] - num listeners: 0 -[state-dump] - cumulative location updates: 0 -[state-dump] - num location updates per second: 0.000 -[state-dump] - num location lookups per second: 0.000 -[state-dump] - num locations added per second: 0.000 -[state-dump] - num locations removed per second: 0.000 -[state-dump] BufferPool: -[state-dump] - create buffer state map size: 0 -[state-dump] PullManager: -[state-dump] - num bytes available for pulled objects: 18256424140 -[state-dump] - num bytes being pulled (all): 0 -[state-dump] - num bytes being pulled / pinned: 0 -[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable} -[state-dump] - first get request bundle: N/A -[state-dump] - first wait request bundle: N/A -[state-dump] - first task request bundle: N/A -[state-dump] - num objects queued: 0 -[state-dump] - num objects actively pulled (all): 0 -[state-dump] - num objects actively pulled / pinned: 0 -[state-dump] - num bundles being pulled: 0 -[state-dump] - num pull retries: 0 -[state-dump] - max timeout seconds: 0 -[state-dump] - max timeout request is already processed. No entry. -[state-dump] -[state-dump] WorkerPool: -[state-dump] - registered jobs: 0 -[state-dump] - process_failed_job_config_missing: 0 -[state-dump] - process_failed_rate_limited: 0 -[state-dump] - process_failed_pending_registration: 0 -[state-dump] - process_failed_runtime_env_setup_failed: 0 -[state-dump] - num PYTHON workers: 0 -[state-dump] - num PYTHON drivers: 0 -[state-dump] - num object spill callbacks queued: 0 -[state-dump] - num object restore queued: 0 -[state-dump] - num util functions queued: 0 -[state-dump] - num idle workers: 0 -[state-dump] TaskDependencyManager: -[state-dump] - task deps map size: 0 -[state-dump] - get req map size: 0 -[state-dump] - wait req map size: 0 -[state-dump] - local objects map size: 0 -[state-dump] WaitManager: -[state-dump] - num active wait requests: 0 -[state-dump] Subscriber: -[state-dump] Channel WORKER_OBJECT_EVICTION -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_REF_REMOVED_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL -[state-dump] - cumulative subscribe requests: 0 -[state-dump] - cumulative unsubscribe requests: 0 -[state-dump] - active subscribed publishers: 0 -[state-dump] - cumulative published messages: 0 -[state-dump] - cumulative processed messages: 0 -[state-dump] num async plasma notifications: 0 -[state-dump] Remote node managers: -[state-dump] Event stats: -[state-dump] Global stats: 27 total (13 active) -[state-dump] Queueing time: mean = 857.013 us, max = 6.631 ms, min = 5.910 us, total = 23.139 ms -[state-dump] Execution time: mean = 453.772 us, total = 12.252 ms -[state-dump] Event stats: -[state-dump] PeriodicalRunner.RunFnPeriodically - 11 total (2 active, 1 running), Execution time: mean = 79.603 us, total = 875.630 us, Queueing time: mean = 2.102 ms, max = 6.631 ms, min = 13.870 us, total = 23.127 ms -[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.flush_free_objects - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 507.130 us, total = 507.130 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 10.138 ms, total = 10.138 ms, Queueing time: mean = 6.040 us, max = 6.040 us, min = 6.040 us, total = 6.040 us -[state-dump] NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 351.110 us, total = 351.110 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 130.460 us, total = 130.460 us, Queueing time: mean = 5.910 us, max = 5.910 us, min = 5.910 us, total = 5.910 us -[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.record_metrics - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 249.870 us, total = 249.870 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] NodeManager.deadline_timer.debug_state_dump - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s -[state-dump] DebugString() time ms: 0 -[state-dump] -[state-dump] -[2024-05-15 09:23:16,339 I 30434 30434] (raylet) accessor.cc:627: Received notification for node id = bb0b88761cf79bcb704cc6189d30e057a690327f14681f11b16e7eb0, IsAlive = 1 -[2024-05-15 09:23:16,444 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30516, the token is 0 -[2024-05-15 09:23:16,446 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30517, the token is 1 -[2024-05-15 09:23:16,447 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30518, the token is 2 -[2024-05-15 09:23:16,449 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30519, the token is 3 -[2024-05-15 09:23:16,451 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30520, the token is 4 -[2024-05-15 09:23:16,453 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30521, the token is 5 -[2024-05-15 09:23:16,455 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30522, the token is 6 -[2024-05-15 09:23:16,456 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30523, the token is 7 -[2024-05-15 09:23:16,458 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30524, the token is 8 -[2024-05-15 09:23:16,467 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30525, the token is 9 -[2024-05-15 09:23:16,469 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30526, the token is 10 -[2024-05-15 09:23:16,478 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 30527, the token is 11 -[2024-05-15 09:23:16,807 I 30434 30462] (raylet) object_store.cc:35: Object store current usage 8e-09 / 18.2564 GB. -[2024-05-15 09:23:16,919 I 30434 30434] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 30255 is dead: 0 driver address: 192.168.1.56 -[2024-05-15 09:23:16,919 I 30434 30434] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool. -[2024-05-15 09:23:17,132 I 30434 30434] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:23:17,134 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 31255, the token is 12 -[2024-05-15 09:23:17,944 I 30434 30434] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:23:17,945 I 30434 30434] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:23:17,945 I 30434 30434] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:23:17,946 I 30434 30434] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:23:18,148 I 30434 30434] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false -[2024-05-15 09:23:19,046 I 30434 30434] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:23:19,049 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 31343, the token is 13 -[2024-05-15 09:23:19,049 I 30434 30434] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:23:19,051 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 31344, the token is 14 -[2024-05-15 09:23:19,051 I 30434 30434] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:23:19,053 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 31345, the token is 15 -[2024-05-15 09:23:19,054 I 30434 30434] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000 -[2024-05-15 09:23:19,056 I 30434 30434] (raylet) worker_pool.cc:494: Started worker process with pid 31346, the token is 16 -[2024-05-15 09:23:21,340 W 30434 30434] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/runtime_env_agent.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/runtime_env_agent.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/runtime_env_agent.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/runtime_env_agent.out deleted file mode 100644 index 84fc7b0..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/runtime_env_agent.out +++ /dev/null @@ -1,2 +0,0 @@ -======== Running on http://0.0.0.0:56423 ======== -(Press CTRL+C to quit) diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-4027c0356a8058939630c02c83f140e58f665370dccef28ec27232bb-ffffffff-30522.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-4027c0356a8058939630c02c83f140e58f665370dccef28ec27232bb-ffffffff-30522.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-4027c0356a8058939630c02c83f140e58f665370dccef28ec27232bb-ffffffff-30522.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-4027c0356a8058939630c02c83f140e58f665370dccef28ec27232bb-ffffffff-30522.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-470cc11134b5185b08ccb39eb35538c57bdff5da02ade1ac9a28dc2e-01000000-31346.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-470cc11134b5185b08ccb39eb35538c57bdff5da02ade1ac9a28dc2e-01000000-31346.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-470cc11134b5185b08ccb39eb35538c57bdff5da02ade1ac9a28dc2e-01000000-31346.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-470cc11134b5185b08ccb39eb35538c57bdff5da02ade1ac9a28dc2e-01000000-31346.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-470cc11134b5185b08ccb39eb35538c57bdff5da02ade1ac9a28dc2e-01000000-31346.out deleted file mode 100644 index 9587b73..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-470cc11134b5185b08ccb39eb35538c57bdff5da02ade1ac9a28dc2e-01000000-31346.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11012 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-59e32db94ae26fdee6783938e219800d0c9d01ece2ed0d53ab775e97-ffffffff-30523.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-59e32db94ae26fdee6783938e219800d0c9d01ece2ed0d53ab775e97-ffffffff-30523.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-59e32db94ae26fdee6783938e219800d0c9d01ece2ed0d53ab775e97-ffffffff-30523.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-59e32db94ae26fdee6783938e219800d0c9d01ece2ed0d53ab775e97-ffffffff-30523.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-5f50e0dc40d805e37bfdb20fb0c5a003cb7882059f3eaca5c920e58f-ffffffff-30526.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-5f50e0dc40d805e37bfdb20fb0c5a003cb7882059f3eaca5c920e58f-ffffffff-30526.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-5f50e0dc40d805e37bfdb20fb0c5a003cb7882059f3eaca5c920e58f-ffffffff-30526.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-5f50e0dc40d805e37bfdb20fb0c5a003cb7882059f3eaca5c920e58f-ffffffff-30526.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-637f3a2677e8ae8805c6719f23410b57166c3f363b57fdf8dc071841-01000000-31255.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-637f3a2677e8ae8805c6719f23410b57166c3f363b57fdf8dc071841-01000000-31255.err deleted file mode 100644 index eb33dcf..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-637f3a2677e8ae8805c6719f23410b57166c3f363b57fdf8dc071841-01000000-31255.err +++ /dev/null @@ -1,5 +0,0 @@ -:job_id:01000000 -:actor_name:PPO -2024-05-15 09:23:18,995 WARNING deprecation.py:50 -- DeprecationWarning: `AlgorithmConfig.num_envs_per_worker` has been deprecated. Use `AlgorithmConfig.num_envs_per_env_runner` instead. This will raise an error in the future! -2024-05-15 09:23:19,000 WARNING deprecation.py:50 -- DeprecationWarning: `WorkerSet(num_workers=... OR local_worker=...)` has been deprecated. Use `EnvRunnerGroup(num_env_runners=... AND local_env_runner=...)` instead. This will raise an error in the future! -2024-05-15 09:23:19,000 WARNING deprecation.py:50 -- DeprecationWarning: `max_num_worker_restarts` has been deprecated. Use `AlgorithmConfig.max_num_env_runner_restarts` instead. This will raise an error in the future! diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-637f3a2677e8ae8805c6719f23410b57166c3f363b57fdf8dc071841-01000000-31255.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-637f3a2677e8ae8805c6719f23410b57166c3f363b57fdf8dc071841-01000000-31255.out deleted file mode 100644 index 1124c85..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-637f3a2677e8ae8805c6719f23410b57166c3f363b57fdf8dc071841-01000000-31255.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:actor_name:PPO diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-689e562c89c96eaa5f9963f525dc7385ce90b6d51659edd437fe1b36-ffffffff-30516.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-689e562c89c96eaa5f9963f525dc7385ce90b6d51659edd437fe1b36-ffffffff-30516.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-689e562c89c96eaa5f9963f525dc7385ce90b6d51659edd437fe1b36-ffffffff-30516.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-689e562c89c96eaa5f9963f525dc7385ce90b6d51659edd437fe1b36-ffffffff-30516.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-6c394590748634c65f828b8780152978379dcbb2dfa93fd2c8e9c177-01000000-31345.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-6c394590748634c65f828b8780152978379dcbb2dfa93fd2c8e9c177-01000000-31345.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-6c394590748634c65f828b8780152978379dcbb2dfa93fd2c8e9c177-01000000-31345.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-6c394590748634c65f828b8780152978379dcbb2dfa93fd2c8e9c177-01000000-31345.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-6c394590748634c65f828b8780152978379dcbb2dfa93fd2c8e9c177-01000000-31345.out deleted file mode 100644 index 01371fe..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-6c394590748634c65f828b8780152978379dcbb2dfa93fd2c8e9c177-01000000-31345.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11011 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-798dbab68448e3660dc975a040d57da090b207025a1f6d2027ad1733-ffffffff-30525.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-798dbab68448e3660dc975a040d57da090b207025a1f6d2027ad1733-ffffffff-30525.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-798dbab68448e3660dc975a040d57da090b207025a1f6d2027ad1733-ffffffff-30525.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-798dbab68448e3660dc975a040d57da090b207025a1f6d2027ad1733-ffffffff-30525.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-7d73b4c103f497a36aa1c772e8651273eda26a9f46515c31d4ba70ee-ffffffff-30517.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-7d73b4c103f497a36aa1c772e8651273eda26a9f46515c31d4ba70ee-ffffffff-30517.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-7d73b4c103f497a36aa1c772e8651273eda26a9f46515c31d4ba70ee-ffffffff-30517.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-7d73b4c103f497a36aa1c772e8651273eda26a9f46515c31d4ba70ee-ffffffff-30517.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-80c6aad82ad8c3f13e8b5c00d09684aaeb5291ee2ab0c0cfb7548148-ffffffff-30518.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-80c6aad82ad8c3f13e8b5c00d09684aaeb5291ee2ab0c0cfb7548148-ffffffff-30518.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-80c6aad82ad8c3f13e8b5c00d09684aaeb5291ee2ab0c0cfb7548148-ffffffff-30518.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-80c6aad82ad8c3f13e8b5c00d09684aaeb5291ee2ab0c0cfb7548148-ffffffff-30518.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-a45ede6c8706319d0f1e9b3f5611e246cbdbe7a068f0cd9be680571f-ffffffff-30524.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-a45ede6c8706319d0f1e9b3f5611e246cbdbe7a068f0cd9be680571f-ffffffff-30524.err deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-a45ede6c8706319d0f1e9b3f5611e246cbdbe7a068f0cd9be680571f-ffffffff-30524.err +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-a45ede6c8706319d0f1e9b3f5611e246cbdbe7a068f0cd9be680571f-ffffffff-30524.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-a45ede6c8706319d0f1e9b3f5611e246cbdbe7a068f0cd9be680571f-ffffffff-30524.out deleted file mode 100644 index 9340eb4..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-a45ede6c8706319d0f1e9b3f5611e246cbdbe7a068f0cd9be680571f-ffffffff-30524.out +++ /dev/null @@ -1,2 +0,0 @@ -:job_id:01000000 -:task_name:bundle_reservation_check_func diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-bbe9a3160168b6b9b5a13e780997491b65ae96842afcf7d41f5b5dc7-ffffffff-30527.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-bbe9a3160168b6b9b5a13e780997491b65ae96842afcf7d41f5b5dc7-ffffffff-30527.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-bbe9a3160168b6b9b5a13e780997491b65ae96842afcf7d41f5b5dc7-ffffffff-30527.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-bbe9a3160168b6b9b5a13e780997491b65ae96842afcf7d41f5b5dc7-ffffffff-30527.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-bca74c2f50fdd136ee6fcb928cfa83f0a82aab4a2955979a121ffaf5-ffffffff-30520.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-bca74c2f50fdd136ee6fcb928cfa83f0a82aab4a2955979a121ffaf5-ffffffff-30520.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-bca74c2f50fdd136ee6fcb928cfa83f0a82aab4a2955979a121ffaf5-ffffffff-30520.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-bca74c2f50fdd136ee6fcb928cfa83f0a82aab4a2955979a121ffaf5-ffffffff-30520.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-d34ed35583a5c164ec58fcbdfaf3fae61607238063df062abb1e62e3-01000000-31344.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-d34ed35583a5c164ec58fcbdfaf3fae61607238063df062abb1e62e3-01000000-31344.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-d34ed35583a5c164ec58fcbdfaf3fae61607238063df062abb1e62e3-01000000-31344.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-d34ed35583a5c164ec58fcbdfaf3fae61607238063df062abb1e62e3-01000000-31344.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-d34ed35583a5c164ec58fcbdfaf3fae61607238063df062abb1e62e3-01000000-31344.out deleted file mode 100644 index 80b5e41..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-d34ed35583a5c164ec58fcbdfaf3fae61607238063df062abb1e62e3-01000000-31344.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11010 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e56e6fb20e31fac4f33cbea6b3f727c3f794f6951e14f6fd53d22a24-01000000-31343.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e56e6fb20e31fac4f33cbea6b3f727c3f794f6951e14f6fd53d22a24-01000000-31343.err deleted file mode 100644 index 2a6ef07..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e56e6fb20e31fac4f33cbea6b3f727c3f794f6951e14f6fd53d22a24-01000000-31343.err +++ /dev/null @@ -1,36 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -libfontconfig.so.1: cannot open shared object file: No such file or directory -libX11.so.6: cannot open shared object file: No such file or directory -ERROR: Can't load Xlib dynamically. - at: DisplayServerX11 (platform/linuxbsd/x11/display_server_x11.cpp:5753) -ERROR: Unable to create DisplayServer, all display drivers failed. - at: setup2 (main/main.cpp:2438) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketLargeE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools12BucketMediumE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: Pages in use exist at exit in PagedAllocator: N7Variant5Pools11BucketSmallE - at: ~PagedAllocator (./core/templates/paged_allocator.h:170) -ERROR: BUG: Unreferenced static string to 0: TextServerDummy - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: IPUnix - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatImporterSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterMJPEG - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: interface_added - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: TextServerAdvanced - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ImageFormatLoader - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatSaver - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: MovieWriterPNGWAV - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ScriptLanguage - at: unref (core/string/string_name.cpp:129) -ERROR: BUG: Unreferenced static string to 0: ResourceFormatLoader - at: unref (core/string/string_name.cpp:129) diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e56e6fb20e31fac4f33cbea6b3f727c3f794f6951e14f6fd53d22a24-01000000-31343.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e56e6fb20e31fac4f33cbea6b3f727c3f794f6951e14f6fd53d22a24-01000000-31343.out deleted file mode 100644 index 6628553..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e56e6fb20e31fac4f33cbea6b3f727c3f794f6951e14f6fd53d22a24-01000000-31343.out +++ /dev/null @@ -1,14 +0,0 @@ -:job_id:01000000 -:actor_name:RolloutWorker -waiting for remote GODOT connection on port 11009 -Godot Engine v4.2.2.stable.mono.official.15073afe3 - https://godotengine.org -Your video card drivers seem not to support the required Vulkan version. - -If possible, consider updating your video card drivers or using the OpenGL 3 driver. - -You can enable the OpenGL 3 driver by starting the engine from the -command line with the command: - - "pneuma.x86_64" --rendering-driver opengl3 - -If you recently updated your video card drivers, try rebooting. diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e91111a761a318af2ed2164189320c4dc4e28fbd5bdd07d1e15df467-ffffffff-30521.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e91111a761a318af2ed2164189320c4dc4e28fbd5bdd07d1e15df467-ffffffff-30521.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e91111a761a318af2ed2164189320c4dc4e28fbd5bdd07d1e15df467-ffffffff-30521.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-e91111a761a318af2ed2164189320c4dc4e28fbd5bdd07d1e15df467-ffffffff-30521.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-fdf7fdbedf3321d0dda54f46b7e1c36a00d154adaeb099c5a11917b1-ffffffff-30519.err b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-fdf7fdbedf3321d0dda54f46b7e1c36a00d154adaeb099c5a11917b1-ffffffff-30519.err deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-fdf7fdbedf3321d0dda54f46b7e1c36a00d154adaeb099c5a11917b1-ffffffff-30519.out b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/logs/worker-fdf7fdbedf3321d0dda54f46b7e1c36a00d154adaeb099c5a11917b1-ffffffff-30519.out deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/node_ip_address.json b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/node_ip_address.json deleted file mode 100644 index 8d998d8..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/node_ip_address.json +++ /dev/null @@ -1 +0,0 @@ -{"node_ip_address": "192.168.1.56"} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/node_ip_address.json.lock b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/node_ip_address.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/ports_by_node.json b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/ports_by_node.json deleted file mode 100644 index f81fdd6..0000000 --- a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/ports_by_node.json +++ /dev/null @@ -1 +0,0 @@ -{"192.168.1.56:/home/valapeos/Projects/pneumarl/logs/rllib/session_2024-05-15_09-23-14_915555_30255/sockets/plasma_store": {"metrics_agent_port": 65072, "metrics_export_port": 59826, "dashboard_agent_listen_port": 52365, "runtime_env_agent_port": 56423, "gcs_server_port": 64798}} \ No newline at end of file diff --git a/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/ports_by_node.json.lock b/Godot/logs/rllib/session_2024-05-15_09-23-14_915555_30255/ports_by_node.json.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Godot/logs/rllib/session_latest b/Godot/logs/rllib/session_latest deleted file mode 120000 index dd4e755..0000000 --- a/Godot/logs/rllib/session_latest +++ /dev/null @@ -1 +0,0 @@ -/home/valapeos/Projects/pneumarl/logs/rllib/session_2024-05-15_09-23-14_915555_30255 \ No newline at end of file diff --git a/Godot/logs/sb3/experiment_13/events.out.tfevents.1715954191.valanixos.15690.0 b/Godot/logs/sb3/experiment_13/events.out.tfevents.1715954191.valanixos.15690.0 new file mode 100644 index 0000000..8f3fb69 Binary files /dev/null and b/Godot/logs/sb3/experiment_13/events.out.tfevents.1715954191.valanixos.15690.0 differ diff --git a/Godot/logs/sb3/experiment_14/events.out.tfevents.1715970707.valanixos.41832.0 b/Godot/logs/sb3/experiment_14/events.out.tfevents.1715970707.valanixos.41832.0 new file mode 100644 index 0000000..df1891e Binary files /dev/null and b/Godot/logs/sb3/experiment_14/events.out.tfevents.1715970707.valanixos.41832.0 differ diff --git a/Godot/logs/sb3/experiment_15/events.out.tfevents.1715970870.valanixos.43900.0 b/Godot/logs/sb3/experiment_15/events.out.tfevents.1715970870.valanixos.43900.0 new file mode 100644 index 0000000..f3a6e20 Binary files /dev/null and b/Godot/logs/sb3/experiment_15/events.out.tfevents.1715970870.valanixos.43900.0 differ diff --git a/Godot/main.py b/Godot/main.py index 5f5ef4d..9a0abb1 100644 --- a/Godot/main.py +++ b/Godot/main.py @@ -1,6 +1,8 @@ import args import os import pathlib + +import torch as T import torch.nn as nn from typing import Callable @@ -92,16 +94,16 @@ def linear_schedule(initial_value: float) -> Callable[[float], float]: return func policy_kwargs = dict( - activation_fn=nn.Tanh(), - net_atch = dict( - pi=[1024, 1024, 1024, 1024], - vf = [4096, 4096, 4096, 4096, 4096, 4096] - ) -) - -optimizer_kwargs = dict( - betas=(0.9, 0.9), - eps=1e-5 + activation_fn=nn.Tanh, + net_arch = dict( + pi=[256], + vf = [2048, 2048]#, 4096, 4096, 4096, 4096] + ), + optimizer_class = T.optim.Adam, + optimizer_kwargs = dict( + betas=(0.9, 0.9), + eps=1e-5 + ), ) if args.resume_model_path is None: @@ -111,6 +113,7 @@ if args.resume_model_path is None: linear_schedule(0.0003) model: PPO = PPO( + # 'MultiInputPolicy' serves as an alias for MultiInputActorCriticPolicy "MultiInputPolicy", env, batch_size=64, @@ -119,6 +122,8 @@ if args.resume_model_path is None: n_steps=256, tensorboard_log=args.exper_dir, learning_rate=learning_rate, + policy_kwargs=policy_kwargs, + # optimizer_kwargs=optimizer_kwargs, ) else: path_zip = pathlib.Path(args.resume_model_path) diff --git a/Godot/poetry.lock b/Godot/poetry.lock index b08be1e..cd5ded8 100644 --- a/Godot/poetry.lock +++ b/Godot/poetry.lock @@ -2204,4 +2204,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "35cc92cf532f4a39b77dae861ba87f98a29b61f5ec07589b3b968e710e5b4c38" +content-hash = "2d575ca622be4eb61953716f1811accd9330cdbd4ad0ceee31700058e486e7e6" diff --git a/Godot/pyproject.toml b/Godot/pyproject.toml index 33ccead..d49e673 100644 --- a/Godot/pyproject.toml +++ b/Godot/pyproject.toml @@ -14,6 +14,7 @@ scikit-image = "^0.23.2" lz4 = "^4.3.3" godot-rl = {url = "https://github.com/edbeeching/godot_rl_agents/archive/refs/heads/main.zip"} pettingzoo = "^1.24.3" +tensorboard = "^2.16.2" [build-system] diff --git a/Godot/scenes/main.tscn b/Godot/scenes/main.tscn index d03fe0d..c7acd44 100644 --- a/Godot/scenes/main.tscn +++ b/Godot/scenes/main.tscn @@ -1356,7 +1356,7 @@ position = Vector2(758, 192) position = Vector2(763, -713) [node name="Bamboo11" parent="Bamboos" instance=ExtResource("4_8jerw")] -position = Vector2(-45, -136) +position = Vector2(-158, -96) [node name="Bamboo12" parent="Bamboos" instance=ExtResource("4_8jerw")] position = Vector2(967, -238) @@ -1364,6 +1364,9 @@ position = Vector2(967, -238) [node name="Bamboo13" parent="Bamboos" instance=ExtResource("4_8jerw")] position = Vector2(294, -594) +[node name="Bamboo14" parent="Bamboos" instance=ExtResource("4_8jerw")] +position = Vector2(560, -141) + [node name="Killzone" parent="." instance=ExtResource("5_3xrri")] [node name="CollisionShape2D" type="CollisionShape2D" parent="Killzone"] diff --git a/Godot/shell.nix b/Godot/shell.nix index e21e2ea..d59ccc7 100644 --- a/Godot/shell.nix +++ b/Godot/shell.nix @@ -1,22 +1,33 @@ with import {}; -mkShell { - NIX_LD_LIBRARY_PATH = lib.makeLibraryPath [ +pkgs.mkShell rec { + + dotnetPkg = (with dotnetCorePackages; combinePackages [ + sdk_9_0 + ]); + + NIX_LD_LIBRARY_PATH = lib.makeLibraryPath ([ stdenv.cc.cc - ]; - NIX_LD = lib.fileContents "${stdenv.cc}/nix-support/dynamic-linker"; + ] ++ deps); + + NIX_LD = "${pkgs.stdenv.cc.libc_bin}/bin/ld.so"; + + nativeBuildInputs = [] ++ deps; + shellHook = '' export LD_LIBRARY_PATH=$NIX_LD_LIBRARY_PATH + DOTNET_ROOT="${dotnetPkg}" ''; packages = [ - (with dotnetCorePackages; combinePackages [ - sdk_6_0 - sdk_7_0 - sdk_8_0 - sdk_9_0 - ]) dotnet-sdk xorg.libX11 ]; + + deps = [ + zlib + zlib.dev + openssl + dotnetPkg + ]; }