Updated icon
This commit is contained in:
Vasilis Valatsos
parent
17c08b7a81
commit
4788dd7783
363 changed files with 84 additions and 7495 deletions
107
Godot/icon.svg
107
Godot/icon.svg
|
|
@ -1,66 +1,43 @@
|
|||
<?xml version="1.0" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
|
||||
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
|
||||
<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
|
||||
width="288.000000pt" height="288.000000pt" viewBox="0 0 288.000000 288.000000"
|
||||
preserveAspectRatio="xMidYMid meet">
|
||||
|
||||
<g transform="translate(0.000000,288.000000) scale(0.100000,-0.100000)"
|
||||
fill="#000000" stroke="none">
|
||||
<path d="M1265 2856 c-339 -51 -600 -180 -831 -410 -486 -487 -559 -1224 -177
|
||||
-1800 132 -199 361 -397 574 -494 401 -185 817 -185 1218 0 281 129 550 398
|
||||
679 679 185 401 185 817 0 1218 -129 282 -397 550 -679 679 -251 116 -545 164
|
||||
-784 128z m480 -40 c55 -13 134 -37 175 -52 73 -26 250 -110 250 -118 0 -3
|
||||
-33 11 -74 29 -176 82 -320 115 -532 122 -190 6 -308 -9 -470 -62 -372 -122
|
||||
-700 -407 -850 -741 -21 -45 -42 -80 -47 -77 -6 4 -8 -3 -5 -15 4 -12 2 -20
|
||||
-2 -17 -22 13 -70 -299 -70 -450 1 -96 23 -263 46 -345 4 -14 4 -22 0 -18 -13
|
||||
12 -54 174 -66 267 -17 121 -7 357 19 471 11 47 34 126 51 175 120 336 337
|
||||
572 670 728 131 62 166 68 50 10 -148 -75 -245 -146 -365 -267 -114 -115 -181
|
||||
-206 -247 -336 -43 -84 -81 -180 -74 -187 2 -2 10 13 16 34 52 157 176 351
|
||||
314 488 196 194 420 314 704 376 126 28 366 21 507 -15z m50 -51 c95 -21 215
|
||||
-61 304 -101 40 -19 78 -31 84 -27 6 3 7 1 3 -5 -4 -7 7 -18 26 -28 196 -101
|
||||
383 -311 501 -564 62 -131 68 -166 10 -50 -75 148 -146 245 -267 365 -115 114
|
||||
-206 181 -336 247 -84 43 -180 81 -187 74 -2 -2 13 -10 34 -16 157 -52 351
|
||||
-176 488 -314 199 -200 326 -444 380 -731 35 -184 8 -438 -71 -655 -27 -77
|
||||
-110 -250 -119 -250 -3 0 4 18 15 39 27 54 80 189 80 204 -1 6 -16 -27 -34
|
||||
-74 -19 -47 -44 -105 -55 -128 -12 -23 -19 -47 -15 -54 4 -7 4 -9 -1 -5 -4 4
|
||||
-35 -33 -67 -83 -120 -186 -297 -334 -528 -442 -109 -51 -167 -71 -110 -37 40
|
||||
24 38 23 -47 -9 -36 -13 -60 -28 -56 -34 3 -6 1 -7 -5 -3 -7 4 -60 -6 -119
|
||||
-21 -99 -26 -122 -28 -283 -27 -191 0 -272 14 -450 76 -83 30 -260 113 -260
|
||||
123 0 3 26 -8 58 -24 109 -55 328 -122 392 -120 14 0 -16 9 -67 20 -171 36
|
||||
-352 111 -497 207 -172 113 -326 301 -429 522 -62 131 -68 166 -10 50 73 -145
|
||||
146 -245 262 -361 217 -217 463 -339 786 -389 262 -40 495 -16 735 77 8 3 12
|
||||
2 8 -2 -4 -4 -53 -25 -110 -46 -57 -21 -98 -38 -91 -39 12 0 148 49 195 70 15
|
||||
7 25 17 21 23 -3 6 -1 7 5 3 17 -10 130 56 242 140 183 138 326 311 421 506
|
||||
45 93 43 70 -3 -37 -115 -267 -310 -480 -584 -641 -43 -25 -70 -43 -59 -40 31
|
||||
10 167 94 238 147 89 67 233 221 296 315 115 174 186 365 221 589 34 221 22
|
||||
426 -35 612 -13 42 -22 79 -20 81 7 7 74 -216 89 -294 7 -44 14 -68 15 -54 3
|
||||
81 -97 399 -120 384 -5 -3 -6 2 -3 10 3 8 -9 41 -27 73 -159 283 -390 504
|
||||
-645 619 -45 21 -80 42 -77 47 4 6 -3 8 -15 5 -12 -4 -20 -2 -17 2 6 10 -100
|
||||
34 -240 55 -186 28 -400 14 -565 -35 -35 -11 -66 -18 -68 -15 -11 10 172 58
|
||||
338 89 55 10 377 -3 445 -19z m-85 -53 c635 -148 1057 -694 1027 -1332 -34
|
||||
-719 -639 -1270 -1357 -1237 -719 34 -1270 639 -1237 1357 24 501 326 938 787
|
||||
1135 89 38 233 79 335 95 96 14 347 4 445 -18z m-1499 -599 c-37 -74 -85 -219
|
||||
-108 -325 -14 -64 -18 -127 -18 -278 1 -177 3 -205 27 -300 69 -275 193 -496
|
||||
393 -696 117 -119 223 -195 367 -265 93 -45 70 -43 -37 3 -410 177 -703 553
|
||||
-790 1013 -35 184 -8 438 71 655 27 77 110 250 119 250 3 0 -8 -26 -24 -57z
|
||||
m1669 -2013 c-8 -5 -22 -9 -30 -9 -10 0 -8 3 5 9 27 12 43 12 25 0z"/>
|
||||
<path d="M2794 1365 c0 -88 2 -123 3 -77 2 46 2 118 0 160 -1 42 -3 5 -3 -83z"/>
|
||||
<path d="M2782 1170 c0 -19 2 -27 5 -17 2 9 2 25 0 35 -3 9 -5 1 -5 -18z"/>
|
||||
<path d="M2758 1053 c-10 -42 -16 -78 -14 -80 5 -6 39 134 34 147 -1 5 -11
|
||||
-24 -20 -67z"/>
|
||||
<path d="M1625 109 c-64 -15 -70 -18 -29 -14 28 3 70 11 95 19 61 19 30 17
|
||||
-66 -5z"/>
|
||||
<path d="M1213 83 c15 -2 37 -2 50 0 12 2 0 4 -28 4 -27 0 -38 -2 -22 -4z"/>
|
||||
<path d="M1365 80 l-80 -6 70 -1 c39 0 97 3 130 7 l60 7 -50 0 c-27 -1 -86 -4
|
||||
-130 -7z"/>
|
||||
<path d="M1773 2453 l47 -48 -57 -57 c-32 -32 -52 -58 -46 -58 6 0 34 24 62
|
||||
52 l50 53 51 -50 c27 -27 50 -45 50 -39 0 12 -182 194 -195 194 -5 0 12 -21
|
||||
38 -47z"/>
|
||||
<path d="M1800 1848 l-325 -321 -5 294 -5 294 -25 0 -25 0 -3 -325 -2 -325
|
||||
-67 -67 -67 -68 -237 236 c-136 135 -243 234 -251 232 -40 -14 -15 -46 228
|
||||
-289 137 -137 254 -249 259 -249 6 0 37 27 70 60 l60 61 5 -418 5 -418 25 0
|
||||
25 0 5 450 5 450 345 342 c295 293 343 344 334 359 -6 10 -15 19 -20 20 -5 2
|
||||
-155 -141 -334 -318z"/>
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="1080" height="1080" viewBox="0 0 1080 1080" xml:space="preserve">
|
||||
<desc>Created with Fabric.js 5.2.4</desc>
|
||||
<defs>
|
||||
</defs>
|
||||
<g transform="matrix(1 0 0 1 540 540)" id="bbbd9393-16c7-44f0-8b5c-16bfd131ab9a" >
|
||||
<rect style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(255,255,255); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" x="-540" y="-540" rx="0" ry="0" width="1080" height="1080" />
|
||||
</g>
|
||||
</svg>
|
||||
<g transform="matrix(Infinity NaN NaN Infinity 0 0)" id="2c258675-4de0-49f7-91dd-b55f919563be" >
|
||||
</g>
|
||||
<g transform="matrix(2.5 0 0 2.5 540 540)" >
|
||||
<g style="" vector-effect="non-scaling-stroke" >
|
||||
<g transform="matrix(0.13 0 0 -0.13 0 0)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-1440.01, -1440.49)" d="M 1265 2856 C 926 2805 665 2676 434 2446 C -52 1959 -125 1222 257 646 C 389 447 618 249 831 152 C 1232 -33 1648 -33 2049 152 C 2330 281 2599 550 2728 831 C 2913 1232 2913 1648 2728 2049 C 2599 2331 2331 2599 2049 2728 C 1798 2844 1504 2892 1265 2856 z M 1745 2816 C 1800 2803 1879 2779 1920 2764 C 1993 2738 2170 2654 2170 2646 C 2170 2643 2137 2657 2096 2675 C 1920 2757 1776 2790 1564 2797 C 1374 2803 1256 2788 1094 2735 C 722 2613 394 2328 244 1994 C 223 1949 202 1914 197 1917 C 191 1921 189 1914 192 1902 C 196 1890 194 1882 190 1885 C 168 1898 120 1586 120 1435 C 121 1339 143 1172 166 1090 C 170 1076 170 1068 166 1072 C 153 1084 112 1246 100 1339 C 83 1460 93 1696 119 1810 C 130 1857 153 1936 170 1985 C 290 2321 507 2557 840 2713 C 971 2775 1006 2781 890 2723 C 742 2648 645 2577 525 2456 C 411 2341 344 2250 278 2120 C 235 2036 197 1940 204 1933 C 206 1931 214 1946 220 1967 C 272 2124 396 2318 534 2455 C 730 2649 954 2769 1238 2831 C 1364 2859 1604 2852 1745 2816 z M 1795 2765 C 1890 2744 2010 2704 2099 2664 C 2139 2645 2177 2633 2183 2637 C 2189 2640 2190 2638 2186 2632 C 2182 2625 2193 2614 2212 2604 C 2408 2503 2595 2293 2713 2040 C 2775 1909 2781 1874 2723 1990 C 2648 2138 2577 2235 2456 2355 C 2341 2469 2250 2536 2120 2602 C 2036 2645 1940 2683 1933 2676 C 1931 2674 1946 2666 1967 2660 C 2124 2608 2318 2484 2455 2346 C 2654 2146 2781 1902 2835 1615 C 2870 1431 2843 1177 2764 960 C 2737 883 2654 710 2645 710 C 2642 710 2649 728 2660 749 C 2687 803 2740 938 2740 953 C 2739 959 2724 926 2706 879 C 2687 832 2662 774 2651 751 C 2639 728 2632 704 2636 697 C 2640 690 2640 688 2635 692 C 2631 696 2600 659 2568 609 C 2448 423 2271 275 2040 167 C 1931 116 1873 96 1930 130 C 1970 154 1968 153 1883 121 C 1847 108 1823 93 1827 87 C 1830 81 1828 80 1822 84 C 1815 88 1762 78 1703 63 C 1604 37 1581 35 1420 36 C 1229 36 1148 50 970 112 C 887 142 710 225 710 235 C 710 238 736 227 768 211 C 877 156 1096 89 1160 91 C 1174 91 1144 100 1093 111 C 922 147 741 222 596 318 C 424 431 270 619 167 840 C 105 971 99 1006 157 890 C 230 745 303 645 419 529 C 636 312 882 190 1205 140 C 1467 100 1700 124 1940 217 C 1948 220 1952 219 1948 215 C 1944 211 1895 190 1838 169 C 1781 148 1740 131 1747 130 C 1759 130 1895 179 1942 200 C 1957 207 1967 217 1963 223 C 1960 229 1962 230 1968 226 C 1985 216 2098 282 2210 366 C 2393 504 2536 677 2631 872 C 2676 965 2674 942 2628 835 C 2513 568 2318 355 2044 194 C 2001 169 1974 151 1985 154 C 2016 164 2152 248 2223 301 C 2312 368 2456 522 2519 616 C 2634 790 2705 981 2740 1205 C 2774 1426 2762 1631 2705 1817 C 2692 1859 2683 1896 2685 1898 C 2692 1905 2759 1682 2774 1604 C 2781 1560 2788 1536 2789 1550 C 2792 1631 2692 1949 2669 1934 C 2664 1931 2663 1936 2666 1944 C 2669 1952 2657 1985 2639 2017 C 2480 2300 2249 2521 1994 2636 C 1949 2657 1914 2678 1917 2683 C 1921 2689 1914 2691 1902 2688 C 1890 2684 1882 2686 1885 2690 C 1891 2700 1785 2724 1645 2745 C 1459 2773 1245 2759 1080 2710 C 1045 2699 1014 2692 1012 2695 C 1001 2705 1184 2753 1350 2784 C 1405 2794 1727 2781 1795 2765 z M 1710 2712 C 2345 2564 2767 2018 2737 1380 C 2703 661 2098 110 1380 143 C 661 177 110 782 143 1500 C 167 2001 469 2438 930 2635 C 1019 2673 1163 2714 1265 2730 C 1361 2744 1612 2734 1710 2712 z M 211 2113 C 174 2039 126 1894 103 1788 C 89 1724 85 1661 85 1510 C 86 1333 88 1305 112 1210 C 181 935 305 714 505 514 C 622 395 728 319 872 249 C 965 204 942 206 835 252 C 425 429 132 805 45 1265 C 10 1449 37 1703 116 1920 C 143 1997 226 2170 235 2170 C 238 2170 227 2144 211 2113 z M 1880 100 C 1872 95 1858 91 1850 91 C 1840 91 1842 94 1855 100 C 1882 112 1898 112 1880 100 z" stroke-linecap="round" />
|
||||
</g>
|
||||
<g transform="matrix(0.13 0 0 -0.13 180.83 9.86)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-2796.25, -1366.51)" d="M 2794 1365 C 2794 1277 2796 1242 2797 1288 C 2799 1334 2799 1406 2797 1448 C 2796 1490 2794 1453 2794 1365 z" stroke-linecap="round" />
|
||||
</g>
|
||||
<g transform="matrix(0.13 0 0 -0.13 179.37 36.04)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-2785.25, -1170.18)" d="M 2782 1170 C 2782 1151 2784 1143 2787 1153 C 2789 1162 2789 1178 2787 1188 C 2784 1197 2782 1189 2782 1170 z" stroke-linecap="round" />
|
||||
</g>
|
||||
<g transform="matrix(0.13 0 0 -0.13 176.14 52.51)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-2761.06, -1046.69)" d="M 2758 1053 C 2748 1011 2742 975 2744 973 C 2749 967 2783 1107 2778 1120 C 2777 1125 2767 1096 2758 1053 z" stroke-linecap="round" />
|
||||
</g>
|
||||
<g transform="matrix(0.13 0 0 -0.13 27.49 177.37)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-1646.21, -110.19)" d="M 1625 109 C 1561 94 1555 91 1596 95 C 1624 98 1666 106 1691 114 C 1752 133 1721 131 1625 109 z" stroke-linecap="round" />
|
||||
</g>
|
||||
<g transform="matrix(0.13 0 0 -0.13 -27.12 180.83)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-1236.6, -84.25)" d="M 1213 83 C 1228 81 1250 81 1263 83 C 1275 85 1263 87 1235 87 C 1208 87 1197 85 1213 83 z" stroke-linecap="round" />
|
||||
</g>
|
||||
<g transform="matrix(0.13 0 0 -0.13 -3.33 181.4)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-1415, -80)" d="M 1365 80 L 1285 74 L 1355 73 C 1394 73 1452 76 1485 80 L 1545 87 L 1495 87 C 1468 86 1409 83 1365 80 z" stroke-linecap="round" />
|
||||
</g>
|
||||
<g transform="matrix(0.13 0 0 -0.13 51.06 -127.27)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-1822.97, -2395)" d="M 1773 2453 L 1820 2405 L 1763 2348 C 1731 2316 1711 2290 1717 2290 C 1723 2290 1751 2314 1779 2342 L 1829 2395 L 1880 2345 C 1907 2318 1930 2300 1930 2306 C 1930 2318 1748 2500 1735 2500 C 1730 2500 1747 2479 1773 2453 z" stroke-linecap="round" />
|
||||
</g>
|
||||
<g transform="matrix(0.13 0 0 -0.13 3.17 11.33)" >
|
||||
<path style="stroke: none; stroke-width: 1; stroke-dasharray: none; stroke-linecap: butt; stroke-dashoffset: 0; stroke-linejoin: miter; stroke-miterlimit: 4; fill: rgb(0,0,0); fill-rule: nonzero; opacity: 1;" vector-effect="non-scaling-stroke" transform=" translate(-1463.77, -1355.51)" d="M 1800 1848 L 1475 1527 L 1470 1821 L 1465 2115 L 1440 2115 L 1415 2115 L 1412 1790 L 1410 1465 L 1343 1398 L 1276 1330 L 1039 1566 C 903 1701 796 1800 788 1798 C 748 1784 773 1752 1016 1509 C 1153 1372 1270 1260 1275 1260 C 1281 1260 1312 1287 1345 1320 L 1405 1381 L 1410 963 L 1415 545 L 1440 545 L 1465 545 L 1470 995 L 1475 1445 L 1820 1787 C 2115 2080 2163 2131 2154 2146 C 2148 2156 2139 2165 2134 2166 C 2129 2168 1979 2025 1800 1848 z" stroke-linecap="round" />
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 4.1 KiB After Width: | Height: | Size: 9.1 KiB |
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
|
|
@ -1,51 +0,0 @@
|
|||
Failure # 1 (occurred at 2024-05-15_09-06-07)
|
||||
The actor died because of an error raised in its creation task, [36mray::PPO.__init__()[39m (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO)
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 239, in _setup
|
||||
self.add_workers(
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 754, in add_workers
|
||||
raise result.get()
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/utils/actor_manager.py", line 497, in _fetch_result
|
||||
result = ray.get(r)
|
||||
^^^^^^^^^^
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, [36mray::RolloutWorker.__init__()[39m (pid=13721, ip=192.168.1.56, actor_id=cf76be01afb9954787ceb84801000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7fd32a36abd0>)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/evaluation/rollout_worker.py", line 407, in __init__
|
||||
self.env = env_creator(copy.deepcopy(self.env_context))
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/valapeos/Projects/pneumarl/rllib_example.py", line 48, in env_creator
|
||||
return RayVectorGodotEnv(config=env_config, port=port, seed=seed)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/wrappers/ray_wrapper.py", line 22, in __init__
|
||||
self._env = GodotEnv(
|
||||
^^^^^^^^^
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 59, in __init__
|
||||
self.connection = self._start_server()
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/godot_rl/core/godot_env.py", line 317, in _start_server
|
||||
connection, client_address = sock.accept()
|
||||
^^^^^^^^^^^^^
|
||||
File "/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/python3.11/socket.py", line 294, in accept
|
||||
fd, addr = self._accept()
|
||||
^^^^^^^^^^^^^^
|
||||
TimeoutError: timed out
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
[36mray::PPO.__init__()[39m (pid=13649, ip=192.168.1.56, actor_id=0b0450d90a301440d0afa9f001000000, repr=PPO)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 554, in __init__
|
||||
super().__init__(
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__
|
||||
self.setup(copy.deepcopy(self.config))
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py", line 640, in setup
|
||||
self.workers = EnvRunnerGroup(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/rllib/env/env_runner_group.py", line 191, in __init__
|
||||
raise e.args[0].args[2]
|
||||
TimeoutError: timed out
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +0,0 @@
|
|||
Traceback (most recent call last):
|
||||
File "python/ray/_raylet.pyx", line 3270, in ray._raylet.check_health
|
||||
File "python/ray/_raylet.pyx", line 583, in ray._raylet.check_status
|
||||
ray.exceptions.RpcError: failed to connect to all addresses; last error: UNKNOWN: ipv4:192.168.1.56:56287: Failed to connect to remote host: Connection refused
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/dashboard/dashboard.py:236: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead
|
||||
logger.warn("Exiting with SIGTERM immediately...")
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
NodeManager:
|
||||
Node ID: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
Node name: 192.168.1.56
|
||||
InitialConfigResources: {node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 177374392320000, CPU: 120000, memory: 354748784640000}
|
||||
ClusterTaskManager:
|
||||
========== Node: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a =================
|
||||
Infeasible queue length: 0
|
||||
Schedule queue length: 0
|
||||
Dispatch queue length: 0
|
||||
num_waiting_for_resource: 0
|
||||
num_waiting_for_plasma_memory: 0
|
||||
num_waiting_for_remote_node_resources: 0
|
||||
num_worker_not_started_by_job_config_not_exist: 0
|
||||
num_worker_not_started_by_registration_timeout: 0
|
||||
num_tasks_waiting_for_workers: 0
|
||||
num_cancelled_tasks: 0
|
||||
cluster_resource_scheduler state:
|
||||
Local id: 3934636312908726399 Local resources: {"total":{object_store_memory: [177374392320000], bundle_group_1_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_e13059e3372a28e5a1b0393dba1401000000: [50000], bundle_group_3_e13059e3372a28e5a1b0393dba1401000000: [10000000], bundle_group_4_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_2_e13059e3372a28e5a1b0393dba1401000000: [10000], memory: [354748784640000], bundle_group_e13059e3372a28e5a1b0393dba1401000000: [50000000], bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_0_e13059e3372a28e5a1b0393dba1401000000: [10000], CPU_group_3_e13059e3372a28e5a1b0393dba1401000000: [10000], node:__internal_head__: [10000], CPU: [120000], node:192.168.1.56: [10000], CPU_group_4_e13059e3372a28e5a1b0393dba1401000000: [10000], bundle_group_2_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_1_e13059e3372a28e5a1b0393dba1401000000: [10000]}}, "available": {object_store_memory: [177374392320000], bundle_group_1_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_e13059e3372a28e5a1b0393dba1401000000: [0], bundle_group_3_e13059e3372a28e5a1b0393dba1401000000: [10000000], bundle_group_4_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_2_e13059e3372a28e5a1b0393dba1401000000: [10000], memory: [354748784640000], bundle_group_e13059e3372a28e5a1b0393dba1401000000: [49999950], bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: [9999990], CPU_group_0_e13059e3372a28e5a1b0393dba1401000000: [0], CPU_group_3_e13059e3372a28e5a1b0393dba1401000000: [10000], node:__internal_head__: [10000], CPU: [70000], node:192.168.1.56: [10000], CPU_group_4_e13059e3372a28e5a1b0393dba1401000000: [10000], bundle_group_2_e13059e3372a28e5a1b0393dba1401000000: [10000000], CPU_group_1_e13059e3372a28e5a1b0393dba1401000000: [10000]}}, "labels":{"ray.io/node_id":"5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a",} is_draining: 0 is_idle: 0 Cluster resources: node id: 3934636312908726399{"total":{bundle_group_2_e13059e3372a28e5a1b0393dba1401000000: 10000000, CPU_group_2_e13059e3372a28e5a1b0393dba1401000000: 10000, CPU_group_1_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_1_e13059e3372a28e5a1b0393dba1401000000: 10000000, object_store_memory: 177374392320000, node:192.168.1.56: 10000, CPU_group_4_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 50000000, memory: 354748784640000, CPU_group_0_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: 10000000, node:__internal_head__: 10000, CPU_group_3_e13059e3372a28e5a1b0393dba1401000000: 10000, CPU: 120000, bundle_group_4_e13059e3372a28e5a1b0393dba1401000000: 10000000, CPU_group_e13059e3372a28e5a1b0393dba1401000000: 50000, bundle_group_3_e13059e3372a28e5a1b0393dba1401000000: 10000000}}, "available": {bundle_group_2_e13059e3372a28e5a1b0393dba1401000000: 10000000, CPU_group_2_e13059e3372a28e5a1b0393dba1401000000: 10000, CPU_group_1_e13059e3372a28e5a1b0393dba1401000000: 10000, object_store_memory: 177374392320000, bundle_group_1_e13059e3372a28e5a1b0393dba1401000000: 10000000, node:192.168.1.56: 10000, CPU_group_4_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 49999950, memory: 354748784640000, bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: 9999990, node:__internal_head__: 10000, CPU_group_3_e13059e3372a28e5a1b0393dba1401000000: 10000, CPU: 70000, bundle_group_4_e13059e3372a28e5a1b0393dba1401000000: 10000000, bundle_group_3_e13059e3372a28e5a1b0393dba1401000000: 10000000}}, "labels":{"ray.io/node_id":"5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []}
|
||||
Waiting tasks size: 0
|
||||
Number of executing tasks: 5
|
||||
Number of pinned task arguments: 0
|
||||
Number of total spilled tasks: 0
|
||||
Number of spilled waiting tasks: 0
|
||||
Number of spilled unschedulable tasks: 0
|
||||
Resource usage {
|
||||
- (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=257429): {CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10}
|
||||
- (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=257432): {CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10}
|
||||
- (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=257431): {CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10}
|
||||
- (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=257430): {bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10, CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000}
|
||||
- (language=PYTHON actor_or_task=PPO.__init__ pid=257357): {bundle_group_e13059e3372a28e5a1b0393dba1401000000: 10, CPU_group_0_e13059e3372a28e5a1b0393dba1401000000: 10000, bundle_group_0_e13059e3372a28e5a1b0393dba1401000000: 10, CPU_group_e13059e3372a28e5a1b0393dba1401000000: 10000}
|
||||
}
|
||||
Running tasks by scheduling class:
|
||||
- {depth=1 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.algorithms.ppo.ppo, class_name=PPO, function_name=__init__, function_hash=81e047a6019e40b6ac8e70bc624f8725} scheduling_strategy=placement_group_scheduling_strategy {
|
||||
placement_group_id: "\3410Y\3437*(\345\241\2609=\272\024\001\000\000\000"
|
||||
placement_group_capture_child_tasks: true
|
||||
}
|
||||
resource_set={bundle_group_e13059e3372a28e5a1b0393dba1401000000 : 0.001, bundle_group_0_e13059e3372a28e5a1b0393dba1401000000 : 0.001, CPU_group_e13059e3372a28e5a1b0393dba1401000000 : 1, CPU_group_0_e13059e3372a28e5a1b0393dba1401000000 : 1, }}: 1/18446744073709551615
|
||||
- {depth=2 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.evaluation.rollout_worker, class_name=RolloutWorker, function_name=__init__, function_hash=2934c96b687748fb9b4a35f16299e44e} scheduling_strategy=placement_group_scheduling_strategy {
|
||||
placement_group_id: "\3410Y\3437*(\345\241\2609=\272\024\001\000\000\000"
|
||||
placement_group_bundle_index: -1
|
||||
placement_group_capture_child_tasks: true
|
||||
}
|
||||
resource_set={CPU_group_e13059e3372a28e5a1b0393dba1401000000 : 1, bundle_group_e13059e3372a28e5a1b0393dba1401000000 : 0.001, }}: 4/18446744073709551615
|
||||
==================================================
|
||||
|
||||
ClusterResources:
|
||||
LocalObjectManager:
|
||||
- num pinned objects: 0
|
||||
- pinned objects size: 0
|
||||
- num objects pending restore: 0
|
||||
- num objects pending spill: 0
|
||||
- num bytes pending spill: 0
|
||||
- num bytes currently spilled: 0
|
||||
- cumulative spill requests: 0
|
||||
- cumulative restore requests: 0
|
||||
- spilled objects pending delete: 0
|
||||
|
||||
ObjectManager:
|
||||
- num local objects: 0
|
||||
- num unfulfilled push requests: 0
|
||||
- num object pull requests: 0
|
||||
- num chunks received total: 0
|
||||
- num chunks received failed (all): 0
|
||||
- num chunks received failed / cancelled: 0
|
||||
- num chunks received failed / plasma error: 0
|
||||
Event stats:
|
||||
Global stats: 0 total (0 active)
|
||||
Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
Execution time: mean = -nan s, total = 0.000 s
|
||||
Event stats:
|
||||
PushManager:
|
||||
- num pushes in flight: 0
|
||||
- num chunks in flight: 0
|
||||
- num chunks remaining: 0
|
||||
- max chunks allowed: 409
|
||||
OwnershipBasedObjectDirectory:
|
||||
- num listeners: 0
|
||||
- cumulative location updates: 0
|
||||
- num location updates per second: 0.000
|
||||
- num location lookups per second: 0.000
|
||||
- num locations added per second: 0.000
|
||||
- num locations removed per second: 0.000
|
||||
BufferPool:
|
||||
- create buffer state map size: 0
|
||||
PullManager:
|
||||
- num bytes available for pulled objects: 17737439232
|
||||
- num bytes being pulled (all): 0
|
||||
- num bytes being pulled / pinned: 0
|
||||
- get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
- wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
- task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
- first get request bundle: N/A
|
||||
- first wait request bundle: N/A
|
||||
- first task request bundle: N/A
|
||||
- num objects queued: 0
|
||||
- num objects actively pulled (all): 0
|
||||
- num objects actively pulled / pinned: 0
|
||||
- num bundles being pulled: 0
|
||||
- num pull retries: 0
|
||||
- max timeout seconds: 0
|
||||
- max timeout request is already processed. No entry.
|
||||
|
||||
WorkerPool:
|
||||
- registered jobs: 1
|
||||
- process_failed_job_config_missing: 0
|
||||
- process_failed_rate_limited: 0
|
||||
- process_failed_pending_registration: 0
|
||||
- process_failed_runtime_env_setup_failed: 0
|
||||
- num PYTHON workers: 12
|
||||
- num PYTHON drivers: 1
|
||||
- num object spill callbacks queued: 0
|
||||
- num object restore queued: 0
|
||||
- num util functions queued: 0
|
||||
- num idle workers: 7
|
||||
TaskDependencyManager:
|
||||
- task deps map size: 0
|
||||
- get req map size: 0
|
||||
- wait req map size: 0
|
||||
- local objects map size: 0
|
||||
WaitManager:
|
||||
- num active wait requests: 0
|
||||
Subscriber:
|
||||
Channel WORKER_OBJECT_LOCATIONS_CHANNEL
|
||||
- cumulative subscribe requests: 0
|
||||
- cumulative unsubscribe requests: 0
|
||||
- active subscribed publishers: 0
|
||||
- cumulative published messages: 0
|
||||
- cumulative processed messages: 0
|
||||
Channel WORKER_REF_REMOVED_CHANNEL
|
||||
- cumulative subscribe requests: 0
|
||||
- cumulative unsubscribe requests: 0
|
||||
- active subscribed publishers: 0
|
||||
- cumulative published messages: 0
|
||||
- cumulative processed messages: 0
|
||||
Channel WORKER_OBJECT_EVICTION
|
||||
- cumulative subscribe requests: 0
|
||||
- cumulative unsubscribe requests: 0
|
||||
- active subscribed publishers: 0
|
||||
- cumulative published messages: 0
|
||||
- cumulative processed messages: 0
|
||||
num async plasma notifications: 0
|
||||
Remote node managers:
|
||||
Event stats:
|
||||
Global stats: 1047 total (27 active)
|
||||
Queueing time: mean = 6.266 ms, max = 1.316 s, min = 80.000 ns, total = 6.561 s
|
||||
Execution time: mean = 3.897 ms, total = 4.080 s
|
||||
Event stats:
|
||||
NodeManagerService.grpc_server.ReportWorkerBacklog - 127 total (0 active), Execution time: mean = 95.444 us, total = 12.121 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_server.ReportWorkerBacklog.HandleRequestImpl - 127 total (0 active), Execution time: mean = 10.989 us, total = 1.396 ms, Queueing time: mean = 11.534 us, max = 90.610 us, min = 1.950 us, total = 1.465 ms
|
||||
ObjectManager.UpdateAvailableMemory - 100 total (0 active), Execution time: mean = 1.307 us, total = 130.690 us, Queueing time: mean = 7.801 us, max = 14.780 us, min = 4.940 us, total = 780.089 us
|
||||
NodeManager.CheckGC - 100 total (1 active), Execution time: mean = 947.400 ns, total = 94.740 us, Queueing time: mean = 42.552 us, max = 1.551 ms, min = 4.834 us, total = 4.255 ms
|
||||
RaySyncer.OnDemandBroadcasting - 100 total (1 active), Execution time: mean = 8.816 us, total = 881.598 us, Queueing time: mean = 35.004 us, max = 1.541 ms, min = 5.143 us, total = 3.500 ms
|
||||
ClientConnection.async_read.ProcessMessageHeader - 83 total (13 active), Execution time: mean = 2.709 us, total = 224.870 us, Queueing time: mean = 78.468 ms, max = 1.316 s, min = 8.730 us, total = 6.513 s
|
||||
ClientConnection.async_read.ProcessMessage - 70 total (0 active), Execution time: mean = 384.076 us, total = 26.885 ms, Queueing time: mean = 12.194 us, max = 459.398 us, min = 2.170 us, total = 853.597 us
|
||||
RayletWorkerPool.deadline_timer.kill_idle_workers - 50 total (1 active), Execution time: mean = 29.997 us, total = 1.500 ms, Queueing time: mean = 84.166 us, max = 3.581 ms, min = 6.447 us, total = 4.208 ms
|
||||
MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 40 total (1 active), Execution time: mean = 84.302 us, total = 3.372 ms, Queueing time: mean = 13.585 us, max = 35.718 us, min = 4.118 us, total = 543.386 us
|
||||
ClientConnection.async_write.DoAsyncWrites - 19 total (0 active), Execution time: mean = 454.737 ns, total = 8.640 us, Queueing time: mean = 10.924 us, max = 37.650 us, min = 5.510 us, total = 207.560 us
|
||||
ObjectManager.ObjectDeleted - 18 total (0 active), Execution time: mean = 6.061 us, total = 109.090 us, Queueing time: mean = 29.991 us, max = 143.339 us, min = 14.230 us, total = 539.839 us
|
||||
NodeManagerService.grpc_server.GetSystemConfig.HandleRequestImpl - 18 total (0 active), Execution time: mean = 31.315 us, total = 563.669 us, Queueing time: mean = 10.577 us, max = 22.040 us, min = 5.000 us, total = 190.380 us
|
||||
ObjectManager.ObjectAdded - 18 total (0 active), Execution time: mean = 5.523 us, total = 99.410 us, Queueing time: mean = 12.605 us, max = 30.460 us, min = 4.340 us, total = 226.890 us
|
||||
NodeManagerService.grpc_server.GetSystemConfig - 18 total (0 active), Execution time: mean = 149.376 us, total = 2.689 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
PeriodicalRunner.RunFnPeriodically - 13 total (0 active), Execution time: mean = 90.959 us, total = 1.182 ms, Queueing time: mean = 1.958 ms, max = 6.687 ms, min = 17.970 us, total = 25.448 ms
|
||||
NodeManager.ScheduleAndDispatchTasks - 11 total (1 active), Execution time: mean = 5.335 us, total = 58.680 us, Queueing time: mean = 9.394 us, max = 17.863 us, min = 5.637 us, total = 103.337 us
|
||||
NodeManager.deadline_timer.flush_free_objects - 10 total (1 active), Execution time: mean = 2.068 us, total = 20.680 us, Queueing time: mean = 55.932 us, max = 483.012 us, min = 6.183 us, total = 559.321 us
|
||||
NodeManagerService.grpc_server.GetResourceLoad - 10 total (0 active), Execution time: mean = 139.640 us, total = 1.396 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManager.deadline_timer.spill_objects_when_over_threshold - 10 total (1 active), Execution time: mean = 827.000 ns, total = 8.270 us, Queueing time: mean = 57.125 us, max = 481.982 us, min = 7.613 us, total = 571.251 us
|
||||
NodeManagerService.grpc_server.GetResourceLoad.HandleRequestImpl - 10 total (0 active), Execution time: mean = 52.351 us, total = 523.509 us, Queueing time: mean = 10.185 us, max = 16.690 us, min = 7.210 us, total = 101.849 us
|
||||
NodeManagerService.grpc_server.RequestWorkerLease - 6 total (0 active), Execution time: mean = 364.345 ms, total = 2.186 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_server.RequestWorkerLease.HandleRequestImpl - 6 total (0 active), Execution time: mean = 113.930 us, total = 683.579 us, Queueing time: mean = 239.866 us, max = 751.998 us, min = 6.810 us, total = 1.439 ms
|
||||
CoreWorkerService.grpc_client.Exit.OnReplyReceived - 5 total (0 active), Execution time: mean = 12.074 us, total = 60.370 us, Queueing time: mean = 10.302 us, max = 15.150 us, min = 7.200 us, total = 51.509 us
|
||||
InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 5 total (1 active), Execution time: mean = 362.904 ms, total = 1.815 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
RaySyncer.BroadcastMessage - 5 total (0 active), Execution time: mean = 92.108 us, total = 460.540 us, Queueing time: mean = 130.000 ns, max = 180.000 ns, min = 80.000 ns, total = 650.000 ns
|
||||
WorkerInfoGcsService.grpc_client.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 1.079 ms, total = 5.393 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
Subscriber.HandlePublishedMessage_GCS_WORKER_DELTA_CHANNEL - 5 total (0 active), Execution time: mean = 1.728 us, total = 8.640 us, Queueing time: mean = 61.160 us, max = 72.490 us, min = 56.720 us, total = 305.799 us
|
||||
- 5 total (0 active), Execution time: mean = 132.000 ns, total = 660.000 ns, Queueing time: mean = 8.438 us, max = 10.689 us, min = 7.000 us, total = 42.189 us
|
||||
WorkerInfoGcsService.grpc_client.ReportWorkerFailure.OnReplyReceived - 5 total (0 active), Execution time: mean = 9.860 us, total = 49.300 us, Queueing time: mean = 149.821 us, max = 249.979 us, min = 8.350 us, total = 749.107 us
|
||||
CoreWorkerService.grpc_client.Exit - 5 total (0 active), Execution time: mean = 842.498 us, total = 4.212 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalPubSubGcsService.grpc_client.GcsSubscriberPoll.OnReplyReceived - 4 total (0 active), Execution time: mean = 69.392 us, total = 277.568 us, Queueing time: mean = 55.710 us, max = 177.129 us, min = 7.700 us, total = 222.839 us
|
||||
ClusterResourceManager.ResetRemoteNodeView - 4 total (1 active), Execution time: mean = 1.492 us, total = 5.970 us, Queueing time: mean = 8.781 us, max = 17.279 us, min = 8.823 us, total = 35.124 us
|
||||
NodeInfoGcsService.grpc_client.CheckAlive - 2 total (0 active), Execution time: mean = 299.624 us, total = 599.248 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
RaySyncerRegister - 2 total (0 active), Execution time: mean = 845.000 ns, total = 1.690 us, Queueing time: mean = 235.000 ns, max = 390.000 ns, min = 80.000 ns, total = 470.000 ns
|
||||
NodeManager.GcsCheckAlive - 2 total (1 active), Execution time: mean = 60.120 us, total = 120.240 us, Queueing time: mean = 149.742 us, max = 299.484 us, min = 299.484 us, total = 299.484 us
|
||||
NodeManager.deadline_timer.record_metrics - 2 total (1 active), Execution time: mean = 257.514 us, total = 515.028 us, Queueing time: mean = 8.553 us, max = 17.105 us, min = 17.105 us, total = 17.105 us
|
||||
InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 2 total (0 active), Execution time: mean = 239.190 us, total = 478.379 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.CheckAlive.OnReplyReceived - 2 total (0 active), Execution time: mean = 13.740 us, total = 27.480 us, Queueing time: mean = 10.925 us, max = 11.030 us, min = 10.820 us, total = 21.850 us
|
||||
InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 2 total (0 active), Execution time: mean = 75.605 us, total = 151.210 us, Queueing time: mean = 500.413 us, max = 913.367 us, min = 87.460 us, total = 1.001 ms
|
||||
NodeManager.deadline_timer.debug_state_dump - 1 total (1 active, 1 running), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_server.ReturnWorker - 1 total (0 active), Execution time: mean = 127.369 us, total = 127.369 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_client.GetAllJobInfo - 1 total (0 active), Execution time: mean = 301.119 us, total = 301.119 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
Subscriber.HandlePublishedMessage_GCS_JOB_CHANNEL - 1 total (0 active), Execution time: mean = 22.890 us, total = 22.890 us, Queueing time: mean = 83.339 us, max = 83.339 us, min = 83.339 us, total = 83.339 us
|
||||
NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 10.159 ms, total = 10.159 ms, Queueing time: mean = 6.700 us, max = 6.700 us, min = 6.700 us, total = 6.700 us
|
||||
JobInfoGcsService.grpc_client.GetAllJobInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 7.430 us, total = 7.430 us, Queueing time: mean = 4.300 us, max = 4.300 us, min = 4.300 us, total = 4.300 us
|
||||
NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 595.718 us, total = 595.718 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.GetAllNodeInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 83.820 us, total = 83.820 us, Queueing time: mean = 5.360 us, max = 5.360 us, min = 5.360 us, total = 5.360 us
|
||||
NodeManagerService.grpc_server.PrepareBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 216.439 us, total = 216.439 us, Queueing time: mean = 8.990 us, max = 8.990 us, min = 8.990 us, total = 8.990 us
|
||||
NodeManager.deadline_timer.print_event_loop_stats - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 359.519 us, total = 359.519 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.GetAllNodeInfo - 1 total (0 active), Execution time: mean = 278.129 us, total = 278.129 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 129.890 us, total = 129.890 us, Queueing time: mean = 5.080 us, max = 5.080 us, min = 5.080 us, total = 5.080 us
|
||||
NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_server.ReturnWorker.HandleRequestImpl - 1 total (0 active), Execution time: mean = 42.810 us, total = 42.810 us, Queueing time: mean = 11.950 us, max = 11.950 us, min = 11.950 us, total = 11.950 us
|
||||
NodeManagerService.grpc_server.PrepareBundleResources - 1 total (0 active), Execution time: mean = 279.579 us, total = 279.579 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_server.CommitBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 173.739 us, total = 173.739 us, Queueing time: mean = 6.420 us, max = 6.420 us, min = 6.420 us, total = 6.420 us
|
||||
JobInfoGcsService.grpc_client.AddJob.OnReplyReceived - 1 total (0 active), Execution time: mean = 15.700 us, total = 15.700 us, Queueing time: mean = 69.289 us, max = 69.289 us, min = 69.289 us, total = 69.289 us
|
||||
WorkerPool.PopWorkerCallback - 1 total (0 active), Execution time: mean = 11.110 us, total = 11.110 us, Queueing time: mean = 5.349 us, max = 5.349 us, min = 5.349 us, total = 5.349 us
|
||||
NodeManagerService.grpc_server.CommitBundleResources - 1 total (0 active), Execution time: mean = 223.729 us, total = 223.729 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_client.AddJob - 1 total (0 active), Execution time: mean = 353.939 us, total = 353.939 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
DebugString() time ms: 0
|
||||
|
|
@ -1,150 +0,0 @@
|
|||
GcsNodeManager:
|
||||
- RegisterNode request count: 1
|
||||
- DrainNode request count: 0
|
||||
- GetAllNodeInfo request count: 26
|
||||
- GetInternalConfig request count: 1
|
||||
|
||||
GcsActorManager:
|
||||
- RegisterActor request count: 5
|
||||
- CreateActor request count: 5
|
||||
- GetActorInfo request count: 5
|
||||
- GetNamedActorInfo request count: 0
|
||||
- GetAllActorInfo request count: 0
|
||||
- KillActor request count: 0
|
||||
- ListNamedActors request count: 0
|
||||
- Registered actors count: 5
|
||||
- Destroyed actors count: 0
|
||||
- Named actors count: 0
|
||||
- Unresolved actors count: 0
|
||||
- Pending actors count: 0
|
||||
- Created actors count: 0
|
||||
- owners_: 1
|
||||
- actor_to_register_callbacks_: 0
|
||||
- actor_to_create_callbacks_: 5
|
||||
- sorted_destroyed_actor_list_: 0
|
||||
|
||||
GcsResourceManager:
|
||||
- GetAllAvailableResources request count0
|
||||
- GetAllResourceUsage request count: 2
|
||||
|
||||
GcsPlacementGroupManager:
|
||||
- CreatePlacementGroup request count: 1
|
||||
- RemovePlacementGroup request count: 0
|
||||
- GetPlacementGroup request count: 5
|
||||
- GetAllPlacementGroup request count: 0
|
||||
- WaitPlacementGroupUntilReady request count: 0
|
||||
- GetNamedPlacementGroup request count: 0
|
||||
- Scheduling pending placement group count: 1
|
||||
- Registered placement groups count: 1
|
||||
- Named placement group count: 0
|
||||
- Pending placement groups count: 0
|
||||
- Infeasible placement groups count: 0
|
||||
|
||||
Publisher:
|
||||
RAY_LOG_CHANNEL
|
||||
- cumulative published messages: 7
|
||||
- cumulative published bytes: 4754
|
||||
- current buffered bytes: 378
|
||||
GCS_NODE_INFO_CHANNEL
|
||||
- cumulative published messages: 1
|
||||
- cumulative published bytes: 559
|
||||
- current buffered bytes: 0
|
||||
GCS_WORKER_DELTA_CHANNEL
|
||||
- cumulative published messages: 5
|
||||
- cumulative published bytes: 485
|
||||
- current buffered bytes: 97
|
||||
GCS_ACTOR_CHANNEL
|
||||
- cumulative published messages: 10
|
||||
- cumulative published bytes: 5864
|
||||
- current buffered bytes: 665
|
||||
GCS_JOB_CHANNEL
|
||||
- cumulative published messages: 1
|
||||
- cumulative published bytes: 280
|
||||
- current buffered bytes: 280
|
||||
|
||||
[runtime env manager] ID to URIs table:
|
||||
[runtime env manager] URIs reference table:
|
||||
|
||||
GcsTaskManager:
|
||||
-Total num task events reported: 25
|
||||
-Total num status task events dropped: 0
|
||||
-Total num profile events dropped: 0
|
||||
-Current num of task events stored: 13
|
||||
-Total num of actor creation tasks: 5
|
||||
-Total num of actor tasks: 6
|
||||
-Total num of normal tasks: 1
|
||||
-Total num of driver tasks: 1
|
||||
|
||||
|
||||
|
||||
|
||||
Global stats: 736 total (19 active)
|
||||
Queueing time: mean = 121.367 us, max = 26.350 ms, min = 200.000 ns, total = 89.326 ms
|
||||
Execution time: mean = 3.099 ms, total = 2.281 s
|
||||
Event stats:
|
||||
GcsInMemoryStore.Put - 111 total (0 active), Execution time: mean = 263.758 us, total = 29.277 ms, Queueing time: mean = 252.445 us, max = 26.181 ms, min = 600.000 ns, total = 28.021 ms
|
||||
GcsInMemoryStore.Get - 75 total (0 active), Execution time: mean = 35.806 us, total = 2.685 ms, Queueing time: mean = 6.298 us, max = 296.869 us, min = 560.000 ns, total = 472.329 us
|
||||
InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 68 total (0 active), Execution time: mean = 8.690 us, total = 590.939 us, Queueing time: mean = 13.208 us, max = 106.509 us, min = 3.160 us, total = 898.166 us
|
||||
InternalKVGcsService.grpc_server.InternalKVGet - 68 total (0 active), Execution time: mean = 120.321 us, total = 8.182 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalKVGcsService.grpc_server.InternalKVPut - 65 total (0 active), Execution time: mean = 126.788 us, total = 8.241 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 65 total (0 active), Execution time: mean = 7.386 us, total = 480.069 us, Queueing time: mean = 33.550 us, max = 206.769 us, min = 3.520 us, total = 2.181 ms
|
||||
NodeInfoGcsService.grpc_server.GetAllNodeInfo - 26 total (0 active), Execution time: mean = 147.213 us, total = 3.828 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 26 total (0 active), Execution time: mean = 19.258 us, total = 500.709 us, Queueing time: mean = 13.521 us, max = 55.990 us, min = 3.980 us, total = 351.549 us
|
||||
WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 159.297 us, total = 2.867 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 14.164 us, total = 254.960 us, Queueing time: mean = 29.422 us, max = 151.780 us, min = 5.450 us, total = 529.589 us
|
||||
InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 12 total (0 active), Execution time: mean = 3.667 us, total = 44.000 us, Queueing time: mean = 12.629 us, max = 27.090 us, min = 2.110 us, total = 151.550 us
|
||||
InternalKVGcsService.grpc_client.InternalKVPut - 12 total (0 active), Execution time: mean = 663.662 us, total = 7.964 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
RayletLoadPulled - 10 total (1 active), Execution time: mean = 57.438 us, total = 574.377 us, Queueing time: mean = 33.745 us, max = 275.254 us, min = 5.826 us, total = 337.454 us
|
||||
NodeManagerService.grpc_client.GetResourceLoad - 8 total (0 active), Execution time: mean = 358.958 us, total = 2.872 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 9.004 us, total = 72.029 us, Queueing time: mean = 165.235 us, max = 1.232 ms, min = 7.310 us, total = 1.322 ms
|
||||
NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 1.371 ms, total = 10.967 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 8 total (0 active), Execution time: mean = 20.915 us, total = 167.320 us, Queueing time: mean = 8.619 us, max = 11.950 us, min = 6.340 us, total = 68.950 us
|
||||
GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 5.349 us, total = 37.440 us, Queueing time: mean = 13.557 us, max = 19.810 us, min = 1.770 us, total = 94.900 us
|
||||
ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 98.212 us, total = 491.059 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 173.023 us, total = 865.117 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 6.072 us, total = 30.360 us, Queueing time: mean = 29.798 us, max = 89.750 us, min = 6.250 us, total = 148.990 us
|
||||
PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 9.720 us, total = 48.599 us, Queueing time: mean = 108.618 us, max = 268.199 us, min = 5.670 us, total = 543.088 us
|
||||
WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 891.008 us, total = 4.455 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 19.798 us, total = 98.990 us, Queueing time: mean = 122.620 us, max = 557.449 us, min = 10.130 us, total = 613.099 us
|
||||
CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 297.391 us, total = 1.487 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 137.746 us, total = 688.728 us, Queueing time: mean = 77.952 us, max = 185.980 us, min = 7.420 us, total = 389.759 us
|
||||
CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsResourceManager::Update - 5 total (0 active), Execution time: mean = 41.756 us, total = 208.780 us, Queueing time: mean = 9.402 us, max = 14.300 us, min = 6.420 us, total = 47.008 us
|
||||
ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 202.172 us, total = 1.011 ms, Queueing time: mean = 7.404 us, max = 9.850 us, min = 6.490 us, total = 37.020 us
|
||||
ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 115.754 us, total = 578.769 us, Queueing time: mean = 7.054 us, max = 9.110 us, min = 6.150 us, total = 35.270 us
|
||||
NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 437.626 ms, total = 2.188 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
ClusterResourceManager.ResetRemoteNodeView - 4 total (1 active), Execution time: mean = 2.167 us, total = 8.670 us, Queueing time: mean = 6.939 us, max = 13.297 us, min = 5.866 us, total = 27.754 us
|
||||
NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 4 total (0 active), Execution time: mean = 6.803 us, total = 27.210 us, Queueing time: mean = 9.760 us, max = 13.870 us, min = 5.530 us, total = 39.040 us
|
||||
NodeInfoGcsService.grpc_server.CheckAlive - 4 total (0 active), Execution time: mean = 94.700 us, total = 378.799 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 65.217 us, total = 260.869 us, Queueing time: mean = 13.203 ms, max = 26.350 ms, min = 11.020 us, total = 52.810 ms
|
||||
HealthCheck - 2 total (0 active), Execution time: mean = 2.305 us, total = 4.610 us, Queueing time: mean = 11.560 us, max = 11.900 us, min = 11.220 us, total = 23.120 us
|
||||
NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 2 total (0 active), Execution time: mean = 18.790 us, total = 37.580 us, Queueing time: mean = 11.985 us, max = 13.820 us, min = 10.150 us, total = 23.970 us
|
||||
NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 2 total (0 active), Execution time: mean = 111.195 us, total = 222.390 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 4.210 us, total = 4.210 us, Queueing time: mean = 2.350 us, max = 2.350 us, min = 2.350 us, total = 2.350 us
|
||||
NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 45.140 us, total = 45.140 us, Queueing time: mean = 6.580 us, max = 6.580 us, min = 6.580 us, total = 6.580 us
|
||||
NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 275.589 us, total = 275.589 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 78.419 us, total = 78.419 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 94.919 us, total = 94.919 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 73.759 us, total = 73.759 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 70.000 ns, total = 70.000 ns, Queueing time: mean = 76.150 us, max = 76.150 us, min = 76.150 us, total = 76.150 us
|
||||
InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 21.909 us, total = 21.909 us, Queueing time: mean = 8.590 us, max = 8.590 us, min = 8.590 us, total = 8.590 us
|
||||
PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 38.550 us, total = 38.550 us, Queueing time: mean = 8.370 us, max = 8.370 us, min = 8.370 us, total = 8.370 us
|
||||
JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 212.909 us, total = 212.909 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 46.590 us, total = 46.590 us, Queueing time: mean = 7.540 us, max = 7.540 us, min = 7.540 us, total = 7.540 us
|
||||
NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 53.830 us, total = 53.830 us, Queueing time: mean = 8.540 us, max = 8.540 us, min = 8.540 us, total = 8.540 us
|
||||
NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 39.100 us, total = 39.100 us, Queueing time: mean = 5.740 us, max = 5.740 us, min = 5.740 us, total = 5.740 us
|
||||
NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 749.138 us, total = 749.138 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 7.509 us, total = 7.509 us, Queueing time: mean = 12.700 us, max = 12.700 us, min = 12.700 us, total = 12.700 us
|
||||
NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 6.690 us, total = 6.690 us, Queueing time: mean = 8.720 us, max = 8.720 us, min = 8.720 us, total = 8.720 us
|
||||
GCSServer.deadline_timer.debug_state_dump - 1 total (1 active, 1 running), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 338.789 us, total = 338.789 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 104.050 us, total = 104.050 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 1.970 us, total = 1.970 us, Queueing time: mean = 1.020 us, max = 1.020 us, min = 1.020 us, total = 1.020 us
|
||||
JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 4.780 us, total = 4.780 us, Queueing time: mean = 7.830 us, max = 7.830 us, min = 7.830 us, total = 7.830 us
|
||||
InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 55.380 us, total = 55.380 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 6.250 us, total = 6.250 us, Queueing time: mean = 200.000 ns, max = 200.000 ns, min = 200.000 ns, total = 200.000 ns
|
||||
InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.780 us, total = 9.780 us, Queueing time: mean = 4.750 us, max = 4.750 us, min = 4.750 us, total = 4.750 us
|
||||
NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 98.280 us, total = 98.280 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
|
|
@ -1,184 +0,0 @@
|
|||
[2024-05-15 08:21:38,980 I 256391 256391] (gcs_server) io_service_pool.cc:35: IOServicePool is running with 1 io_service.
|
||||
[2024-05-15 08:21:38,981 I 256391 256391] (gcs_server) event.cc:234: Set ray event level to warning
|
||||
[2024-05-15 08:21:38,981 I 256391 256391] (gcs_server) event.cc:342: Ray Event initialized for GCS
|
||||
[2024-05-15 08:21:38,981 I 256391 256391] (gcs_server) gcs_server.cc:74: GCS storage type is StorageType::IN_MEMORY
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:42: Loading job table data.
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:54: Loading node table data.
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:80: Loading actor table data.
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:93: Loading actor task spec table data.
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:66: Loading placement group table data.
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:46: Finished loading job table data, size = 0
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:58: Finished loading node table data, size = 0
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:84: Finished loading actor table data, size = 0
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:97: Finished loading actor task spec table data, size = 0
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_init_data.cc:71: Finished loading placement group table data, size = 0
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_server.cc:162: No existing server cluster ID found. Generating new ID: 535ba0eef614eb34440db6e780ab29a9f7318b794e022f2f96a3d065
|
||||
[2024-05-15 08:21:38,982 I 256391 256391] (gcs_server) gcs_server.cc:648: Autoscaler V2 enabled: 0
|
||||
[2024-05-15 08:21:38,983 I 256391 256391] (gcs_server) grpc_server.cc:134: GcsServer server started, listening on port 56287.
|
||||
[2024-05-15 08:21:39,008 I 256391 256391] (gcs_server) gcs_server.cc:250: GcsNodeManager:
|
||||
- RegisterNode request count: 0
|
||||
- DrainNode request count: 0
|
||||
- GetAllNodeInfo request count: 0
|
||||
- GetInternalConfig request count: 0
|
||||
|
||||
GcsActorManager:
|
||||
- RegisterActor request count: 0
|
||||
- CreateActor request count: 0
|
||||
- GetActorInfo request count: 0
|
||||
- GetNamedActorInfo request count: 0
|
||||
- GetAllActorInfo request count: 0
|
||||
- KillActor request count: 0
|
||||
- ListNamedActors request count: 0
|
||||
- Registered actors count: 0
|
||||
- Destroyed actors count: 0
|
||||
- Named actors count: 0
|
||||
- Unresolved actors count: 0
|
||||
- Pending actors count: 0
|
||||
- Created actors count: 0
|
||||
- owners_: 0
|
||||
- actor_to_register_callbacks_: 0
|
||||
- actor_to_create_callbacks_: 0
|
||||
- sorted_destroyed_actor_list_: 0
|
||||
|
||||
GcsResourceManager:
|
||||
- GetAllAvailableResources request count0
|
||||
- GetAllResourceUsage request count: 0
|
||||
|
||||
GcsPlacementGroupManager:
|
||||
- CreatePlacementGroup request count: 0
|
||||
- RemovePlacementGroup request count: 0
|
||||
- GetPlacementGroup request count: 0
|
||||
- GetAllPlacementGroup request count: 0
|
||||
- WaitPlacementGroupUntilReady request count: 0
|
||||
- GetNamedPlacementGroup request count: 0
|
||||
- Scheduling pending placement group count: 0
|
||||
- Registered placement groups count: 0
|
||||
- Named placement group count: 0
|
||||
- Pending placement groups count: 0
|
||||
- Infeasible placement groups count: 0
|
||||
|
||||
Publisher:
|
||||
|
||||
[runtime env manager] ID to URIs table:
|
||||
[runtime env manager] URIs reference table:
|
||||
|
||||
GcsTaskManager:
|
||||
-Total num task events reported: 0
|
||||
-Total num status task events dropped: 0
|
||||
-Total num profile events dropped: 0
|
||||
-Current num of task events stored: 0
|
||||
-Total num of actor creation tasks: 0
|
||||
-Total num of actor tasks: 0
|
||||
-Total num of normal tasks: 0
|
||||
-Total num of driver tasks: 0
|
||||
|
||||
|
||||
[2024-05-15 08:21:39,008 I 256391 256391] (gcs_server) gcs_server.cc:844: Event stats:
|
||||
|
||||
|
||||
Global stats: 21 total (10 active)
|
||||
Queueing time: mean = 3.759 ms, max = 26.350 ms, min = 1.000 us, total = 78.947 ms
|
||||
Execution time: mean = 1.257 ms, total = 26.400 ms
|
||||
Event stats:
|
||||
InternalKVGcsService.grpc_client.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsInMemoryStore.GetAll - 5 total (0 active), Execution time: mean = 4.548 us, total = 22.740 us, Queueing time: mean = 18.250 us, max = 19.810 us, min = 17.030 us, total = 91.250 us
|
||||
PeriodicalRunner.RunFnPeriodically - 4 total (2 active, 1 running), Execution time: mean = 645.000 ns, total = 2.580 us, Queueing time: mean = 13.168 ms, max = 26.350 ms, min = 26.320 ms, total = 52.670 ms
|
||||
GcsInMemoryStore.Put - 3 total (0 active), Execution time: mean = 8.790 ms, total = 26.369 ms, Queueing time: mean = 8.728 ms, max = 26.181 ms, min = 1.000 us, total = 26.184 ms
|
||||
GcsInMemoryStore.Get - 1 total (0 active), Execution time: mean = 5.480 us, total = 5.480 us, Queueing time: mean = 1.290 us, max = 1.290 us, min = 1.290 us, total = 1.290 us
|
||||
ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
RayletLoadPulled - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
|
||||
|
||||
[2024-05-15 08:21:39,008 I 256391 256391] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats:
|
||||
|
||||
|
||||
Global stats: 2 total (1 active)
|
||||
Queueing time: mean = 31.364 us, max = 62.729 us, min = 62.729 us, total = 62.729 us
|
||||
Execution time: mean = 10.225 us, total = 20.450 us
|
||||
Event stats:
|
||||
PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 20.450 us, total = 20.450 us, Queueing time: mean = 62.729 us, max = 62.729 us, min = 62.729 us, total = 62.729 us
|
||||
GcsTaskManager.GcJobSummary - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
|
||||
|
||||
[2024-05-15 08:21:40,379 I 256391 256391] (gcs_server) gcs_node_manager.cc:55: Registering node info, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, address = 192.168.1.56, node name = 192.168.1.56
|
||||
[2024-05-15 08:21:40,379 I 256391 256391] (gcs_server) gcs_node_manager.cc:61: Finished registering node info, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, address = 192.168.1.56, node name = 192.168.1.56
|
||||
[2024-05-15 08:21:40,379 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:797: A new node: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a registered, will try to reschedule all the infeasible placement groups.
|
||||
[2024-05-15 08:21:40,963 I 256391 256391] (gcs_server) gcs_job_manager.cc:42: Adding job, job id = 01000000, driver pid = 256306
|
||||
[2024-05-15 08:21:40,963 I 256391 256391] (gcs_server) gcs_job_manager.cc:57: Finished adding job, job id = 01000000, driver pid = 256306
|
||||
[2024-05-15 08:21:40,999 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:432: Registering placement group, placement group id = e13059e3372a28e5a1b0393dba1401000000, name = , strategy = 0
|
||||
[2024-05-15 08:21:40,999 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:436: Finished registering placement group, placement group id = e13059e3372a28e5a1b0393dba1401000000, name = , strategy = 0
|
||||
[2024-05-15 08:21:40,999 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:185: Preparing resource from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for bundles: {placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4}},
|
||||
[2024-05-15 08:21:41,000 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:195: Finished leasing resource from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for bundles: {placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4}},
|
||||
[2024-05-15 08:21:41,000 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:213: Committing resource to a node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for bundles: {placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2}},
|
||||
[2024-05-15 08:21:41,000 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:220: Finished committing resource to 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for bundles: {placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4}},{placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2}},
|
||||
[2024-05-15 08:21:41,000 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:321: Successfully created placement group , id: e13059e3372a28e5a1b0393dba1401000000
|
||||
[2024-05-15 08:21:41,175 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = fdf624732d0a9f104204059a01000000
|
||||
[2024-05-15 08:21:41,176 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = fdf624732d0a9f104204059a01000000
|
||||
[2024-05-15 08:21:41,177 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = fdf624732d0a9f104204059a01000000
|
||||
[2024-05-15 08:21:41,177 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor fdf624732d0a9f104204059a01000000, job id = 01000000
|
||||
[2024-05-15 08:21:41,580 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor fdf624732d0a9f104204059a01000000, job id = 01000000
|
||||
[2024-05-15 08:21:41,580 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor fdf624732d0a9f104204059a01000000 on worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000
|
||||
[2024-05-15 08:21:43,096 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = ba75f4cef86c74fb1b900c5a01000000
|
||||
[2024-05-15 08:21:43,096 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = ba75f4cef86c74fb1b900c5a01000000
|
||||
[2024-05-15 08:21:43,096 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = ba75f4cef86c74fb1b900c5a01000000
|
||||
[2024-05-15 08:21:43,096 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ba75f4cef86c74fb1b900c5a01000000, job id = 01000000
|
||||
[2024-05-15 08:21:43,097 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = ad0947e6b294e1fc2aa241ba01000000
|
||||
[2024-05-15 08:21:43,097 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = ad0947e6b294e1fc2aa241ba01000000
|
||||
[2024-05-15 08:21:43,097 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = ad0947e6b294e1fc2aa241ba01000000
|
||||
[2024-05-15 08:21:43,097 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ad0947e6b294e1fc2aa241ba01000000, job id = 01000000
|
||||
[2024-05-15 08:21:43,098 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 28c480d27043ee6d59fd4ecb01000000
|
||||
[2024-05-15 08:21:43,098 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 28c480d27043ee6d59fd4ecb01000000
|
||||
[2024-05-15 08:21:43,098 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 28c480d27043ee6d59fd4ecb01000000
|
||||
[2024-05-15 08:21:43,098 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 28c480d27043ee6d59fd4ecb01000000, job id = 01000000
|
||||
[2024-05-15 08:21:43,099 I 256391 256391] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 1c936e2ca7df5e9888d17b2701000000
|
||||
[2024-05-15 08:21:43,099 I 256391 256391] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 1c936e2ca7df5e9888d17b2701000000
|
||||
[2024-05-15 08:21:43,099 I 256391 256391] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 1c936e2ca7df5e9888d17b2701000000
|
||||
[2024-05-15 08:21:43,099 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 1c936e2ca7df5e9888d17b2701000000, job id = 01000000
|
||||
[2024-05-15 08:21:43,518 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ad0947e6b294e1fc2aa241ba01000000, job id = 01000000
|
||||
[2024-05-15 08:21:43,518 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 28c480d27043ee6d59fd4ecb01000000, job id = 01000000
|
||||
[2024-05-15 08:21:43,518 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor ad0947e6b294e1fc2aa241ba01000000 on worker 4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000
|
||||
[2024-05-15 08:21:43,518 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 28c480d27043ee6d59fd4ecb01000000 on worker 8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000
|
||||
[2024-05-15 08:21:43,571 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 1c936e2ca7df5e9888d17b2701000000, job id = 01000000
|
||||
[2024-05-15 08:21:43,571 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ba75f4cef86c74fb1b900c5a01000000, job id = 01000000
|
||||
[2024-05-15 08:21:43,571 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 1c936e2ca7df5e9888d17b2701000000 on worker 1c55cb5b210853988205f9bc82cfcfa1dc3961d0edbf43051c4c3d5f at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000
|
||||
[2024-05-15 08:21:43,571 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor ba75f4cef86c74fb1b900c5a01000000 on worker 3a95f67a1d239748b37bd2874e2ec5ee7411384a73ed36570ee1d355 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, job id = 01000000
|
||||
[2024-05-15 08:21:48,989 W 256391 256412] (gcs_server) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:56844: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster.
|
||||
[2024-05-15 08:21:54,092 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = fdf624732d0a9f104204059a01000000, job id = 01000000
|
||||
[2024-05-15 08:21:54,092 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = fdf624732d0a9f104204059a01000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:21:54,092 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:607: Cancelling all committed bundles of a placement group, id is e13059e3372a28e5a1b0393dba1401000000
|
||||
[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:458: Placement group of an id, e13059e3372a28e5a1b0393dba1401000000 is removed successfully.
|
||||
[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_manager.cc:1167: Actor ad0947e6b294e1fc2aa241ba01000000 is failed on worker 4a99d489eb5618111256e3f4fbfd2ed5242185937a205e2409a271e3 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000
|
||||
[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor ad0947e6b294e1fc2aa241ba01000000, job id = 01000000
|
||||
[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_manager.cc:1167: Actor 28c480d27043ee6d59fd4ecb01000000 is failed on worker 8bbf27df8846320051bf79c3cbd314fc0e569b6d0876c1ec2a1f9ea0 at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000
|
||||
[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a for actor 28c480d27043ee6d59fd4ecb01000000, job id = 01000000
|
||||
[2024-05-15 08:21:54,093 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = ba75f4cef86c74fb1b900c5a01000000, job id = 01000000
|
||||
[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = ba75f4cef86c74fb1b900c5a01000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 28c480d27043ee6d59fd4ecb01000000, job id = 01000000
|
||||
[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 28c480d27043ee6d59fd4ecb01000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 1c936e2ca7df5e9888d17b2701000000, job id = 01000000
|
||||
[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 1c936e2ca7df5e9888d17b2701000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = ad0947e6b294e1fc2aa241ba01000000, job id = 01000000
|
||||
[2024-05-15 08:21:54,094 I 256391 256391] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = ad0947e6b294e1fc2aa241ba01000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={3} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={0} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_actor_manager.cc:798: Actor fdf624732d0a9f104204059a01000000 is out of scope, destroying actor, job id = 01000000
|
||||
[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={1} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={4} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:54,095 I 256391 256391] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={e13059e3372a28e5a1b0393dba1401000000}, bundle index={2} at node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:54,129 I 256391 256391] (gcs_server) gcs_actor_manager.cc:794: Worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:21:54,130 I 256391 256391] (gcs_server) gcs_actor_manager.cc:794: Worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:21:54,130 I 256391 256391] (gcs_server) gcs_actor_manager.cc:794: Worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:21:54,130 I 256391 256391] (gcs_server) gcs_actor_manager.cc:794: Worker f4daee8bf765e282c30ae3579ff222db46925352ea2ca1d6a8bd5d17 failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:21:54,602 I 256391 256391] (gcs_server) gcs_job_manager.cc:87: Finished marking job state, job id = 01000000
|
||||
[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_node_manager.cc:128: Shutting down raylet, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_node_manager.cc:252: Removing node, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, node name = 192.168.1.56
|
||||
[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_placement_group_manager.cc:767: Node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a failed, rescheduling the placement groups on the dead node.
|
||||
[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_actor_manager.cc:1048: Node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a failed, reconstructing actors.
|
||||
[2024-05-15 08:21:54,797 I 256391 256391] (gcs_server) gcs_job_manager.cc:302: Node 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a failed, mark all jobs from this node as finished
|
||||
[2024-05-15 08:21:54,985 I 256391 256391] (gcs_server) gcs_node_manager.cc:170: Raylet 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a is drained. Status GrpcUnavailable: RPC Error message: Cancelling all calls; RPC Error details: . The information will be published to the cluster.
|
||||
[2024-05-15 08:21:54,997 I 256391 256439] (gcs_server) ray_syncer-inl.h:318: Failed to read the message from: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:55,010 I 256391 256391] (gcs_server) gcs_server_main.cc:111: GCS server received SIGTERM, shutting down...
|
||||
[2024-05-15 08:21:55,010 I 256391 256391] (gcs_server) gcs_server.cc:272: Stopping GCS server.
|
||||
[2024-05-15 08:21:55,015 I 256391 256391] (gcs_server) gcs_server.cc:292: GCS server stopped.
|
||||
[2024-05-15 08:21:55,015 I 256391 256391] (gcs_server) io_service_pool.cc:47: IOServicePool is stopped.
|
||||
[2024-05-15 08:21:55,092 I 256391 256391] (gcs_server) stats.h:120: Stats module has shutdown.
|
||||
|
|
@ -1,255 +0,0 @@
|
|||
[2024-05-15 08:21:40,367 I 256534 256534] (raylet) main.cc:179: Setting cluster ID to: 535ba0eef614eb34440db6e780ab29a9f7318b794e022f2f96a3d065
|
||||
[2024-05-15 08:21:40,369 I 256534 256534] (raylet) main.cc:242: Raylet is not set to kill unknown children.
|
||||
[2024-05-15 08:21:40,369 I 256534 256534] (raylet) io_service_pool.cc:35: IOServicePool is running with 1 io_service.
|
||||
[2024-05-15 08:21:40,369 I 256534 256534] (raylet) main.cc:371: Setting node ID to: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:40,369 I 256534 256534] (raylet) store_runner.cc:32: Allowing the Plasma store to use up to 17.7374GB of memory.
|
||||
[2024-05-15 08:21:40,369 I 256534 256534] (raylet) store_runner.cc:48: Starting object store with directory /dev/shm, fallback /home/valapeos/Projects/pneumarl/logs/rllib, and huge page support disabled
|
||||
[2024-05-15 08:21:40,370 I 256534 256562] (raylet) dlmalloc.cc:154: create_and_mmap_buffer(17737449480, /dev/shm/plasmaXXXXXX)
|
||||
[2024-05-15 08:21:40,370 I 256534 256562] (raylet) store.cc:564: ========== Plasma store: =================
|
||||
Current usage: 0 / 17.7374 GB
|
||||
- num bytes created total: 0
|
||||
0 pending objects of total size 0MB
|
||||
- objects spillable: 0
|
||||
- bytes spillable: 0
|
||||
- objects unsealed: 0
|
||||
- bytes unsealed: 0
|
||||
- objects in use: 0
|
||||
- bytes in use: 0
|
||||
- objects evictable: 0
|
||||
- bytes evictable: 0
|
||||
|
||||
- objects created by worker: 0
|
||||
- bytes created by worker: 0
|
||||
- objects restored: 0
|
||||
- bytes restored: 0
|
||||
- objects received: 0
|
||||
- bytes received: 0
|
||||
- objects errored: 0
|
||||
- bytes errored: 0
|
||||
|
||||
[2024-05-15 08:21:40,371 I 256534 256534] (raylet) grpc_server.cc:134: ObjectManager server started, listening on port 35145.
|
||||
[2024-05-15 08:21:40,372 I 256534 256534] (raylet) worker_killing_policy.cc:101: Running GroupByOwner policy.
|
||||
[2024-05-15 08:21:40,373 W 256534 256534] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:21:40,373 I 256534 256534] (raylet) memory_monitor.cc:47: MemoryMonitor initialized with usage threshold at 63982133248 bytes (0.95 system memory), total system memory bytes: 67349614592
|
||||
[2024-05-15 08:21:40,373 I 256534 256534] (raylet) node_manager.cc:285: Initializing NodeManager with ID 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:40,373 I 256534 256534] (raylet) grpc_server.cc:134: NodeManager server started, listening on port 45647.
|
||||
[2024-05-15 08:21:40,379 I 256534 256596] (raylet) agent_manager.cc:78: Monitor agent process with name dashboard_agent/424238335
|
||||
[2024-05-15 08:21:40,379 I 256534 256598] (raylet) agent_manager.cc:78: Monitor agent process with name runtime_env_agent
|
||||
[2024-05-15 08:21:40,379 I 256534 256534] (raylet) event.cc:234: Set ray event level to warning
|
||||
[2024-05-15 08:21:40,379 I 256534 256534] (raylet) event.cc:342: Ray Event initialized for RAYLET
|
||||
[2024-05-15 08:21:40,380 I 256534 256534] (raylet) raylet.cc:128: Raylet of id, 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a started. Raylet consists of node_manager and object_manager. node_manager address: 192.168.1.56:45647 object_manager address: 192.168.1.56:35145 hostname: valanixos
|
||||
[2024-05-15 08:21:40,381 I 256534 256534] (raylet) node_manager.cc:521: [state-dump] NodeManager:
|
||||
[state-dump] Node ID: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[state-dump] Node name: 192.168.1.56
|
||||
[state-dump] InitialConfigResources: {node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 177374392320000, CPU: 120000, memory: 354748784640000}
|
||||
[state-dump] ClusterTaskManager:
|
||||
[state-dump] ========== Node: 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a =================
|
||||
[state-dump] Infeasible queue length: 0
|
||||
[state-dump] Schedule queue length: 0
|
||||
[state-dump] Dispatch queue length: 0
|
||||
[state-dump] num_waiting_for_resource: 0
|
||||
[state-dump] num_waiting_for_plasma_memory: 0
|
||||
[state-dump] num_waiting_for_remote_node_resources: 0
|
||||
[state-dump] num_worker_not_started_by_job_config_not_exist: 0
|
||||
[state-dump] num_worker_not_started_by_registration_timeout: 0
|
||||
[state-dump] num_tasks_waiting_for_workers: 0
|
||||
[state-dump] num_cancelled_tasks: 0
|
||||
[state-dump] cluster_resource_scheduler state:
|
||||
[state-dump] Local id: 3934636312908726399 Local resources: {"total":{CPU: [120000], memory: [354748784640000], node:192.168.1.56: [10000], node:__internal_head__: [10000], object_store_memory: [177374392320000]}}, "available": {CPU: [120000], memory: [354748784640000], node:192.168.1.56: [10000], node:__internal_head__: [10000], object_store_memory: [177374392320000]}}, "labels":{"ray.io/node_id":"5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a",} is_draining: 0 is_idle: 1 Cluster resources: node id: 3934636312908726399{"total":{node:__internal_head__: 10000, memory: 354748784640000, CPU: 120000, object_store_memory: 177374392320000, node:192.168.1.56: 10000}}, "available": {node:__internal_head__: 10000, memory: 354748784640000, CPU: 120000, object_store_memory: 177374392320000, node:192.168.1.56: 10000}}, "labels":{"ray.io/node_id":"5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []}
|
||||
[state-dump] Waiting tasks size: 0
|
||||
[state-dump] Number of executing tasks: 0
|
||||
[state-dump] Number of pinned task arguments: 0
|
||||
[state-dump] Number of total spilled tasks: 0
|
||||
[state-dump] Number of spilled waiting tasks: 0
|
||||
[state-dump] Number of spilled unschedulable tasks: 0
|
||||
[state-dump] Resource usage {
|
||||
[state-dump] }
|
||||
[state-dump] Running tasks by scheduling class:
|
||||
[state-dump] ==================================================
|
||||
[state-dump]
|
||||
[state-dump] ClusterResources:
|
||||
[state-dump] LocalObjectManager:
|
||||
[state-dump] - num pinned objects: 0
|
||||
[state-dump] - pinned objects size: 0
|
||||
[state-dump] - num objects pending restore: 0
|
||||
[state-dump] - num objects pending spill: 0
|
||||
[state-dump] - num bytes pending spill: 0
|
||||
[state-dump] - num bytes currently spilled: 0
|
||||
[state-dump] - cumulative spill requests: 0
|
||||
[state-dump] - cumulative restore requests: 0
|
||||
[state-dump] - spilled objects pending delete: 0
|
||||
[state-dump]
|
||||
[state-dump] ObjectManager:
|
||||
[state-dump] - num local objects: 0
|
||||
[state-dump] - num unfulfilled push requests: 0
|
||||
[state-dump] - num object pull requests: 0
|
||||
[state-dump] - num chunks received total: 0
|
||||
[state-dump] - num chunks received failed (all): 0
|
||||
[state-dump] - num chunks received failed / cancelled: 0
|
||||
[state-dump] - num chunks received failed / plasma error: 0
|
||||
[state-dump] Event stats:
|
||||
[state-dump] Global stats: 0 total (0 active)
|
||||
[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] Execution time: mean = -nan s, total = 0.000 s
|
||||
[state-dump] Event stats:
|
||||
[state-dump] PushManager:
|
||||
[state-dump] - num pushes in flight: 0
|
||||
[state-dump] - num chunks in flight: 0
|
||||
[state-dump] - num chunks remaining: 0
|
||||
[state-dump] - max chunks allowed: 409
|
||||
[state-dump] OwnershipBasedObjectDirectory:
|
||||
[state-dump] - num listeners: 0
|
||||
[state-dump] - cumulative location updates: 0
|
||||
[state-dump] - num location updates per second: 0.000
|
||||
[state-dump] - num location lookups per second: 0.000
|
||||
[state-dump] - num locations added per second: 0.000
|
||||
[state-dump] - num locations removed per second: 0.000
|
||||
[state-dump] BufferPool:
|
||||
[state-dump] - create buffer state map size: 0
|
||||
[state-dump] PullManager:
|
||||
[state-dump] - num bytes available for pulled objects: 17737439232
|
||||
[state-dump] - num bytes being pulled (all): 0
|
||||
[state-dump] - num bytes being pulled / pinned: 0
|
||||
[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
[state-dump] - first get request bundle: N/A
|
||||
[state-dump] - first wait request bundle: N/A
|
||||
[state-dump] - first task request bundle: N/A
|
||||
[state-dump] - num objects queued: 0
|
||||
[state-dump] - num objects actively pulled (all): 0
|
||||
[state-dump] - num objects actively pulled / pinned: 0
|
||||
[state-dump] - num bundles being pulled: 0
|
||||
[state-dump] - num pull retries: 0
|
||||
[state-dump] - max timeout seconds: 0
|
||||
[state-dump] - max timeout request is already processed. No entry.
|
||||
[state-dump]
|
||||
[state-dump] WorkerPool:
|
||||
[state-dump] - registered jobs: 0
|
||||
[state-dump] - process_failed_job_config_missing: 0
|
||||
[state-dump] - process_failed_rate_limited: 0
|
||||
[state-dump] - process_failed_pending_registration: 0
|
||||
[state-dump] - process_failed_runtime_env_setup_failed: 0
|
||||
[state-dump] - num PYTHON workers: 0
|
||||
[state-dump] - num PYTHON drivers: 0
|
||||
[state-dump] - num object spill callbacks queued: 0
|
||||
[state-dump] - num object restore queued: 0
|
||||
[state-dump] - num util functions queued: 0
|
||||
[state-dump] - num idle workers: 0
|
||||
[state-dump] TaskDependencyManager:
|
||||
[state-dump] - task deps map size: 0
|
||||
[state-dump] - get req map size: 0
|
||||
[state-dump] - wait req map size: 0
|
||||
[state-dump] - local objects map size: 0
|
||||
[state-dump] WaitManager:
|
||||
[state-dump] - num active wait requests: 0
|
||||
[state-dump] Subscriber:
|
||||
[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL
|
||||
[state-dump] - cumulative subscribe requests: 0
|
||||
[state-dump] - cumulative unsubscribe requests: 0
|
||||
[state-dump] - active subscribed publishers: 0
|
||||
[state-dump] - cumulative published messages: 0
|
||||
[state-dump] - cumulative processed messages: 0
|
||||
[state-dump] Channel WORKER_REF_REMOVED_CHANNEL
|
||||
[state-dump] - cumulative subscribe requests: 0
|
||||
[state-dump] - cumulative unsubscribe requests: 0
|
||||
[state-dump] - active subscribed publishers: 0
|
||||
[state-dump] - cumulative published messages: 0
|
||||
[state-dump] - cumulative processed messages: 0
|
||||
[state-dump] Channel WORKER_OBJECT_EVICTION
|
||||
[state-dump] - cumulative subscribe requests: 0
|
||||
[state-dump] - cumulative unsubscribe requests: 0
|
||||
[state-dump] - active subscribed publishers: 0
|
||||
[state-dump] - cumulative published messages: 0
|
||||
[state-dump] - cumulative processed messages: 0
|
||||
[state-dump] num async plasma notifications: 0
|
||||
[state-dump] Remote node managers:
|
||||
[state-dump] Event stats:
|
||||
[state-dump] Global stats: 27 total (13 active)
|
||||
[state-dump] Queueing time: mean = 866.145 us, max = 6.687 ms, min = 5.080 us, total = 23.386 ms
|
||||
[state-dump] Execution time: mean = 458.777 us, total = 12.387 ms
|
||||
[state-dump] Event stats:
|
||||
[state-dump] PeriodicalRunner.RunFnPeriodically - 11 total (2 active, 1 running), Execution time: mean = 80.443 us, total = 884.877 us, Queueing time: mean = 2.125 ms, max = 6.687 ms, min = 17.970 us, total = 23.374 ms
|
||||
[state-dump] NodeManager.deadline_timer.debug_state_dump - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeManager.deadline_timer.flush_free_objects - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 10.159 ms, total = 10.159 ms, Queueing time: mean = 6.700 us, max = 6.700 us, min = 6.700 us, total = 6.700 us
|
||||
[state-dump] NodeManager.deadline_timer.record_metrics - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 595.718 us, total = 595.718 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 359.519 us, total = 359.519 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 258.210 us, total = 258.210 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 129.890 us, total = 129.890 us, Queueing time: mean = 5.080 us, max = 5.080 us, min = 5.080 us, total = 5.080 us
|
||||
[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] DebugString() time ms: 0
|
||||
[state-dump]
|
||||
[state-dump]
|
||||
[2024-05-15 08:21:40,381 I 256534 256534] (raylet) accessor.cc:627: Received notification for node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a, IsAlive = 1
|
||||
[2024-05-15 08:21:40,486 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256618, the token is 0
|
||||
[2024-05-15 08:21:40,488 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256619, the token is 1
|
||||
[2024-05-15 08:21:40,489 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256620, the token is 2
|
||||
[2024-05-15 08:21:40,491 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256621, the token is 3
|
||||
[2024-05-15 08:21:40,493 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256622, the token is 4
|
||||
[2024-05-15 08:21:40,495 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256623, the token is 5
|
||||
[2024-05-15 08:21:40,497 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256624, the token is 6
|
||||
[2024-05-15 08:21:40,499 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256625, the token is 7
|
||||
[2024-05-15 08:21:40,501 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256626, the token is 8
|
||||
[2024-05-15 08:21:40,503 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256627, the token is 9
|
||||
[2024-05-15 08:21:40,505 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256628, the token is 10
|
||||
[2024-05-15 08:21:40,507 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 256629, the token is 11
|
||||
[2024-05-15 08:21:40,844 I 256534 256562] (raylet) object_store.cc:35: Object store current usage 8e-09 / 17.7374 GB.
|
||||
[2024-05-15 08:21:40,964 I 256534 256534] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 256306 is dead: 0 driver address: 192.168.1.56
|
||||
[2024-05-15 08:21:40,964 I 256534 256534] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool.
|
||||
[2024-05-15 08:21:41,179 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:21:41,181 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257357, the token is 12
|
||||
[2024-05-15 08:21:41,991 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:41,991 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:41,992 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:41,992 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:42,194 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:43,098 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:21:43,100 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257429, the token is 13
|
||||
[2024-05-15 08:21:43,100 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:21:43,102 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257430, the token is 14
|
||||
[2024-05-15 08:21:43,103 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:21:43,104 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257431, the token is 15
|
||||
[2024-05-15 08:21:43,104 I 256534 256534] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:21:43,106 I 256534 256534] (raylet) worker_pool.cc:494: Started worker process with pid 257432, the token is 16
|
||||
[2024-05-15 08:21:45,381 W 256534 256534] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:21:50,379 W 256534 256556] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:56844: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster.
|
||||
[2024-05-15 08:21:50,383 W 256534 256534] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:54,093 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:21:54,094 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:21:54,095 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:21:54,095 I 256534 256534] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:21:54,602 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=3, has creation task exception = false
|
||||
[2024-05-15 08:21:54,602 I 256534 256534] (raylet) node_manager.cc:1541: Driver (pid=256306) is disconnected. job_id: 01000000
|
||||
[2024-05-15 08:21:54,603 I 256534 256534] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 256306 is dead: 1 driver address: 192.168.1.56
|
||||
[2024-05-15 08:21:54,603 I 256534 256534] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool.
|
||||
[2024-05-15 08:21:54,785 I 256534 256534] (raylet) worker_pool.cc:1107: Force exiting worker whose job has exited a424db48d9263a3a0f640c7168dc3404402e2f6e7d613ff172defb0c
|
||||
[2024-05-15 08:21:54,795 I 256534 256534] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:21:54,797 I 256534 256534] (raylet) main.cc:413: Raylet received SIGTERM, shutting down...
|
||||
[2024-05-15 08:21:54,797 I 256534 256534] (raylet) accessor.cc:451: Unregistering node info, node id = 5796b7c556a32eb747fc415a8d49599b0b1bd85a4db6cf0ca573197a
|
||||
[2024-05-15 08:21:54,797 I 256534 256534] (raylet) agent_manager.cc:109: Killing agent dashboard_agent/424238335, pid 256593.
|
||||
[2024-05-15 08:21:54,802 I 256534 256596] (raylet) agent_manager.cc:80: Agent process with name dashboard_agent/424238335 exited, exit code 0.
|
||||
[2024-05-15 08:21:54,802 I 256534 256534] (raylet) agent_manager.cc:109: Killing agent runtime_env_agent, pid 256597.
|
||||
[2024-05-15 08:21:54,806 I 256534 256598] (raylet) agent_manager.cc:80: Agent process with name runtime_env_agent exited, exit code 0.
|
||||
[2024-05-15 08:21:54,807 I 256534 256534] (raylet) io_service_pool.cc:47: IOServicePool is stopped.
|
||||
[2024-05-15 08:21:54,982 I 256534 256534] (raylet) stats.h:120: Stats module has shutdown.
|
||||
[2024-05-15 08:21:54,982 I 256534 256600] (raylet) experimental_mutable_object_provider.cc:200: Core worker channel io service stopped.
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
======== Running on http://0.0.0.0:58296 ========
|
||||
(Press CTRL+C to quit)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:RolloutWorker
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:RolloutWorker
|
||||
No game binary has been provided, please press PLAY in the Godot editor
|
||||
waiting for remote GODOT connection on port 11012
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:RolloutWorker
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:RolloutWorker
|
||||
No game binary has been provided, please press PLAY in the Godot editor
|
||||
waiting for remote GODOT connection on port 11009
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:RolloutWorker
|
||||
[2024-05-15 08:21:54,138 C 257430 257430] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe
|
||||
*** StackTrace Information ***
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7ff3f600dbba] ray::operator<<()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7ff3f600f477] ray::SpdLogMessage::Flush()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7ff3f600f917] ray::RayLog::~RayLog()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7ff3f57d2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7ff3f57d319a] std::_Function_handler<>::_M_invoke()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7ff3f57ea0de] ray::core::InboundRequest::Accept()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7ff3f57bc440] ray::core::NormalSchedulingQueue::ScheduleRequests()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7ff3f5a4e71e] EventTracker::RecordExecution()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7ff3f5a47b0e] std::_Function_handler<>::_M_invoke()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7ff3f5a47f86] boost::asio::detail::completion_handler<>::do_complete()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7ff3f60f78eb] boost::asio::detail::scheduler::do_run_one()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7ff3f60f9269] boost::asio::detail::scheduler::run()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7ff3f60f9972] boost::asio::io_context::run()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7ff3f57713fd] ray::core::CoreWorker::RunTaskExecutionLoop()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7ff3f57b5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7ff3f57b60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7ff3f55bd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop()
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7ff3f74413c2] method_vectorcall_NOARGS
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7ff3f745ad93] PyObject_Vectorcall
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7ff3f72f7c5c] _PyEval_EvalFrameDefault
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7ff3f752bc9c] _PyEval_Vector.constprop.0
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7ff3f752be3a] PyEval_EvalCode
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7ff3f7556a10] run_mod
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7ff3f7576752] _PyRun_SimpleFileObject
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7ff3f7576ff1] _PyRun_AnyFileObject
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7ff3f757990f] Py_RunMain
|
||||
/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7ff3f703d10e] __libc_start_call_main
|
||||
/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7ff3f703d1c9] __libc_start_main@GLIBC_2.2.5
|
||||
ray::RolloutWorker(_start+0x25) [0x401075] _start
|
||||
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:RolloutWorker
|
||||
No game binary has been provided, please press PLAY in the Godot editor
|
||||
waiting for remote GODOT connection on port 11010
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:RolloutWorker
|
||||
[2024-05-15 08:21:54,138 C 257431 257431] direct_actor_transport.cc:187: Check failed: _s.ok() Bad status: IOError: Broken pipe
|
||||
*** StackTrace Information ***
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100dbba) [0x7f0e8860dbba] ray::operator<<()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x100f477) [0x7f0e8860f477] ray::SpdLogMessage::Flush()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x37) [0x7f0e8860f917] ray::RayLog::~RayLog()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d2bbd) [0x7f0e87dd2bbd] ray::core::CoreWorkerDirectTaskReceiver::HandleTask()::{lambda()#1}::operator()()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7d319a) [0x7f0e87dd319a] std::_Function_handler<>::_M_invoke()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7ea0de) [0x7f0e87dea0de] ray::core::InboundRequest::Accept()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x7bc440) [0x7f0e87dbc440] ray::core::NormalSchedulingQueue::ScheduleRequests()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa4e71e) [0x7f0e8804e71e] EventTracker::RecordExecution()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47b0e) [0x7f0e88047b0e] std::_Function_handler<>::_M_invoke()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0xa47f86) [0x7f0e88047f86] boost::asio::detail::completion_handler<>::do_complete()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f78eb) [0x7f0e886f78eb] boost::asio::detail::scheduler::do_run_one()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9269) [0x7f0e886f9269] boost::asio::detail::scheduler::run()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x10f9972) [0x7f0e886f9972] boost::asio::io_context::run()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker20RunTaskExecutionLoopEv+0xcd) [0x7f0e87d713fd] ray::core::CoreWorker::RunTaskExecutionLoop()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core21CoreWorkerProcessImpl26RunWorkerTaskExecutionLoopEv+0x8c) [0x7f0e87db5f1c] ray::core::CoreWorkerProcessImpl::RunWorkerTaskExecutionLoop()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core17CoreWorkerProcess20RunTaskExecutionLoopEv+0x1d) [0x7f0e87db60cd] ray::core::CoreWorkerProcess::RunTaskExecutionLoop()
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/_raylet.so(+0x5bd4b7) [0x7f0e87bbd4b7] __pyx_pw_3ray_7_raylet_10CoreWorker_7run_task_loop()
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x2413c2) [0x7f0e89c413c2] method_vectorcall_NOARGS
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyObject_Vectorcall+0x33) [0x7f0e89c5ad93] PyObject_Vectorcall
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyEval_EvalFrameDefault+0x33cc) [0x7f0e89af7c5c] _PyEval_EvalFrameDefault
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x32bc9c) [0x7f0e89d2bc9c] _PyEval_Vector.constprop.0
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(PyEval_EvalCode+0x14a) [0x7f0e89d2be3a] PyEval_EvalCode
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(+0x356a10) [0x7f0e89d56a10] run_mod
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_SimpleFileObject+0x222) [0x7f0e89d76752] _PyRun_SimpleFileObject
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(_PyRun_AnyFileObject+0xa1) [0x7f0e89d76ff1] _PyRun_AnyFileObject
|
||||
/nix/store/glfr70gi7hfaj50mwj2431p8bg60fhqw-python3-3.11.9/lib/libpython3.11.so.1.0(Py_RunMain+0x94f) [0x7f0e89d7990f] Py_RunMain
|
||||
/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(+0x2a10e) [0x7f0e8975a10e] __libc_start_call_main
|
||||
/nix/store/35pq4hr29c3sl79lgfwgsvd9nwzyp4am-glibc-2.39-5/lib/libc.so.6(__libc_start_main+0x89) [0x7f0e8975a1c9] __libc_start_main@GLIBC_2.2.5
|
||||
ray::RolloutWorker(_start+0x25) [0x401075] _start
|
||||
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:RolloutWorker
|
||||
No game binary has been provided, please press PLAY in the Godot editor
|
||||
waiting for remote GODOT connection on port 11011
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
:job_id:01000000
|
||||
:task_name:bundle_reservation_check_func
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
:job_id:01000000
|
||||
:task_name:bundle_reservation_check_func
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:PPO
|
||||
2024-05-15 08:21:43,045 WARNING deprecation.py:50 -- DeprecationWarning: `AlgorithmConfig.num_envs_per_worker` has been deprecated. Use `AlgorithmConfig.num_envs_per_env_runner` instead. This will raise an error in the future!
|
||||
2024-05-15 08:21:43,051 WARNING deprecation.py:50 -- DeprecationWarning: `WorkerSet(num_workers=... OR local_worker=...)` has been deprecated. Use `EnvRunnerGroup(num_env_runners=... AND local_env_runner=...)` instead. This will raise an error in the future!
|
||||
2024-05-15 08:21:43,051 WARNING deprecation.py:50 -- DeprecationWarning: `max_num_worker_restarts` has been deprecated. Use `AlgorithmConfig.max_num_env_runner_restarts` instead. This will raise an error in the future!
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:PPO
|
||||
|
|
@ -1 +0,0 @@
|
|||
{"node_ip_address": "192.168.1.56"}
|
||||
|
|
@ -1 +0,0 @@
|
|||
{"192.168.1.56:/home/valapeos/Projects/pneumarl/logs/rllib/session_2024-05-15_08-21-38_956751_256306/sockets/plasma_store": {"metrics_agent_port": 56844, "metrics_export_port": 42241, "dashboard_agent_listen_port": 52365, "runtime_env_agent_port": 58296, "gcs_server_port": 56287}}
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +0,0 @@
|
|||
Traceback (most recent call last):
|
||||
File "python/ray/_raylet.pyx", line 3270, in ray._raylet.check_health
|
||||
File "python/ray/_raylet.pyx", line 583, in ray._raylet.check_status
|
||||
ray.exceptions.RpcError: failed to connect to all addresses; last error: UNKNOWN: ipv4:192.168.1.56:47881: Failed to connect to remote host: Connection refused
|
||||
/home/valapeos/.cache/pypoetry/virtualenvs/pneumarl-4s6AwDM4-py3.11/lib/python3.11/site-packages/ray/dashboard/dashboard.py:236: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead
|
||||
logger.warn("Exiting with SIGTERM immediately...")
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
NodeManager:
|
||||
Node ID: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
Node name: 192.168.1.56
|
||||
InitialConfigResources: {CPU: 120000, memory: 353902559240000, node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 176951279610000}
|
||||
ClusterTaskManager:
|
||||
========== Node: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 =================
|
||||
Infeasible queue length: 0
|
||||
Schedule queue length: 0
|
||||
Dispatch queue length: 0
|
||||
num_waiting_for_resource: 0
|
||||
num_waiting_for_plasma_memory: 0
|
||||
num_waiting_for_remote_node_resources: 0
|
||||
num_worker_not_started_by_job_config_not_exist: 0
|
||||
num_worker_not_started_by_registration_timeout: 0
|
||||
num_tasks_waiting_for_workers: 0
|
||||
num_cancelled_tasks: 0
|
||||
cluster_resource_scheduler state:
|
||||
Local id: -1885364415256342489 Local resources: {"total":{node:__internal_head__: [10000], bundle_group_81c6c20970b7e7ed4f550507158301000000: [50000000], bundle_group_4_81c6c20970b7e7ed4f550507158301000000: [10000000], bundle_group_3_81c6c20970b7e7ed4f550507158301000000: [10000000], bundle_group_2_81c6c20970b7e7ed4f550507158301000000: [10000000], CPU: [120000], CPU_group_2_81c6c20970b7e7ed4f550507158301000000: [10000], CPU_group_3_81c6c20970b7e7ed4f550507158301000000: [10000], bundle_group_1_81c6c20970b7e7ed4f550507158301000000: [10000000], CPU_group_0_81c6c20970b7e7ed4f550507158301000000: [10000], bundle_group_0_81c6c20970b7e7ed4f550507158301000000: [10000000], memory: [353902559240000], CPU_group_4_81c6c20970b7e7ed4f550507158301000000: [10000], CPU_group_81c6c20970b7e7ed4f550507158301000000: [50000], CPU_group_1_81c6c20970b7e7ed4f550507158301000000: [10000], object_store_memory: [176951279610000], node:192.168.1.56: [10000]}}, "available": {node:__internal_head__: [10000], bundle_group_81c6c20970b7e7ed4f550507158301000000: [49999950], bundle_group_4_81c6c20970b7e7ed4f550507158301000000: [10000000], bundle_group_3_81c6c20970b7e7ed4f550507158301000000: [10000000], bundle_group_2_81c6c20970b7e7ed4f550507158301000000: [10000000], CPU: [70000], CPU_group_2_81c6c20970b7e7ed4f550507158301000000: [10000], CPU_group_3_81c6c20970b7e7ed4f550507158301000000: [10000], bundle_group_1_81c6c20970b7e7ed4f550507158301000000: [10000000], CPU_group_0_81c6c20970b7e7ed4f550507158301000000: [0], bundle_group_0_81c6c20970b7e7ed4f550507158301000000: [9999990], memory: [353902559240000], CPU_group_4_81c6c20970b7e7ed4f550507158301000000: [10000], CPU_group_81c6c20970b7e7ed4f550507158301000000: [0], CPU_group_1_81c6c20970b7e7ed4f550507158301000000: [10000], object_store_memory: [176951279610000], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450",} is_draining: 0 is_idle: 0 Cluster resources: node id: -1885364415256342489{"total":{memory: 353902559240000, CPU_group_4_81c6c20970b7e7ed4f550507158301000000: 10000, CPU_group_0_81c6c20970b7e7ed4f550507158301000000: 10000, CPU_group_1_81c6c20970b7e7ed4f550507158301000000: 10000, CPU_group_81c6c20970b7e7ed4f550507158301000000: 50000, node:192.168.1.56: 10000, object_store_memory: 176951279610000, node:__internal_head__: 10000, bundle_group_81c6c20970b7e7ed4f550507158301000000: 50000000, CPU_group_3_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_3_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_4_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_2_81c6c20970b7e7ed4f550507158301000000: 10000000, CPU: 120000, CPU_group_2_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_1_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_0_81c6c20970b7e7ed4f550507158301000000: 10000000}}, "available": {object_store_memory: 176951279610000, node:192.168.1.56: 10000, CPU_group_1_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_0_81c6c20970b7e7ed4f550507158301000000: 9999990, CPU_group_4_81c6c20970b7e7ed4f550507158301000000: 10000, memory: 353902559240000, bundle_group_1_81c6c20970b7e7ed4f550507158301000000: 10000000, CPU_group_2_81c6c20970b7e7ed4f550507158301000000: 10000, CPU: 70000, bundle_group_2_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_3_81c6c20970b7e7ed4f550507158301000000: 10000000, CPU_group_3_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_4_81c6c20970b7e7ed4f550507158301000000: 10000000, bundle_group_81c6c20970b7e7ed4f550507158301000000: 49999950, node:__internal_head__: 10000}}, "labels":{"ray.io/node_id":"a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []}
|
||||
Waiting tasks size: 0
|
||||
Number of executing tasks: 5
|
||||
Number of pinned task arguments: 0
|
||||
Number of total spilled tasks: 0
|
||||
Number of spilled waiting tasks: 0
|
||||
Number of spilled unschedulable tasks: 0
|
||||
Resource usage {
|
||||
- (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=259038): {bundle_group_81c6c20970b7e7ed4f550507158301000000: 10, CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000}
|
||||
- (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=259036): {CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_81c6c20970b7e7ed4f550507158301000000: 10}
|
||||
- (language=PYTHON actor_or_task=PPO.__init__ pid=258963): {CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000, CPU_group_0_81c6c20970b7e7ed4f550507158301000000: 10000, bundle_group_81c6c20970b7e7ed4f550507158301000000: 10, bundle_group_0_81c6c20970b7e7ed4f550507158301000000: 10}
|
||||
- (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=259039): {bundle_group_81c6c20970b7e7ed4f550507158301000000: 10, CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000}
|
||||
- (language=PYTHON actor_or_task=RolloutWorker.__init__ pid=259037): {bundle_group_81c6c20970b7e7ed4f550507158301000000: 10, CPU_group_81c6c20970b7e7ed4f550507158301000000: 10000}
|
||||
}
|
||||
Running tasks by scheduling class:
|
||||
- {depth=1 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.algorithms.ppo.ppo, class_name=PPO, function_name=__init__, function_hash=af33c904067c44ebb8a2cc1b903cfde9} scheduling_strategy=placement_group_scheduling_strategy {
|
||||
placement_group_id: "\201\306\302\tp\267\347\355OU\005\007\025\203\001\000\000\000"
|
||||
placement_group_capture_child_tasks: true
|
||||
}
|
||||
resource_set={bundle_group_81c6c20970b7e7ed4f550507158301000000 : 0.001, CPU_group_81c6c20970b7e7ed4f550507158301000000 : 1, CPU_group_0_81c6c20970b7e7ed4f550507158301000000 : 1, bundle_group_0_81c6c20970b7e7ed4f550507158301000000 : 0.001, }}: 1/18446744073709551615
|
||||
- {depth=2 function_descriptor={type=PythonFunctionDescriptor, module_name=ray.rllib.evaluation.rollout_worker, class_name=RolloutWorker, function_name=__init__, function_hash=3e0ea47c3fc64ebd817dc76e776d34cf} scheduling_strategy=placement_group_scheduling_strategy {
|
||||
placement_group_id: "\201\306\302\tp\267\347\355OU\005\007\025\203\001\000\000\000"
|
||||
placement_group_bundle_index: -1
|
||||
placement_group_capture_child_tasks: true
|
||||
}
|
||||
resource_set={bundle_group_81c6c20970b7e7ed4f550507158301000000 : 0.001, CPU_group_81c6c20970b7e7ed4f550507158301000000 : 1, }}: 4/18446744073709551615
|
||||
==================================================
|
||||
|
||||
ClusterResources:
|
||||
LocalObjectManager:
|
||||
- num pinned objects: 0
|
||||
- pinned objects size: 0
|
||||
- num objects pending restore: 0
|
||||
- num objects pending spill: 0
|
||||
- num bytes pending spill: 0
|
||||
- num bytes currently spilled: 0
|
||||
- cumulative spill requests: 0
|
||||
- cumulative restore requests: 0
|
||||
- spilled objects pending delete: 0
|
||||
|
||||
ObjectManager:
|
||||
- num local objects: 0
|
||||
- num unfulfilled push requests: 0
|
||||
- num object pull requests: 0
|
||||
- num chunks received total: 0
|
||||
- num chunks received failed (all): 0
|
||||
- num chunks received failed / cancelled: 0
|
||||
- num chunks received failed / plasma error: 0
|
||||
Event stats:
|
||||
Global stats: 0 total (0 active)
|
||||
Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
Execution time: mean = -nan s, total = 0.000 s
|
||||
Event stats:
|
||||
PushManager:
|
||||
- num pushes in flight: 0
|
||||
- num chunks in flight: 0
|
||||
- num chunks remaining: 0
|
||||
- max chunks allowed: 409
|
||||
OwnershipBasedObjectDirectory:
|
||||
- num listeners: 0
|
||||
- cumulative location updates: 0
|
||||
- num location updates per second: 0.000
|
||||
- num location lookups per second: 0.000
|
||||
- num locations added per second: 0.000
|
||||
- num locations removed per second: 0.000
|
||||
BufferPool:
|
||||
- create buffer state map size: 0
|
||||
PullManager:
|
||||
- num bytes available for pulled objects: 17695127961
|
||||
- num bytes being pulled (all): 0
|
||||
- num bytes being pulled / pinned: 0
|
||||
- get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
- wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
- task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
- first get request bundle: N/A
|
||||
- first wait request bundle: N/A
|
||||
- first task request bundle: N/A
|
||||
- num objects queued: 0
|
||||
- num objects actively pulled (all): 0
|
||||
- num objects actively pulled / pinned: 0
|
||||
- num bundles being pulled: 0
|
||||
- num pull retries: 0
|
||||
- max timeout seconds: 0
|
||||
- max timeout request is already processed. No entry.
|
||||
|
||||
WorkerPool:
|
||||
- registered jobs: 1
|
||||
- process_failed_job_config_missing: 0
|
||||
- process_failed_rate_limited: 0
|
||||
- process_failed_pending_registration: 0
|
||||
- process_failed_runtime_env_setup_failed: 0
|
||||
- num PYTHON workers: 12
|
||||
- num PYTHON drivers: 1
|
||||
- num object spill callbacks queued: 0
|
||||
- num object restore queued: 0
|
||||
- num util functions queued: 0
|
||||
- num idle workers: 7
|
||||
TaskDependencyManager:
|
||||
- task deps map size: 0
|
||||
- get req map size: 0
|
||||
- wait req map size: 0
|
||||
- local objects map size: 0
|
||||
WaitManager:
|
||||
- num active wait requests: 0
|
||||
Subscriber:
|
||||
Channel WORKER_OBJECT_LOCATIONS_CHANNEL
|
||||
- cumulative subscribe requests: 0
|
||||
- cumulative unsubscribe requests: 0
|
||||
- active subscribed publishers: 0
|
||||
- cumulative published messages: 0
|
||||
- cumulative processed messages: 0
|
||||
Channel WORKER_OBJECT_EVICTION
|
||||
- cumulative subscribe requests: 0
|
||||
- cumulative unsubscribe requests: 0
|
||||
- active subscribed publishers: 0
|
||||
- cumulative published messages: 0
|
||||
- cumulative processed messages: 0
|
||||
Channel WORKER_REF_REMOVED_CHANNEL
|
||||
- cumulative subscribe requests: 0
|
||||
- cumulative unsubscribe requests: 0
|
||||
- active subscribed publishers: 0
|
||||
- cumulative published messages: 0
|
||||
- cumulative processed messages: 0
|
||||
num async plasma notifications: 0
|
||||
Remote node managers:
|
||||
Event stats:
|
||||
Global stats: 2474 total (27 active)
|
||||
Queueing time: mean = 2.790 ms, max = 1.347 s, min = 70.000 ns, total = 6.903 s
|
||||
Execution time: mean = 1.664 ms, total = 4.117 s
|
||||
Event stats:
|
||||
NodeManagerService.grpc_server.ReportWorkerBacklog.HandleRequestImpl - 387 total (0 active), Execution time: mean = 9.061 us, total = 3.507 ms, Queueing time: mean = 10.254 us, max = 414.788 us, min = 4.500 us, total = 3.968 ms
|
||||
NodeManagerService.grpc_server.ReportWorkerBacklog - 387 total (0 active), Execution time: mean = 87.039 us, total = 33.684 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
RaySyncer.OnDemandBroadcasting - 300 total (1 active), Execution time: mean = 5.138 us, total = 1.541 ms, Queueing time: mean = 10.532 us, max = 61.158 us, min = 5.749 us, total = 3.160 ms
|
||||
ObjectManager.UpdateAvailableMemory - 300 total (0 active), Execution time: mean = 978.530 ns, total = 293.559 us, Queueing time: mean = 6.798 us, max = 27.920 us, min = 4.160 us, total = 2.040 ms
|
||||
NodeManager.CheckGC - 300 total (1 active), Execution time: mean = 687.800 ns, total = 206.340 us, Queueing time: mean = 14.601 us, max = 225.787 us, min = 5.938 us, total = 4.380 ms
|
||||
RayletWorkerPool.deadline_timer.kill_idle_workers - 150 total (1 active), Execution time: mean = 11.656 us, total = 1.748 ms, Queueing time: mean = 11.804 us, max = 247.415 us, min = 5.048 us, total = 1.771 ms
|
||||
MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 120 total (1 active), Execution time: mean = 80.284 us, total = 9.634 ms, Queueing time: mean = 10.150 us, max = 25.878 us, min = 3.129 us, total = 1.218 ms
|
||||
ClientConnection.async_read.ProcessMessageHeader - 83 total (13 active), Execution time: mean = 2.708 us, total = 224.740 us, Queueing time: mean = 82.469 ms, max = 1.347 s, min = 9.280 us, total = 6.845 s
|
||||
ClientConnection.async_read.ProcessMessage - 70 total (0 active), Execution time: mean = 443.031 us, total = 31.012 ms, Queueing time: mean = 53.817 us, max = 3.130 ms, min = 1.950 us, total = 3.767 ms
|
||||
NodeManager.ScheduleAndDispatchTasks - 31 total (1 active), Execution time: mean = 4.692 us, total = 145.450 us, Queueing time: mean = 8.759 us, max = 18.074 us, min = 6.457 us, total = 271.529 us
|
||||
NodeManagerService.grpc_server.GetResourceLoad.HandleRequestImpl - 30 total (0 active), Execution time: mean = 50.454 us, total = 1.514 ms, Queueing time: mean = 9.796 us, max = 16.609 us, min = 6.390 us, total = 293.868 us
|
||||
NodeManager.deadline_timer.flush_free_objects - 30 total (1 active), Execution time: mean = 1.674 us, total = 50.230 us, Queueing time: mean = 40.812 us, max = 324.394 us, min = 2.845 us, total = 1.224 ms
|
||||
NodeManagerService.grpc_server.GetResourceLoad - 30 total (0 active), Execution time: mean = 136.638 us, total = 4.099 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManager.deadline_timer.spill_objects_when_over_threshold - 30 total (1 active), Execution time: mean = 1.005 us, total = 30.150 us, Queueing time: mean = 41.323 us, max = 323.594 us, min = 3.375 us, total = 1.240 ms
|
||||
ClientConnection.async_write.DoAsyncWrites - 19 total (0 active), Execution time: mean = 681.053 ns, total = 12.940 us, Queueing time: mean = 10.481 us, max = 15.320 us, min = 6.630 us, total = 199.130 us
|
||||
ObjectManager.ObjectDeleted - 18 total (0 active), Execution time: mean = 8.087 us, total = 145.560 us, Queueing time: mean = 47.910 us, max = 424.198 us, min = 12.300 us, total = 862.376 us
|
||||
NodeManagerService.grpc_server.GetSystemConfig.HandleRequestImpl - 18 total (0 active), Execution time: mean = 25.750 us, total = 463.508 us, Queueing time: mean = 10.899 us, max = 16.410 us, min = 6.660 us, total = 196.189 us
|
||||
ObjectManager.ObjectAdded - 18 total (0 active), Execution time: mean = 7.946 us, total = 143.020 us, Queueing time: mean = 36.346 us, max = 359.769 us, min = 5.730 us, total = 654.237 us
|
||||
NodeManagerService.grpc_server.GetSystemConfig - 18 total (0 active), Execution time: mean = 138.977 us, total = 2.502 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
PeriodicalRunner.RunFnPeriodically - 13 total (0 active), Execution time: mean = 108.665 us, total = 1.413 ms, Queueing time: mean = 2.209 ms, max = 7.494 ms, min = 18.500 us, total = 28.713 ms
|
||||
ClusterResourceManager.ResetRemoteNodeView - 11 total (1 active), Execution time: mean = 1.784 us, total = 19.620 us, Queueing time: mean = 8.919 us, max = 11.237 us, min = 7.955 us, total = 98.109 us
|
||||
RaySyncer.BroadcastMessage - 6 total (0 active), Execution time: mean = 111.837 us, total = 671.020 us, Queueing time: mean = 171.667 ns, max = 210.000 ns, min = 100.000 ns, total = 1.030 us
|
||||
- 6 total (0 active), Execution time: mean = 103.333 ns, total = 620.000 ns, Queueing time: mean = 8.868 us, max = 11.030 us, min = 7.790 us, total = 53.210 us
|
||||
NodeManager.GcsCheckAlive - 6 total (1 active), Execution time: mean = 84.400 us, total = 506.398 us, Queueing time: mean = 100.684 us, max = 158.511 us, min = 42.914 us, total = 604.106 us
|
||||
NodeManager.deadline_timer.record_metrics - 6 total (1 active), Execution time: mean = 191.353 us, total = 1.148 ms, Queueing time: mean = 13.328 us, max = 45.615 us, min = 6.792 us, total = 79.968 us
|
||||
NodeManagerService.grpc_server.RequestWorkerLease.HandleRequestImpl - 6 total (0 active), Execution time: mean = 122.608 us, total = 735.649 us, Queueing time: mean = 260.461 us, max = 829.938 us, min = 5.530 us, total = 1.563 ms
|
||||
NodeManagerService.grpc_server.RequestWorkerLease - 6 total (0 active), Execution time: mean = 364.173 ms, total = 2.185 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.CheckAlive.OnReplyReceived - 6 total (0 active), Execution time: mean = 11.150 us, total = 66.900 us, Queueing time: mean = 8.055 us, max = 9.550 us, min = 5.810 us, total = 48.330 us
|
||||
NodeInfoGcsService.grpc_client.CheckAlive - 6 total (0 active), Execution time: mean = 314.641 us, total = 1.888 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
Subscriber.HandlePublishedMessage_GCS_WORKER_DELTA_CHANNEL - 5 total (0 active), Execution time: mean = 2.344 us, total = 11.720 us, Queueing time: mean = 51.394 us, max = 66.610 us, min = 45.690 us, total = 256.970 us
|
||||
CoreWorkerService.grpc_client.Exit.OnReplyReceived - 5 total (0 active), Execution time: mean = 11.538 us, total = 57.690 us, Queueing time: mean = 9.718 us, max = 20.000 us, min = 5.040 us, total = 48.590 us
|
||||
CoreWorkerService.grpc_client.Exit - 5 total (0 active), Execution time: mean = 800.304 us, total = 4.002 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
WorkerInfoGcsService.grpc_client.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 1.124 ms, total = 5.622 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
WorkerInfoGcsService.grpc_client.ReportWorkerFailure.OnReplyReceived - 5 total (0 active), Execution time: mean = 7.830 us, total = 39.149 us, Queueing time: mean = 46.599 us, max = 72.909 us, min = 9.690 us, total = 232.997 us
|
||||
InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 5 total (1 active), Execution time: mean = 361.620 ms, total = 1.808 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalPubSubGcsService.grpc_client.GcsSubscriberPoll.OnReplyReceived - 4 total (0 active), Execution time: mean = 60.217 us, total = 240.869 us, Queueing time: mean = 7.112 us, max = 10.609 us, min = 4.280 us, total = 28.449 us
|
||||
NodeManager.deadline_timer.debug_state_dump - 3 total (1 active, 1 running), Execution time: mean = 398.066 us, total = 1.194 ms, Queueing time: mean = 5.335 us, max = 10.340 us, min = 5.666 us, total = 16.006 us
|
||||
RaySyncerRegister - 2 total (0 active), Execution time: mean = 815.000 ns, total = 1.630 us, Queueing time: mean = 165.000 ns, max = 260.000 ns, min = 70.000 ns, total = 330.000 ns
|
||||
InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 2 total (0 active), Execution time: mean = 276.159 us, total = 552.319 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 2 total (0 active), Execution time: mean = 68.675 us, total = 137.350 us, Queueing time: mean = 540.198 us, max = 1.004 ms, min = 76.220 us, total = 1.080 ms
|
||||
NodeManager.deadline_timer.print_event_loop_stats - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_client.AddJob - 1 total (0 active), Execution time: mean = 377.469 us, total = 377.469 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.GetAllNodeInfo - 1 total (0 active), Execution time: mean = 301.139 us, total = 301.139 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_client.GetAllJobInfo - 1 total (0 active), Execution time: mean = 289.119 us, total = 289.119 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 147.790 us, total = 147.790 us, Queueing time: mean = 8.099 us, max = 8.099 us, min = 8.099 us, total = 8.099 us
|
||||
NodeManagerService.grpc_server.CommitBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 219.390 us, total = 219.390 us, Queueing time: mean = 6.870 us, max = 6.870 us, min = 6.870 us, total = 6.870 us
|
||||
Subscriber.HandlePublishedMessage_GCS_JOB_CHANNEL - 1 total (0 active), Execution time: mean = 23.820 us, total = 23.820 us, Queueing time: mean = 56.110 us, max = 56.110 us, min = 56.110 us, total = 56.110 us
|
||||
NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 730.548 us, total = 730.548 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 11.090 ms, total = 11.090 ms, Queueing time: mean = 8.630 us, max = 8.630 us, min = 8.630 us, total = 8.630 us
|
||||
JobInfoGcsService.grpc_client.GetAllJobInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 5.820 us, total = 5.820 us, Queueing time: mean = 4.820 us, max = 4.820 us, min = 4.820 us, total = 4.820 us
|
||||
NodeManagerService.grpc_server.PrepareBundleResources - 1 total (0 active), Execution time: mean = 293.789 us, total = 293.789 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_client.GetAllNodeInfo.OnReplyReceived - 1 total (0 active), Execution time: mean = 73.390 us, total = 73.390 us, Queueing time: mean = 5.250 us, max = 5.250 us, min = 5.250 us, total = 5.250 us
|
||||
NodeManagerService.grpc_server.ReturnWorker - 1 total (0 active), Execution time: mean = 121.460 us, total = 121.460 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
WorkerPool.PopWorkerCallback - 1 total (0 active), Execution time: mean = 12.860 us, total = 12.860 us, Queueing time: mean = 6.460 us, max = 6.460 us, min = 6.460 us, total = 6.460 us
|
||||
NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 429.879 us, total = 429.879 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_server.CommitBundleResources - 1 total (0 active), Execution time: mean = 275.420 us, total = 275.420 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_server.PrepareBundleResources.HandleRequestImpl - 1 total (0 active), Execution time: mean = 234.479 us, total = 234.479 us, Queueing time: mean = 6.390 us, max = 6.390 us, min = 6.390 us, total = 6.390 us
|
||||
JobInfoGcsService.grpc_client.AddJob.OnReplyReceived - 1 total (0 active), Execution time: mean = 16.820 us, total = 16.820 us, Queueing time: mean = 4.910 us, max = 4.910 us, min = 4.910 us, total = 4.910 us
|
||||
NodeManagerService.grpc_server.ReturnWorker.HandleRequestImpl - 1 total (0 active), Execution time: mean = 37.550 us, total = 37.550 us, Queueing time: mean = 9.240 us, max = 9.240 us, min = 9.240 us, total = 9.240 us
|
||||
DebugString() time ms: 0
|
||||
|
|
@ -1,150 +0,0 @@
|
|||
GcsNodeManager:
|
||||
- RegisterNode request count: 1
|
||||
- DrainNode request count: 0
|
||||
- GetAllNodeInfo request count: 28
|
||||
- GetInternalConfig request count: 1
|
||||
|
||||
GcsActorManager:
|
||||
- RegisterActor request count: 5
|
||||
- CreateActor request count: 5
|
||||
- GetActorInfo request count: 5
|
||||
- GetNamedActorInfo request count: 0
|
||||
- GetAllActorInfo request count: 0
|
||||
- KillActor request count: 0
|
||||
- ListNamedActors request count: 0
|
||||
- Registered actors count: 5
|
||||
- Destroyed actors count: 0
|
||||
- Named actors count: 0
|
||||
- Unresolved actors count: 0
|
||||
- Pending actors count: 0
|
||||
- Created actors count: 0
|
||||
- owners_: 1
|
||||
- actor_to_register_callbacks_: 0
|
||||
- actor_to_create_callbacks_: 5
|
||||
- sorted_destroyed_actor_list_: 0
|
||||
|
||||
GcsResourceManager:
|
||||
- GetAllAvailableResources request count0
|
||||
- GetAllResourceUsage request count: 8
|
||||
|
||||
GcsPlacementGroupManager:
|
||||
- CreatePlacementGroup request count: 1
|
||||
- RemovePlacementGroup request count: 0
|
||||
- GetPlacementGroup request count: 5
|
||||
- GetAllPlacementGroup request count: 0
|
||||
- WaitPlacementGroupUntilReady request count: 0
|
||||
- GetNamedPlacementGroup request count: 0
|
||||
- Scheduling pending placement group count: 1
|
||||
- Registered placement groups count: 1
|
||||
- Named placement group count: 0
|
||||
- Pending placement groups count: 0
|
||||
- Infeasible placement groups count: 0
|
||||
|
||||
Publisher:
|
||||
RAY_LOG_CHANNEL
|
||||
- cumulative published messages: 13
|
||||
- cumulative published bytes: 9872
|
||||
- current buffered bytes: 853
|
||||
GCS_JOB_CHANNEL
|
||||
- cumulative published messages: 1
|
||||
- cumulative published bytes: 280
|
||||
- current buffered bytes: 280
|
||||
GCS_ACTOR_CHANNEL
|
||||
- cumulative published messages: 10
|
||||
- cumulative published bytes: 5864
|
||||
- current buffered bytes: 1328
|
||||
GCS_WORKER_DELTA_CHANNEL
|
||||
- cumulative published messages: 5
|
||||
- cumulative published bytes: 485
|
||||
- current buffered bytes: 97
|
||||
GCS_NODE_INFO_CHANNEL
|
||||
- cumulative published messages: 1
|
||||
- cumulative published bytes: 559
|
||||
- current buffered bytes: 0
|
||||
|
||||
[runtime env manager] ID to URIs table:
|
||||
[runtime env manager] URIs reference table:
|
||||
|
||||
GcsTaskManager:
|
||||
-Total num task events reported: 25
|
||||
-Total num status task events dropped: 0
|
||||
-Total num profile events dropped: 0
|
||||
-Current num of task events stored: 13
|
||||
-Total num of actor creation tasks: 5
|
||||
-Total num of actor tasks: 6
|
||||
-Total num of normal tasks: 1
|
||||
-Total num of driver tasks: 1
|
||||
|
||||
|
||||
|
||||
|
||||
Global stats: 1124 total (19 active)
|
||||
Queueing time: mean = 82.405 us, max = 26.288 ms, min = 230.000 ns, total = 92.623 ms
|
||||
Execution time: mean = 2.055 ms, total = 2.310 s
|
||||
Event stats:
|
||||
GcsInMemoryStore.Put - 159 total (0 active), Execution time: mean = 185.082 us, total = 29.428 ms, Queueing time: mean = 173.789 us, max = 26.115 ms, min = 680.000 ns, total = 27.632 ms
|
||||
InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 113 total (0 active), Execution time: mean = 6.158 us, total = 695.898 us, Queueing time: mean = 35.882 us, max = 229.260 us, min = 2.960 us, total = 4.055 ms
|
||||
InternalKVGcsService.grpc_server.InternalKVPut - 113 total (0 active), Execution time: mean = 122.098 us, total = 13.797 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsInMemoryStore.Get - 81 total (0 active), Execution time: mean = 18.103 us, total = 1.466 ms, Queueing time: mean = 11.716 us, max = 272.030 us, min = 620.000 ns, total = 948.999 us
|
||||
InternalKVGcsService.grpc_server.InternalKVGet.HandleRequestImpl - 74 total (0 active), Execution time: mean = 10.299 us, total = 762.127 us, Queueing time: mean = 9.189 us, max = 33.660 us, min = 4.360 us, total = 679.978 us
|
||||
InternalKVGcsService.grpc_server.InternalKVGet - 74 total (0 active), Execution time: mean = 148.802 us, total = 11.011 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalKVGcsService.grpc_client.InternalKVPut.OnReplyReceived - 48 total (0 active), Execution time: mean = 3.526 us, total = 169.250 us, Queueing time: mean = 12.323 us, max = 27.350 us, min = 4.810 us, total = 591.518 us
|
||||
InternalKVGcsService.grpc_client.InternalKVPut - 48 total (0 active), Execution time: mean = 383.844 us, total = 18.425 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
RayletLoadPulled - 40 total (1 active), Execution time: mean = 74.947 us, total = 2.998 ms, Queueing time: mean = 18.572 us, max = 277.794 us, min = 4.442 us, total = 742.873 us
|
||||
NodeManagerService.grpc_client.GetResourceLoad - 38 total (0 active), Execution time: mean = 337.670 us, total = 12.831 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_client.GetResourceLoad.OnReplyReceived - 38 total (0 active), Execution time: mean = 25.848 us, total = 982.226 us, Queueing time: mean = 8.813 us, max = 21.630 us, min = 5.180 us, total = 334.899 us
|
||||
NodeInfoGcsService.grpc_server.GetAllNodeInfo.HandleRequestImpl - 28 total (0 active), Execution time: mean = 12.195 us, total = 341.460 us, Queueing time: mean = 20.317 us, max = 361.159 us, min = 5.360 us, total = 568.879 us
|
||||
NodeInfoGcsService.grpc_server.GetAllNodeInfo - 28 total (0 active), Execution time: mean = 100.779 us, total = 2.822 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
WorkerInfoGcsService.grpc_server.AddWorkerInfo.HandleRequestImpl - 18 total (0 active), Execution time: mean = 18.321 us, total = 329.770 us, Queueing time: mean = 36.146 us, max = 275.469 us, min = 6.010 us, total = 650.628 us
|
||||
WorkerInfoGcsService.grpc_server.AddWorkerInfo - 18 total (0 active), Execution time: mean = 157.753 us, total = 2.840 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_server.CheckAlive - 16 total (0 active), Execution time: mean = 88.642 us, total = 1.418 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_server.CheckAlive.HandleRequestImpl - 16 total (0 active), Execution time: mean = 6.994 us, total = 111.910 us, Queueing time: mean = 9.734 us, max = 16.280 us, min = 6.620 us, total = 155.739 us
|
||||
ClusterResourceManager.ResetRemoteNodeView - 14 total (1 active), Execution time: mean = 2.224 us, total = 31.130 us, Queueing time: mean = 6.328 us, max = 11.648 us, min = 4.711 us, total = 88.591 us
|
||||
HealthCheck - 12 total (0 active), Execution time: mean = 2.049 us, total = 24.590 us, Queueing time: mean = 9.494 us, max = 18.440 us, min = 7.090 us, total = 113.929 us
|
||||
NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage - 8 total (0 active), Execution time: mean = 115.109 us, total = 920.869 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeResourceInfoGcsService.grpc_server.GetAllResourceUsage.HandleRequestImpl - 8 total (0 active), Execution time: mean = 25.117 us, total = 200.940 us, Queueing time: mean = 10.002 us, max = 11.780 us, min = 7.730 us, total = 80.020 us
|
||||
NodeInfoGcsService.grpc_server.GetClusterId.HandleRequestImpl - 8 total (0 active), Execution time: mean = 14.996 us, total = 119.970 us, Queueing time: mean = 65.355 us, max = 436.549 us, min = 4.770 us, total = 522.839 us
|
||||
NodeInfoGcsService.grpc_server.GetClusterId - 8 total (0 active), Execution time: mean = 712.702 us, total = 5.702 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsInMemoryStore.GetAll - 7 total (0 active), Execution time: mean = 5.966 us, total = 41.760 us, Queueing time: mean = 12.703 us, max = 17.850 us, min = 1.920 us, total = 88.920 us
|
||||
GcsResourceManager::Update - 6 total (0 active), Execution time: mean = 42.716 us, total = 256.298 us, Queueing time: mean = 9.708 us, max = 13.170 us, min = 5.790 us, total = 58.249 us
|
||||
WorkerInfoGcsService.grpc_server.ReportWorkerFailure.HandleRequestImpl - 5 total (0 active), Execution time: mean = 17.160 us, total = 85.799 us, Queueing time: mean = 282.695 us, max = 692.049 us, min = 10.030 us, total = 1.413 ms
|
||||
ActorInfoGcsService.grpc_server.GetActorInfo - 5 total (0 active), Execution time: mean = 126.632 us, total = 633.159 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
ActorInfoGcsService.grpc_server.RegisterActor - 5 total (0 active), Execution time: mean = 325.337 us, total = 1.627 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_client.RequestWorkerLease.OnReplyReceived - 5 total (0 active), Execution time: mean = 166.314 us, total = 831.568 us, Queueing time: mean = 31.304 us, max = 105.160 us, min = 4.820 us, total = 156.520 us
|
||||
PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup - 5 total (0 active), Execution time: mean = 191.969 us, total = 959.847 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
ActorInfoGcsService.grpc_server.GetActorInfo.HandleRequestImpl - 5 total (0 active), Execution time: mean = 6.290 us, total = 31.450 us, Queueing time: mean = 31.650 us, max = 75.050 us, min = 6.040 us, total = 158.250 us
|
||||
NodeManagerService.grpc_client.RequestWorkerLease - 5 total (0 active), Execution time: mean = 437.588 ms, total = 2.188 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
WorkerInfoGcsService.grpc_server.ReportWorkerFailure - 5 total (0 active), Execution time: mean = 833.848 us, total = 4.169 ms, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
CoreWorkerService.grpc_client.WaitForActorOutOfScope - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
ActorInfoGcsService.grpc_server.CreateActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 131.430 us, total = 657.148 us, Queueing time: mean = 7.282 us, max = 11.420 us, min = 5.470 us, total = 36.410 us
|
||||
CoreWorkerService.grpc_client.PushTask - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
ActorInfoGcsService.grpc_server.CreateActor - 5 total (5 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
PlacementGroupInfoGcsService.grpc_server.GetPlacementGroup.HandleRequestImpl - 5 total (0 active), Execution time: mean = 10.648 us, total = 53.240 us, Queueing time: mean = 121.932 us, max = 289.339 us, min = 14.550 us, total = 609.658 us
|
||||
ActorInfoGcsService.grpc_server.RegisterActor.HandleRequestImpl - 5 total (0 active), Execution time: mean = 217.513 us, total = 1.088 ms, Queueing time: mean = 7.596 us, max = 8.430 us, min = 6.900 us, total = 37.980 us
|
||||
PeriodicalRunner.RunFnPeriodically - 4 total (0 active), Execution time: mean = 53.635 us, total = 214.540 us, Queueing time: mean = 13.173 ms, max = 26.288 ms, min = 14.480 us, total = 52.692 ms
|
||||
GCSServer.deadline_timer.debug_state_dump - 4 total (1 active, 1 running), Execution time: mean = 338.224 us, total = 1.353 ms, Queueing time: mean = 4.808 us, max = 6.686 us, min = 6.045 us, total = 19.233 us
|
||||
InternalKVGcsService.grpc_server.InternalKVDel.HandleRequestImpl - 1 total (0 active), Execution time: mean = 9.290 us, total = 9.290 us, Queueing time: mean = 4.890 us, max = 4.890 us, min = 4.890 us, total = 4.890 us
|
||||
NodeInfoGcsService.grpc_server.GetInternalConfig - 1 total (0 active), Execution time: mean = 113.319 us, total = 113.319 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup - 1 total (0 active), Execution time: mean = 100.969 us, total = 100.969 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_server.RegisterNode.HandleRequestImpl - 1 total (0 active), Execution time: mean = 68.430 us, total = 68.430 us, Queueing time: mean = 17.470 us, max = 17.470 us, min = 17.470 us, total = 17.470 us
|
||||
NodeManagerService.grpc_client.CommitBundleResources - 1 total (0 active), Execution time: mean = 407.498 us, total = 407.498 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_server.GetInternalConfig.HandleRequestImpl - 1 total (0 active), Execution time: mean = 11.320 us, total = 11.320 us, Queueing time: mean = 13.690 us, max = 13.690 us, min = 13.690 us, total = 13.690 us
|
||||
NodeManagerService.grpc_client.PrepareBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 42.040 us, total = 42.040 us, Queueing time: mean = 7.230 us, max = 7.230 us, min = 7.230 us, total = 7.230 us
|
||||
JobInfoGcsService.grpc_server.AddJob.HandleRequestImpl - 1 total (0 active), Execution time: mean = 47.930 us, total = 47.930 us, Queueing time: mean = 8.830 us, max = 8.830 us, min = 8.830 us, total = 8.830 us
|
||||
JobInfoGcsService.grpc_server.GetNextJobID.HandleRequestImpl - 1 total (0 active), Execution time: mean = 4.790 us, total = 4.790 us, Queueing time: mean = 7.410 us, max = 7.410 us, min = 7.410 us, total = 7.410 us
|
||||
InternalKVGcsService.grpc_server.InternalKVExists - 1 total (0 active), Execution time: mean = 78.379 us, total = 78.379 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
PlacementGroupInfoGcsService.grpc_server.CreatePlacementGroup.HandleRequestImpl - 1 total (0 active), Execution time: mean = 42.060 us, total = 42.060 us, Queueing time: mean = 6.720 us, max = 6.720 us, min = 6.720 us, total = 6.720 us
|
||||
GcsInMemoryStore.Delete - 1 total (0 active), Execution time: mean = 2.030 us, total = 2.030 us, Queueing time: mean = 1.070 us, max = 1.070 us, min = 1.070 us, total = 1.070 us
|
||||
JobInfoGcsService.grpc_server.GetAllJobInfo.HandleRequestImpl - 1 total (0 active), Execution time: mean = 7.970 us, total = 7.970 us, Queueing time: mean = 6.560 us, max = 6.560 us, min = 6.560 us, total = 6.560 us
|
||||
GcsInMemoryStore.Exists - 1 total (0 active), Execution time: mean = 2.930 us, total = 2.930 us, Queueing time: mean = 1.850 us, max = 1.850 us, min = 1.850 us, total = 1.850 us
|
||||
NodeManagerService.grpc_client.PrepareBundleResources - 1 total (0 active), Execution time: mean = 793.648 us, total = 793.648 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsPlacementGroupManager.SchedulePendingPlacementGroups - 1 total (0 active), Execution time: mean = 80.000 ns, total = 80.000 ns, Queueing time: mean = 95.950 us, max = 95.950 us, min = 95.950 us, total = 95.950 us
|
||||
InternalKVGcsService.grpc_server.InternalKVDel - 1 total (0 active), Execution time: mean = 49.420 us, total = 49.420 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_server.GetAllJobInfo - 1 total (0 active), Execution time: mean = 95.100 us, total = 95.100 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_server.GetNextJobID - 1 total (0 active), Execution time: mean = 70.760 us, total = 70.760 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeManagerService.grpc_client.CommitBundleResources.OnReplyReceived - 1 total (0 active), Execution time: mean = 53.540 us, total = 53.540 us, Queueing time: mean = 7.710 us, max = 7.710 us, min = 7.710 us, total = 7.710 us
|
||||
GcsHealthCheckManager::AddNode - 1 total (0 active), Execution time: mean = 6.850 us, total = 6.850 us, Queueing time: mean = 230.000 ns, max = 230.000 ns, min = 230.000 ns, total = 230.000 ns
|
||||
GCSServer.deadline_timer.debug_state_event_stats_print - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
NodeInfoGcsService.grpc_server.RegisterNode - 1 total (0 active), Execution time: mean = 329.699 us, total = 329.699 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
JobInfoGcsService.grpc_server.AddJob - 1 total (0 active), Execution time: mean = 141.810 us, total = 141.810 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalKVGcsService.grpc_server.InternalKVExists.HandleRequestImpl - 1 total (0 active), Execution time: mean = 16.150 us, total = 16.150 us, Queueing time: mean = 6.570 us, max = 6.570 us, min = 6.570 us, total = 6.570 us
|
||||
|
|
@ -1,190 +0,0 @@
|
|||
[2024-05-15 08:22:48,114 I 257996 257996] (gcs_server) io_service_pool.cc:35: IOServicePool is running with 1 io_service.
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) event.cc:234: Set ray event level to warning
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) event.cc:342: Ray Event initialized for GCS
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_server.cc:74: GCS storage type is StorageType::IN_MEMORY
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:42: Loading job table data.
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:54: Loading node table data.
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:80: Loading actor table data.
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:93: Loading actor task spec table data.
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:66: Loading placement group table data.
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:46: Finished loading job table data, size = 0
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:58: Finished loading node table data, size = 0
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:84: Finished loading actor table data, size = 0
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:97: Finished loading actor task spec table data, size = 0
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_init_data.cc:71: Finished loading placement group table data, size = 0
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_server.cc:162: No existing server cluster ID found. Generating new ID: 08d8dd0a3ae9d38657267cfdd0aaa4f146ec1f5e86c8c03b04920aec
|
||||
[2024-05-15 08:22:48,115 I 257996 257996] (gcs_server) gcs_server.cc:648: Autoscaler V2 enabled: 0
|
||||
[2024-05-15 08:22:48,116 I 257996 257996] (gcs_server) grpc_server.cc:134: GcsServer server started, listening on port 47881.
|
||||
[2024-05-15 08:22:48,142 I 257996 257996] (gcs_server) gcs_server.cc:250: GcsNodeManager:
|
||||
- RegisterNode request count: 0
|
||||
- DrainNode request count: 0
|
||||
- GetAllNodeInfo request count: 0
|
||||
- GetInternalConfig request count: 0
|
||||
|
||||
GcsActorManager:
|
||||
- RegisterActor request count: 0
|
||||
- CreateActor request count: 0
|
||||
- GetActorInfo request count: 0
|
||||
- GetNamedActorInfo request count: 0
|
||||
- GetAllActorInfo request count: 0
|
||||
- KillActor request count: 0
|
||||
- ListNamedActors request count: 0
|
||||
- Registered actors count: 0
|
||||
- Destroyed actors count: 0
|
||||
- Named actors count: 0
|
||||
- Unresolved actors count: 0
|
||||
- Pending actors count: 0
|
||||
- Created actors count: 0
|
||||
- owners_: 0
|
||||
- actor_to_register_callbacks_: 0
|
||||
- actor_to_create_callbacks_: 0
|
||||
- sorted_destroyed_actor_list_: 0
|
||||
|
||||
GcsResourceManager:
|
||||
- GetAllAvailableResources request count0
|
||||
- GetAllResourceUsage request count: 0
|
||||
|
||||
GcsPlacementGroupManager:
|
||||
- CreatePlacementGroup request count: 0
|
||||
- RemovePlacementGroup request count: 0
|
||||
- GetPlacementGroup request count: 0
|
||||
- GetAllPlacementGroup request count: 0
|
||||
- WaitPlacementGroupUntilReady request count: 0
|
||||
- GetNamedPlacementGroup request count: 0
|
||||
- Scheduling pending placement group count: 0
|
||||
- Registered placement groups count: 0
|
||||
- Named placement group count: 0
|
||||
- Pending placement groups count: 0
|
||||
- Infeasible placement groups count: 0
|
||||
|
||||
Publisher:
|
||||
|
||||
[runtime env manager] ID to URIs table:
|
||||
[runtime env manager] URIs reference table:
|
||||
|
||||
GcsTaskManager:
|
||||
-Total num task events reported: 0
|
||||
-Total num status task events dropped: 0
|
||||
-Total num profile events dropped: 0
|
||||
-Current num of task events stored: 0
|
||||
-Total num of actor creation tasks: 0
|
||||
-Total num of actor tasks: 0
|
||||
-Total num of normal tasks: 0
|
||||
-Total num of driver tasks: 0
|
||||
|
||||
|
||||
[2024-05-15 08:22:48,142 I 257996 257996] (gcs_server) gcs_server.cc:844: Event stats:
|
||||
|
||||
|
||||
Global stats: 29 total (18 active)
|
||||
Queueing time: mean = 2.716 ms, max = 26.288 ms, min = 1.000 us, total = 78.756 ms
|
||||
Execution time: mean = 907.843 us, total = 26.327 ms
|
||||
Event stats:
|
||||
InternalKVGcsService.grpc_client.InternalKVPut - 6 total (6 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsInMemoryStore.GetAll - 5 total (0 active), Execution time: mean = 3.798 us, total = 18.990 us, Queueing time: mean = 16.800 us, max = 17.850 us, min = 15.510 us, total = 84.000 us
|
||||
PeriodicalRunner.RunFnPeriodically - 4 total (2 active, 1 running), Execution time: mean = 1.025 us, total = 4.100 us, Queueing time: mean = 13.138 ms, max = 26.288 ms, min = 26.264 ms, total = 52.552 ms
|
||||
InternalKVGcsService.grpc_server.InternalKVPut - 4 total (4 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
InternalKVGcsService.grpc_server.InternalKVPut.HandleRequestImpl - 4 total (4 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
GcsInMemoryStore.Put - 3 total (0 active), Execution time: mean = 8.766 ms, total = 26.299 ms, Queueing time: mean = 8.706 ms, max = 26.115 ms, min = 1.000 us, total = 26.118 ms
|
||||
GcsInMemoryStore.Get - 1 total (0 active), Execution time: mean = 5.370 us, total = 5.370 us, Queueing time: mean = 1.240 us, max = 1.240 us, min = 1.240 us, total = 1.240 us
|
||||
ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
RayletLoadPulled - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
|
||||
|
||||
[2024-05-15 08:22:48,142 I 257996 257996] (gcs_server) gcs_server.cc:845: GcsTaskManager Event stats:
|
||||
|
||||
|
||||
Global stats: 2 total (1 active)
|
||||
Queueing time: mean = 26.755 us, max = 53.509 us, min = 53.509 us, total = 53.509 us
|
||||
Execution time: mean = 10.900 us, total = 21.800 us
|
||||
Event stats:
|
||||
PeriodicalRunner.RunFnPeriodically - 1 total (0 active), Execution time: mean = 21.800 us, total = 21.800 us, Queueing time: mean = 53.509 us, max = 53.509 us, min = 53.509 us, total = 53.509 us
|
||||
GcsTaskManager.GcJobSummary - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
|
||||
|
||||
[2024-05-15 08:22:49,503 I 257996 257996] (gcs_server) gcs_node_manager.cc:55: Registering node info, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, address = 192.168.1.56, node name = 192.168.1.56
|
||||
[2024-05-15 08:22:49,503 I 257996 257996] (gcs_server) gcs_node_manager.cc:61: Finished registering node info, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, address = 192.168.1.56, node name = 192.168.1.56
|
||||
[2024-05-15 08:22:49,503 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:797: A new node: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 registered, will try to reschedule all the infeasible placement groups.
|
||||
[2024-05-15 08:22:50,064 I 257996 257996] (gcs_server) gcs_job_manager.cc:42: Adding job, job id = 01000000, driver pid = 257884
|
||||
[2024-05-15 08:22:50,064 I 257996 257996] (gcs_server) gcs_job_manager.cc:57: Finished adding job, job id = 01000000, driver pid = 257884
|
||||
[2024-05-15 08:22:50,097 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:432: Registering placement group, placement group id = 81c6c20970b7e7ed4f550507158301000000, name = , strategy = 0
|
||||
[2024-05-15 08:22:50,097 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:436: Finished registering placement group, placement group id = 81c6c20970b7e7ed4f550507158301000000, name = , strategy = 0
|
||||
[2024-05-15 08:22:50,097 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:185: Preparing resource from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for bundles: {placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4}},
|
||||
[2024-05-15 08:22:50,098 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:195: Finished leasing resource from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for bundles: {placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4}},
|
||||
[2024-05-15 08:22:50,098 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:213: Committing resource to a node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for bundles: {placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3}},
|
||||
[2024-05-15 08:22:50,098 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:220: Finished committing resource to a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for bundles: {placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2}},{placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3}},
|
||||
[2024-05-15 08:22:50,098 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:321: Successfully created placement group , id: 81c6c20970b7e7ed4f550507158301000000
|
||||
[2024-05-15 08:22:50,280 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = bdd59c451e6d1ad3c5498e6901000000
|
||||
[2024-05-15 08:22:50,281 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = bdd59c451e6d1ad3c5498e6901000000
|
||||
[2024-05-15 08:22:50,281 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = bdd59c451e6d1ad3c5498e6901000000
|
||||
[2024-05-15 08:22:50,281 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor bdd59c451e6d1ad3c5498e6901000000, job id = 01000000
|
||||
[2024-05-15 08:22:50,690 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor bdd59c451e6d1ad3c5498e6901000000, job id = 01000000
|
||||
[2024-05-15 08:22:50,690 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor bdd59c451e6d1ad3c5498e6901000000 on worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000
|
||||
[2024-05-15 08:22:52,262 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = f4ac3f8db7fdd2b856cf9a4d01000000
|
||||
[2024-05-15 08:22:52,262 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = f4ac3f8db7fdd2b856cf9a4d01000000
|
||||
[2024-05-15 08:22:52,263 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = f4ac3f8db7fdd2b856cf9a4d01000000
|
||||
[2024-05-15 08:22:52,263 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor f4ac3f8db7fdd2b856cf9a4d01000000, job id = 01000000
|
||||
[2024-05-15 08:22:52,263 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 81bf39d29ed6424b18900e4501000000
|
||||
[2024-05-15 08:22:52,263 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 81bf39d29ed6424b18900e4501000000
|
||||
[2024-05-15 08:22:52,264 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 81bf39d29ed6424b18900e4501000000
|
||||
[2024-05-15 08:22:52,264 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 81bf39d29ed6424b18900e4501000000, job id = 01000000
|
||||
[2024-05-15 08:22:52,264 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 228710b5ac6babc91120ca1101000000
|
||||
[2024-05-15 08:22:52,264 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 228710b5ac6babc91120ca1101000000
|
||||
[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 228710b5ac6babc91120ca1101000000
|
||||
[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 228710b5ac6babc91120ca1101000000, job id = 01000000
|
||||
[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_manager.cc:262: Registering actor, job id = 01000000, actor id = 0206158a753b8236d33a875801000000
|
||||
[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_manager.cc:268: Registered actor, job id = 01000000, actor id = 0206158a753b8236d33a875801000000
|
||||
[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_manager.cc:287: Creating actor, job id = 01000000, actor id = 0206158a753b8236d33a875801000000
|
||||
[2024-05-15 08:22:52,265 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 0206158a753b8236d33a875801000000, job id = 01000000
|
||||
[2024-05-15 08:22:52,695 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor f4ac3f8db7fdd2b856cf9a4d01000000, job id = 01000000
|
||||
[2024-05-15 08:22:52,695 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor f4ac3f8db7fdd2b856cf9a4d01000000 on worker 6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000
|
||||
[2024-05-15 08:22:52,697 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 0206158a753b8236d33a875801000000, job id = 01000000
|
||||
[2024-05-15 08:22:52,697 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 0206158a753b8236d33a875801000000 on worker d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000
|
||||
[2024-05-15 08:22:52,698 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 228710b5ac6babc91120ca1101000000, job id = 01000000
|
||||
[2024-05-15 08:22:52,698 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 228710b5ac6babc91120ca1101000000 on worker 15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000
|
||||
[2024-05-15 08:22:52,747 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:633: Finished leasing worker from a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 81bf39d29ed6424b18900e4501000000, job id = 01000000
|
||||
[2024-05-15 08:22:52,747 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:447: Start creating actor 81bf39d29ed6424b18900e4501000000 on worker 7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, job id = 01000000
|
||||
[2024-05-15 08:22:58,128 W 257996 258017] (gcs_server) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:59951: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster.
|
||||
[2024-05-15 08:23:28,795 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = bdd59c451e6d1ad3c5498e6901000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,795 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = bdd59c451e6d1ad3c5498e6901000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:23:28,795 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 01000000ffffffffffffffffffffffffffffffffffffffffffffffff failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:23:28,795 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:607: Cancelling all committed bundles of a placement group, id is 81c6c20970b7e7ed4f550507158301000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_job_manager.cc:87: Finished marking job state, job id = 01000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:830: Placement group of an id, 81c6c20970b7e7ed4f550507158301000000 is successfully removed because the job died.
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1167: Actor f4ac3f8db7fdd2b856cf9a4d01000000 is failed on worker 6fd0fa521cf8934f3c3851aba2729538d972a156aff6831f7dd6b986 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor f4ac3f8db7fdd2b856cf9a4d01000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1167: Actor 228710b5ac6babc91120ca1101000000 is failed on worker 15ca4ae25a45979cdda69039cb13dc75761088f7253fc8239633db60 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 228710b5ac6babc91120ca1101000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1167: Actor 0206158a753b8236d33a875801000000 is failed on worker d6f4499a7b15bb33ca872fedc419bd7cddae05d4e54a34a62db450ad at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 0206158a753b8236d33a875801000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={4} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={0} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={1} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={2} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1167: Actor 81bf39d29ed6424b18900e4501000000 is failed on worker 7507ee7ba4849f965294757e0366344ef6c15e496b0887a3d11a1d79 at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, need_reschedule = 1, death context type = ActorDiedErrorContext, remaining_restarts = 1000, job id = 01000000
|
||||
[2024-05-15 08:23:28,796 I 257996 257996] (gcs_server) gcs_actor_scheduler.cc:312: Start leasing worker from node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 for actor 81bf39d29ed6424b18900e4501000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = f4ac3f8db7fdd2b856cf9a4d01000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = f4ac3f8db7fdd2b856cf9a4d01000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 0206158a753b8236d33a875801000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 0206158a753b8236d33a875801000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 228710b5ac6babc91120ca1101000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 228710b5ac6babc91120ca1101000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:816: Destroying actor, actor id = 81bf39d29ed6424b18900e4501000000, job id = 01000000
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_actor_manager.cc:303: Finished creating actor, job id = 01000000, actor id = 81bf39d29ed6424b18900e4501000000, status = SchedulingCancelled: Actor creation cancelled.
|
||||
[2024-05-15 08:23:28,797 I 257996 257996] (gcs_server) gcs_placement_group_scheduler.cc:261: Finished cancelling the resource reserved for bundle: placement group id={81c6c20970b7e7ed4f550507158301000000}, bundle index={3} at node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:23:28,828 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:23:28,828 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:23:28,828 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:23:28,828 I 257996 257996] (gcs_server) gcs_actor_manager.cc:794: Worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 failed, destroying actor child, job id = 01000000
|
||||
[2024-05-15 08:23:28,838 I 257996 257996] (gcs_server) gcs_node_manager.cc:128: Shutting down raylet, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:23:28,839 I 257996 257996] (gcs_server) gcs_node_manager.cc:252: Removing node, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, node name = 192.168.1.56
|
||||
[2024-05-15 08:23:28,839 I 257996 257996] (gcs_server) gcs_placement_group_manager.cc:767: Node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 failed, rescheduling the placement groups on the dead node.
|
||||
[2024-05-15 08:23:28,839 I 257996 257996] (gcs_server) gcs_actor_manager.cc:1048: Node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 failed, reconstructing actors.
|
||||
[2024-05-15 08:23:28,839 I 257996 257996] (gcs_server) gcs_job_manager.cc:302: Node a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 failed, mark all jobs from this node as finished
|
||||
[2024-05-15 08:23:28,921 I 257996 257996] (gcs_server) gcs_node_manager.cc:170: Raylet a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 is drained. Status GrpcUnavailable: RPC Error message: Cancelling all calls; RPC Error details: . The information will be published to the cluster.
|
||||
[2024-05-15 08:23:28,934 I 257996 258044] (gcs_server) ray_syncer-inl.h:318: Failed to read the message from: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:23:28,952 I 257996 257996] (gcs_server) gcs_server_main.cc:111: GCS server received SIGTERM, shutting down...
|
||||
[2024-05-15 08:23:28,952 I 257996 257996] (gcs_server) gcs_server.cc:272: Stopping GCS server.
|
||||
[2024-05-15 08:23:28,957 I 257996 257996] (gcs_server) gcs_server.cc:292: GCS server stopped.
|
||||
[2024-05-15 08:23:28,957 I 257996 257996] (gcs_server) io_service_pool.cc:47: IOServicePool is stopped.
|
||||
[2024-05-15 08:23:29,046 I 257996 257996] (gcs_server) stats.h:120: Stats module has shutdown.
|
||||
|
|
@ -1,259 +0,0 @@
|
|||
[2024-05-15 08:22:49,490 I 258144 258144] (raylet) main.cc:179: Setting cluster ID to: 08d8dd0a3ae9d38657267cfdd0aaa4f146ec1f5e86c8c03b04920aec
|
||||
[2024-05-15 08:22:49,492 I 258144 258144] (raylet) main.cc:242: Raylet is not set to kill unknown children.
|
||||
[2024-05-15 08:22:49,492 I 258144 258144] (raylet) io_service_pool.cc:35: IOServicePool is running with 1 io_service.
|
||||
[2024-05-15 08:22:49,492 I 258144 258144] (raylet) main.cc:371: Setting node ID to: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:22:49,492 I 258144 258144] (raylet) store_runner.cc:32: Allowing the Plasma store to use up to 17.6951GB of memory.
|
||||
[2024-05-15 08:22:49,492 I 258144 258144] (raylet) store_runner.cc:48: Starting object store with directory /dev/shm, fallback /home/valapeos/Projects/pneumarl/logs/rllib, and huge page support disabled
|
||||
[2024-05-15 08:22:49,492 I 258144 258172] (raylet) dlmalloc.cc:154: create_and_mmap_buffer(17695244296, /dev/shm/plasmaXXXXXX)
|
||||
[2024-05-15 08:22:49,492 I 258144 258172] (raylet) store.cc:564: ========== Plasma store: =================
|
||||
Current usage: 0 / 17.6951 GB
|
||||
- num bytes created total: 0
|
||||
0 pending objects of total size 0MB
|
||||
- objects spillable: 0
|
||||
- bytes spillable: 0
|
||||
- objects unsealed: 0
|
||||
- bytes unsealed: 0
|
||||
- objects in use: 0
|
||||
- bytes in use: 0
|
||||
- objects evictable: 0
|
||||
- bytes evictable: 0
|
||||
|
||||
- objects created by worker: 0
|
||||
- bytes created by worker: 0
|
||||
- objects restored: 0
|
||||
- bytes restored: 0
|
||||
- objects received: 0
|
||||
- bytes received: 0
|
||||
- objects errored: 0
|
||||
- bytes errored: 0
|
||||
|
||||
[2024-05-15 08:22:49,494 I 258144 258144] (raylet) grpc_server.cc:134: ObjectManager server started, listening on port 35889.
|
||||
[2024-05-15 08:22:49,495 I 258144 258144] (raylet) worker_killing_policy.cc:101: Running GroupByOwner policy.
|
||||
[2024-05-15 08:22:49,495 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:22:49,495 I 258144 258144] (raylet) memory_monitor.cc:47: MemoryMonitor initialized with usage threshold at 63982133248 bytes (0.95 system memory), total system memory bytes: 67349614592
|
||||
[2024-05-15 08:22:49,495 I 258144 258144] (raylet) node_manager.cc:285: Initializing NodeManager with ID a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:22:49,495 I 258144 258144] (raylet) grpc_server.cc:134: NodeManager server started, listening on port 46465.
|
||||
[2024-05-15 08:22:49,501 I 258144 258204] (raylet) agent_manager.cc:78: Monitor agent process with name dashboard_agent/424238335
|
||||
[2024-05-15 08:22:49,502 I 258144 258206] (raylet) agent_manager.cc:78: Monitor agent process with name runtime_env_agent
|
||||
[2024-05-15 08:22:49,502 I 258144 258144] (raylet) event.cc:234: Set ray event level to warning
|
||||
[2024-05-15 08:22:49,502 I 258144 258144] (raylet) event.cc:342: Ray Event initialized for RAYLET
|
||||
[2024-05-15 08:22:49,503 I 258144 258144] (raylet) raylet.cc:128: Raylet of id, a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 started. Raylet consists of node_manager and object_manager. node_manager address: 192.168.1.56:46465 object_manager address: 192.168.1.56:35889 hostname: valanixos
|
||||
[2024-05-15 08:22:49,504 I 258144 258144] (raylet) node_manager.cc:521: [state-dump] NodeManager:
|
||||
[state-dump] Node ID: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[state-dump] Node name: 192.168.1.56
|
||||
[state-dump] InitialConfigResources: {CPU: 120000, memory: 353902559240000, node:__internal_head__: 10000, node:192.168.1.56: 10000, object_store_memory: 176951279610000}
|
||||
[state-dump] ClusterTaskManager:
|
||||
[state-dump] ========== Node: a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450 =================
|
||||
[state-dump] Infeasible queue length: 0
|
||||
[state-dump] Schedule queue length: 0
|
||||
[state-dump] Dispatch queue length: 0
|
||||
[state-dump] num_waiting_for_resource: 0
|
||||
[state-dump] num_waiting_for_plasma_memory: 0
|
||||
[state-dump] num_waiting_for_remote_node_resources: 0
|
||||
[state-dump] num_worker_not_started_by_job_config_not_exist: 0
|
||||
[state-dump] num_worker_not_started_by_registration_timeout: 0
|
||||
[state-dump] num_tasks_waiting_for_workers: 0
|
||||
[state-dump] num_cancelled_tasks: 0
|
||||
[state-dump] cluster_resource_scheduler state:
|
||||
[state-dump] Local id: -1885364415256342489 Local resources: {"total":{node:__internal_head__: [10000], CPU: [120000], memory: [353902559240000], object_store_memory: [176951279610000], node:192.168.1.56: [10000]}}, "available": {node:__internal_head__: [10000], CPU: [120000], memory: [353902559240000], object_store_memory: [176951279610000], node:192.168.1.56: [10000]}}, "labels":{"ray.io/node_id":"a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450",} is_draining: 0 is_idle: 1 Cluster resources: node id: -1885364415256342489{"total":{node:192.168.1.56: 10000, node:__internal_head__: 10000, memory: 353902559240000, object_store_memory: 176951279610000, CPU: 120000}}, "available": {node:192.168.1.56: 10000, node:__internal_head__: 10000, memory: 353902559240000, object_store_memory: 176951279610000, CPU: 120000}}, "labels":{"ray.io/node_id":"a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450",}, "is_draining": 0, "draining_deadline_timestamp_ms": -1} { "placment group locations": [], "node to bundles": []}
|
||||
[state-dump] Waiting tasks size: 0
|
||||
[state-dump] Number of executing tasks: 0
|
||||
[state-dump] Number of pinned task arguments: 0
|
||||
[state-dump] Number of total spilled tasks: 0
|
||||
[state-dump] Number of spilled waiting tasks: 0
|
||||
[state-dump] Number of spilled unschedulable tasks: 0
|
||||
[state-dump] Resource usage {
|
||||
[state-dump] }
|
||||
[state-dump] Running tasks by scheduling class:
|
||||
[state-dump] ==================================================
|
||||
[state-dump]
|
||||
[state-dump] ClusterResources:
|
||||
[state-dump] LocalObjectManager:
|
||||
[state-dump] - num pinned objects: 0
|
||||
[state-dump] - pinned objects size: 0
|
||||
[state-dump] - num objects pending restore: 0
|
||||
[state-dump] - num objects pending spill: 0
|
||||
[state-dump] - num bytes pending spill: 0
|
||||
[state-dump] - num bytes currently spilled: 0
|
||||
[state-dump] - cumulative spill requests: 0
|
||||
[state-dump] - cumulative restore requests: 0
|
||||
[state-dump] - spilled objects pending delete: 0
|
||||
[state-dump]
|
||||
[state-dump] ObjectManager:
|
||||
[state-dump] - num local objects: 0
|
||||
[state-dump] - num unfulfilled push requests: 0
|
||||
[state-dump] - num object pull requests: 0
|
||||
[state-dump] - num chunks received total: 0
|
||||
[state-dump] - num chunks received failed (all): 0
|
||||
[state-dump] - num chunks received failed / cancelled: 0
|
||||
[state-dump] - num chunks received failed / plasma error: 0
|
||||
[state-dump] Event stats:
|
||||
[state-dump] Global stats: 0 total (0 active)
|
||||
[state-dump] Queueing time: mean = -nan s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] Execution time: mean = -nan s, total = 0.000 s
|
||||
[state-dump] Event stats:
|
||||
[state-dump] PushManager:
|
||||
[state-dump] - num pushes in flight: 0
|
||||
[state-dump] - num chunks in flight: 0
|
||||
[state-dump] - num chunks remaining: 0
|
||||
[state-dump] - max chunks allowed: 409
|
||||
[state-dump] OwnershipBasedObjectDirectory:
|
||||
[state-dump] - num listeners: 0
|
||||
[state-dump] - cumulative location updates: 0
|
||||
[state-dump] - num location updates per second: 0.000
|
||||
[state-dump] - num location lookups per second: 0.000
|
||||
[state-dump] - num locations added per second: 0.000
|
||||
[state-dump] - num locations removed per second: 0.000
|
||||
[state-dump] BufferPool:
|
||||
[state-dump] - create buffer state map size: 0
|
||||
[state-dump] PullManager:
|
||||
[state-dump] - num bytes available for pulled objects: 17695127961
|
||||
[state-dump] - num bytes being pulled (all): 0
|
||||
[state-dump] - num bytes being pulled / pinned: 0
|
||||
[state-dump] - get request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
[state-dump] - wait request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
[state-dump] - task request bundles: BundlePullRequestQueue{0 total, 0 active, 0 inactive, 0 unpullable}
|
||||
[state-dump] - first get request bundle: N/A
|
||||
[state-dump] - first wait request bundle: N/A
|
||||
[state-dump] - first task request bundle: N/A
|
||||
[state-dump] - num objects queued: 0
|
||||
[state-dump] - num objects actively pulled (all): 0
|
||||
[state-dump] - num objects actively pulled / pinned: 0
|
||||
[state-dump] - num bundles being pulled: 0
|
||||
[state-dump] - num pull retries: 0
|
||||
[state-dump] - max timeout seconds: 0
|
||||
[state-dump] - max timeout request is already processed. No entry.
|
||||
[state-dump]
|
||||
[state-dump] WorkerPool:
|
||||
[state-dump] - registered jobs: 0
|
||||
[state-dump] - process_failed_job_config_missing: 0
|
||||
[state-dump] - process_failed_rate_limited: 0
|
||||
[state-dump] - process_failed_pending_registration: 0
|
||||
[state-dump] - process_failed_runtime_env_setup_failed: 0
|
||||
[state-dump] - num PYTHON workers: 0
|
||||
[state-dump] - num PYTHON drivers: 0
|
||||
[state-dump] - num object spill callbacks queued: 0
|
||||
[state-dump] - num object restore queued: 0
|
||||
[state-dump] - num util functions queued: 0
|
||||
[state-dump] - num idle workers: 0
|
||||
[state-dump] TaskDependencyManager:
|
||||
[state-dump] - task deps map size: 0
|
||||
[state-dump] - get req map size: 0
|
||||
[state-dump] - wait req map size: 0
|
||||
[state-dump] - local objects map size: 0
|
||||
[state-dump] WaitManager:
|
||||
[state-dump] - num active wait requests: 0
|
||||
[state-dump] Subscriber:
|
||||
[state-dump] Channel WORKER_OBJECT_LOCATIONS_CHANNEL
|
||||
[state-dump] - cumulative subscribe requests: 0
|
||||
[state-dump] - cumulative unsubscribe requests: 0
|
||||
[state-dump] - active subscribed publishers: 0
|
||||
[state-dump] - cumulative published messages: 0
|
||||
[state-dump] - cumulative processed messages: 0
|
||||
[state-dump] Channel WORKER_OBJECT_EVICTION
|
||||
[state-dump] - cumulative subscribe requests: 0
|
||||
[state-dump] - cumulative unsubscribe requests: 0
|
||||
[state-dump] - active subscribed publishers: 0
|
||||
[state-dump] - cumulative published messages: 0
|
||||
[state-dump] - cumulative processed messages: 0
|
||||
[state-dump] Channel WORKER_REF_REMOVED_CHANNEL
|
||||
[state-dump] - cumulative subscribe requests: 0
|
||||
[state-dump] - cumulative unsubscribe requests: 0
|
||||
[state-dump] - active subscribed publishers: 0
|
||||
[state-dump] - cumulative published messages: 0
|
||||
[state-dump] - cumulative processed messages: 0
|
||||
[state-dump] num async plasma notifications: 0
|
||||
[state-dump] Remote node managers:
|
||||
[state-dump] Event stats:
|
||||
[state-dump] Global stats: 27 total (13 active)
|
||||
[state-dump] Queueing time: mean = 976.153 us, max = 7.494 ms, min = 8.099 us, total = 26.356 ms
|
||||
[state-dump] Execution time: mean = 506.838 us, total = 13.685 ms
|
||||
[state-dump] Event stats:
|
||||
[state-dump] PeriodicalRunner.RunFnPeriodically - 11 total (2 active, 1 running), Execution time: mean = 90.688 us, total = 997.568 us, Queueing time: mean = 2.394 ms, max = 7.494 ms, min = 18.500 us, total = 26.339 ms
|
||||
[state-dump] NodeManager.deadline_timer.flush_free_objects - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeInfoGcsService.grpc_client.RegisterNode - 1 total (0 active), Execution time: mean = 730.548 us, total = 730.548 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeManager.deadline_timer.debug_state_dump - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] ClusterResourceManager.ResetRemoteNodeView - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch - 1 total (0 active), Execution time: mean = 288.960 us, total = 288.960 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeManager.ScheduleAndDispatchTasks - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] RayletWorkerPool.deadline_timer.kill_idle_workers - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig.OnReplyReceived - 1 total (0 active), Execution time: mean = 11.090 ms, total = 11.090 ms, Queueing time: mean = 8.630 us, max = 8.630 us, min = 8.630 us, total = 8.630 us
|
||||
[state-dump] NodeManager.GCTaskFailureReason - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] MemoryMonitor.CheckIsMemoryUsageAboveThreshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeInfoGcsService.grpc_client.GetInternalConfig - 1 total (0 active), Execution time: mean = 429.879 us, total = 429.879 us, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberCommandBatch.OnReplyReceived - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeInfoGcsService.grpc_client.RegisterNode.OnReplyReceived - 1 total (0 active), Execution time: mean = 147.790 us, total = 147.790 us, Queueing time: mean = 8.099 us, max = 8.099 us, min = 8.099 us, total = 8.099 us
|
||||
[state-dump] NodeManager.deadline_timer.record_metrics - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] InternalPubSubGcsService.grpc_client.GcsSubscriberPoll - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] NodeManager.deadline_timer.spill_objects_when_over_threshold - 1 total (1 active), Execution time: mean = 0.000 s, total = 0.000 s, Queueing time: mean = 0.000 s, max = -0.000 s, min = 9223372036.855 s, total = 0.000 s
|
||||
[state-dump] DebugString() time ms: 0
|
||||
[state-dump]
|
||||
[state-dump]
|
||||
[2024-05-15 08:22:49,505 I 258144 258144] (raylet) accessor.cc:627: Received notification for node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450, IsAlive = 1
|
||||
[2024-05-15 08:22:49,514 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258226, the token is 0
|
||||
[2024-05-15 08:22:49,516 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258227, the token is 1
|
||||
[2024-05-15 08:22:49,517 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258228, the token is 2
|
||||
[2024-05-15 08:22:49,519 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258229, the token is 3
|
||||
[2024-05-15 08:22:49,521 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258230, the token is 4
|
||||
[2024-05-15 08:22:49,523 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258231, the token is 5
|
||||
[2024-05-15 08:22:49,525 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258232, the token is 6
|
||||
[2024-05-15 08:22:49,526 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258233, the token is 7
|
||||
[2024-05-15 08:22:49,528 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258234, the token is 8
|
||||
[2024-05-15 08:22:49,530 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258235, the token is 9
|
||||
[2024-05-15 08:22:49,532 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258236, the token is 10
|
||||
[2024-05-15 08:22:49,535 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258237, the token is 11
|
||||
[2024-05-15 08:22:49,883 I 258144 258172] (raylet) object_store.cc:35: Object store current usage 8e-09 / 17.6951 GB.
|
||||
[2024-05-15 08:22:50,065 I 258144 258144] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 257884 is dead: 0 driver address: 192.168.1.56
|
||||
[2024-05-15 08:22:50,065 I 258144 258144] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool.
|
||||
[2024-05-15 08:22:50,284 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:22:50,285 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 258963, the token is 12
|
||||
[2024-05-15 08:22:51,110 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:22:51,111 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:22:51,111 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:22:51,111 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:22:51,311 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:22:52,264 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:22:52,267 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 259036, the token is 13
|
||||
[2024-05-15 08:22:52,267 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:22:52,269 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 259037, the token is 14
|
||||
[2024-05-15 08:22:52,270 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:22:52,271 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 259038, the token is 15
|
||||
[2024-05-15 08:22:52,271 I 258144 258144] (raylet) runtime_env_agent_client.cc:369: Create runtime env for job 01000000
|
||||
[2024-05-15 08:22:52,273 I 258144 258144] (raylet) worker_pool.cc:494: Started worker process with pid 259039, the token is 16
|
||||
[2024-05-15 08:22:54,505 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:22:59,502 W 258144 258166] (raylet) metric_exporter.cc:105: [1] Export metrics to agent failed: GrpcUnavailable: RPC Error message: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:59951: Failed to connect to remote host: Connection refused; RPC Error details: . This won't affect Ray, but you can lose metrics from the cluster.
|
||||
[2024-05-15 08:22:59,507 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:23:04,508 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:23:09,510 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:23:14,512 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:23:19,513 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:23:24,515 W 258144 258144] (raylet) memory_monitor.cc:197: Got negative used memory for cgroup -1, setting it to zero
|
||||
[2024-05-15 08:23:28,795 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=3, has creation task exception = false
|
||||
[2024-05-15 08:23:28,795 I 258144 258144] (raylet) node_manager.cc:1541: Driver (pid=257884) is disconnected. job_id: 01000000
|
||||
[2024-05-15 08:23:28,795 I 258144 258144] (raylet) node_manager.cc:1066: The leased worker 0ba2c6d5a2f06001fc22f656e19669240434e6c9312566757dc53c01 is killed because the owner process 01000000ffffffffffffffffffffffffffffffffffffffffffffffff died.
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=1, has creation task exception = false
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:23:28,796 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1436: NodeManager::DisconnectClient, disconnect_type=0, has creation task exception = false
|
||||
[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:1450: Ignoring client disconnect because the client has already been disconnected.
|
||||
[2024-05-15 08:23:28,797 I 258144 258144] (raylet) node_manager.cc:606: New job has started. Job id 01000000 Driver pid 257884 is dead: 1 driver address: 192.168.1.56
|
||||
[2024-05-15 08:23:28,797 I 258144 258144] (raylet) worker_pool.cc:677: Job 01000000 already started in worker pool.
|
||||
[2024-05-15 08:23:28,838 I 258144 258144] (raylet) main.cc:413: Raylet received SIGTERM, shutting down...
|
||||
[2024-05-15 08:23:28,838 I 258144 258144] (raylet) accessor.cc:451: Unregistering node info, node id = a84309f38ccf6a95bc1fc0113c11ec4b884bd4e6b34f85fbcbfdc450
|
||||
[2024-05-15 08:23:28,839 I 258144 258144] (raylet) agent_manager.cc:109: Killing agent dashboard_agent/424238335, pid 258203.
|
||||
[2024-05-15 08:23:28,848 I 258144 258204] (raylet) agent_manager.cc:80: Agent process with name dashboard_agent/424238335 exited, exit code 0.
|
||||
[2024-05-15 08:23:28,848 I 258144 258144] (raylet) agent_manager.cc:109: Killing agent runtime_env_agent, pid 258205.
|
||||
[2024-05-15 08:23:28,853 I 258144 258206] (raylet) agent_manager.cc:80: Agent process with name runtime_env_agent exited, exit code 0.
|
||||
[2024-05-15 08:23:28,853 I 258144 258144] (raylet) io_service_pool.cc:47: IOServicePool is stopped.
|
||||
[2024-05-15 08:23:28,919 I 258144 258144] (raylet) stats.h:120: Stats module has shutdown.
|
||||
[2024-05-15 08:23:28,919 I 258144 258208] (raylet) experimental_mutable_object_provider.cc:200: Core worker channel io service stopped.
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
======== Running on http://0.0.0.0:43711 ========
|
||||
(Press CTRL+C to quit)
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:PPO
|
||||
2024-05-15 08:22:52,212 WARNING deprecation.py:50 -- DeprecationWarning: `AlgorithmConfig.num_envs_per_worker` has been deprecated. Use `AlgorithmConfig.num_envs_per_env_runner` instead. This will raise an error in the future!
|
||||
2024-05-15 08:22:52,217 WARNING deprecation.py:50 -- DeprecationWarning: `WorkerSet(num_workers=... OR local_worker=...)` has been deprecated. Use `EnvRunnerGroup(num_env_runners=... AND local_env_runner=...)` instead. This will raise an error in the future!
|
||||
2024-05-15 08:22:52,217 WARNING deprecation.py:50 -- DeprecationWarning: `max_num_worker_restarts` has been deprecated. Use `AlgorithmConfig.max_num_env_runner_restarts` instead. This will raise an error in the future!
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
:job_id:01000000
|
||||
:actor_name:PPO
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue