{"id":3221,"date":"2026-06-23T17:11:39","date_gmt":"2026-06-23T09:11:39","guid":{"rendered":"https:\/\/linuxjk.cn\/?p=3221"},"modified":"2026-08-13T14:33:13","modified_gmt":"2026-08-13T06:33:13","slug":"dask-%e5%8f%8c%e6%9c%ba%e9%9b%86%e7%be%a4%e5%ae%8c%e6%95%b4%e9%83%a8%e7%bd%b2%e6%8c%87%e5%8d%97","status":"publish","type":"post","link":"https:\/\/linuxjk.cn\/?p=3221","title":{"rendered":"Dask \u53cc\u673a\u96c6\u7fa4\u5b8c\u6574\u90e8\u7f72\u6307\u5357"},"content":{"rendered":"\n<p class=\"wp-block-paragraph\">\u4e0b\u9762\u662f\u6839\u636e<strong>\u5b9e\u9645\u9a8c\u8bc1\u901a\u8fc7\u7684\u7248\u672c<\/strong>\uff08dask\/distributed 2024.6.2 + pandas 2.2.3 + numpy 1.26.4 + pyarrow 21.0.0\uff09\u6574\u7406\u7684\u5b8c\u6574\u90e8\u7f72\u6559\u7a0b\u3002<\/p>\n\n\n\n<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_88 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E9%83%A8%E7%BD%B2%E5%89%8D%E6%8F%90\" >\u90e8\u7f72\u524d\u63d0<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E5%85%B3%E9%94%AE%E5%8C%85%E7%89%88%E6%9C%AC%E9%80%9F%E6%9F%A5%EF%BC%88%E4%B8%89%E7%AB%AF%E5%BF%85%E9%A1%BB%E4%B8%80%E8%87%B4%EF%BC%89\" >\u5173\u952e\u5305\u7248\u672c\u901f\u67e5\uff08\u4e09\u7aef\u5fc5\u987b\u4e00\u81f4\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_0_%E9%98%B6%E6%AE%B5%EF%BC%9A%E7%8E%AF%E5%A2%83%E6%A3%80%E6%9F%A5%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\" >\u7b2c 0 \u9636\u6bb5\uff1a\u73af\u5883\u68c0\u67e5\uff08\u4e24\u53f0\u90fd\u505a\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_1_%E9%98%B6%E6%AE%B5%EF%BC%9APython_%E7%8E%AF%E5%A2%83%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\" >\u7b2c 1 \u9636\u6bb5\uff1aPython \u73af\u5883\uff08\u4e24\u53f0\u90fd\u505a\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_2_%E9%98%B6%E6%AE%B5%EF%BC%9A%E5%AE%89%E8%A3%85_Dask_%E9%9B%86%E7%BE%A4%E7%AB%AF%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\" >\u7b2c 2 \u9636\u6bb5\uff1a\u5b89\u88c5 Dask \u96c6\u7fa4\u7aef\uff08\u4e24\u53f0\u90fd\u505a\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_3_%E9%98%B6%E6%AE%B5%EF%BC%9ADask_%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\" >\u7b2c 3 \u9636\u6bb5\uff1aDask \u914d\u7f6e\u6587\u4ef6\uff08\u4e24\u53f0\u90fd\u505a\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_4_%E9%98%B6%E6%AE%B5%EF%BC%9A%E5%87%86%E5%A4%87%E7%9B%AE%E5%BD%95%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\" >\u7b2c 4 \u9636\u6bb5\uff1a\u51c6\u5907\u76ee\u5f55\uff08\u4e24\u53f0\u90fd\u505a\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_5_%E9%98%B6%E6%AE%B5%EF%BC%9A%E5%89%8D%E5%8F%B0%E5%90%AF%E5%8A%A8%E6%B5%8B%E8%AF%95%EF%BC%88%E5%85%88%E9%AA%8C%E8%AF%81%E8%83%BD%E8%B7%91%EF%BC%89\" >\u7b2c 5 \u9636\u6bb5\uff1a\u524d\u53f0\u542f\u52a8\u6d4b\u8bd5\uff08\u5148\u9a8c\u8bc1\u80fd\u8dd1\uff09<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#51_%E6%9C%BA%E5%99%A8_38%EF%BC%9A%E5%90%AF%E5%8A%A8_Scheduler\" >5.1 \u673a\u5668 38\uff1a\u542f\u52a8 Scheduler<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#52_%E6%9C%BA%E5%99%A8_38%EF%BC%9A%E5%90%AF%E5%8A%A8_4_%E4%B8%AA%E4%B8%BB%E5%8A%9B_Worker\" >5.2 \u673a\u5668 38\uff1a\u542f\u52a8 4 \u4e2a\u4e3b\u529b Worker<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#53_%E6%9C%BA%E5%99%A8_39%EF%BC%9A%E5%90%AF%E5%8A%A8_Secondary_Worker\" >5.3 \u673a\u5668 39\uff1a\u542f\u52a8 Secondary Worker<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#54_%E9%AA%8C%E8%AF%81%E9%9B%86%E7%BE%A4\" >5.4 \u9a8c\u8bc1\u96c6\u7fa4<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-13\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#55_%E7%AE%80%E5%8D%95%E8%AE%A1%E7%AE%97%E6%B5%8B%E8%AF%95\" >5.5 \u7b80\u5355\u8ba1\u7b97\u6d4b\u8bd5<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-14\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_6_%E9%98%B6%E6%AE%B5%EF%BC%9Asystemd_%E6%9C%8D%E5%8A%A1%E9%83%A8%E7%BD%B2\" >\u7b2c 6 \u9636\u6bb5\uff1asystemd \u670d\u52a1\u90e8\u7f72<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-15\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#61_%E6%9C%BA%E5%99%A8_38%EF%BC%9AScheduler_Service\" >6.1 \u673a\u5668 38\uff1aScheduler Service<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-16\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#62_%E6%9C%BA%E5%99%A8_38%EF%BC%9AWorker_Service\" >6.2 \u673a\u5668 38\uff1aWorker Service<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-17\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#63_%E6%9C%BA%E5%99%A8_39%EF%BC%9AWorker_Service\" >6.3 \u673a\u5668 39\uff1aWorker Service<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-18\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#64_%E5%90%AF%E5%8A%A8%E6%9C%8D%E5%8A%A1\" >6.4 \u542f\u52a8\u670d\u52a1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-19\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#65_%E9%AA%8C%E8%AF%81%E6%9C%8D%E5%8A%A1%E5%8C%96%E9%9B%86%E7%BE%A4\" >6.5 \u9a8c\u8bc1\u670d\u52a1\u5316\u96c6\u7fa4<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-20\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_7_%E9%98%B6%E6%AE%B5%EF%BC%9ADSS_Code_Env_%E9%9B%86%E6%88%90\" >\u7b2c 7 \u9636\u6bb5\uff1aDSS Code Env \u96c6\u6210<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-21\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#71_%E6%96%B9%E5%BC%8F_A%EF%BC%9A%E4%BD%BF%E7%94%A8_DSS_%E5%86%85%E7%BD%AE_pyenv%EF%BC%88%E6%8E%A8%E8%8D%90%EF%BC%89\" >7.1 \u65b9\u5f0f A\uff1a\u4f7f\u7528 DSS \u5185\u7f6e pyenv\uff08\u63a8\u8350\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-22\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#72_%E6%96%B9%E5%BC%8F_B%EF%BC%9A%E9%80%9A%E8%BF%87_DSS_Code_Env_%E7%AE%A1%E7%90%86%E7%95%8C%E9%9D%A2\" >7.2 \u65b9\u5f0f B\uff1a\u901a\u8fc7 DSS Code Env \u7ba1\u7406\u754c\u9762<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-23\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#73_%E5%A4%84%E7%90%86_GitHub_%E4%BE%9D%E8%B5%96%E6%8B%89%E5%8F%96%E5%A4%B1%E8%B4%A5%EF%BC%88%E5%A6%82%E9%81%87%E5%88%B0%EF%BC%89\" >7.3 \u5904\u7406 GitHub \u4f9d\u8d56\u62c9\u53d6\u5931\u8d25\uff08\u5982\u9047\u5230\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-24\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#74_%E6%B5%8B%E8%AF%95_DSS_%E5%88%B0%E9%9B%86%E7%BE%A4%E7%9A%84%E8%BF%9E%E6%8E%A5\" >7.4 \u6d4b\u8bd5 DSS \u5230\u96c6\u7fa4\u7684\u8fde\u63a5<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-25\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_8_%E9%98%B6%E6%AE%B5%EF%BC%9A%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4\" >\u7b2c 8 \u9636\u6bb5\uff1a\u65e5\u5e38\u8fd0\u7ef4<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-26\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E5%90%AF%E5%8A%A8_%E5%81%9C%E6%AD%A2_%E9%87%8D%E5%90%AF\" >\u542f\u52a8 \/ \u505c\u6b62 \/ \u91cd\u542f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-27\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E6%9F%A5%E7%9C%8B%E6%97%A5%E5%BF%97\" >\u67e5\u770b\u65e5\u5fd7<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-28\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#Dashboard_%E7%9B%91%E6%8E%A7\" >Dashboard \u76d1\u63a7<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-29\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E5%AE%8C%E6%95%B4%E9%9B%86%E7%BE%A4%E9%87%8D%E5%90%AF\" >\u5b8c\u6574\u96c6\u7fa4\u91cd\u542f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-30\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_9_%E9%98%B6%E6%AE%B5%EF%BC%9ADSS_Recipe_%E6%A8%A1%E6%9D%BF%EF%BC%88Job_%E7%BA%A7%E5%88%AB%E6%9C%BA%E5%99%A8%E4%BA%B2%E5%92%8C%E6%80%A7%E8%B0%83%E5%BA%A6%EF%BC%89\" >\u7b2c 9 \u9636\u6bb5\uff1aDSS Recipe \u6a21\u677f\uff08Job \u7ea7\u522b\u673a\u5668\u4eb2\u548c\u6027\u8c03\u5ea6\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-31\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_10_%E9%98%B6%E6%AE%B5%EF%BC%9A%E6%95%85%E9%9A%9C%E6%8E%92%E6%9F%A5%E9%80%9F%E6%9F%A5%E8%A1%A8\" >\u7b2c 10 \u9636\u6bb5\uff1a\u6545\u969c\u6392\u67e5\u901f\u67e5\u8868<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-32\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_11_%E9%98%B6%E6%AE%B5%EF%BC%9A%E9%98%BF%E9%87%8C%E4%BA%91%E5%AE%89%E5%85%A8%E7%BB%84\" >\u7b2c 11 \u9636\u6bb5\uff1a\u963f\u91cc\u4e91\u5b89\u5168\u7ec4<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-33\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E6%9C%80%E7%BB%88%E9%AA%8C%E8%AF%81%E6%B8%85%E5%8D%95\" >\u6700\u7ec8\u9a8c\u8bc1\u6e05\u5355<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-34\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E5%85%B3%E9%94%AE%E6%B3%A8%E6%84%8F%E4%BA%8B%E9%A1%B9%E6%B1%87%E6%80%BB\" >\u5173\u952e\u6ce8\u610f\u4e8b\u9879\u6c47\u603b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-35\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#%E7%AC%AC_12_%E9%98%B6%E6%AE%B5%EF%BC%9A%E7%94%9F%E4%BA%A7%E5%86%85%E5%AD%98%E4%BF%9D%E6%8A%A4%E4%B8%8E%E7%9B%91%E6%8E%A7%E5%8A%A0%E5%9B%BA\" >\u7b2c 12 \u9636\u6bb5\uff1a\u751f\u4ea7\u5185\u5b58\u4fdd\u62a4\u4e0e\u76d1\u63a7\u52a0\u56fa<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-36\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#121_%E6%95%85%E9%9A%9C%E8%83%8C%E6%99%AF\" >12.1 \u6545\u969c\u80cc\u666f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-37\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#122_%E7%BB%9F%E4%B8%80%E5%90%8E%E7%9A%84%E7%94%9F%E4%BA%A7%E9%85%8D%E7%BD%AE\" >12.2 \u7edf\u4e00\u540e\u7684\u751f\u4ea7\u914d\u7f6e<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-38\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#123_Spill_%E7%9B%AE%E5%BD%95\" >12.3 Spill \u76ee\u5f55<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-39\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#124_systemd_%E5%86%85%E5%AD%98%E4%BF%9D%E6%8A%A4\" >12.4 systemd \u5185\u5b58\u4fdd\u62a4<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-40\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#125_38_%E8%8A%82%E7%82%B9_Worker_%E9%85%8D%E7%BD%AE\" >12.5 38 \u8282\u70b9 Worker \u914d\u7f6e<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-41\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#126_39_%E8%8A%82%E7%82%B9_Worker_%E9%85%8D%E7%BD%AE\" >12.6 39 \u8282\u70b9 Worker \u914d\u7f6e<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-42\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#127_Dask_%E5%86%85%E5%AD%98%E9%98%88%E5%80%BC\" >12.7 Dask \u5185\u5b58\u9608\u503c<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-43\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#128_%E6%9C%8D%E5%8A%A1%E9%87%8D%E8%BD%BD%E4%B8%8E%E9%AA%8C%E8%AF%81\" >12.8 \u670d\u52a1\u91cd\u8f7d\u4e0e\u9a8c\u8bc1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-44\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#129_%E4%B8%BB%E6%9C%BA%E7%9B%91%E6%8E%A7%E8%84%9A%E6%9C%AC\" >12.9 \u4e3b\u673a\u76d1\u63a7\u811a\u672c<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-45\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#1210_%E7%9B%91%E6%8E%A7%E6%97%A5%E5%BF%97%E8%BD%AE%E8%BD%AC\" >12.10 \u76d1\u63a7\u65e5\u5fd7\u8f6e\u8f6c<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-46\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#1211_%E6%97%A5%E5%B8%B8%E6%A3%80%E6%9F%A5%E5%91%BD%E4%BB%A4\" >12.11 \u65e5\u5e38\u68c0\u67e5\u547d\u4ee4<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-47\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#1212_%E5%9B%9E%E6%BB%9A%E6%96%B9%E6%A1%88\" >12.12 \u56de\u6eda\u65b9\u6848<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-48\" href=\"https:\/\/linuxjk.cn\/?p=3221\/#1213_%E5%BD%93%E5%89%8D%E9%AA%8C%E8%AF%81%E7%BB%93%E8%AE%BA\" >12.13 \u5f53\u524d\u9a8c\u8bc1\u7ed3\u8bba<\/a><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E9%83%A8%E7%BD%B2%E5%89%8D%E6%8F%90\"><\/span>\u90e8\u7f72\u524d\u63d0<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u4e24\u53f0\u673a\u5668\uff1a<code>10.21.80.38<\/code>\uff08DSS \u4e3b\u673a\uff0cUPCDIPPDTKW02\uff09\u3001<code>10.21.80.39<\/code>\uff08\u526f\u8282\u70b9\uff0cUPCDIPPDTKW01\uff09<\/li>\n\n\n\n<li>\u7cfb\u7edf\uff1aUbuntu 22.04<\/li>\n\n\n\n<li>\u7528\u6237\uff1a<code>pcd_pdtk_op<\/code>\uff08\u4e3b\u7ec4 <code>users<\/code>\uff0cgid=100\uff09<\/li>\n\n\n\n<li>\u963f\u91cc\u4e91\u540c VPC \u540c\u5b89\u5168\u7ec4\uff0c\u5185\u7f51\u5e26\u5bbd 2.7 Gbps<\/li>\n\n\n\n<li>DSS 13.3.2 \u5df2\u90e8\u7f72\u5728 38\uff0cpyenv \u4e3a Python 3.10\uff08pandas 2.2.3\uff09<\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%85%B3%E9%94%AE%E5%8C%85%E7%89%88%E6%9C%AC%E9%80%9F%E6%9F%A5%EF%BC%88%E4%B8%89%E7%AB%AF%E5%BF%85%E9%A1%BB%E4%B8%80%E8%87%B4%EF%BC%89\"><\/span>\u5173\u952e\u5305\u7248\u672c\u901f\u67e5\uff08\u4e09\u7aef\u5fc5\u987b\u4e00\u81f4\uff09<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th>\u5305<\/th><th>\u7248\u672c<\/th><th>\u8bf4\u660e<\/th><\/tr><\/thead><tbody><tr><td>dask<\/td><td><strong>2024.6.2<\/strong><\/td><td>\u6838\u5fc3\uff0c\u652f\u6301 pandas 2.x<\/td><\/tr><tr><td>distributed<\/td><td><strong>2024.6.2<\/strong><\/td><td>\u6838\u5fc3<\/td><\/tr><tr><td>cloudpickle<\/td><td><strong>3.1.2<\/strong><\/td><td>\u5e8f\u5217\u5316\u6838\u5fc3\uff0c\u5fc5\u987b\u663e\u5f0f\u9489\u7248\u672c<\/td><\/tr><tr><td>msgpack<\/td><td><strong>1.1.2<\/strong><\/td><td>\u901a\u4fe1\u534f\u8bae\u5e8f\u5217\u5316<\/td><\/tr><tr><td>lz4<\/td><td><strong>3.1.3<\/strong><\/td><td>\u901a\u4fe1\u538b\u7f29<\/td><\/tr><tr><td>pandas<\/td><td><strong>2.2.3<\/strong><\/td><td>\u6570\u636e\u5e8f\u5217\u5316<\/td><\/tr><tr><td>numpy<\/td><td><strong>1.26.4<\/strong><\/td><td>\u6570\u636e\u5e8f\u5217\u5316<\/td><\/tr><tr><td>pyarrow<\/td><td><strong>21.0.0<\/strong><\/td><td>\u5217\u5f0f\u5b58\u50a8<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_0_%E9%98%B6%E6%AE%B5%EF%BC%9A%E7%8E%AF%E5%A2%83%E6%A3%80%E6%9F%A5%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\"><\/span>\u7b2c 0 \u9636\u6bb5\uff1a\u73af\u5883\u68c0\u67e5\uff08\u4e24\u53f0\u90fd\u505a\uff09<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<pre class=\"wp-block-preformatted\"># \u7cfb\u7edf\u7248\u672c<br>lsb_release -a<br>\u200b<br># \u5185\u5b58\u5927\u5c0f<br>free -h<br>\u200b<br># \u78c1\u76d8\u7a7a\u95f4<br>df -h \/<br>\u200b<br># \u4e92\u901a\u9a8c\u8bc1<br>ping -c 3 10.21.80.38 &nbsp; &nbsp; # \u5728 39 \u4e0a\u6267\u884c<br>ping -c 3 10.21.80.39 &nbsp; &nbsp; # \u5728 38 \u4e0a\u6267\u884c<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u8bb0\u4e0b\u4e24\u53f0\u7684\u5b9e\u9645\u5185\u5b58\uff0c\u51b3\u5b9a\u540e\u9762 worker \u5185\u5b58\u9650\u989d\u3002<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_1_%E9%98%B6%E6%AE%B5%EF%BC%9APython_%E7%8E%AF%E5%A2%83%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\"><\/span>\u7b2c 1 \u9636\u6bb5\uff1aPython \u73af\u5883\uff08\u4e24\u53f0\u90fd\u505a\uff09<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">DSS pyenv \u662f Python 3.10\uff0cworker \u7aef\u4f7f\u7528 Python 3.9\uff08\u5c0f\u7248\u672c\u5dee\u5f02\u517c\u5bb9\uff0c\u5e8f\u5217\u5316\u65e0\u5f71\u54cd\uff09\u3002<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\"># \u68c0\u67e5 Python 3.9 \u662f\u5426\u53ef\u7528<br>python3.9 --version<br># \u5e94\u8f93\u51fa: Python 3.9.x<\/pre>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_2_%E9%98%B6%E6%AE%B5%EF%BC%9A%E5%AE%89%E8%A3%85_Dask_%E9%9B%86%E7%BE%A4%E7%AB%AF%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\"><\/span>\u7b2c 2 \u9636\u6bb5\uff1a\u5b89\u88c5 Dask \u96c6\u7fa4\u7aef\uff08\u4e24\u53f0\u90fd\u505a\uff09<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<pre class=\"wp-block-preformatted\"># 1. \u521b\u5efa\u865a\u62df\u73af\u5883<br>python3.9 -m venv ~\/dask-env<br>\u200b<br># 2. \u6fc0\u6d3b<br>source ~\/dask-env\/bin\/activate<br>\u200b<br># 3. \u5347\u7ea7 pip + \u914d\u7f6e\u963f\u91cc\u4e91\u955c\u50cf\uff08\u53ef\u9009\uff09<br>pip install --upgrade pip -i https:\/\/mirrors.aliyun.com\/pypi\/simple\/<br>pip config set global.index-url https:\/\/mirrors.aliyun.com\/pypi\/simple\/<br>\u200b<br># 4. \u5b89\u88c5\u4e0e DSS pyenv \u5bf9\u9f50\u7684\u7248\u672c<br>pip install \\<br> &nbsp; &nbsp;\"dask==2024.6.2\" \\<br> &nbsp; &nbsp;\"distributed==2024.6.2\" \\<br> &nbsp; &nbsp;\"cloudpickle==3.1.2\" \\<br> &nbsp; &nbsp;\"msgpack==1.1.2\" \\<br> &nbsp; &nbsp;\"lz4==3.1.3\" \\<br> &nbsp; &nbsp;\"pandas==2.2.3\" \\<br> &nbsp; &nbsp;\"numpy==1.26.4\" \\<br> &nbsp; &nbsp;\"pyarrow==21.0.0\" \\<br> &nbsp; &nbsp;\"bokeh==2.4.3\" \\<br> &nbsp; &nbsp;\"sqlalchemy==2.0.30\"<br>\u200b<br># 5. \u9a8c\u8bc1\u7248\u672c\uff08\u4e24\u53f0\u8f93\u51fa\u5fc5\u987b\u5b8c\u5168\u4e00\u81f4\uff09<br>pip list 2&gt;\/dev\/null | grep -iE \"^dask |^distributed|^pandas |^pyarrow|^numpy |^cloudpickle|^msgpack|^lz4 \"<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9884\u671f\u8f93\u51fa\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">cloudpickle &nbsp; &nbsp; 3.1.2<br>dask &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;  2024.6.2<br>distributed &nbsp; &nbsp; 2024.6.2<br>lz4 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; 3.1.3<br>msgpack &nbsp; &nbsp; &nbsp; &nbsp; 1.1.2<br>numpy &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; 1.26.4<br>pandas &nbsp; &nbsp; &nbsp; &nbsp;  2.2.3<br>pyarrow &nbsp; &nbsp; &nbsp; &nbsp; 21.0.0<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>\u91cd\u8981<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u8dd1 <code>pip check<\/code>\uff0c\u786e\u8ba4 <code>No broken requirements found.<\/code><\/li>\n\n\n\n<li><code>cloudpickle<\/code> \u5fc5\u987b\u663e\u5f0f\u6307\u5b9a 3.1.2\uff0c\u4e0d\u6307\u5b9a\u7684\u8bdd pip \u53ef\u80fd\u88c5\u5176\u4ed6\u7248\u672c\uff0c\u4e0e DSS \u7aef\u4e0d\u4e00\u81f4\u4f1a\u5bfc\u81f4 KilledWorker<\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_3_%E9%98%B6%E6%AE%B5%EF%BC%9ADask_%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\"><\/span>\u7b2c 3 \u9636\u6bb5\uff1aDask \u914d\u7f6e\u6587\u4ef6\uff08\u4e24\u53f0\u90fd\u505a\uff09<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<pre class=\"wp-block-preformatted\"># \u521b\u5efa\u914d\u7f6e\u76ee\u5f55<br>mkdir -p ~\/.config\/dask<br>\u200b<br># \u5199\u5165\u914d\u7f6e<br>cat &gt; ~\/.config\/dask\/distributed.yaml &lt;&lt;'EOF'<br>distributed:<br>  scheduler:<br> &nbsp;  work-stealing: false &nbsp; &nbsp; &nbsp;# \u5173\u95ed work-stealing\uff0c\u914d\u5408\u673a\u5668\u4eb2\u548c\u6027\u8c03\u5ea6<br>\u200b<br>  worker:<br> &nbsp;  memory:<br> &nbsp; &nbsp;  target: 0.75 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;# 75% \u89e6\u53d1\u4e3b\u52a8\u6ea2\u51fa<br> &nbsp; &nbsp;  spill: 0.85 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; # 85% \u5f3a\u5236\u6ea2\u51fa\u5230\u78c1\u76d8<br> &nbsp; &nbsp;  pause: 0.90 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; # 90% \u6682\u505c\u63a5\u65b0\u4efb\u52a1<br> &nbsp; &nbsp;  terminate: 0.98 &nbsp; &nbsp; &nbsp; &nbsp; # 98% \u6740\u8fdb\u7a0b<br>\u200b<br>  comm:<br> &nbsp;  compression: lz4 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;# \u8de8\u673a\u4f20\u8f93\u7528 lz4 \u538b\u7f29<br> &nbsp;  timeouts:<br> &nbsp; &nbsp;  connect: 30s<br> &nbsp; &nbsp;  tcp: 30s<br>EOF<br>\u200b<br># \u9a8c\u8bc1<br>cat ~\/.config\/dask\/distributed.yaml<\/pre>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_4_%E9%98%B6%E6%AE%B5%EF%BC%9A%E5%87%86%E5%A4%87%E7%9B%AE%E5%BD%95%EF%BC%88%E4%B8%A4%E5%8F%B0%E9%83%BD%E5%81%9A%EF%BC%89\"><\/span>\u7b2c 4 \u9636\u6bb5\uff1a\u51c6\u5907\u76ee\u5f55\uff08\u4e24\u53f0\u90fd\u505a\uff09<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<pre class=\"wp-block-preformatted\"># Spill \u76ee\u5f55\uff08\u5982\u679c \/data \u6709\u8db3\u591f\u7a7a\u95f4\u7528 \/data\uff0c\u5426\u5219\u7528 \/tmp\uff09<br>mkdir -p \/tmp\/dask-spill<br>\u200b<br># \u65e5\u5fd7\u76ee\u5f55<br>sudo mkdir -p \/data\/dask<br>sudo chown pcd_pdtk_op:users \/data\/dask<br>sudo chmod 755 \/data\/dask<\/pre>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_5_%E9%98%B6%E6%AE%B5%EF%BC%9A%E5%89%8D%E5%8F%B0%E5%90%AF%E5%8A%A8%E6%B5%8B%E8%AF%95%EF%BC%88%E5%85%88%E9%AA%8C%E8%AF%81%E8%83%BD%E8%B7%91%EF%BC%89\"><\/span>\u7b2c 5 \u9636\u6bb5\uff1a\u524d\u53f0\u542f\u52a8\u6d4b\u8bd5\uff08\u5148\u9a8c\u8bc1\u80fd\u8dd1\uff09<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"51_%E6%9C%BA%E5%99%A8_38%EF%BC%9A%E5%90%AF%E5%8A%A8_Scheduler\"><\/span>5.1 \u673a\u5668 38\uff1a\u542f\u52a8 Scheduler<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">SSH \u5230 38\uff0c\u5f00 tmux\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">tmux new -s dask-scheduler<br>source ~\/dask-env\/bin\/activate<br>dask scheduler --host 0.0.0.0 --port 8786 --dashboard-address :8787<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>dask 2024.x \u63a8\u8350\u7528 <code>dask scheduler<\/code>\uff08\u7a7a\u683c\uff09\uff0c<code>dask-scheduler<\/code>\uff08\u8fde\u5b57\u7b26\uff09\u4e5f\u4ecd\u7136\u53ef\u7528<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u770b\u5230\u5982\u4e0b\u8f93\u51fa\u8bf4\u660e\u6210\u529f\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">distributed.scheduler - INFO - &nbsp; Scheduler at:  tcp:\/\/10.21.80.38:8786<br>distributed.scheduler - INFO - &nbsp; dashboard at: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; :8787<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u6309 <code>Ctrl+B<\/code> \u7136\u540e <code>D<\/code> \u8131\u79bb tmux\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"52_%E6%9C%BA%E5%99%A8_38%EF%BC%9A%E5%90%AF%E5%8A%A8_4_%E4%B8%AA%E4%B8%BB%E5%8A%9B_Worker\"><\/span>5.2 \u673a\u5668 38\uff1a\u542f\u52a8 4 \u4e2a\u4e3b\u529b Worker<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u65b0\u5f00\u7ec8\u7aef\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">tmux new -s dask-worker<br>source ~\/dask-env\/bin\/activate<br>\u200b<br>dask worker tcp:\/\/10.21.80.38:8786 \\<br> &nbsp; &nbsp;--nworkers 4 \\<br> &nbsp; &nbsp;--nthreads 4 \\<br> &nbsp; &nbsp;--memory-limit 60GB \\<br> &nbsp; &nbsp;--local-directory \/tmp\/dask-spill \\<br> &nbsp; &nbsp;--resources \"primary=1 machine_38=1\" \\<br> &nbsp; &nbsp;--name primary<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>\u5173\u952e\u70b9<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>dask 2024.x \u7528 <code>dask worker<\/code>\uff08\u7a7a\u683c\uff09\u6216 <code>dask-worker<\/code>\uff08\u8fde\u5b57\u7b26\uff09\uff0c\u4e24\u8005\u90fd\u53ef\u7528<\/li>\n\n\n\n<li>\u5fc5\u987b\u7528 <code>--nworkers<\/code>\uff08<code>--nprocs<\/code> \u5728 2024.x \u4e2d\u5df2\u79fb\u9664\uff09<\/li>\n\n\n\n<li><code>--name primary<\/code>\uff08\u81ea\u52a8\u8ffd\u52a0 <code>-0<\/code>\u3001<code>-1<\/code>\u3001<code>-2<\/code>\u3001<code>-3<\/code>\uff09<\/li>\n\n\n\n<li><code>--resources \"primary=1 machine_38=1\"<\/code>\uff1a<code>machine_38<\/code> \u6807\u7b7e\u7528\u4e8e Job \u7ea7\u522b\u673a\u5668\u4eb2\u548c\u6027\u8c03\u5ea6<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u770b\u5230 4 \u884c <code>Registered to: tcp:\/\/10.21.80.38:8786<\/code> \u8bf4\u660e OK\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"53_%E6%9C%BA%E5%99%A8_39%EF%BC%9A%E5%90%AF%E5%8A%A8_Secondary_Worker\"><\/span>5.3 \u673a\u5668 39\uff1a\u542f\u52a8 Secondary Worker<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">SSH \u5230 39\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">tmux new -s dask-worker<br>source ~\/dask-env\/bin\/activate<br>\u200b<br>dask worker tcp:\/\/10.21.80.38:8786 \\<br> &nbsp; &nbsp;--nworkers 1 \\<br> &nbsp; &nbsp;--nthreads 4 \\<br> &nbsp; &nbsp;--memory-limit 50GB \\<br> &nbsp; &nbsp;--local-directory \/tmp\/dask-spill \\<br> &nbsp; &nbsp;--resources \"secondary=1 machine_39=1\" \\<br> &nbsp; &nbsp;--name secondary-39<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"54_%E9%AA%8C%E8%AF%81%E9%9B%86%E7%BE%A4\"><\/span>5.4 \u9a8c\u8bc1\u96c6\u7fa4<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u56de\u5230 38 \u4efb\u610f\u7ec8\u7aef\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">source ~\/dask-env\/bin\/activate<br>python3 -c \"<br>from dask.distributed import Client<br>c = Client('tcp:\/\/10.21.80.38:8786')<br>info = c.scheduler_info()<br>print(f'Workers: {len(info[\\\"workers\\\"])}\/5')<br>for w in info['workers'].values():<br> &nbsp;  res = w.get('resources', {})<br> &nbsp;  mem = w['memory_limit']\/1e9<br> &nbsp;  print(f'  {w[\\\"name\\\"]}: nthreads={w[\\\"nthreads\\\"]}, mem={mem:.0f}GB, resources={res}')<br>\"<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9884\u671f\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">Workers: 5\/5<br>  primary-0: nthreads=4, mem=60GB, resources={'primary': 1.0, 'machine_38': 1.0}<br>  primary-1: nthreads=4, mem=60GB, resources={'primary': 1.0, 'machine_38': 1.0}<br>  primary-2: nthreads=4, mem=60GB, resources={'primary': 1.0, 'machine_38': 1.0}<br>  primary-3: nthreads=4, mem=60GB, resources={'primary': 1.0, 'machine_38': 1.0}<br>  secondary-39: nthreads=4, mem=XXGB, resources={'secondary': 1.0, 'machine_39': 1.0}<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"55_%E7%AE%80%E5%8D%95%E8%AE%A1%E7%AE%97%E6%B5%8B%E8%AF%95\"><\/span>5.5 \u7b80\u5355\u8ba1\u7b97\u6d4b\u8bd5<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\">import dask.array as da<br>from dask.distributed import Client<br>from dask import annotate<br>\u200b<br>c = Client(\"tcp:\/\/10.21.80.38:8786\")<br>with annotate(resources={'primary': 1}):<br> &nbsp; &nbsp;x = da.random.random((50000, 25000), chunks=(5000, 5000))<br> &nbsp; &nbsp;result = x.mean().compute()<br>print(f\"Mean: {result}\")<br>print(\"\u2713 \u96c6\u7fa4\u5de5\u4f5c\u6b63\u5e38\")<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u8dd1\u901a\u540e<strong>\u8fdb\u5165 tmux \u5404\u7a97\u53e3\u6309 Ctrl+C \u5173\u6389\u524d\u53f0\u8fdb\u7a0b<\/strong>\uff08\u6216 <code>tmux kill-server<\/code>\uff09\uff0c\u51c6\u5907\u505a systemd \u90e8\u7f72\u3002<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_6_%E9%98%B6%E6%AE%B5%EF%BC%9Asystemd_%E6%9C%8D%E5%8A%A1%E9%83%A8%E7%BD%B2\"><\/span>\u7b2c 6 \u9636\u6bb5\uff1asystemd \u670d\u52a1\u90e8\u7f72<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"61_%E6%9C%BA%E5%99%A8_38%EF%BC%9AScheduler_Service\"><\/span>6.1 \u673a\u5668 38\uff1aScheduler Service<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo tee \/etc\/systemd\/system\/dask-scheduler.service &gt; \/dev\/null &lt;&lt;'EOF'<br>[Unit]<br>Description=Dask Scheduler<br>After=network.target<br>Wants=network-online.target<br>\u200b<br>[Service]<br>Type=simple<br>User=pcd_pdtk_op<br>Group=users<br>WorkingDirectory=\/home\/pcd_pdtk_op<br>Environment=\"PATH=\/home\/pcd_pdtk_op\/dask-env\/bin:\/usr\/local\/bin:\/usr\/bin:\/bin\"<br>ExecStart=\/home\/pcd_pdtk_op\/dask-env\/bin\/dask-scheduler \\<br> &nbsp; &nbsp;--host 0.0.0.0 \\<br> &nbsp; &nbsp;--port 8786 \\<br> &nbsp; &nbsp;--dashboard-address :8787<br>Restart=always<br>RestartSec=10<br>StandardOutput=append:\/data\/dask\/scheduler.log<br>StandardError=append:\/data\/dask\/scheduler.err<br>\u200b<br>[Install]<br>WantedBy=multi-user.target<br>EOF<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>\u5173\u952e\u70b9<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><code>Group=users<\/code>\uff08\u4e0d\u662f <code>Group=pcd_pdtk_op<\/code>\uff09<\/li>\n\n\n\n<li>systemd ExecStart \u5efa\u8bae\u7528 <code>dask-scheduler<\/code>\uff08\u8fde\u5b57\u7b26\uff09\uff0c\u76f4\u63a5\u6307\u5411 bin \u4e0b\u7684\u53ef\u6267\u884c\u6587\u4ef6<\/li>\n\n\n\n<li>\u65e5\u5fd7\u8def\u5f84 <code>\/data\/dask\/<\/code><\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"62_%E6%9C%BA%E5%99%A8_38%EF%BC%9AWorker_Service\"><\/span>6.2 \u673a\u5668 38\uff1aWorker Service<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo tee \/etc\/systemd\/system\/dask-worker.service &gt; \/dev\/null &lt;&lt;'EOF'<br>[Unit]<br>Description=Dask Worker (Primary on 38)<br>After=network.target dask-scheduler.service<br>Requires=dask-scheduler.service<br>\u200b<br>[Service]<br>Type=simple<br>User=pcd_pdtk_op<br>Group=users<br>WorkingDirectory=\/home\/pcd_pdtk_op<br>Environment=\"PATH=\/home\/pcd_pdtk_op\/dask-env\/bin:\/usr\/local\/bin:\/usr\/bin:\/bin\"<br>ExecStart=\/home\/pcd_pdtk_op\/dask-env\/bin\/dask-worker tcp:\/\/10.21.80.38:8786 \\<br> &nbsp; &nbsp;--nworkers 4 \\<br> &nbsp; &nbsp;--nthreads 4 \\<br> &nbsp; &nbsp;--memory-limit 60GB \\<br> &nbsp; &nbsp;--local-directory \/tmp\/dask-spill \\<br> &nbsp; &nbsp;--resources \"primary=1 machine_38=1\" \\<br> &nbsp; &nbsp;--name primary<br>Restart=always<br>RestartSec=15<br>StandardOutput=append:\/data\/dask\/worker.log<br>StandardError=append:\/data\/dask\/worker.err<br>LimitNOFILE=65535<br>\u200b<br>[Install]<br>WantedBy=multi-user.target<br>EOF<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>dask 2024.x \u5fc5\u987b\u7528 <code>--nworkers<\/code><\/strong>\uff0c<code>--nprocs<\/code> \u5df2\u88ab\u79fb\u9664\uff0c\u4f1a\u5bfc\u81f4 exit code 2\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"63_%E6%9C%BA%E5%99%A8_39%EF%BC%9AWorker_Service\"><\/span>6.3 \u673a\u5668 39\uff1aWorker Service<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">SSH \u5230 39\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo tee \/etc\/systemd\/system\/dask-worker.service &gt; \/dev\/null &lt;&lt;'EOF'<br>[Unit]<br>Description=Dask Worker (Secondary on 39)<br>After=network.target<br>Wants=network-online.target<br>\u200b<br>[Service]<br>Type=simple<br>User=pcd_pdtk_op<br>Group=users<br>WorkingDirectory=\/home\/pcd_pdtk_op<br>Environment=\"PATH=\/home\/pcd_pdtk_op\/dask-env\/bin:\/usr\/local\/bin:\/usr\/bin:\/bin\"<br>ExecStart=\/home\/pcd_pdtk_op\/dask-env\/bin\/dask-worker tcp:\/\/10.21.80.38:8786 \\<br> &nbsp; &nbsp;--nworkers 1 \\<br> &nbsp; &nbsp;--nthreads 4 \\<br> &nbsp; &nbsp;--memory-limit 50GB \\<br> &nbsp; &nbsp;--local-directory \/tmp\/dask-spill \\<br> &nbsp; &nbsp;--resources \"secondary=1 machine_39=1\" \\<br> &nbsp; &nbsp;--name secondary-39<br>Restart=always<br>RestartSec=15<br>StandardOutput=append:\/data\/dask\/worker.log<br>StandardError=append:\/data\/dask\/worker.err<br>LimitNOFILE=65535<br>\u200b<br>[Install]<br>WantedBy=multi-user.target<br>EOF<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"64_%E5%90%AF%E5%8A%A8%E6%9C%8D%E5%8A%A1\"><\/span>6.4 \u542f\u52a8\u670d\u52a1<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u673a\u5668 38<\/strong>\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo systemctl daemon-reload<br>sudo systemctl enable dask-scheduler dask-worker<br>sudo systemctl start dask-scheduler<br>sleep 5<br>sudo systemctl start dask-worker<br>\u200b<br># \u67e5\u770b\u72b6\u6001<br>sudo systemctl status dask-scheduler --no-pager<br>sudo systemctl status dask-worker --no-pager<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u673a\u5668 39<\/strong>\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo systemctl daemon-reload<br>sudo systemctl enable dask-worker<br>sudo systemctl start dask-worker<br>\u200b<br>sudo systemctl status dask-worker --no-pager<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u5747\u5e94\u770b\u5230 <code>Active: active (running)<\/code>\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"65_%E9%AA%8C%E8%AF%81%E6%9C%8D%E5%8A%A1%E5%8C%96%E9%9B%86%E7%BE%A4\"><\/span>6.5 \u9a8c\u8bc1\u670d\u52a1\u5316\u96c6\u7fa4<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\">source ~\/dask-env\/bin\/activate<br>python3 -c \"<br>from dask.distributed import Client<br>c = Client('tcp:\/\/10.21.80.38:8786')<br>info = c.scheduler_info()<br>print(f'Workers: {len(info[\\\"workers\\\"])}\/5')<br>for w in info['workers'].values():<br> &nbsp;  print(f'  {w[\\\"name\\\"]}: {w[\\\"memory_limit\\\"]\/1e9:.0f}GB, {w.get(\\\"resources\\\")}')<br>\"<\/pre>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_7_%E9%98%B6%E6%AE%B5%EF%BC%9ADSS_Code_Env_%E9%9B%86%E6%88%90\"><\/span>\u7b2c 7 \u9636\u6bb5\uff1aDSS Code Env \u96c6\u6210<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"71_%E6%96%B9%E5%BC%8F_A%EF%BC%9A%E4%BD%BF%E7%94%A8_DSS_%E5%86%85%E7%BD%AE_pyenv%EF%BC%88%E6%8E%A8%E8%8D%90%EF%BC%89\"><\/span>7.1 \u65b9\u5f0f A\uff1a\u4f7f\u7528 DSS \u5185\u7f6e pyenv\uff08\u63a8\u8350\uff09<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u5982\u679c DSS \u7684 pyenv\uff08<code>\/data\/DATA_DIR\/pyenv<\/code>\uff09\u5df2\u5305\u542b pandas 2.2.3\u3001numpy 1.26.4\uff0c\u53ea\u9700\u786e\u8ba4 dask \u76f8\u5173\u5305\u5df2\u5b89\u88c5\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/data\/DATA_DIR\/pyenv\/bin\/pip install \\<br> &nbsp; &nbsp;\"dask==2024.6.2\" \\<br> &nbsp; &nbsp;\"distributed==2024.6.2\" \\<br> &nbsp; &nbsp;\"cloudpickle==3.1.2\" \\<br> &nbsp; &nbsp;\"msgpack==1.1.2\" \\<br> &nbsp; &nbsp;\"lz4==3.1.3\"<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"72_%E6%96%B9%E5%BC%8F_B%EF%BC%9A%E9%80%9A%E8%BF%87_DSS_Code_Env_%E7%AE%A1%E7%90%86%E7%95%8C%E9%9D%A2\"><\/span>7.2 \u65b9\u5f0f B\uff1a\u901a\u8fc7 DSS Code Env \u7ba1\u7406\u754c\u9762<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Administration \u2192 Code envs \u2192 \u5bf9\u5e94 code env \u2192 Packages to install<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728\u5df2\u6709 requirements <strong>\u672b\u5c3e\u8ffd\u52a0<\/strong>\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">dask==2024.6.2<br>distributed==2024.6.2<br>cloudpickle==3.1.2<br>msgpack==1.1.2<br>lz4==3.1.3<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>\u5fc5\u987b\u663e\u5f0f\u5199 cloudpickle\u3001msgpack\u3001lz4<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><code>cloudpickle<\/code> \u4e0d\u9489\u7248\u672c\u7684\u8bdd pip \u53ef\u80fd\u89e3\u6790\u5230\u5176\u4ed6\u7248\u672c\uff0c\u4e0e worker \u7aef\u4e0d\u4e00\u81f4\u4f1a\u5bfc\u81f4 KilledWorker<\/li>\n\n\n\n<li><code>msgpack<\/code> \u548c <code>lz4<\/code> \u540c\u7406\uff0c\u9489\u6b7b\u786e\u4fdd\u4e09\u7aef\u4e00\u81f4<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>\u4e0d\u8981\u5199<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><code>pandas<\/code> \/ <code>pyarrow<\/code> \/ <code>numpy<\/code>\uff08\u5df2\u7531 DSS \u73af\u5883\u9501\u5b9a\uff0c\u4e0d\u8981\u8986\u76d6\uff09<\/li>\n\n\n\n<li><code>bokeh<\/code>\uff08client \u7aef\u4e0d\u9700\u8981\uff09<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u70b9 <strong>Save and update<\/strong>\uff0c\u7b49 2-5 \u5206\u949f\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"73_%E5%A4%84%E7%90%86_GitHub_%E4%BE%9D%E8%B5%96%E6%8B%89%E5%8F%96%E5%A4%B1%E8%B4%A5%EF%BC%88%E5%A6%82%E9%81%87%E5%88%B0%EF%BC%89\"><\/span>7.3 \u5904\u7406 GitHub \u4f9d\u8d56\u62c9\u53d6\u5931\u8d25\uff08\u5982\u9047\u5230\uff09<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u82e5 code env \u542b GitHub \u79c1\u6709\u5e93\u4f9d\u8d56\uff08\u5982 <code>lvmh-group\/atom-ds-core-library<\/code>\uff09\uff0c\u53ef\u80fd\u56e0 GnuTLS \u62a5\u9519\u3002\u5904\u7406\u65b9\u6848\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\"># \u65b9\u6848 A\uff1a\u91cd\u8bd5\u51e0\u6b21<br># \u65b9\u6848 B\uff1a\u7ed9 git \u914d\u4ee3\u7406\uff08DSS \u8fd0\u884c\u7528\u6237\u8eab\u4efd\u6267\u884c\uff09<br>git config --global http.proxy http:\/\/127.0.0.1:7890<br>git config --global https.proxy http:\/\/127.0.0.1:7890<br>\u200b<br># \u65b9\u6848 C\uff1a\u7528 ghproxy \u955c\u50cf<br>git config --global url.\"https:\/\/ghproxy.com\/https:\/\/github.com\/\".insteadOf \"https:\/\/github.com\/\"<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"74_%E6%B5%8B%E8%AF%95_DSS_%E5%88%B0%E9%9B%86%E7%BE%A4%E7%9A%84%E8%BF%9E%E6%8E%A5\"><\/span>7.4 \u6d4b\u8bd5 DSS \u5230\u96c6\u7fa4\u7684\u8fde\u63a5<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">DSS \u91cc\u65b0\u5efa Python recipe\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">import pandas, dask, distributed, cloudpickle, msgpack<br>print(f\"pandas: &nbsp; &nbsp; &nbsp;{pandas.__version__} &nbsp; &nbsp; &nbsp;  (\u671f\u671b 2.2.3)\")<br>print(f\"numpy: &nbsp; &nbsp; &nbsp; {__import__('numpy').__version__} &nbsp; &nbsp; &nbsp; (\u671f\u671b 1.26.4)\")<br>print(f\"dask: &nbsp; &nbsp; &nbsp; &nbsp;{dask.__version__} &nbsp; &nbsp; &nbsp; &nbsp;  (\u671f\u671b 2024.6.2)\")<br>print(f\"distributed: {distributed.__version__} &nbsp; (\u671f\u671b 2024.6.2)\")<br>print(f\"cloudpickle: {cloudpickle.__version__} &nbsp; (\u671f\u671b 3.1.2)\")<br>print(f\"msgpack: &nbsp; &nbsp; {msgpack.version} &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; (\u671f\u671b (1, 1, 2))\")<br>\u200b<br>from dask.distributed import Client<br>c = Client(\"tcp:\/\/10.21.80.38:8786\")<br>print(c)<br>\u200b<br>info = c.scheduler_info()<br>print(f\"\\nWorkers: {len(info['workers'])}\/5\")<br>for w in info['workers'].values():<br> &nbsp; &nbsp;print(f\" &nbsp;{w['name']}: {w['memory_limit']\/1e9:.0f}GB, {w.get('resources')}\")<br>\u200b<br>c.close()<\/pre>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_8_%E9%98%B6%E6%AE%B5%EF%BC%9A%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4\"><\/span>\u7b2c 8 \u9636\u6bb5\uff1a\u65e5\u5e38\u8fd0\u7ef4<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%90%AF%E5%8A%A8_%E5%81%9C%E6%AD%A2_%E9%87%8D%E5%90%AF\"><\/span>\u542f\u52a8 \/ \u505c\u6b62 \/ \u91cd\u542f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\"># 38 \u4e0a<br>sudo systemctl restart dask-scheduler<br>sudo systemctl restart dask-worker<br>sudo systemctl stop dask-worker<br>\u200b<br># 39 \u4e0a<br>sudo systemctl restart dask-worker<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%9F%A5%E7%9C%8B%E6%97%A5%E5%BF%97\"><\/span>\u67e5\u770b\u65e5\u5fd7<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\"># systemd \u65e5\u5fd7<br>sudo journalctl -u dask-scheduler -f<br>sudo journalctl -u dask-worker -f<br>\u200b<br># \u5e94\u7528\u65e5\u5fd7\u6587\u4ef6<br>tail -f \/data\/dask\/scheduler.log<br>tail -f \/data\/dask\/worker.log<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Dashboard_%E7%9B%91%E6%8E%A7\"><\/span>Dashboard \u76d1\u63a7<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u6d4f\u89c8\u5668\u8bbf\u95ee <code>http:\/\/10.21.80.38:8787<\/code>\uff08\u9700\u963f\u91cc\u4e91\u5b89\u5168\u7ec4\u653e\u901a 8787 \u5165\u65b9\u5411\uff09<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u672c\u5730\u65e0\u6cd5\u76f4\u63a5\u8bbf\u95ee\u53ef\u7528 SSH \u7aef\u53e3\u8f6c\u53d1\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">ssh -L 8787:10.21.80.38:8787 pcd_pdtk_op@10.21.80.38<br># \u6d4f\u89c8\u5668\u6253\u5f00 http:\/\/localhost:8787<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%AE%8C%E6%95%B4%E9%9B%86%E7%BE%A4%E9%87%8D%E5%90%AF\"><\/span>\u5b8c\u6574\u96c6\u7fa4\u91cd\u542f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\"># \u5728 39 \u4e0a\u505c worker<br>ssh pcd_pdtk_op@10.21.80.39 \"sudo systemctl stop dask-worker\"<br>\u200b<br># \u5728 38 \u4e0a\u91cd\u542f<br>sudo systemctl restart dask-scheduler<br>sleep 5<br>sudo systemctl restart dask-worker<br>\u200b<br># \u5728 39 \u4e0a\u542f worker<br>ssh pcd_pdtk_op@10.21.80.39 \"sudo systemctl start dask-worker\"<\/pre>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_9_%E9%98%B6%E6%AE%B5%EF%BC%9ADSS_Recipe_%E6%A8%A1%E6%9D%BF%EF%BC%88Job_%E7%BA%A7%E5%88%AB%E6%9C%BA%E5%99%A8%E4%BA%B2%E5%92%8C%E6%80%A7%E8%B0%83%E5%BA%A6%EF%BC%89\"><\/span>\u7b2c 9 \u9636\u6bb5\uff1aDSS Recipe \u6a21\u677f\uff08Job \u7ea7\u522b\u673a\u5668\u4eb2\u548c\u6027\u8c03\u5ea6\uff09<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<pre class=\"wp-block-preformatted\">\"\"\"<br>Dask on DSS \u2014 Job \u7ea7\u522b\u673a\u5668\u4eb2\u548c\u6027\u8c03\u5ea6<br>====================================<br>\u8c03\u5ea6\u903b\u8f91\uff1a<br>  - \u6bcf\u4e2a Job \u81ea\u52a8\u9009\u6700\u7a7a\u95f2\u7684\u673a\u5668<br>  - scatter + annotate \u53cc\u91cd\u9489\u673a\u5668\uff0c\u6570\u636e\u96f6\u8de8\u673a\u4f20\u8f93<br>  - 2 \u4e2a Job \u5e76\u884c\u65f6\u5404\u5360\u4e00\u53f0<br>\"\"\"<br>import dataiku<br>import dask<br>import dask.dataframe as dd<br>import pandas as pd<br>import numpy as np<br>from dask.distributed import Client, wait<br>from dask import delayed<br>\u200b<br># ============ \u914d\u7f6e ============<br>SCHEDULER_ADDR = \"tcp:\/\/10.21.80.38:8786\"<br>MACHINE_TAGS &nbsp; = [\"machine_38\", \"machine_39\"]<br>OUTPUT_DATASET = \"your_output_dataset\" &nbsp;# \u2190 \u6539\u6210\u4f60 recipe \u7684 output dataset \u540d<br>NUM_TEST_ROWS &nbsp;= 2_000_000 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;# \u6d4b\u8bd5\u6570\u636e\u884c\u6570\uff0c\u6b63\u5f0f\u4f7f\u7528\u65f6\u66ff\u6362\u4e3a DSS \u6570\u636e\u8bfb\u53d6<br>CHUNK_SIZE &nbsp; &nbsp; = 500_000<br>\u200b<br># ============ 1. \u8fde\u63a5 ============<br>client = Client(SCHEDULER_ADDR)<br>print(client)<br>\u200b<br># ============ 2. \u7248\u672c\u6821\u9a8c ============<br>def get_worker_versions():<br> &nbsp; &nbsp;import sys, pandas, numpy, dask, distributed, cloudpickle<br> &nbsp; &nbsp;return {<br> &nbsp; &nbsp; &nbsp; &nbsp;\"python\": f\"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\",<br> &nbsp; &nbsp; &nbsp; &nbsp;\"pandas\": pandas.__version__,<br> &nbsp; &nbsp; &nbsp; &nbsp;\"numpy\": numpy.__version__,<br> &nbsp; &nbsp; &nbsp; &nbsp;\"dask\": dask.__version__,<br> &nbsp; &nbsp; &nbsp; &nbsp;\"distributed\": distributed.__version__,<br> &nbsp; &nbsp; &nbsp; &nbsp;\"cloudpickle\": cloudpickle.__version__,<br> &nbsp;  }<br>\u200b<br>import sys<br>client_ver = {<br> &nbsp; &nbsp;\"python\": f\"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\",<br> &nbsp; &nbsp;\"pandas\": pd.__version__, \"numpy\": np.__version__,<br> &nbsp; &nbsp;\"dask\": dask.__version__, \"distributed\": __import__('distributed').__version__,<br> &nbsp; &nbsp;\"cloudpickle\": __import__('cloudpickle').__version__,<br>}<br>\u200b<br>worker_results = client.run(get_worker_versions)<br>if not worker_results:<br> &nbsp; &nbsp;raise RuntimeError(\"\u6ca1\u6709 worker \u8fde\u63a5\u5230\u96c6\u7fa4\uff0c\u8bf7\u68c0\u67e5 worker \u670d\u52a1\u72b6\u6001\")<br>worker_ver = list(worker_results.values())[0]<br>\u200b<br>print(f\"\\n{'Package':&lt;15} {'Client':&lt;18} {'Worker':&lt;18} {'Match'}\")<br>print(\"-\" * 60)<br>for pkg in client_ver:<br> &nbsp; &nbsp;cv, wv = client_ver[pkg], worker_ver.get(pkg, \"?\")<br> &nbsp; &nbsp;print(f\"{pkg:&lt;15} {cv:&lt;18} {wv:&lt;18} {'\u2713' if cv == wv else '\u2717'}\")<br>print()<br>\u200b<br># ============ 3. \u9009\u673a\u5668 ============<br>def pick_machine(client, machine_tags):<br> &nbsp; &nbsp;workers = client.scheduler_info()[\"workers\"]<br> &nbsp; &nbsp;machine_stats = {}<br> &nbsp; &nbsp;for tag in machine_tags:<br> &nbsp; &nbsp; &nbsp; &nbsp;addrs, free_mem = [], 0<br> &nbsp; &nbsp; &nbsp; &nbsp;for addr, info in workers.items():<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;if info.get(\"resources\", {}).get(tag, 0) &gt; 0:<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;addrs.append(addr)<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;ml = info.get(\"memory_limit\", 0)<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;m = info.get(\"metrics\", {})<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;free_mem += max(0, ml - m.get(\"managed_bytes\", 0) - m.get(\"unmanaged_old_bytes\", 0))<br> &nbsp; &nbsp; &nbsp; &nbsp;if addrs:<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;machine_stats[tag] = {\"addrs\": addrs, \"free_mem\": free_mem}<br>\u200b<br> &nbsp; &nbsp;if not machine_stats:<br> &nbsp; &nbsp; &nbsp; &nbsp;raise RuntimeError(f\"\u6ca1\u6709\u5e26 {machine_tags} \u6807\u7b7e\u7684 worker\")<br>\u200b<br> &nbsp; &nbsp;for tag, s in machine_stats.items():<br> &nbsp; &nbsp; &nbsp; &nbsp;print(f\" &nbsp;{tag}: {len(s['addrs'])} workers, \u53ef\u7528 {s['free_mem']\/1e9:.1f} GB\")<br>\u200b<br> &nbsp; &nbsp;chosen_tag = max(machine_stats, key=lambda t: machine_stats[t][\"free_mem\"])<br> &nbsp; &nbsp;chosen = machine_stats[chosen_tag]<br> &nbsp; &nbsp;print(f\"\u2713 \u9009\u5b9a: {chosen_tag} ({len(chosen['addrs'])} workers, {chosen['free_mem']\/1e9:.1f} GB)\\n\")<br> &nbsp; &nbsp;return chosen_tag, chosen[\"addrs\"]<br>\u200b<br>machine_tag, target_workers = pick_machine(client, MACHINE_TAGS)<br>\u200b<br># ============ 4A. \u751f\u6210\u6d4b\u8bd5\u6570\u636e\uff08\u9a8c\u8bc1\u7528\uff0c\u6b63\u5f0f\u4f7f\u7528\u65f6\u66ff\u6362\u4e3a 4B\uff09 ============<br>print(f\"\u751f\u6210 {NUM_TEST_ROWS:,} \u884c\u6d4b\u8bd5\u6570\u636e...\")<br>rng = np.random.RandomState(42)<br>\u200b<br>futures = []<br>rows_done = 0<br>chunk_id = 0<br>while rows_done &lt; NUM_TEST_ROWS:<br> &nbsp; &nbsp;n = min(CHUNK_SIZE, NUM_TEST_ROWS - rows_done)<br> &nbsp; &nbsp;chunk = pd.DataFrame({<br> &nbsp; &nbsp; &nbsp; &nbsp;\"store_id\": rng.randint(1, 101, size=n),<br> &nbsp; &nbsp; &nbsp; &nbsp;\"amount\": &nbsp; rng.uniform(10, 500, size=n).round(2),<br> &nbsp; &nbsp; &nbsp; &nbsp;\"qty\": &nbsp; &nbsp; &nbsp;rng.randint(1, 20, size=n),<br> &nbsp; &nbsp; &nbsp; &nbsp;\"date\": &nbsp; &nbsp; pd.Timestamp(\"2025-01-01\") + pd.to_timedelta(rng.randint(0, 365, size=n), unit=\"D\"),<br> &nbsp;  })<br> &nbsp; &nbsp;future = client.scatter(chunk, workers=target_workers, broadcast=False)<br> &nbsp; &nbsp;futures.append(future)<br> &nbsp; &nbsp;rows_done += n<br> &nbsp; &nbsp;chunk_id += 1<br> &nbsp; &nbsp;del chunk<br>\u200b<br># ============ 4B. \u4ece DSS Dataset \u8bfb\u53d6\uff08\u6b63\u5f0f\u4f7f\u7528\u65f6\u53d6\u6d88\u6ce8\u91ca\uff0c\u5220\u9664 4A\uff09 ============<br># INPUT_DATASET = \"your_input_dataset\"  # \u2190 \u6539\u6210\u4f60 recipe \u7684 input dataset \u540d<br># input_ds = dataiku.Dataset(INPUT_DATASET)<br># futures = []<br># chunk_id = 0<br># for chunk in input_ds.iter_dataframes(chunksize=CHUNK_SIZE):<br># &nbsp; &nbsp; future = client.scatter(chunk, workers=target_workers, broadcast=False)<br># &nbsp; &nbsp; futures.append(future)<br># &nbsp; &nbsp; chunk_id += 1<br># &nbsp; &nbsp; del chunk<br>\u200b<br>wait(futures)<br>print(f\"\u2713 scatter \u5b8c\u6210: {chunk_id} chunks \u2192 {len(target_workers)} workers\")<br>\u200b<br># \u9a8c\u8bc1\u6570\u636e\u5206\u5e03<br>workers_info = client.scheduler_info()[\"workers\"]<br>w2m = {}<br>for tag in MACHINE_TAGS:<br> &nbsp; &nbsp;for addr, info in workers_info.items():<br> &nbsp; &nbsp; &nbsp; &nbsp;if info.get(\"resources\", {}).get(tag, 0) &gt; 0:<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;w2m[addr] = tag<br>\u200b<br>tag_counts = {t: 0 for t in MACHINE_TAGS}<br>for f in futures:<br> &nbsp; &nbsp;for key, addrs in client.who_has(f).items():<br> &nbsp; &nbsp; &nbsp; &nbsp;for a in addrs:<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;t = w2m.get(a, \"unknown\")<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;if t in tag_counts:<br> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;tag_counts[t] += 1<br>\u200b<br>for t, c in tag_counts.items():<br> &nbsp; &nbsp;print(f\" &nbsp;{t}: {c} partitions\")<br>\u200b<br># \u6784\u5efa dask DataFrame<br>meta = futures[0].result().iloc[:0]<br>ddf = dd.from_delayed([delayed(lambda f: f)(f) for f in futures], meta=meta)<br>print(f\"\u2713 dask DataFrame: {ddf.npartitions} partitions\\n\")<br>\u200b<br># ============ 5. \u4e1a\u52a1\u8ba1\u7b97\uff08\u9489\u5728\u76ee\u6807\u673a\u5668\uff09 ============<br>print(\"\u5f00\u59cb\u8ba1\u7b97...\")<br>with dask.annotate(resources={machine_tag: 1}):<br> &nbsp; &nbsp;result = ddf.groupby(\"store_id\").agg({<br> &nbsp; &nbsp; &nbsp; &nbsp;\"amount\": \"sum\",<br> &nbsp; &nbsp; &nbsp; &nbsp;\"qty\": &nbsp; &nbsp;\"sum\",<br> &nbsp;  }).reset_index()<br>\u200b<br># ============ 6. \u5199\u51fa\u5230 DSS ============<br># \u65b9\u5f0f A\uff1a\u7ed3\u679c\u8f83\u5c0f\uff08&lt; 1GB\uff09\uff0c\u76f4\u63a5 compute \u56de\u672c\u5730\u5199\u5165<br>with dask.annotate(resources={machine_tag: 1}):<br> &nbsp; &nbsp;result_pd = result.compute()<br>\u200b<br>print(f\"\u2713 \u8ba1\u7b97\u5b8c\u6210: {len(result_pd)} \u884c, \"<br> &nbsp; &nbsp; &nbsp;f\"{result_pd.memory_usage(deep=True).sum()\/1e6:.1f} MB\")<br>print(result_pd.head(10))<br>\u200b<br>output_ds = dataiku.Dataset(OUTPUT_DATASET)<br>output_ds.write_with_schema(result_pd)<br>print(f\"\\n\u2713 \u5df2\u5199\u5165 {OUTPUT_DATASET}\")<br>\u200b<br># \u65b9\u5f0f B\uff1a\u7ed3\u679c\u8f83\u5927\uff08&gt; 1GB\uff09\uff0c\u5206\u6279\u5199\uff0c\u907f\u514d DSS recipe \u8fdb\u7a0b OOM<br># output_ds = dataiku.Dataset(OUTPUT_DATASET)<br># output_ds.write_schema_from_dataframe(result.head(100))<br># with output_ds.get_writer() as writer:<br># &nbsp; &nbsp; for part in result.to_delayed():<br># &nbsp; &nbsp; &nbsp; &nbsp; with dask.annotate(resources={machine_tag: 1}):<br># &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; part_pd = part.compute()<br># &nbsp; &nbsp; &nbsp; &nbsp; writer.write_dataframe(part_pd)<br># &nbsp; &nbsp; &nbsp; &nbsp; del part_pd<br>\u200b<br>client.close()<br>print(\"\u2713 Done\")<\/pre>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_10_%E9%98%B6%E6%AE%B5%EF%BC%9A%E6%95%85%E9%9A%9C%E6%8E%92%E6%9F%A5%E9%80%9F%E6%9F%A5%E8%A1%A8\"><\/span>\u7b2c 10 \u9636\u6bb5\uff1a\u6545\u969c\u6392\u67e5\u901f\u67e5\u8868<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th>\u75c7\u72b6<\/th><th>\u68c0\u67e5\/\u5904\u7406<\/th><\/tr><\/thead><tbody><tr><td>systemd \u542f\u52a8 status=216\/GROUP<\/td><td>Group \u4e0d\u5b58\u5728\uff0c\u6539 <code>Group=users<\/code><\/td><\/tr><tr><td>Worker exit code=2<\/td><td><code>--nprocs<\/code> \u5728 2024.x \u4e2d\u5df2\u79fb\u9664\uff0c\u6539\u6210 <code>--nworkers<\/code><\/td><\/tr><tr><td>Worker \u8fde\u4e0d\u4e0a scheduler<\/td><td><code>sudo systemctl status dask-scheduler<\/code> + \u963f\u91cc\u4e91\u5b89\u5168\u7ec4 8786 \u7aef\u53e3<\/td><\/tr><tr><td>KilledWorker \/ \u7248\u672c\u534f\u8bae\u9519\u8bef<\/td><td><code>pip list<\/code> \u5bf9\u6bd4 cloudpickle\u3001msgpack\u3001lz4 \u4e09\u7aef\u5fc5\u987b\u5b8c\u5168\u4e00\u81f4<\/td><\/tr><tr><td><code>StringMethods<\/code> AttributeError<\/td><td>dask \u7248\u672c\u8fc7\u65e7\u4e0d\u517c\u5bb9 pandas 2.x\uff0c\u5347\u7ea7\u5230 dask 2024.1.0+<\/td><\/tr><tr><td><code>IndexError: list index out of range<\/code> (\u7248\u672c\u6821\u9a8c\u65f6)<\/td><td>\u6ca1\u6709 worker \u8fde\u4e0a\u6765\uff0c\u5148\u68c0\u67e5 scheduler \u548c worker \u670d\u52a1\u72b6\u6001<\/td><\/tr><tr><td>Worker \u9891\u7e41\u91cd\u542f<\/td><td><code>sudo journalctl -u dask-worker -n 100<\/code> \u770b\u9519\u8bef<\/td><\/tr><tr><td>annotate \u540e task \u6302\u8d77\u4e0d\u6267\u884c<\/td><td>worker <code>--resources<\/code> \u7f3a\u5c11 machine tag\uff0c\u68c0\u67e5\u662f\u5426\u6709 <code>machine_38=1<\/code> \/ <code>machine_39=1<\/code><\/td><\/tr><tr><td>\u4efb\u52a1\u5728 39 \u4e0a\u8dd1\u4e86<\/td><td>38 \u5185\u5b58\u6ee1\u4e86\u6216 partition \u8c03\u5ea6\u4e0d\u5e73\u8861\uff0c\u662f\u6b63\u5e38\u6ea2\u51fa\u884c\u4e3a<\/td><\/tr><tr><td>\u4efb\u52a1\u5168\u5728 38 \u4e0a\u8dd1\uff0c39 \u4e0d\u52a8<\/td><td>\u6b63\u5e38\uff0c39 \u662f\u6ea2\u51fa\u8865\u5145\u7528\u7684<\/td><\/tr><tr><td>Dashboard \u6253\u4e0d\u5f00<\/td><td>\u963f\u91cc\u4e91\u5b89\u5168\u7ec4\u653e\u901a 8787 \u5165\u65b9\u5411<\/td><\/tr><tr><td><code>compute()<\/code> \u65f6 DSS OOM<\/td><td>\u7ed3\u679c\u592a\u5927\uff0c\u6539\u7528\u5206\u6279\u5199\uff08\u65b9\u5f0f B\uff09\u6216 <code>to_parquet<\/code> \u5199\u76d8<\/td><\/tr><tr><td>Dataset xxx cannot be used<\/td><td>dataset \u540d\u6ca1\u5728 recipe \u7684 Input\/Output \u9762\u677f\u91cc\u58f0\u660e\uff0c\u6216\u540d\u5b57\u62fc\u9519\u4e86<\/td><\/tr><tr><td>\u63d2\u4ef6 GitHub \u4f9d\u8d56\u62c9\u53d6\u5931\u8d25<\/td><td>\u914d git proxy \u6216\u7528 ghproxy \u955c\u50cf<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_11_%E9%98%B6%E6%AE%B5%EF%BC%9A%E9%98%BF%E9%87%8C%E4%BA%91%E5%AE%89%E5%85%A8%E7%BB%84\"><\/span>\u7b2c 11 \u9636\u6bb5\uff1a\u963f\u91cc\u4e91\u5b89\u5168\u7ec4<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e24\u53f0 ECS \u52a0\u5165\u540c\u4e00\u5b89\u5168\u7ec4\uff0c\u7ec4\u5185\u89c4\u5219\u653e\u901a\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th>\u534f\u8bae<\/th><th>\u7aef\u53e3<\/th><th>\u6388\u6743\u5bf9\u8c61<\/th><\/tr><\/thead><tbody><tr><td>TCP<\/td><td>8786\/8787<\/td><td>\u540c\u5b89\u5168\u7ec4<\/td><\/tr><tr><td>TCP<\/td><td>1024-65535<\/td><td>\u540c\u5b89\u5168\u7ec4\uff08worker \u968f\u673a\u7aef\u53e3\uff09<\/td><\/tr><tr><td>ICMP<\/td><td>\u5168\u90e8<\/td><td>\u540c\u5b89\u5168\u7ec4<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%9C%80%E7%BB%88%E9%AA%8C%E8%AF%81%E6%B8%85%E5%8D%95\"><\/span>\u6700\u7ec8\u9a8c\u8bc1\u6e05\u5355<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u4e24\u53f0 <code>pip list<\/code> \u5173\u952e\u5305\u7248\u672c\u5b8c\u5168\u4e00\u81f4\uff08dask \/ distributed \/ cloudpickle \/ msgpack \/ lz4 \/ pandas \/ numpy \/ pyarrow\uff09<\/li>\n\n\n\n<li><code>~\/.config\/dask\/distributed.yaml<\/code> \u4e24\u53f0\u5b58\u5728\u4e14\u4e00\u81f4<\/li>\n\n\n\n<li>38 \u4e0a <code>dask-scheduler.service<\/code> \u548c <code>dask-worker.service<\/code> running<\/li>\n\n\n\n<li>39 \u4e0a <code>dask-worker.service<\/code> running<\/li>\n\n\n\n<li>Worker \u4f7f\u7528 <code>--nworkers<\/code>\uff08\u4e0d\u662f <code>--nprocs<\/code>\uff09<\/li>\n\n\n\n<li>Worker resources \u5305\u542b machine tag\uff08<code>machine_38=1<\/code> \/ <code>machine_39=1<\/code>\uff09<\/li>\n\n\n\n<li>Python \u5ba2\u6237\u7aef\u80fd\u770b\u5230 5 \u4e2a worker<\/li>\n\n\n\n<li>Dashboard <code>http:\/\/10.21.80.38:8787<\/code> \u80fd\u8bbf\u95ee<\/li>\n\n\n\n<li>DSS code env \u88c5\u597d dask \/ distributed \/ cloudpickle \/ msgpack \/ lz4<\/li>\n\n\n\n<li>DSS recipe \u91cc\u80fd <code>Client(\"tcp:\/\/10.21.80.38:8786\")<\/code> \u6210\u529f<\/li>\n\n\n\n<li>\u91cd\u542f\u670d\u52a1\u540e\u80fd\u81ea\u52a8\u6ce8\u518c\u5230\u96c6\u7fa4<\/li>\n<\/ul>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%85%B3%E9%94%AE%E6%B3%A8%E6%84%8F%E4%BA%8B%E9%A1%B9%E6%B1%87%E6%80%BB\"><\/span>\u5173\u952e\u6ce8\u610f\u4e8b\u9879\u6c47\u603b<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>dask 2024.6.2 \u76f8\u6bd4 2022.5.0 \u7684\u53d8\u5316<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><code>--nprocs<\/code> \u5df2\u79fb\u9664\uff0c\u5fc5\u987b\u7528 <code>--nworkers<\/code><\/li>\n\n\n\n<li><code>dask scheduler<\/code> \/ <code>dask worker<\/code>\uff08\u7a7a\u683c\u547d\u4ee4\uff09\u662f\u63a8\u8350\u5199\u6cd5\uff0c\u8fde\u5b57\u7b26\u5199\u6cd5\u4ecd\u517c\u5bb9<\/li>\n\n\n\n<li>\u539f\u751f\u652f\u6301 pandas 2.x\uff0c\u4e0d\u518d\u6709 <code>StringMethods<\/code> \u517c\u5bb9\u6027\u95ee\u9898<\/li>\n\n\n\n<li>\u652f\u6301 P2P shuffle\uff0c\u5927\u8868 groupby\/join \u6027\u80fd\u66f4\u597d<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>DSS Code Env \u7684\u7ea6\u675f<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>cloudpickle==3.1.2\u3001msgpack==1.1.2\u3001lz4==3.1.3 \u5fc5\u987b\u663e\u5f0f\u9489\u7248\u672c<\/li>\n\n\n\n<li>pandas \/ numpy \/ pyarrow \u7531 DSS \u73af\u5883\u7ba1\u7406\uff0c\u4e0d\u8981\u5728 dask \u914d\u7f6e\u4e2d\u8986\u76d6<\/li>\n\n\n\n<li>\u6570\u636e\u8bfb\u53d6\u8d70 DSS API + scatter\uff0c\u4e0d\u8d70 <code>dd.read_sql_*<\/code><\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>\u673a\u5668\u4eb2\u548c\u6027\u8c03\u5ea6<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>Worker \u5fc5\u987b\u5e26 <code>machine_38=1<\/code> \/ <code>machine_39=1<\/code> \u8d44\u6e90\u6807\u7b7e<\/li>\n\n\n\n<li>Recipe \u4ee3\u7801\u901a\u8fc7 <code>MACHINE_TAGS<\/code> \u5f15\u7528\u8fd9\u4e9b\u6807\u7b7e<\/li>\n\n\n\n<li><code>scatter(workers=target_workers)<\/code> \u9489\u6570\u636e\u4f4d\u7f6e<\/li>\n\n\n\n<li><code>dask.annotate(resources={machine_tag: 1})<\/code> \u9489\u8ba1\u7b97\u4f4d\u7f6e<\/li>\n\n\n\n<li>\u4e24\u4e2a Job \u5e76\u884c\u65f6\u81ea\u52a8\u6309\u53ef\u7528\u5185\u5b58\u5206\u914d\u5230\u4e0d\u540c\u673a\u5668<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u26a0\ufe0f <strong>\u4f18\u5316\u5efa\u8bae<\/strong>\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>SQL \u7aef\u9884\u805a\u5408\u4f18\u5148\u4e8e Dask shuffle<\/li>\n\n\n\n<li>partition \u5927\u5c0f\u63a7\u5236\u5728 100-500 MB<\/li>\n\n\n\n<li>\u7ed3\u679c\u5927\u4e8e 1GB \u7528\u5206\u6279\u5199\uff08\u65b9\u5f0f B\uff09\u800c\u4e0d\u662f <code>compute()<\/code> \u6536\u56de client<\/li>\n<\/ul>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AC%AC_12_%E9%98%B6%E6%AE%B5%EF%BC%9A%E7%94%9F%E4%BA%A7%E5%86%85%E5%AD%98%E4%BF%9D%E6%8A%A4%E4%B8%8E%E7%9B%91%E6%8E%A7%E5%8A%A0%E5%9B%BA\"><\/span>\u7b2c 12 \u9636\u6bb5\uff1a\u751f\u4ea7\u5185\u5b58\u4fdd\u62a4\u4e0e\u76d1\u63a7\u52a0\u56fa<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"121_%E6%95%85%E9%9A%9C%E8%83%8C%E6%99%AF\"><\/span>12.1 \u6545\u969c\u80cc\u666f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">2026-08-12\uff0c<code>10.21.80.38<\/code> \u53d1\u751f Linux \u5168\u5c40 OOM\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u591a\u4e2a UID <code>1005<\/code> \u7684 Python\/Dask \u8fdb\u7a0b\u5360\u7528\u5927\u91cf\u5185\u5b58\uff1b<\/li>\n\n\n\n<li>\u5185\u6838\u591a\u6b21\u6267\u884c <code>Out of memory: Killed process<\/code>\uff1b<\/li>\n\n\n\n<li>Dask Worker Event Loop \u957f\u65f6\u95f4\u65e0\u54cd\u5e94\uff1b<\/li>\n\n\n\n<li>Dataiku \u7aef\u53e3 <code>11000<\/code> \u5065\u5eb7\u68c0\u67e5\u5931\u8d25\uff1b<\/li>\n\n\n\n<li>\u91cd\u542f\u670d\u52a1\u5668\u540e Dataiku \u6062\u590d\u3002<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u539f\u914d\u7f6e\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">38\uff1a4 workers \u00d7 60GB<br>39\uff1a1 worker \u00d7 50GB<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">Dask \u7684 <code>--memory-limit<\/code> \u6309\u6bcf\u4e2a Worker \u8ba1\u7b97\uff0c38 \u8282\u70b9\u7406\u8bba\u4e0a\u53ef\u4f7f\u7528\u7ea6 240GB\uff0c\u4e0e Dataiku JVM\u3001\u7cfb\u7edf\u53ca\u5176\u4ed6\u8fdb\u7a0b\u53d1\u751f\u5185\u5b58\u4e89\u62a2\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"122_%E7%BB%9F%E4%B8%80%E5%90%8E%E7%9A%84%E7%94%9F%E4%BA%A7%E9%85%8D%E7%BD%AE\"><\/span>12.2 \u7edf\u4e00\u540e\u7684\u751f\u4ea7\u914d\u7f6e<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e24\u53f0\u670d\u52a1\u5668\u7edf\u4e00\u4f7f\u7528\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">Dask \u7248\u672c\uff1a2024.6.2<br>Workers\uff1a4<br>Threads per worker\uff1a4<br>Memory limit per worker\uff1a40GB<br>\u6bcf\u53f0\u670d\u52a1\u5668 Dask \u603b\u76ee\u6807\u5185\u5b58\uff1a4 \u00d7 40GB = 160GB<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">Dask \u65e5\u5fd7\u4e2d\u6bcf\u4e2a Worker \u663e\u793a\u7ea6\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">37.25GiB<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u8282\u70b9\u914d\u7f6e\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th>\u8282\u70b9<\/th><th>\u5730\u5740<\/th><th>Worker \u540d\u79f0<\/th><th>\u8d44\u6e90\u6807\u7b7e<\/th><\/tr><\/thead><tbody><tr><td>\u4e3b\u8282\u70b9<\/td><td>10.21.80.38<\/td><td>primary-0 ~ primary-3<\/td><td><code>primary=1 machine_38=1<\/code><\/td><\/tr><tr><td>\u526f\u8282\u70b9<\/td><td>10.21.80.39<\/td><td>secondary-39-0 ~ secondary-39-3<\/td><td><code>secondary=1 machine_39=1<\/code><\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\">\u96c6\u7fa4\u603b\u72b6\u6001\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">8 workers<br>32 threads<br>\u6bcf\u4e2a Worker 37.25GiB<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"123_Spill_%E7%9B%AE%E5%BD%95\"><\/span>12.3 Spill \u76ee\u5f55<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e24\u53f0\u670d\u52a1\u5668\u5747\u4f7f\u7528\u6570\u636e\u76d8\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo install -d -o pcd_pdtk_op -g users -m 0750 \/data\/dask-spill<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">Worker \u542f\u52a8\u53c2\u6570\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">--local-directory \/data\/dask-spill<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e0d\u518d\u4f7f\u7528\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/tmp\/dask-spill<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"124_systemd_%E5%86%85%E5%AD%98%E4%BF%9D%E6%8A%A4\"><\/span>12.4 systemd \u5185\u5b58\u4fdd\u62a4<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e24\u53f0\u670d\u52a1\u5668\u7684 <code>dask-worker.service<\/code> \u5747\u589e\u52a0\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">MemoryAccounting=yes<br>MemoryHigh=160G<br>MemoryMax=175G<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u8bf4\u660e\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><code>MemoryHigh=160G<\/code>\uff1a\u8fbe\u5230\u540e\u89e6\u53d1\u5185\u5b58\u56de\u6536\u538b\u529b\uff1b<\/li>\n\n\n\n<li><code>MemoryMax=175G<\/code>\uff1aDask \u670d\u52a1\u786c\u9650\u5236\uff1b<\/li>\n\n\n\n<li>\u9650\u5236\u4f5c\u7528\u4e8e\u6574\u4e2a Dask Worker \u670d\u52a1\uff1b<\/li>\n\n\n\n<li>\u9632\u6b62\u5f02\u5e38\u4efb\u52a1\u62d6\u57ae\u6574\u53f0\u670d\u52a1\u5668\u3002<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u67e5\u770b\u914d\u7f6e\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo systemctl show dask-worker.service \\<br>  -p MemoryAccounting -p MemoryHigh -p MemoryMax<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9884\u671f\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">MemoryAccounting=yes<br>MemoryHigh=171798691840<br>MemoryMax=187904819200<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"125_38_%E8%8A%82%E7%82%B9_Worker_%E9%85%8D%E7%BD%AE\"><\/span>12.5 38 \u8282\u70b9 Worker \u914d\u7f6e<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\">ExecStart=\/home\/pcd_pdtk_op\/dask-env\/bin\/dask-worker tcp:\/\/10.21.80.38:8786 --nworkers 4 --nthreads 4 --memory-limit 40GB --local-directory \/data\/dask-spill --resources \"primary=1 machine_38=1\" --name primary<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"126_39_%E8%8A%82%E7%82%B9_Worker_%E9%85%8D%E7%BD%AE\"><\/span>12.6 39 \u8282\u70b9 Worker \u914d\u7f6e<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\">ExecStart=\/home\/pcd_pdtk_op\/dask-env\/bin\/dask-worker tcp:\/\/10.21.80.38:8786 --nworkers 4 --nthreads 4 --memory-limit 40GB --local-directory \/data\/dask-spill --resources \"secondary=1 machine_39=1\" --name secondary-39<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"127_Dask_%E5%86%85%E5%AD%98%E9%98%88%E5%80%BC\"><\/span>12.7 Dask \u5185\u5b58\u9608\u503c<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e24\u53f0\u670d\u52a1\u5668\u7684\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/home\/pcd_pdtk_op\/.config\/dask\/distributed.yaml<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u7edf\u4e00\u914d\u7f6e\u4e3a\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">distributed:<br>  scheduler:<br> &nbsp;  work-stealing: false<br>\u200b<br>  worker:<br> &nbsp;  memory:<br> &nbsp; &nbsp;  target: 0.75<br> &nbsp; &nbsp;  spill: 0.85<br> &nbsp; &nbsp;  pause: 0.90<br> &nbsp; &nbsp;  terminate: 0.98<br>\u200b<br>  comm:<br> &nbsp;  compression: lz4<br> &nbsp;  timeouts:<br> &nbsp; &nbsp;  connect: 30s<br> &nbsp; &nbsp;  tcp: 30s<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"128_%E6%9C%8D%E5%8A%A1%E9%87%8D%E8%BD%BD%E4%B8%8E%E9%AA%8C%E8%AF%81\"><\/span>12.8 \u670d\u52a1\u91cd\u8f7d\u4e0e\u9a8c\u8bc1<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo systemctl daemon-reload<br>sudo systemctl restart dask-worker.service<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728 38 \u4e0a\u9a8c\u8bc1\u96c6\u7fa4\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/home\/pcd_pdtk_op\/dask-env\/bin\/python - &lt;&lt;'PY'<br>from dask.distributed import Client<br>\u200b<br>c = Client(\"tcp:\/\/10.21.80.38:8786\", timeout=\"10s\")<br>info = c.scheduler_info()<br>print(\"workers:\", len(info[\"workers\"]))<br>\u200b<br>for address, w in sorted(info[\"workers\"].items(), key=lambda x: x[1][\"name\"]):<br> &nbsp;  print(<br> &nbsp; &nbsp; &nbsp;  f'{w[\"name\"]}: '<br> &nbsp; &nbsp; &nbsp;  f'threads={w[\"nthreads\"]}, '<br> &nbsp; &nbsp; &nbsp;  f'memory={w[\"memory_limit\"]\/2**30:.2f}GiB, '<br> &nbsp; &nbsp; &nbsp;  f'resources={w.get(\"resources\", {})}'<br> &nbsp;  )<br>\u200b<br>c.close()<br>PY<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9884\u671f\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">workers: 8<br>\u6bcf\u4e2a Worker: threads=4<br>\u6bcf\u4e2a Worker: memory=37.25GiB<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9a8c\u8bc1 Dataiku\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">curl -sS -o \/dev\/null -w '11000 HTTP %{http_code}\\n' \\<br>  http:\/\/10.21.80.38:11000\/<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9884\u671f\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">11000 HTTP 200<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9a8c\u8bc1 spill \u8def\u5f84\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo grep -E 'Worker name:|Memory:|Local Directory:' \\<br>  \/data\/dask\/worker.err | tail -24<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"129_%E4%B8%BB%E6%9C%BA%E7%9B%91%E6%8E%A7%E8%84%9A%E6%9C%AC\"><\/span>12.9 \u4e3b\u673a\u76d1\u63a7\u811a\u672c<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u76d1\u63a7\u811a\u672c\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/data\/check.sh<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u76d1\u63a7\u65e5\u5fd7\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/data\/daily_check.log<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u76d1\u63a7\u5185\u5bb9\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>CPU \u4f7f\u7528\u7387\uff1b<\/li>\n\n\n\n<li>\u7cfb\u7edf\u8d1f\u8f7d\uff1b<\/li>\n\n\n\n<li>\u5185\u5b58\u4f7f\u7528\u7387\uff1b<\/li>\n\n\n\n<li>Swap \u4f7f\u7528\u7387\uff1b<\/li>\n\n\n\n<li>\u5185\u5b58\u5360\u7528\u6700\u9ad8\u8fdb\u7a0b\uff1b<\/li>\n\n\n\n<li>CPU \u5360\u7528\u6700\u9ad8\u8fdb\u7a0b\uff1b<\/li>\n\n\n\n<li>Python \u8fdb\u7a0b\uff1b<\/li>\n\n\n\n<li>\u6700\u8fd1 10 \u5206\u949f OOM \u65e5\u5fd7\u3002<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">Cron \u914d\u7f6e\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">*\/5 * * * * \/data\/check.sh<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u811a\u672c\u4f7f\u7528\u9501\u6587\u4ef6\u9632\u6b62\u91cd\u590d\u6267\u884c\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/tmp\/daily_check.lock<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u786e\u8ba4\u4efb\u52a1\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo crontab -u pcd_pdtk_op -l<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1210_%E7%9B%91%E6%8E%A7%E6%97%A5%E5%BF%97%E8%BD%AE%E8%BD%AC\"><\/span>12.10 \u76d1\u63a7\u65e5\u5fd7\u8f6e\u8f6c<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u914d\u7f6e\u6587\u4ef6\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/etc\/logrotate.d\/daily-check<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u5185\u5bb9\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">\/data\/daily_check.log {<br> &nbsp;  daily<br> &nbsp;  rotate 14<br> &nbsp;  maxsize 100M<br> &nbsp;  compress<br> &nbsp;  delaycompress<br> &nbsp;  missingok<br> &nbsp;  notifempty<br> &nbsp;  copytruncate<br>}<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u8bf4\u660e\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u6bcf\u5929\u68c0\u67e5\u4e00\u6b21\uff1b<\/li>\n\n\n\n<li>\u65e5\u5fd7\u8d85\u8fc7 100MB \u65f6\u63d0\u524d\u8f6e\u8f6c\uff1b<\/li>\n\n\n\n<li>\u4fdd\u7559 14 \u4efd\uff1b<\/li>\n\n\n\n<li>\u5386\u53f2\u65e5\u5fd7\u538b\u7f29\uff1b<\/li>\n\n\n\n<li>\u4e0d\u9700\u8981\u91cd\u542f\u76d1\u63a7\u811a\u672c\u3002<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u9a8c\u8bc1\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo logrotate -d \/etc\/logrotate.d\/daily-check<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1211_%E6%97%A5%E5%B8%B8%E6%A3%80%E6%9F%A5%E5%91%BD%E4%BB%A4\"><\/span>12.11 \u65e5\u5e38\u68c0\u67e5\u547d\u4ee4<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-preformatted\">free -h<br>sudo journalctl -k --since '1 hour ago' --no-pager | \\<br>grep -Ei 'oom|out of memory|killed process'<br>ps -eo pid,ppid,user,%cpu,%mem,rss,etime,cmd --sort=-rss | head -30<br>sudo systemctl status dask-worker.service --no-pager<br>sudo grep -Ei 'signal 9|signal 15|memory|spill|pause|terminate|unresponsive' \\<br>  \/data\/dask\/worker.err | tail -100<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1212_%E5%9B%9E%E6%BB%9A%E6%96%B9%E6%A1%88\"><\/span>12.12 \u56de\u6eda\u65b9\u6848<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u5982\u9700\u6062\u590d\u539f Worker \u914d\u7f6e\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo rm -f \/etc\/systemd\/system\/dask-worker.service.d\/override.conf<br>sudo systemctl daemon-reload<br>sudo systemctl restart dask-worker.service<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u5982\u9700\u6062\u590d\u539f\u65e5\u5fd7\u67e5\u8be2\u7a97\u53e3\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">sudo sed -i 's\/--since \"10 minutes ago\"\/--since \"1 hour ago\"\/' \/data\/check.sh<\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1213_%E5%BD%93%E5%89%8D%E9%AA%8C%E8%AF%81%E7%BB%93%E8%AE%BA\"><\/span>12.13 \u5f53\u524d\u9a8c\u8bc1\u7ed3\u8bba<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u5f53\u524d\u53cc\u673a Dask \u96c6\u7fa4\u5df2\u9a8c\u8bc1\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\">38 \u548c 39 \u5171 8 \u4e2a Worker<br>\u6bcf\u4e2a Worker 4 \u4e2a\u7ebf\u7a0b<br>\u6bcf\u4e2a Worker 37.25GiB<br>spill \u4f7f\u7528 \/data\/dask-spill<br>Dask \u670d\u52a1\u5177\u5907 MemoryHigh\/MemoryMax \u4fdd\u62a4<br>Dataiku 11000 \u8fd4\u56de HTTP 200<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u8be5\u914d\u7f6e\u7528\u4e8e\u907f\u514d Dask \u5f02\u5e38\u4efb\u52a1\u518d\u6b21\u89e6\u53d1\u4e3b\u673a\u7ea7 OOM\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u4e0b\u9762\u662f\u6839\u636e\u5b9e\u9645\u9a8c\u8bc1\u901a\u8fc7\u7684\u7248\u672c\uff08dask\/distributed 2024.6.2 + pandas 2.2.3 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"site-sidebar-layout":"default","site-content-layout":"","ast-site-content-layout":"default","site-content-style":"default","site-sidebar-style":"default","ast-global-header-display":"","ast-banner-title-visibility":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","ast-disable-related-posts":"","theme-transparent-header-meta":"","adv-header-id-meta":"","stick-header-meta":"","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":"","astra-migrate-meta-layouts":"default","ast-page-background-enabled":"default","ast-page-background-meta":{"desktop":{"background-color":"var(--ast-global-color-4)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"ast-content-background-meta":{"desktop":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"footnotes":""},"categories":[1],"tags":[],"class_list":["post-3221","post","type-post","status-publish","format-standard","hentry","category-vxhs888p"],"_links":{"self":[{"href":"https:\/\/linuxjk.cn\/index.php?rest_route=\/wp\/v2\/posts\/3221","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/linuxjk.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/linuxjk.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/linuxjk.cn\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/linuxjk.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=3221"}],"version-history":[{"count":2,"href":"https:\/\/linuxjk.cn\/index.php?rest_route=\/wp\/v2\/posts\/3221\/revisions"}],"predecessor-version":[{"id":3238,"href":"https:\/\/linuxjk.cn\/index.php?rest_route=\/wp\/v2\/posts\/3221\/revisions\/3238"}],"wp:attachment":[{"href":"https:\/\/linuxjk.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=3221"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/linuxjk.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=3221"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/linuxjk.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=3221"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}