{"id":1295,"date":"2023-03-19T07:28:54","date_gmt":"2023-03-18T23:28:54","guid":{"rendered":"http:\/\/viplao.com\/?p=1295"},"modified":"2023-03-19T07:42:16","modified_gmt":"2023-03-18T23:42:16","slug":"%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94","status":"publish","type":"post","link":"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/","title":{"rendered":"\u3010\u8fd0\u7ef4\u5de5\u5177\u3011flink sql \u57fa\u7840\u5b9e\u8df5 &#8211; \u5e38\u89c1\u7684\u7591\u95ee\u70b9"},"content":{"rendered":"\n<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_71 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u6587\u7ae0\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 eztoc-toggle-hide-by-default' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1%E5%89%8D%E8%A8%80\" title=\"1.\u524d\u8a00\">1.\u524d\u8a00<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2%E7%8A%B6%E6%80%81%E5%8E%9F%E7%90%86\" title=\"2.\u72b6\u6001\u539f\u7406\">2.\u72b6\u6001\u539f\u7406<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#21%E7%8A%B6%E6%80%81%E3%80%81%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%E3%80%81Checkpoint_%E4%B8%89%E8%80%85%E4%B9%8B%E9%97%B4%E7%9A%84%E5%8C%BA%E5%88%AB%E5%8F%8A%E5%85%B3%E7%B3%BB%EF%BC%9F\" title=\"2.1.\u72b6\u6001\u3001\u72b6\u6001\u540e\u7aef\u3001Checkpoint \u4e09\u8005\u4e4b\u95f4\u7684\u533a\u522b\u53ca\u5173\u7cfb\uff1f\">2.1.\u72b6\u6001\u3001\u72b6\u6001\u540e\u7aef\u3001Checkpoint \u4e09\u8005\u4e4b\u95f4\u7684\u533a\u522b\u53ca\u5173\u7cfb\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#22%E6%8A%8A%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%E4%BB%8E_FileSystem_%E5%8F%98%E4%B8%BA_RocksDB_%E5%90%8E%EF%BC%8CFlink_%E4%BB%BB%E5%8A%A1%E7%8A%B6%E6%80%81%E5%AD%98%E5%82%A8%E4%BC%9A%E5%8F%91%E7%94%9F%E9%82%A3%E4%BA%9B%E5%8F%98%E5%8C%96%EF%BC%9F\" title=\"2.2.\u628a\u72b6\u6001\u540e\u7aef\u4ece FileSystem \u53d8\u4e3a RocksDB \u540e\uff0cFlink \u4efb\u52a1\u72b6\u6001\u5b58\u50a8\u4f1a\u53d1\u751f\u90a3\u4e9b\u53d8\u5316\uff1f\">2.2.\u628a\u72b6\u6001\u540e\u7aef\u4ece FileSystem \u53d8\u4e3a RocksDB \u540e\uff0cFlink \u4efb\u52a1\u72b6\u6001\u5b58\u50a8\u4f1a\u53d1\u751f\u90a3\u4e9b\u53d8\u5316\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#23%E4%BB%80%E4%B9%88%E6%A0%B7%E7%9A%84%E4%B8%9A%E5%8A%A1%E5%9C%BA%E6%99%AF%E4%BD%A0%E4%BC%9A%E9%80%89%E6%8B%A9_filesystem%EF%BC%8C%E4%BB%80%E4%B9%88%E6%A0%B7%E7%9A%84%E4%B8%9A%E5%8A%A1%E5%9C%BA%E6%99%AF%E4%BD%A0%E4%BC%9A%E9%80%89_rocksdb_%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%EF%BC%9F\" title=\"2.3.\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009\u62e9 filesystem\uff0c\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009 rocksdb \u72b6\u6001\u540e\u7aef\uff1f\">2.3.\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009\u62e9 filesystem\uff0c\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009 rocksdb \u72b6\u6001\u540e\u7aef\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#24Flink_SQL_API_State_TTL_%E7%9A%84%E8%BF%87%E6%9C%9F%E6%9C%BA%E5%88%B6%E6%98%AF_onCreateAndUpdate_%E8%BF%98%E6%98%AFonReadAndWrite\" title=\"2.4.Flink SQL API State TTL \u7684\u8fc7\u671f\u673a\u5236\u662f onCreateAndUpdate \u8fd8\u662fonReadAndWrite\">2.4.Flink SQL API State TTL \u7684\u8fc7\u671f\u673a\u5236\u662f onCreateAndUpdate \u8fd8\u662fonReadAndWrite<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3\" title=\"3.\u65f6\u95f4\u7a97\u53e3\">3.\u65f6\u95f4\u7a97\u53e3<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#31watermark_%E5%88%B0%E5%BA%95%E6%98%AF%E5%B9%B2%E5%95%A5%E7%9A%84%EF%BC%9F%E5%BA%94%E7%94%A8%E5%9C%BA%E6%99%AF%EF%BC%9F\" title=\"3.1.watermark \u5230\u5e95\u662f\u5e72\u5565\u7684\uff1f\u5e94\u7528\u573a\u666f\uff1f\">3.1.watermark \u5230\u5e95\u662f\u5e72\u5565\u7684\uff1f\u5e94\u7528\u573a\u666f\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#32%E4%B8%80%E4%B8%AA_Flink_%E4%BB%BB%E5%8A%A1%E4%B8%AD%E5%8F%AF%E4%BB%A5%E6%97%A2%E6%9C%89%E4%BA%8B%E4%BB%B6%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3%EF%BC%8C%E5%8F%88%E6%9C%89%E5%A4%84%E7%90%86%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3%E5%90%97%EF%BC%9F\" title=\"3.2.\u4e00\u4e2a Flink \u4efb\u52a1\u4e2d\u53ef\u4ee5\u65e2\u6709\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\uff0c\u53c8\u6709\u5904\u7406\u65f6\u95f4\u7a97\u53e3\u5417\uff1f\">3.2.\u4e00\u4e2a Flink \u4efb\u52a1\u4e2d\u53ef\u4ee5\u65e2\u6709\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\uff0c\u53c8\u6709\u5904\u7406\u65f6\u95f4\u7a97\u53e3\u5417\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#33window_%E5%90%8E%E9%9D%A2%E8%B7%9F_aggregate_%E5%92%8C_process_%E7%9A%84%E4%B8%A4%E4%B8%AA%E7%AA%97%E5%8F%A3%E8%AE%A1%E7%AE%97%E7%9A%84%E5%8C%BA%E5%88%AB%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F\" title=\"3.3.window \u540e\u9762\u8ddf aggregate \u548c process \u7684\u4e24\u4e2a\u7a97\u53e3\u8ba1\u7b97\u7684\u533a\u522b\u662f\u4ec0\u4e48\uff1f\">3.3.window \u540e\u9762\u8ddf aggregate \u548c process \u7684\u4e24\u4e2a\u7a97\u53e3\u8ba1\u7b97\u7684\u533a\u522b\u662f\u4ec0\u4e48\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#4%E7%BC%96%E7%A8%8B%E6%8A%80%E5%B7%A7\" title=\"4.\u7f16\u7a0b\u6280\u5de7\">4.\u7f16\u7a0b\u6280\u5de7<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#41%E4%B8%BA%E4%BB%80%E4%B9%88_Flink_DataStream_API_%E5%9C%A8%E5%87%BD%E6%95%B0%E5%85%A5%E5%8F%82%E6%88%96%E8%80%85%E5%87%BA%E5%8F%82%E6%9C%89%E6%B3%9B%E5%9E%8B%E6%97%B6%EF%BC%8C%E4%B8%8D%E8%83%BD%E4%BD%BF%E7%94%A8_lambda_%E8%A1%A8%E8%BE%BE%E5%BC%8F%EF%BC%9F\" title=\"4.1.\u4e3a\u4ec0\u4e48 Flink DataStream API \u5728\u51fd\u6570\u5165\u53c2\u6216\u8005\u51fa\u53c2\u6709\u6cdb\u578b\u65f6\uff0c\u4e0d\u80fd\u4f7f\u7528 lambda \u8868\u8fbe\u5f0f\uff1f\">4.1.\u4e3a\u4ec0\u4e48 Flink DataStream API \u5728\u51fd\u6570\u5165\u53c2\u6216\u8005\u51fa\u53c2\u6709\u6cdb\u578b\u65f6\uff0c\u4e0d\u80fd\u4f7f\u7528 lambda \u8868\u8fbe\u5f0f\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-13\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#42Flink_%E4%B8%BA%E4%BB%80%E4%B9%88%E5%BC%BA%E8%B0%83_function_%E5%AE%9E%E7%8E%B0%E6%97%B6%EF%BC%8C%E5%AE%9E%E4%BE%8B%E5%8C%96%E7%9A%84%E5%8F%98%E9%87%8F%E8%A6%81%E5%AE%9E%E7%8E%B0_serializable_%E6%8E%A5%E5%8F%A3%EF%BC%9F\" title=\"4.2.Flink \u4e3a\u4ec0\u4e48\u5f3a\u8c03 function \u5b9e\u73b0\u65f6\uff0c\u5b9e\u4f8b\u5316\u7684\u53d8\u91cf\u8981\u5b9e\u73b0 serializable \u63a5\u53e3\uff1f\">4.2.Flink \u4e3a\u4ec0\u4e48\u5f3a\u8c03 function \u5b9e\u73b0\u65f6\uff0c\u5b9e\u4f8b\u5316\u7684\u53d8\u91cf\u8981\u5b9e\u73b0 serializable \u63a5\u53e3\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-14\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#43Flink_%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%E5%8F%AF%E4%BB%A5%E9%80%9A%E8%BF%87%E5%93%AA%E5%87%A0%E7%A7%8D%E6%96%B9%E5%BC%8F%E8%AE%BE%E7%BD%AE%EF%BC%8C%E4%BC%98%E5%85%88%E7%BA%A7%E5%85%B3%E7%B3%BB%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F\" title=\"4.3.Flink \u7684\u5e76\u884c\u5ea6\u53ef\u4ee5\u901a\u8fc7\u54ea\u51e0\u79cd\u65b9\u5f0f\u8bbe\u7f6e\uff0c\u4f18\u5148\u7ea7\u5173\u7cfb\u662f\u4ec0\u4e48\uff1f\">4.3.Flink \u7684\u5e76\u884c\u5ea6\u53ef\u4ee5\u901a\u8fc7\u54ea\u51e0\u79cd\u65b9\u5f0f\u8bbe\u7f6e\uff0c\u4f18\u5148\u7ea7\u5173\u7cfb\u662f\u4ec0\u4e48\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-15\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#5%E5%AE%9E%E6%88%98%E7%BB%8F%E9%AA%8C\" title=\"5.\u5b9e\u6218\u7ecf\u9a8c\">5.\u5b9e\u6218\u7ecf\u9a8c<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-16\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#51Flink_SQL_%E8%AE%A1%E7%AE%97%E7%94%A8%E6%88%B7%E5%88%86%E5%B8%83\" title=\"5.1.Flink SQL \u8ba1\u7b97\u7528\u6237\u5206\u5e03\">5.1.Flink SQL \u8ba1\u7b97\u7528\u6237\u5206\u5e03<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-17\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#52Flink_SQL_%E8%AE%A1%E7%AE%97_DAU\" title=\"5.2.Flink SQL \u8ba1\u7b97 DAU\">5.2.Flink SQL \u8ba1\u7b97 DAU<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-18\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#53%E4%BD%A0%E6%98%AF%E6%80%8E%E4%B9%88%E5%90%88%E7%90%86%E7%9A%84%E8%AF%84%E4%BC%B0_Flink_%E4%BB%BB%E5%8A%A1%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F\" title=\"5.3.\u4f60\u662f\u600e\u4e48\u5408\u7406\u7684\u8bc4\u4f30 Flink \u4efb\u52a1\u7684\u5e76\u884c\u5ea6\uff1f\">5.3.\u4f60\u662f\u600e\u4e48\u5408\u7406\u7684\u8bc4\u4f30 Flink \u4efb\u52a1\u7684\u5e76\u884c\u5ea6\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-19\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#54%E4%BD%A0%E6%98%AF%E6%80%8E%E4%B9%88%E5%90%88%E7%90%86%E8%AF%84%E4%BC%B0%E4%BB%BB%E5%8A%A1%E6%9C%80%E5%A4%A7%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F\" title=\"5.4.\u4f60\u662f\u600e\u4e48\u5408\u7406\u8bc4\u4f30\u4efb\u52a1\u6700\u5927\u5e76\u884c\u5ea6\uff1f\">5.4.\u4f60\u662f\u600e\u4e48\u5408\u7406\u8bc4\u4f30\u4efb\u52a1\u6700\u5927\u5e76\u884c\u5ea6\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-20\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1%E5%89%8D%E8%A8%80-2\" title=\"1.\u524d\u8a00\">1.\u524d\u8a00<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-21\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2%E7%8A%B6%E6%80%81%E5%8E%9F%E7%90%86-2\" title=\"2.\u72b6\u6001\u539f\u7406\">2.\u72b6\u6001\u539f\u7406<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-22\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#21%E7%8A%B6%E6%80%81%E3%80%81%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%E3%80%81Checkpoint_%E4%B8%89%E8%80%85%E4%B9%8B%E9%97%B4%E7%9A%84%E5%8C%BA%E5%88%AB%E5%8F%8A%E5%85%B3%E7%B3%BB%EF%BC%9F-2\" title=\"2.1.\u72b6\u6001\u3001\u72b6\u6001\u540e\u7aef\u3001Checkpoint \u4e09\u8005\u4e4b\u95f4\u7684\u533a\u522b\u53ca\u5173\u7cfb\uff1f\">2.1.\u72b6\u6001\u3001\u72b6\u6001\u540e\u7aef\u3001Checkpoint \u4e09\u8005\u4e4b\u95f4\u7684\u533a\u522b\u53ca\u5173\u7cfb\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-23\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#22%E6%8A%8A%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%E4%BB%8E_FileSystem_%E5%8F%98%E4%B8%BA_RocksDB_%E5%90%8E%EF%BC%8CFlink_%E4%BB%BB%E5%8A%A1%E7%8A%B6%E6%80%81%E5%AD%98%E5%82%A8%E4%BC%9A%E5%8F%91%E7%94%9F%E9%82%A3%E4%BA%9B%E5%8F%98%E5%8C%96%EF%BC%9F-2\" title=\"2.2.\u628a\u72b6\u6001\u540e\u7aef\u4ece FileSystem \u53d8\u4e3a RocksDB \u540e\uff0cFlink \u4efb\u52a1\u72b6\u6001\u5b58\u50a8\u4f1a\u53d1\u751f\u90a3\u4e9b\u53d8\u5316\uff1f\">2.2.\u628a\u72b6\u6001\u540e\u7aef\u4ece FileSystem \u53d8\u4e3a RocksDB \u540e\uff0cFlink \u4efb\u52a1\u72b6\u6001\u5b58\u50a8\u4f1a\u53d1\u751f\u90a3\u4e9b\u53d8\u5316\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-24\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#23%E4%BB%80%E4%B9%88%E6%A0%B7%E7%9A%84%E4%B8%9A%E5%8A%A1%E5%9C%BA%E6%99%AF%E4%BD%A0%E4%BC%9A%E9%80%89%E6%8B%A9_filesystem%EF%BC%8C%E4%BB%80%E4%B9%88%E6%A0%B7%E7%9A%84%E4%B8%9A%E5%8A%A1%E5%9C%BA%E6%99%AF%E4%BD%A0%E4%BC%9A%E9%80%89_rocksdb_%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%EF%BC%9F-2\" title=\"2.3.\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009\u62e9 filesystem\uff0c\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009 rocksdb \u72b6\u6001\u540e\u7aef\uff1f\">2.3.\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009\u62e9 filesystem\uff0c\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009 rocksdb \u72b6\u6001\u540e\u7aef\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-25\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#24Flink_SQL_API_State_TTL_%E7%9A%84%E8%BF%87%E6%9C%9F%E6%9C%BA%E5%88%B6%E6%98%AF_onCreateAndUpdate_%E8%BF%98%E6%98%AFonReadAndWrite-2\" title=\"2.4.Flink SQL API State TTL \u7684\u8fc7\u671f\u673a\u5236\u662f onCreateAndUpdate \u8fd8\u662fonReadAndWrite\">2.4.Flink SQL API State TTL \u7684\u8fc7\u671f\u673a\u5236\u662f onCreateAndUpdate \u8fd8\u662fonReadAndWrite<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-26\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3-2\" title=\"3.\u65f6\u95f4\u7a97\u53e3\">3.\u65f6\u95f4\u7a97\u53e3<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-27\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#31watermark_%E5%88%B0%E5%BA%95%E6%98%AF%E5%B9%B2%E5%95%A5%E7%9A%84%EF%BC%9F%E5%BA%94%E7%94%A8%E5%9C%BA%E6%99%AF%EF%BC%9F-2\" title=\"3.1.watermark \u5230\u5e95\u662f\u5e72\u5565\u7684\uff1f\u5e94\u7528\u573a\u666f\uff1f\">3.1.watermark \u5230\u5e95\u662f\u5e72\u5565\u7684\uff1f\u5e94\u7528\u573a\u666f\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-28\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#32%E4%B8%80%E4%B8%AA_Flink_%E4%BB%BB%E5%8A%A1%E4%B8%AD%E5%8F%AF%E4%BB%A5%E6%97%A2%E6%9C%89%E4%BA%8B%E4%BB%B6%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3%EF%BC%8C%E5%8F%88%E6%9C%89%E5%A4%84%E7%90%86%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3%E5%90%97%EF%BC%9F-2\" title=\"3.2.\u4e00\u4e2a Flink \u4efb\u52a1\u4e2d\u53ef\u4ee5\u65e2\u6709\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\uff0c\u53c8\u6709\u5904\u7406\u65f6\u95f4\u7a97\u53e3\u5417\uff1f\">3.2.\u4e00\u4e2a Flink \u4efb\u52a1\u4e2d\u53ef\u4ee5\u65e2\u6709\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\uff0c\u53c8\u6709\u5904\u7406\u65f6\u95f4\u7a97\u53e3\u5417\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-29\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#33window_%E5%90%8E%E9%9D%A2%E8%B7%9F_aggregate_%E5%92%8C_process_%E7%9A%84%E4%B8%A4%E4%B8%AA%E7%AA%97%E5%8F%A3%E8%AE%A1%E7%AE%97%E7%9A%84%E5%8C%BA%E5%88%AB%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F-2\" title=\"3.3.window \u540e\u9762\u8ddf aggregate \u548c process \u7684\u4e24\u4e2a\u7a97\u53e3\u8ba1\u7b97\u7684\u533a\u522b\u662f\u4ec0\u4e48\uff1f\">3.3.window \u540e\u9762\u8ddf aggregate \u548c process \u7684\u4e24\u4e2a\u7a97\u53e3\u8ba1\u7b97\u7684\u533a\u522b\u662f\u4ec0\u4e48\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-30\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#4%E7%BC%96%E7%A8%8B%E6%8A%80%E5%B7%A7-2\" title=\"4.\u7f16\u7a0b\u6280\u5de7\">4.\u7f16\u7a0b\u6280\u5de7<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-31\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#41%E4%B8%BA%E4%BB%80%E4%B9%88_Flink_DataStream_API_%E5%9C%A8%E5%87%BD%E6%95%B0%E5%85%A5%E5%8F%82%E6%88%96%E8%80%85%E5%87%BA%E5%8F%82%E6%9C%89%E6%B3%9B%E5%9E%8B%E6%97%B6%EF%BC%8C%E4%B8%8D%E8%83%BD%E4%BD%BF%E7%94%A8_lambda_%E8%A1%A8%E8%BE%BE%E5%BC%8F%EF%BC%9F-2\" title=\"4.1.\u4e3a\u4ec0\u4e48 Flink DataStream API \u5728\u51fd\u6570\u5165\u53c2\u6216\u8005\u51fa\u53c2\u6709\u6cdb\u578b\u65f6\uff0c\u4e0d\u80fd\u4f7f\u7528 lambda \u8868\u8fbe\u5f0f\uff1f\">4.1.\u4e3a\u4ec0\u4e48 Flink DataStream API \u5728\u51fd\u6570\u5165\u53c2\u6216\u8005\u51fa\u53c2\u6709\u6cdb\u578b\u65f6\uff0c\u4e0d\u80fd\u4f7f\u7528 lambda \u8868\u8fbe\u5f0f\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-32\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#42Flink_%E4%B8%BA%E4%BB%80%E4%B9%88%E5%BC%BA%E8%B0%83_function_%E5%AE%9E%E7%8E%B0%E6%97%B6%EF%BC%8C%E5%AE%9E%E4%BE%8B%E5%8C%96%E7%9A%84%E5%8F%98%E9%87%8F%E8%A6%81%E5%AE%9E%E7%8E%B0_serializable_%E6%8E%A5%E5%8F%A3%EF%BC%9F-2\" title=\"4.2.Flink \u4e3a\u4ec0\u4e48\u5f3a\u8c03 function \u5b9e\u73b0\u65f6\uff0c\u5b9e\u4f8b\u5316\u7684\u53d8\u91cf\u8981\u5b9e\u73b0 serializable \u63a5\u53e3\uff1f\">4.2.Flink \u4e3a\u4ec0\u4e48\u5f3a\u8c03 function \u5b9e\u73b0\u65f6\uff0c\u5b9e\u4f8b\u5316\u7684\u53d8\u91cf\u8981\u5b9e\u73b0 serializable \u63a5\u53e3\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-33\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#43Flink_%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%E5%8F%AF%E4%BB%A5%E9%80%9A%E8%BF%87%E5%93%AA%E5%87%A0%E7%A7%8D%E6%96%B9%E5%BC%8F%E8%AE%BE%E7%BD%AE%EF%BC%8C%E4%BC%98%E5%85%88%E7%BA%A7%E5%85%B3%E7%B3%BB%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F-2\" title=\"4.3.Flink \u7684\u5e76\u884c\u5ea6\u53ef\u4ee5\u901a\u8fc7\u54ea\u51e0\u79cd\u65b9\u5f0f\u8bbe\u7f6e\uff0c\u4f18\u5148\u7ea7\u5173\u7cfb\u662f\u4ec0\u4e48\uff1f\">4.3.Flink \u7684\u5e76\u884c\u5ea6\u53ef\u4ee5\u901a\u8fc7\u54ea\u51e0\u79cd\u65b9\u5f0f\u8bbe\u7f6e\uff0c\u4f18\u5148\u7ea7\u5173\u7cfb\u662f\u4ec0\u4e48\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-34\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#5%E5%AE%9E%E6%88%98%E7%BB%8F%E9%AA%8C-2\" title=\"5.\u5b9e\u6218\u7ecf\u9a8c\">5.\u5b9e\u6218\u7ecf\u9a8c<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-35\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#51Flink_SQL_%E8%AE%A1%E7%AE%97%E7%94%A8%E6%88%B7%E5%88%86%E5%B8%83-2\" title=\"5.1.Flink SQL \u8ba1\u7b97\u7528\u6237\u5206\u5e03\">5.1.Flink SQL \u8ba1\u7b97\u7528\u6237\u5206\u5e03<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-36\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#52Flink_SQL_%E8%AE%A1%E7%AE%97_DAU-2\" title=\"5.2.Flink SQL \u8ba1\u7b97 DAU\">5.2.Flink SQL \u8ba1\u7b97 DAU<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-37\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#53%E4%BD%A0%E6%98%AF%E6%80%8E%E4%B9%88%E5%90%88%E7%90%86%E7%9A%84%E8%AF%84%E4%BC%B0_Flink_%E4%BB%BB%E5%8A%A1%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F-2\" title=\"5.3.\u4f60\u662f\u600e\u4e48\u5408\u7406\u7684\u8bc4\u4f30 Flink \u4efb\u52a1\u7684\u5e76\u884c\u5ea6\uff1f\">5.3.\u4f60\u662f\u600e\u4e48\u5408\u7406\u7684\u8bc4\u4f30 Flink \u4efb\u52a1\u7684\u5e76\u884c\u5ea6\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-38\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#54%E4%BD%A0%E6%98%AF%E6%80%8E%E4%B9%88%E5%90%88%E7%90%86%E8%AF%84%E4%BC%B0%E4%BB%BB%E5%8A%A1%E6%9C%80%E5%A4%A7%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F-2\" title=\"5.4.\u4f60\u662f\u600e\u4e48\u5408\u7406\u8bc4\u4f30\u4efb\u52a1\u6700\u5927\u5e76\u884c\u5ea6\uff1f\">5.4.\u4f60\u662f\u600e\u4e48\u5408\u7406\u8bc4\u4f30\u4efb\u52a1\u6700\u5927\u5e76\u884c\u5ea6\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-39\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E4%B8%80_Flink_%E6%8F%90%E4%BA%A4%E7%B3%BB%E5%88%97\" title=\"\u4e00. Flink \u63d0\u4ea4\u7cfb\u5217\">\u4e00. Flink \u63d0\u4ea4\u7cfb\u5217<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-40\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1_Flink%E6%80%8E%E4%B9%88%E6%8F%90%E4%BA%A4\" title=\"1. Flink\u600e\u4e48\u63d0\u4ea4?\">1. Flink\u600e\u4e48\u63d0\u4ea4?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-41\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2_Flink%E9%9B%86%E7%BE%A4%E8%A7%84%E6%A8%A1%EF%BC%9F%E5%9C%A8Flink%E9%A1%B9%E7%9B%AE%E5%81%9A%E4%BA%86%E4%BB%80%E4%B9%88\" title=\"2. Flink\u96c6\u7fa4\u89c4\u6a21\uff1f\u5728Flink\u9879\u76ee\u505a\u4e86\u4ec0\u4e48?\">2. Flink\u96c6\u7fa4\u89c4\u6a21\uff1f\u5728Flink\u9879\u76ee\u505a\u4e86\u4ec0\u4e48?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-42\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3_Flink%E6%8F%90%E4%BA%A4%E4%BD%9C%E4%B8%9A%E7%9A%84%E6%B5%81%E7%A8%8B%EF%BC%8C%E4%BB%A5%E5%8F%8A%E4%B8%8EYarn%E6%80%8E%E4%B9%88%E4%BA%A4%E4%BA%92\" title=\"3. Flink\u63d0\u4ea4\u4f5c\u4e1a\u7684\u6d41\u7a0b\uff0c\u4ee5\u53ca\u4e0eYarn\u600e\u4e48\u4ea4\u4e92?\">3. Flink\u63d0\u4ea4\u4f5c\u4e1a\u7684\u6d41\u7a0b\uff0c\u4ee5\u53ca\u4e0eYarn\u600e\u4e48\u4ea4\u4e92?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-43\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#4_Flink%E6%8F%90%E4%BA%A4Job%E7%9A%84%E6%96%B9%E5%BC%8F%E4%BB%A5%E5%8F%8A%E5%8F%82%E6%95%B0%E8%AE%BE%E7%BD%AE%EF%BC%9F\" title=\"4. Flink\u63d0\u4ea4Job\u7684\u65b9\u5f0f\u4ee5\u53ca\u53c2\u6570\u8bbe\u7f6e\uff1f\">4. Flink\u63d0\u4ea4Job\u7684\u65b9\u5f0f\u4ee5\u53ca\u53c2\u6570\u8bbe\u7f6e\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-44\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#5_Flink%E7%9A%84JobManger_%E6%9C%89%E5%A4%9A%E5%B0%91%E4%B8%AAJobManager\" title=\"5. Flink\u7684JobManger? \u6709\u591a\u5c11\u4e2aJobManager?\">5. Flink\u7684JobManger? \u6709\u591a\u5c11\u4e2aJobManager?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-45\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#JobManger\" title=\"JobManger\">JobManger<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-46\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#6_JobManger_%E5%9C%A8%E9%9B%86%E7%BE%A4%E5%90%AF%E5%8A%A8%E8%BF%87%E7%A8%8B%E4%B8%AD%E8%B5%B7%E5%88%B0%E4%BB%80%E4%B9%88%E4%BD%9C%E7%94%A8%EF%BC%9F\" title=\"6. JobManger \u5728\u96c6\u7fa4\u542f\u52a8\u8fc7\u7a0b\u4e2d\u8d77\u5230\u4ec0\u4e48\u4f5c\u7528\uff1f\">6. JobManger \u5728\u96c6\u7fa4\u542f\u52a8\u8fc7\u7a0b\u4e2d\u8d77\u5230\u4ec0\u4e48\u4f5c\u7528\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-47\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#7_Flink%E7%9A%84TaskManager\" title=\"7. Flink\u7684TaskManager?\">7. Flink\u7684TaskManager?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-48\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#8_%E8%AF%B4%E4%B8%80%E4%B8%8Bslot%EF%BC%8C%E4%B8%9A%E5%8A%A1%E4%B8%AD%E4%B8%80%E4%B8%AATaskManager%E8%AE%BE%E7%BD%AE%E5%87%A0%E4%B8%AAslot%EF%BC%9F\" title=\"8. \u8bf4\u4e00\u4e0bslot\uff0c\u4e1a\u52a1\u4e2d\u4e00\u4e2aTaskManager\u8bbe\u7f6e\u51e0\u4e2aslot\uff1f\">8. \u8bf4\u4e00\u4e0bslot\uff0c\u4e1a\u52a1\u4e2d\u4e00\u4e2aTaskManager\u8bbe\u7f6e\u51e0\u4e2aslot\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-49\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#9_Flink_%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F\" title=\"9. Flink \u7684\u5e76\u884c\u5ea6\uff1f\">9. Flink \u7684\u5e76\u884c\u5ea6\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-50\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#10_Flink_%E8%AE%A1%E7%AE%97%E8%B5%84%E6%BA%90%E7%9A%84%E8%B0%83%E5%BA%A6%E6%98%AF%E5%A6%82%E4%BD%95%E5%AE%9E%E7%8E%B0%E7%9A%84%EF%BC%9F\" title=\"10. Flink \u8ba1\u7b97\u8d44\u6e90\u7684\u8c03\u5ea6\u662f\u5982\u4f55\u5b9e\u73b0\u7684\uff1f\">10. Flink \u8ba1\u7b97\u8d44\u6e90\u7684\u8c03\u5ea6\u662f\u5982\u4f55\u5b9e\u73b0\u7684\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-51\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#11_Flink_%E7%9A%84_Slot_%E5%92%8C_Parallelism\" title=\"11. Flink \u7684 Slot \u548c Parallelism\">11. Flink \u7684 Slot \u548c Parallelism<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-52\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#12_Operator_Chains%E4%BA%86%E8%A7%A3%E5%90%97%EF%BC%9F\" title=\"12. Operator Chains\u4e86\u89e3\u5417\uff1f\">12. Operator Chains\u4e86\u89e3\u5417\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-53\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#13_Flink_%E7%9A%84%E8%BF%90%E8%A1%8C%E5%BF%85%E9%A1%BB%E4%BE%9D%E8%B5%96_Hadoop_%E7%BB%84%E4%BB%B6%E5%90%97%EF%BC%9F\" title=\"13. Flink \u7684\u8fd0\u884c\u5fc5\u987b\u4f9d\u8d56 Hadoop \u7ec4\u4ef6\u5417\uff1f\">13. Flink \u7684\u8fd0\u884c\u5fc5\u987b\u4f9d\u8d56 Hadoop \u7ec4\u4ef6\u5417\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-54\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#14_%E6%80%8E%E4%B9%88%E4%BF%AE%E6%94%B9%E6%AD%A3%E5%9C%A8%E8%BF%90%E8%A1%8C%E7%9A%84Flink%E7%A8%8B%E5%BA%8F%EF%BC%9F%E5%A6%82%E6%9E%9C%E6%9C%89%E6%96%B0%E7%9A%84%E5%AE%9E%E6%97%B6%E6%8C%87%E6%A0%87%E4%BD%A0%E4%BB%AC%E6%98%AF%E6%80%8E%E4%B9%88%E4%B8%8A%E7%BA%BF%E7%9A%84%EF%BC%9F\" title=\"14. \u600e\u4e48\u4fee\u6539\u6b63\u5728\u8fd0\u884c\u7684Flink\u7a0b\u5e8f\uff1f\u5982\u679c\u6709\u65b0\u7684\u5b9e\u65f6\u6307\u6807\u4f60\u4eec\u662f\u600e\u4e48\u4e0a\u7ebf\u7684\uff1f\">14. \u600e\u4e48\u4fee\u6539\u6b63\u5728\u8fd0\u884c\u7684Flink\u7a0b\u5e8f\uff1f\u5982\u679c\u6709\u65b0\u7684\u5b9e\u65f6\u6307\u6807\u4f60\u4eec\u662f\u600e\u4e48\u4e0a\u7ebf\u7684\uff1f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-55\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E4%BA%8C_%E7%8A%B6%E6%80%81%E7%BC%96%E7%A8%8B%E7%B3%BB%E5%88%97\" title=\"\u4e8c. \u72b6\u6001\u7f16\u7a0b\u7cfb\u5217\">\u4e8c. \u72b6\u6001\u7f16\u7a0b\u7cfb\u5217<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-56\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1_%E8%AF%B4%E4%B8%80%E4%B8%8B%E7%8A%B6%E6%80%81%E7%BC%96%E7%A8%8B%EF%BC%88operator_state%EF%BC%8Ckeyed_state%EF%BC%89\" title=\"1. \u8bf4\u4e00\u4e0b\u72b6\u6001\u7f16\u7a0b\uff08operator state\uff0ckeyed state\uff09\">1. \u8bf4\u4e00\u4e0b\u72b6\u6001\u7f16\u7a0b\uff08operator state\uff0ckeyed state\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-57\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E7%AE%97%E5%AD%90%E7%8A%B6%E6%80%81%EF%BC%88Operator_State%EF%BC%89%E5%92%8C%E6%8C%89%E9%94%AE%E5%88%86%E5%8C%BA%E7%8A%B6%E6%80%81\" title=\"\u7b97\u5b50\u72b6\u6001\uff08Operator State\uff09\u548c\u6309\u952e\u5206\u533a\u72b6\u6001\">\u7b97\u5b50\u72b6\u6001\uff08Operator State\uff09\u548c\u6309\u952e\u5206\u533a\u72b6\u6001<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-58\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2_10%E4%B8%AAint%E4%BB%A5%E6%95%B0%E7%BB%84%E7%9A%84%E5%BD%A2%E5%BC%8F%E4%BF%9D%E5%AD%98%E4%BF%9D%E5%AD%98%E5%9C%A8%E4%BB%80%E4%B9%88%E7%8A%B6%E6%80%81%E5%A5%BDVlaueState%E8%BF%98%E6%98%AFListState%E5%AD%98%E5%9C%A8%E5%93%AA%E4%B8%AA%E7%9A%84%E6%80%A7%E8%83%BD%E6%AF%94%E8%BE%83%E5%A5%BD\" title=\"2. 10\u4e2aint\u4ee5\u6570\u7ec4\u7684\u5f62\u5f0f\u4fdd\u5b58,\u4fdd\u5b58\u5728\u4ec0\u4e48\u72b6\u6001\u597d?VlaueState\u8fd8\u662fListState?\u5b58\u5728\u54ea\u4e2a\u7684\u6027\u80fd\u6bd4\u8f83\u597d?\">2. 10\u4e2aint\u4ee5\u6570\u7ec4\u7684\u5f62\u5f0f\u4fdd\u5b58,\u4fdd\u5b58\u5728\u4ec0\u4e48\u72b6\u6001\u597d?VlaueState\u8fd8\u662fListState?\u5b58\u5728\u54ea\u4e2a\u7684\u6027\u80fd\u6bd4\u8f83\u597d?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-59\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3_%E4%BD%BF%E7%94%A8MapStage%EF%BC%8Cgroup_by_id_%E5%A6%82%E4%BD%95%E8%AE%BE%E8%AE%A1%EF%BC%9Fid%E4%B8%8D%E6%94%BE%E5%9C%A8key%E8%A1%8C%E4%B8%8D%E8%A1%8C%EF%BC%9F\" title=\"3. \u4f7f\u7528MapStage\uff0cgroup by id \u5982\u4f55\u8bbe\u8ba1\uff1fid\u4e0d\u653e\u5728key\u884c\u4e0d\u884c\uff1f\">3. \u4f7f\u7528MapStage\uff0cgroup by id \u5982\u4f55\u8bbe\u8ba1\uff1fid\u4e0d\u653e\u5728key\u884c\u4e0d\u884c\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-60\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#4_Flink%E6%98%AF%E5%A6%82%E4%BD%95%E7%AE%A1%E7%90%86kafka%E7%9A%84offset%EF%BC%8C%E4%BD%BF%E7%94%A8%E4%BB%80%E4%B9%88%E7%B1%BB%E5%9E%8B%E7%9A%84%E7%8A%B6%E6%80%81%E4%BF%9D%E5%AD%98offset%EF%BC%9F\" title=\"4. Flink\u662f\u5982\u4f55\u7ba1\u7406kafka\u7684offset\uff0c\u4f7f\u7528\u4ec0\u4e48\u7c7b\u578b\u7684\u72b6\u6001\u4fdd\u5b58offset\uff1f\">4. Flink\u662f\u5982\u4f55\u7ba1\u7406kafka\u7684offset\uff0c\u4f7f\u7528\u4ec0\u4e48\u7c7b\u578b\u7684\u72b6\u6001\u4fdd\u5b58offset\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-61\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#5_%E4%B8%80%E4%B8%AA%E7%AA%97%E5%8F%A3%EF%BC%8C%E7%8E%B0%E5%9C%A8%E5%8F%AA%E5%8F%96%E7%AC%AC%E4%B8%80%E5%B8%A7%E5%92%8C%E6%9C%80%E5%90%8E%E4%B8%80%E5%B8%A7%EF%BC%8C%E6%80%8E%E4%B9%88%E5%81%9A%EF%BC%9F\" title=\"5. \u4e00\u4e2a\u7a97\u53e3\uff0c\u73b0\u5728\u53ea\u53d6\u7b2c\u4e00\u5e27\u548c\u6700\u540e\u4e00\u5e27\uff0c\u600e\u4e48\u505a\uff1f\">5. \u4e00\u4e2a\u7a97\u53e3\uff0c\u73b0\u5728\u53ea\u53d6\u7b2c\u4e00\u5e27\u548c\u6700\u540e\u4e00\u5e27\uff0c\u600e\u4e48\u505a\uff1f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-62\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E4%B8%89_%E5%8F%8D%E5%8E%8B%E4%B8%8E%E9%97%AE%E9%A2%98%E7%B3%BB%E5%88%97\" title=\"\u4e09. \u53cd\u538b\u4e0e\u95ee\u9898\u7cfb\u5217\">\u4e09. \u53cd\u538b\u4e0e\u95ee\u9898\u7cfb\u5217<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-63\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1_Flink%E7%94%A8%E4%BB%80%E4%B9%88%E7%9B%91%E6%8E%A7%EF%BC%8C%E7%9B%91%E6%8E%A7%E4%BB%80%E4%B9%88%EF%BC%9F%E5%A6%82%E4%BD%95%E6%9C%89%E6%95%88%E5%A4%84%E7%90%86%E6%95%B0%E6%8D%AE%E7%A7%AF%E5%8E%8B%EF%BC%9F\" title=\"1. Flink\u7528\u4ec0\u4e48\u76d1\u63a7\uff0c\u76d1\u63a7\u4ec0\u4e48\uff1f\u5982\u4f55\u6709\u6548\u5904\u7406\u6570\u636e\u79ef\u538b\uff1f\">1. Flink\u7528\u4ec0\u4e48\u76d1\u63a7\uff0c\u76d1\u63a7\u4ec0\u4e48\uff1f\u5982\u4f55\u6709\u6548\u5904\u7406\u6570\u636e\u79ef\u538b\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-64\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E8%83%8C%E5%8E%8B\" title=\"\u80cc\u538b\">\u80cc\u538b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-65\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2_%E9%81%87%E5%88%B0Flink%E4%B8%8D%E5%A4%AA%E8%83%BD%E8%A7%A3%E5%86%B3%E7%9A%84%E9%97%AE%E9%A2%98%EF%BC%9F%EF%BC%88PVUV%E6%94%BE%E5%86%85%E5%AD%98%EF%BC%8COOM%E4%BA%86%EF%BC%8C%E5%90%8E%E9%9D%A2%E9%85%8D%E5%90%88redis%E4%BB%A5%E5%8F%8A%E5%B8%83%E9%9A%86%E8%BF%87%E6%BB%A4%E5%99%A8%EF%BC%89\" title=\"2. \u9047\u5230Flink\u4e0d\u592a\u80fd\u89e3\u51b3\u7684\u95ee\u9898\uff1f\uff08PV,UV\u653e\u5185\u5b58\uff0cOOM\u4e86\uff0c\u540e\u9762\u914d\u5408redis\u4ee5\u53ca\u5e03\u9686\u8fc7\u6ee4\u5668\uff09\">2. \u9047\u5230Flink\u4e0d\u592a\u80fd\u89e3\u51b3\u7684\u95ee\u9898\uff1f\uff08PV,UV\u653e\u5185\u5b58\uff0cOOM\u4e86\uff0c\u540e\u9762\u914d\u5408redis\u4ee5\u53ca\u5e03\u9686\u8fc7\u6ee4\u5668\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-66\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3_%E4%BD%BF%E7%94%A8flink%E7%BB%9F%E8%AE%A1%E8%AE%A2%E5%8D%95%E8%A1%A8%E7%9A%84GMV%EF%BC%8C%E5%A6%82%E6%9E%9Cmysql%E4%B8%AD%E7%9A%84%E6%95%B0%E6%8D%AE%E5%87%BA%E7%8E%B0%E9%94%99%E8%AF%AF%EF%BC%8C%E4%B9%8B%E5%90%8E%E5%9C%A8mysql%E4%B8%AD%E5%81%9A%E6%95%B0%E6%8D%AE%E7%9A%84%E4%BF%AE%E6%94%B9%E6%93%8D%E4%BD%9C%EF%BC%8C%E9%82%A3%E4%B9%88flink%E7%A8%8B%E5%BA%8F%E5%A6%82%E4%BD%95%E4%BF%9D%E8%AF%81GMV%E7%9A%84%E6%AD%A3%E7%A1%AE%E6%80%A7%EF%BC%8C%E4%BD%A0%E4%BB%AC%E6%98%AF%E5%A6%82%E4%BD%95%E8%A7%A3%E5%86%B3%EF%BC%9F\" title=\"3. \u4f7f\u7528flink\u7edf\u8ba1\u8ba2\u5355\u8868\u7684GMV\uff0c\u5982\u679cmysql\u4e2d\u7684\u6570\u636e\u51fa\u73b0\u9519\u8bef\uff0c\u4e4b\u540e\u5728mysql\u4e2d\u505a\u6570\u636e\u7684\u4fee\u6539\u64cd\u4f5c\uff0c\u90a3\u4e48flink\u7a0b\u5e8f\u5982\u4f55\u4fdd\u8bc1GMV\u7684\u6b63\u786e\u6027\uff0c\u4f60\u4eec\u662f\u5982\u4f55\u89e3\u51b3\uff1f\">3. \u4f7f\u7528flink\u7edf\u8ba1\u8ba2\u5355\u8868\u7684GMV\uff0c\u5982\u679cmysql\u4e2d\u7684\u6570\u636e\u51fa\u73b0\u9519\u8bef\uff0c\u4e4b\u540e\u5728mysql\u4e2d\u505a\u6570\u636e\u7684\u4fee\u6539\u64cd\u4f5c\uff0c\u90a3\u4e48flink\u7a0b\u5e8f\u5982\u4f55\u4fdd\u8bc1GMV\u7684\u6b63\u786e\u6027\uff0c\u4f60\u4eec\u662f\u5982\u4f55\u89e3\u51b3\uff1f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-67\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E5%9B%9B_Spark_%E4%B8%8E_Flink_%E5%AF%B9%E6%AF%94\" title=\"\u56db. Spark \u4e0e Flink \u5bf9\u6bd4\">\u56db. Spark \u4e0e Flink \u5bf9\u6bd4<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-68\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1_Spark%E4%B8%8EFlink%E5%8C%BA%E5%88%AB\" title=\"1. Spark\u4e0eFlink\u533a\u522b\">1. Spark\u4e0eFlink\u533a\u522b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-69\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2_Flink%E7%9A%84key_By%E5%92%8CSpark%E7%9A%84_group_by%E6%9C%89%E4%BB%80%E4%B9%88%E5%8C%BA%E5%88%AB\" title=\"2. Flink\u7684key By\u548cSpark\u7684 group by\u6709\u4ec0\u4e48\u533a\u522b?\">2. Flink\u7684key By\u548cSpark\u7684 group by\u6709\u4ec0\u4e48\u533a\u522b?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-70\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3_%E4%B8%BA%E4%BB%80%E4%B9%88%E8%A6%81%E7%94%A8Flink%E6%9B%BF%E4%BB%A3SparkStreaming%EF%BC%9F\" title=\"3. \u4e3a\u4ec0\u4e48\u8981\u7528Flink\u66ff\u4ee3SparkStreaming\uff1f\">3. \u4e3a\u4ec0\u4e48\u8981\u7528Flink\u66ff\u4ee3SparkStreaming\uff1f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-71\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E4%BA%94_Checkpoint%E7%B3%BB%E5%88%97\" title=\"\u4e94. Checkpoint\u7cfb\u5217\">\u4e94. Checkpoint\u7cfb\u5217<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-72\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1_Flink_checkpoint%E7%9A%84%E5%AE%9E%E7%8E%B0%E5%8E%9F%E7%90%86\" title=\"1. Flink checkpoint\u7684\u5b9e\u73b0\u539f\u7406\">1. Flink checkpoint\u7684\u5b9e\u73b0\u539f\u7406<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-73\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2_Checkpoint_%E5%AD%98%E5%82%A8\" title=\"2. Checkpoint \u5b58\u50a8\">2. Checkpoint \u5b58\u50a8<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-74\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#4_Flink%E7%9A%84checkpoint%E6%9C%BA%E5%88%B6%E4%BB%A5%E5%8F%8A%E7%B2%BE%E5%87%86%E4%B8%80%E6%AC%A1%E6%80%A7%E6%B6%88%E8%B4%B9%E5%A6%82%E4%BD%95%E5%AE%9E%E7%8E%B0%EF%BC%9F\" title=\"4. Flink\u7684checkpoint\u673a\u5236\u4ee5\u53ca\u7cbe\u51c6\u4e00\u6b21\u6027\u6d88\u8d39\u5982\u4f55\u5b9e\u73b0\uff1f\">4. Flink\u7684checkpoint\u673a\u5236\u4ee5\u53ca\u7cbe\u51c6\u4e00\u6b21\u6027\u6d88\u8d39\u5982\u4f55\u5b9e\u73b0\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-75\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3_%E7%B2%BE%E7%A1%AE%E4%B8%80%E6%AC%A1%EF%BC%8C%E8%87%B3%E5%B0%91%E4%B8%80%E6%AC%A1%E5%AF%B9checkpoint%E6%9C%89%E4%BB%80%E4%B9%88%E5%BD%B1%E5%93%8D%EF%BC%9F\" title=\"3. \u7cbe\u786e\u4e00\u6b21\uff0c\u81f3\u5c11\u4e00\u6b21\u5bf9checkpoint\u6709\u4ec0\u4e48\u5f71\u54cd\uff1f\">3. \u7cbe\u786e\u4e00\u6b21\uff0c\u81f3\u5c11\u4e00\u6b21\u5bf9checkpoint\u6709\u4ec0\u4e48\u5f71\u54cd\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-76\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#4_Savapoint%E4%BA%86%E8%A7%A3%E5%A4%9A%E5%B0%91%EF%BC%9F\" title=\"4. Savapoint\u4e86\u89e3\u591a\u5c11\uff1f\">4. Savapoint\u4e86\u89e3\u591a\u5c11\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-77\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#5_Flink_checkpoint%E7%9A%84%E8%B6%85%E6%97%B6%E9%97%AE%E9%A2%98_%E5%A6%82%E4%BD%95%E8%A7%A3%E5%86%B3\" title=\"5. Flink checkpoint\u7684\u8d85\u65f6\u95ee\u9898 \u5982\u4f55\u89e3\u51b3?\">5. Flink checkpoint\u7684\u8d85\u65f6\u95ee\u9898 \u5982\u4f55\u89e3\u51b3?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-78\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#6_%E4%BD%9C%E4%B8%9A%E6%8C%82%E6%8E%89%E4%BA%86%EF%BC%8C%E6%81%A2%E5%A4%8D%E4%B8%8A%E4%B8%80%E4%B8%AACheckpoint%EF%BC%8C%E7%94%A8%E4%BB%80%E4%B9%88%E5%91%BD%E4%BB%A4%EF%BC%9F\" title=\"6. \u4f5c\u4e1a\u6302\u6389\u4e86\uff0c\u6062\u590d\u4e0a\u4e00\u4e2aCheckpoint\uff0c\u7528\u4ec0\u4e48\u547d\u4ee4\uff1f\">6. \u4f5c\u4e1a\u6302\u6389\u4e86\uff0c\u6062\u590d\u4e0a\u4e00\u4e2aCheckpoint\uff0c\u7528\u4ec0\u4e48\u547d\u4ee4\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-79\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#7_%E4%BB%80%E4%B9%88%E6%98%AFFlink%E7%9A%84%E9%9D%9Ebarrier%E5%AF%B9%E9%BD%90%EF%BC%8C%E5%A6%82%E4%BD%95%E5%AE%9E%E7%8E%B0%EF%BC%9F\" title=\"7. \u4ec0\u4e48\u662fFlink\u7684\u975ebarrier\u5bf9\u9f50\uff0c\u5982\u4f55\u5b9e\u73b0\uff1f\">7. \u4ec0\u4e48\u662fFlink\u7684\u975ebarrier\u5bf9\u9f50\uff0c\u5982\u4f55\u5b9e\u73b0\uff1f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-80\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E5%85%AD_%E7%AA%97%E5%8F%A3_%E4%B8%8E_Watermark%E7%B3%BB%E5%88%97\" title=\"\u516d. \u7a97\u53e3 \u4e0e Watermark\u7cfb\u5217\">\u516d. \u7a97\u53e3 \u4e0e Watermark\u7cfb\u5217<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-81\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1_Flink%E6%97%B6%E9%97%B4%E8%AF%AD%E4%B9%89\" title=\"1. Flink\u65f6\u95f4\u8bed\u4e49\">1. Flink\u65f6\u95f4\u8bed\u4e49<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-82\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2_%E4%BB%80%E4%B9%88%E6%98%AFWatermark%E5%8F%8A%E4%B8%BB%E8%A6%81%E4%BD%9C%E7%94%A8%EF%BC%9F%E4%BB%80%E4%B9%88%E6%97%B6%E5%80%99%E5%8E%BB%E8%A7%A6%E5%8F%91%E8%AE%A1%E7%AE%97\" title=\"2. \u4ec0\u4e48\u662fWatermark\u53ca\u4e3b\u8981\u4f5c\u7528\uff1f\u4ec0\u4e48\u65f6\u5019\u53bb\u89e6\u53d1\u8ba1\u7b97?\">2. \u4ec0\u4e48\u662fWatermark\u53ca\u4e3b\u8981\u4f5c\u7528\uff1f\u4ec0\u4e48\u65f6\u5019\u53bb\u89e6\u53d1\u8ba1\u7b97?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-83\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3_%E6%B6%88%E6%81%AF%E8%B6%85%E8%BF%87watermark%E7%9A%84%E6%97%B6%E9%97%B4%E4%BC%9A%E4%B8%A2%E5%A4%B1%E6%95%B0%E6%8D%AE%E5%90%97\" title=\"3. \u6d88\u606f\u8d85\u8fc7watermark\u7684\u65f6\u95f4\u4f1a\u4e22\u5931\u6570\u636e\u5417?\">3. \u6d88\u606f\u8d85\u8fc7watermark\u7684\u65f6\u95f4\u4f1a\u4e22\u5931\u6570\u636e\u5417?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-84\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#4_%E5%BC%80%E7%AA%97%E5%87%BD%E6%95%B0%E6%9C%89%E5%93%AA%E4%BA%9B%EF%BC%9F\" title=\"4. \u5f00\u7a97\u51fd\u6570\u6709\u54ea\u4e9b\uff1f\">4. \u5f00\u7a97\u51fd\u6570\u6709\u54ea\u4e9b\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-85\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#5_%E6%B2%A1%E6%9C%89%E6%95%B0%E6%8D%AE%E6%B5%81%E7%9A%84%E6%97%B6%E5%80%99%E7%AA%97%E5%8F%A3%E8%BF%99%E4%B8%AA%E7%AA%97%E5%8F%A3%E5%AD%98%E5%9C%A8%E5%90%97\" title=\"5. (\u6ca1\u6709\u6570\u636e\u6d41\u7684\u65f6\u5019)\u7a97\u53e3,\u8fd9\u4e2a\u7a97\u53e3\u5b58\u5728\u5417?\">5. (\u6ca1\u6709\u6570\u636e\u6d41\u7684\u65f6\u5019)\u7a97\u53e3,\u8fd9\u4e2a\u7a97\u53e3\u5b58\u5728\u5417?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-86\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#6_1%E5%B0%8F%E6%97%B6%E7%9A%84%E6%BB%9A%E5%8A%A8%E7%AA%97%E5%8F%A3%E4%B8%80%E5%B0%8F%E6%97%B6%E5%A4%84%E7%90%86%E4%B8%80%E6%AC%A1%E7%9A%84%E5%8E%8B%E5%8A%9B%E6%AF%94%E8%BE%83%E5%A4%A7%E6%83%B3%E8%AE%A9%E4%BB%965%E5%88%86%E9%92%9F%E5%A4%84%E7%90%86%E4%B8%80%E6%AC%A1%E6%80%8E%E4%B9%88%E5%8A%9E\" title=\"6. 1\u5c0f\u65f6\u7684\u6eda\u52a8\u7a97\u53e3,\u4e00\u5c0f\u65f6\u5904\u7406\u4e00\u6b21\u7684\u538b\u529b\u6bd4\u8f83\u5927,\u60f3\u8ba9\u4ed65\u5206\u949f\u5904\u7406\u4e00\u6b21.\u600e\u4e48\u529e?\">6. 1\u5c0f\u65f6\u7684\u6eda\u52a8\u7a97\u53e3,\u4e00\u5c0f\u65f6\u5904\u7406\u4e00\u6b21\u7684\u538b\u529b\u6bd4\u8f83\u5927,\u60f3\u8ba9\u4ed65\u5206\u949f\u5904\u7406\u4e00\u6b21.\u600e\u4e48\u529e?<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-87\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E4%B8%83_Join%E7%B3%BB%E5%88%97\" title=\"\u4e03. Join\u7cfb\u5217\">\u4e03. Join\u7cfb\u5217<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-88\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1_Flink%E7%9A%84%E5%8F%8C%E6%B5%81join%E7%9A%84%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86\" title=\"1. Flink\u7684\u53cc\u6d41join\u7684\u5e95\u5c42\u539f\u7406?\">1. Flink\u7684\u53cc\u6d41join\u7684\u5e95\u5c42\u539f\u7406?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-89\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E5%9F%BA%E4%BA%8EConnect%E7%9A%84%E5%8F%8C%E6%B5%81JOIN%E5%AE%9E%E7%8E%B0%E6%9C%BA%E5%88%B6\" title=\"\u57fa\u4e8eConnect\u7684\u53cc\u6d41JOIN\u5b9e\u73b0\u673a\u5236\">\u57fa\u4e8eConnect\u7684\u53cc\u6d41JOIN\u5b9e\u73b0\u673a\u5236<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-90\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E5%9F%BA%E4%BA%8EWindow_Join%E7%9A%84%E5%8F%8C%E6%B5%81JOIN%E5%AE%9E%E7%8E%B0%E6%9C%BA%E5%88%B6\" title=\"\u57fa\u4e8eWindow Join\u7684\u53cc\u6d41JOIN\u5b9e\u73b0\u673a\u5236\">\u57fa\u4e8eWindow Join\u7684\u53cc\u6d41JOIN\u5b9e\u73b0\u673a\u5236<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-91\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#coGroup%E7%AE%97%E5%AD%90\" title=\"coGroup\u7b97\u5b50\">coGroup\u7b97\u5b50<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-92\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2_A%E8%A1%A8left_join_B%E8%A1%A8\" title=\"2. A\u8868left join B\u8868\">2. A\u8868left join B\u8868<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-93\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3_Flink_%E7%BB%B4%E8%A1%A8%E5%85%B3%E8%81%94%E6%80%8E%E4%B9%88%E5%81%9A%E7%9A%84%EF%BC%9F\" title=\"3. Flink \u7ef4\u8868\u5173\u8054\u600e\u4e48\u505a\u7684\uff1f\">3. Flink \u7ef4\u8868\u5173\u8054\u600e\u4e48\u505a\u7684\uff1f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-94\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E5%85%AB_%E5%85%B6%E4%BB%96\" title=\"\u516b. \u5176\u4ed6\">\u516b. \u5176\u4ed6<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-95\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#1_process%E7%94%A8%E7%9A%84%E7%A7%8D%E7%B1%BB%EF%BC%9F\" title=\"1. process\u7528\u7684\u79cd\u7c7b\uff1f\">1. process\u7528\u7684\u79cd\u7c7b\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-96\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#2_Flink%E7%9A%84%E5%86%85%E5%AD%98%E7%AE%A1%E7%90%86%EF%BC%9F\" title=\"2. Flink\u7684\u5185\u5b58\u7ba1\u7406\uff1f\">2. Flink\u7684\u5185\u5b58\u7ba1\u7406\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-97\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#JobManager_%E5%86%85%E5%AD%98%E6%A8%A1%E5%9E%8B\" title=\"JobManager \u5185\u5b58\u6a21\u578b\">JobManager \u5185\u5b58\u6a21\u578b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-98\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#TaskManager_%E5%86%85%E5%AD%98%E6%A8%A1%E5%9E%8B\" title=\"TaskManager \u5185\u5b58\u6a21\u578b\">TaskManager \u5185\u5b58\u6a21\u578b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-99\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E6%80%BB%E4%BD%93%E5%86%85%E5%AD%98\" title=\"\u603b\u4f53\u5185\u5b58\">\u603b\u4f53\u5185\u5b58<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-100\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#JVM_Heap_JVM_%E5%A0%86%E4%B8%8A%E5%86%85%E5%AD%98\" title=\"JVM Heap (JVM \u5806\u4e0a\u5185\u5b58)\">JVM Heap (JVM \u5806\u4e0a\u5185\u5b58)<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-101\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#Off-Heap_MempryJVM_%E5%A0%86%E5%A4%96%E5%86%85%E5%AD%98\" title=\"Off-Heap Mempry(JVM \u5806\u5916\u5185\u5b58)\">Off-Heap Mempry(JVM \u5806\u5916\u5185\u5b58)<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-102\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#3_Flink%E7%9A%84%E5%BA%8F%E5%88%97%E5%8C%96%E6%9C%BA%E5%88%B6%EF%BC%9F\" title=\"3. Flink\u7684\u5e8f\u5217\u5316\u673a\u5236\uff1f\">3. Flink\u7684\u5e8f\u5217\u5316\u673a\u5236\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-103\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#4_Rich_Functions%E4%B8%8EFunctions%E5%8C%BA%E5%88%AB\" title=\"4. Rich Functions\u4e0eFunctions\u533a\u522b?\">4. Rich Functions\u4e0eFunctions\u533a\u522b?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-104\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#5_Flink_%E5%88%86%E5%8C%BA%E7%AD%96%E7%95%A5%EF%BC%9F\" title=\"5. Flink \u5206\u533a\u7b56\u7565\uff1f\">5. Flink \u5206\u533a\u7b56\u7565\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-105\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#6_Flink_%E4%B8%AD%E7%9A%84%E5%88%86%E5%B8%83%E5%BC%8F%E7%BC%93%E5%AD%98\" title=\"6. Flink \u4e2d\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\">6. Flink \u4e2d\u7684\u5206\u5e03\u5f0f\u7f13\u5b58<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-106\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#7_Flink%E9%87%8C%E9%9D%A2%E5%BC%82%E6%AD%A5IO%E4%BB%A3%E7%A0%81%E5%85%B7%E4%BD%93%E6%80%8E%E4%B9%88%E5%86%99%E7%9A%84\" title=\"7. Flink\u91cc\u9762\u5f02\u6b65IO\u4ee3\u7801\u5177\u4f53\u600e\u4e48\u5199\u7684?\">7. Flink\u91cc\u9762\u5f02\u6b65IO\u4ee3\u7801\u5177\u4f53\u600e\u4e48\u5199\u7684?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-107\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#8_Flink%E7%9A%84%E9%87%8D%E5%90%AF%E7%AD%96%E7%95%A5\" title=\"8. Flink\u7684\u91cd\u542f\u7b56\u7565\">8. Flink\u7684\u91cd\u542f\u7b56\u7565<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-108\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#9_Flink_%E6%95%B0%E6%8D%AE%E5%80%BE%E6%96%9C\" title=\"9. Flink \u6570\u636e\u503e\u659c\">9. Flink \u6570\u636e\u503e\u659c<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-109\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/#%E6%96%B9%E6%A1%88\" title=\"\u65b9\u6848\">\u65b9\u6848<\/a><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1%E5%89%8D%E8%A8%80\"><\/span>1.\u524d\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u672c\u6587\u4e3b\u8981\u662f\u6574\u7406\u535a\u4e3b\u6536\u96c6\u7684 Flink \u9ad8\u9891\u9762\u8bd5\u9898\u3002\u4e4b\u540e<strong>\u6bcf\u5468<\/strong>\u90fd\u4f1a\u6709\u4e00\u7bc7\u3002<\/p>\n\n\n\n<p>\u5176\u4e2d\u4e3b\u8981\u5212\u5206\u4e3a\u4e00\u4e0b 4 \u5927\u4e3b\u9898\uff0c\u9996\u5148\u662f\u524d\u4e24\u4e2a&nbsp;<strong>\u72b6\u6001\u539f\u7406<\/strong>\u3001<strong>\u65f6\u95f4\u7a97\u53e3<\/strong>&nbsp;\u662f\u7528\u4e8e\u8003\u6838\u5019\u9009\u4eba\u5bf9\u4e8e Flink \u57fa\u672c\u539f\u7406\u7684\u7406\u89e3\uff0c<strong>\u7f16\u7a0b\u6280\u5de7\u3001\u5b9e\u6218\u7ecf\u9a8c<\/strong>&nbsp;\u4e3b\u8981\u662f\u8003\u6838\u5019\u9009\u4eba\u4f7f\u7528 Flink \u7684\u7ecf\u9a8c\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic2.zhimg.com\/80\/v2-3e7852146ca46eaa3f65a0b8a58a10e1_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>1<\/p>\n\n\n\n<p>\u5185\u5bb9\u8f83\u591a\uff0c\u5e76\u4e14\u6bd4\u8f83\u8be6\u7ec6\uff0c<strong>\u5efa\u8bae\u83b7\u53d6 PDF \u8d44\u6599<\/strong>\uff0c\u516c\u4f17\u53f7\u52a0\u535a\u4e3b\u5fae\u4fe1\u83b7\u53d6\u3002\u6b22\u8fce\u5927\u5bb6\u6307\u6b63\u9519\u8bef\u4e0e\u4e0d\u8db3\u4e4b\u5904\uff0c\u4e5f\u6b22\u8fce\u5927\u5bb6\u7ee7\u7eed\u8865\u5145\u6b64\u5185\u5bb9\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2%E7%8A%B6%E6%80%81%E5%8E%9F%E7%90%86\"><\/span>2.\u72b6\u6001\u539f\u7406<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"21%E7%8A%B6%E6%80%81%E3%80%81%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%E3%80%81Checkpoint_%E4%B8%89%E8%80%85%E4%B9%8B%E9%97%B4%E7%9A%84%E5%8C%BA%E5%88%AB%E5%8F%8A%E5%85%B3%E7%B3%BB%EF%BC%9F\"><\/span>2.1.\u72b6\u6001\u3001\u72b6\u6001\u540e\u7aef\u3001Checkpoint \u4e09\u8005\u4e4b\u95f4\u7684\u533a\u522b\u53ca\u5173\u7cfb\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p><strong>\u7ed3\u8bba\uff1a\u62ff\u4e94\u4e2a\u5b57\u505a\u6bd4\u55bb\uff1a&#8221;\u94c1\u9505\u7096\u5927\u9e45&#8221;\uff0c\u94c1\u9505\u662f\u72b6\u6001\u540e\u7aef\uff0c\u5927\u9e45\u662f\u72b6\u6001\uff0cCheckpoint \u662f\u7096\u7684\u52a8\u4f5c\u3002<\/strong><\/p>\n\n\n\n<ol>\n<li>\u2b50 \u72b6\u6001\uff1a\u672c\u8d28\u6765\u8bf4\u5c31\u662f\u6570\u636e\uff0c\u5728 Flink \u4e2d\uff0c\u5176\u5b9e\u5c31\u662f Flink \u63d0\u4f9b\u7ed9\u7528\u6237\u7684\u72b6\u6001\u7f16\u7a0b\u63a5\u53e3\u3002\u6bd4\u5982 flink \u4e2d\u7684 MapState\uff0cValueState\uff0cListState\u3002<br><\/li>\n\n\n\n<li>\u2b50 \u72b6\u6001\u540e\u7aef\uff1aFlink \u63d0\u4f9b\u7684\u7528\u4e8e\u7ba1\u7406\u72b6\u6001\u7684\u7ec4\u4ef6\uff0c\u72b6\u6001\u540e\u7aef\u51b3\u5b9a\u4e86\u4ee5\u4ec0\u4e48\u6837\u6570\u636e\u7ed3\u6784\uff0c\u4ec0\u4e48\u6837\u7684\u5b58\u50a8\u65b9\u5f0f\u53bb\u5b58\u50a8\u548c\u7ba1\u7406\u6211\u4eec\u7684\u72b6\u6001\u3002Flink \u76ee\u524d\u5b98\u65b9\u63d0\u4f9b\u4e86 memory\u3001filesystem\uff0crocksdb \u4e09\u79cd\u72b6\u6001\u540e\u7aef\u6765\u5b58\u50a8\u6211\u4eec\u7684\u72b6\u6001\u3002<br><\/li>\n\n\n\n<li>\u2b50 Checkpoint\uff08\u72b6\u6001\u7ba1\u7406\uff09\uff1aFlink \u63d0\u4f9b\u7684\u7528\u4e8e\u5b9a\u65f6\u5c06\u72b6\u6001\u540e\u7aef\u4e2d\u5b58\u50a8\u7684\u72b6\u6001\u540c\u6b65\u5230\u8fdc\u7a0b\u7684\u5b58\u50a8\u7cfb\u7edf\u7684\u7ec4\u4ef6\u6216\u8005\u80fd\u529b\u3002\u4e3a\u4e86\u9632\u6b62 long run \u7684 Flink \u4efb\u52a1\u6302\u4e86\u5bfc\u81f4\u72b6\u6001\u4e22\u5931\uff0c\u4ea7\u751f\u6570\u636e\u8d28\u91cf\u95ee\u9898\uff0cFlink \u63d0\u4f9b\u4e86\u72b6\u6001\u7ba1\u7406\uff08Checkpoint\uff0cSavepoint\uff09\u7684\u80fd\u529b\u628a\u6211\u4eec\u4f7f\u7528\u7684\u72b6\u6001\u7ed9\u7ba1\u7406\u8d77\u6765\uff0c\u5b9a\u65f6\u7684\u4fdd\u5b58\u5230\u8fdc\u7a0b\u3002\u7136\u540e\u53ef\u4ee5\u5728 Flink \u4efb\u52a1 failover \u65f6\uff0c\u4ece\u8fdc\u7a0b\u628a\u72b6\u6001\u6570\u636e\u6062\u590d\u5230 Flink \u4efb\u52a1\u4e2d\uff0c\u4fdd\u969c\u6570\u636e\u8d28\u91cf\u3002<br><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"22%E6%8A%8A%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%E4%BB%8E_FileSystem_%E5%8F%98%E4%B8%BA_RocksDB_%E5%90%8E%EF%BC%8CFlink_%E4%BB%BB%E5%8A%A1%E7%8A%B6%E6%80%81%E5%AD%98%E5%82%A8%E4%BC%9A%E5%8F%91%E7%94%9F%E9%82%A3%E4%BA%9B%E5%8F%98%E5%8C%96%EF%BC%9F\"><\/span>2.2.\u628a\u72b6\u6001\u540e\u7aef\u4ece FileSystem \u53d8\u4e3a RocksDB \u540e\uff0cFlink \u4efb\u52a1\u72b6\u6001\u5b58\u50a8\u4f1a\u53d1\u751f\u90a3\u4e9b\u53d8\u5316\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p><strong>\u7ed3\u8bba\uff1a\u662f\u5426\u4f7f\u7528 RocksDB \u53ea\u4f1a\u5f71\u54cd Flink \u4efb\u52a1\u4e2d keyed-state \u5b58\u50a8\u7684\u65b9\u5f0f\u548c\u5730\u65b9\uff0cFlink \u4efb\u52a1\u4e2d\u7684 operator-state \u4e0d\u4f1a\u53d7\u5230\u5f71\u54cd\u3002<\/strong><\/p>\n\n\n\n<p>\u9996\u5148\u6211\u4eec\u6765\u770b\u770b\uff0cFlink \u4e2d\u7684\u72b6\u6001\u53ea\u4f1a\u5206\u4e3a\u4e24\u7c7b\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 keyed-state\uff1a\u952e\u503c\u72b6\u6001\uff0c\u5982\u5176\u540d\u5b57\uff0c\u6b64\u7c7b\u72b6\u6001\u662f\u4ee5 k-v \u7684\u5f62\u5f0f\u5b58\u50a8\uff0c\u72b6\u6001\u503c\u548c key \u7ed1\u5b9a\u3002Flink \u4e2d\u7684 keyby \u4e4b\u540e\u7d27\u8ddf\u7684\u7b97\u5b50\u7684 state \u5c31\u662f\u952e\u503c\u72b6\u6001\uff1b<\/li>\n\n\n\n<li>\u2b50 operator-state\uff1a\u7b97\u5b50\u72b6\u6001\uff0c\u975e keyed-state \u7684 state \u90fd\u662f\u7b97\u5b50\u72b6\u6001\uff0c\u975e k-v \u7ed3\u6784\uff0c\u72b6\u6001\u503c\u548c\u7b97\u5b50\u7ed1\u5b9a\uff0c\u4e0d\u548c key \u7ed1\u5b9a\u3002Flink \u4e2d\u7684 kafka source \u7b97\u5b50\u4e2d\u7528\u4e8e\u5b58\u50a8 kafka offset \u7684 state \u5c31\u662f\u7b97\u5b50\u72b6\u6001\u3002<\/li>\n<\/ol>\n\n\n\n<p>\u5982\u4e0b\u56fe\u6240\u793a\u662f 3 \u79cd\u72b6\u6001\u540e\u7aef\u548c 2 \u79cd State \u7684\u5bf9\u5e94\u5b58\u50a8\u5173\u7cfb\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic3.zhimg.com\/80\/v2-bb5636c2df2df2e8321dabc686ff88ae_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>2<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u6a2a\u5411\uff08\u884c\uff09\u6765\u770b\uff0c\u5373 Flink \u7684\u72b6\u6001\u5206\u7c7b\u3002\u5206\u4e3a Operator state-backend\u3001Keyed state-backend\uff1b<\/li>\n\n\n\n<li>\u2b50 \u7eb5\u5411\uff08\u5217\uff09\u6765\u770b\uff0c\u5373 Flink \u7684\u72b6\u6001\u540e\u7aef\u5206\u7c7b\u3002\u7528\u6237\u53ef\u4ee5\u914d\u7f6e memory\uff0cfilesystem\uff0crocksdb 3 \u4e2d\u72b6\u6001\u540e\u7aef\uff0c\u5728 Flink \u4efb\u52a1\u4e2d\u751f\u6210 MemoryStateBackend\uff0cFsStateBackend\uff0cRocksdbStateBackend\uff0c\u5176\u58f0\u660e\u4e86\u6574\u4e2a\u4efb\u52a1\u7684\u72b6\u6001\u7ba1\u7406\u540e\u7aef\u7c7b\u578b\uff1b<\/li>\n\n\n\n<li>\u2b50 \u6bcf\u4e2a\u683c\u5b50\u4e2d\u7684\u5185\u5bb9\u5c31\u662f\u7528\u6237\u5728\u914d\u7f6e xx \u72b6\u6001\u540e\u7aef\uff08\u5217\uff09\u65f6\uff0c\u7ed9\u7528\u6237\u4f7f\u7528\u7684\u72b6\u6001\uff08\u884c\uff09\u751f\u6210\u7684\u72b6\u6001\u540e\u7aef\u5b9e\u4f8b\uff0c\u751f\u6210\u7684\u8fd9\u4e2a\u5b9e\u4f8b\u5c31\u662f\u5728 Flink \u4e2d\u5b9e\u9645\u7528\u4e8e\u7ba1\u7406\u7528\u6237\u4f7f\u7528\u7684\u72b6\u6001\u7684\u7ec4\u4ef6\u3002<\/li>\n<\/ol>\n\n\n\n<p>\u56e0\u6b64\u5bf9\u5e94\u7684\u7ed3\u8bba\u5c31\u662f\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 Flink \u4efb\u52a1\u4e2d\u7684 operator-state\u3002\u65e0\u8bba\u7528\u6237\u914d\u7f6e\u54ea\u79cd\u72b6\u6001\u540e\u7aef\uff08\u65e0\u8bba\u662f memory\uff0cfilesystem\uff0crocksdb\uff09\uff0c\u90fd\u662f\u4f7f\u7528 DefaultOperatorStateBackend \u6765\u7ba1\u7406\u7684\uff0c\u72b6\u6001\u6570\u636e\u90fd\u5b58\u50a8\u5728\u5185\u5b58\u4e2d\uff0c\u505a Checkpoint \u65f6\u540c\u6b65\u5230\u8fdc\u7a0b\u6587\u4ef6\u5b58\u50a8\u4e2d\uff08\u6bd4\u5982 HDFS\uff09\u3002<\/li>\n\n\n\n<li>\u2b50 Flink \u4efb\u52a1\u4e2d\u7684 keyed-state\u3002<strong>\u7528\u6237\u5728\u914d\u7f6e rocksdb \u65f6\uff0c\u4f1a\u4f7f\u7528 RocksdbKeyedStateBackend \u53bb\u7ba1\u7406\u72b6\u6001\uff1b\u7528\u6237\u5728\u914d\u7f6e memory\uff0cfilesystem \u65f6\uff0c\u4f1a\u4f7f\u7528 HeapKeyedStateBackend \u53bb\u7ba1\u7406\u72b6\u6001\u3002\u56e0\u6b64\u5c31\u6709\u4e86\u8fd9\u4e2a\u95ee\u9898\u7684\u7ed3\u8bba\uff0c\u914d\u7f6e rocksdb \u53ea\u4f1a\u5f71\u54cd keyed-state \u5b58\u50a8\u7684\u65b9\u5f0f\u548c\u5730\u65b9\uff0coperator-state \u4e0d\u4f1a\u53d7\u5230\u5f71\u54cd\u3002<\/strong><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"23%E4%BB%80%E4%B9%88%E6%A0%B7%E7%9A%84%E4%B8%9A%E5%8A%A1%E5%9C%BA%E6%99%AF%E4%BD%A0%E4%BC%9A%E9%80%89%E6%8B%A9_filesystem%EF%BC%8C%E4%BB%80%E4%B9%88%E6%A0%B7%E7%9A%84%E4%B8%9A%E5%8A%A1%E5%9C%BA%E6%99%AF%E4%BD%A0%E4%BC%9A%E9%80%89_rocksdb_%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%EF%BC%9F\"><\/span>2.3.\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009\u62e9 filesystem\uff0c\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009 rocksdb \u72b6\u6001\u540e\u7aef\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u5728\u56de\u7b54\u8fd9\u4e2a\u95ee\u9898\u524d\uff0c\u6211\u4eec\u5148\u770b\u770b\u6bcf\u79cd\u72b6\u6001\u540e\u7aef\u7684\u7279\u6027\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 MemoryStateBackend<\/li>\n<\/ol>\n\n\n\n<ul>\n<li>\u539f\u7406\uff1a\u8fd0\u884c\u65f6\u6240\u9700\u7684 State \u6570\u636e\u5168\u90e8\u4fdd\u5b58\u5728 TaskManager JVM \u5806\u4e0a\u5185\u5b58\u4e2d\uff0c\u6267\u884c Checkpoint \u7684\u65f6\u5019\uff0c\u4f1a\u628a State \u7684\u5feb\u7167\u6570\u636e\u4fdd\u5b58\u5230 JobManager \u8fdb\u7a0b \u7684\u5185\u5b58\u4e2d\u3002\u6267\u884c Savepoint \u65f6\uff0c\u53ef\u4ee5\u628a State \u5b58\u50a8\u5230\u6587\u4ef6\u7cfb\u7edf\u4e2d\u3002<br><\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic3.zhimg.com\/80\/v2-9afd973aae2e1ab14a5a6ba771fb04ce_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<ul>\n<li>\u9002\u7528\u573a\u666f\uff1a<br><\/li>\n\n\n\n<li>\u57fa\u4e8e\u5185\u5b58\u7684 StateBackend \u5728\u751f\u4ea7\u73af\u5883\u4e0b\u4e0d\u5efa\u8bae\u4f7f\u7528\uff0c\u56e0\u4e3a State \u5927\u5c0f\u8d85\u8fc7 JobManager \u5185\u5b58\u5c31 OOM \u4e86\uff0c<strong>\u6b64\u79cd\u72b6\u6001\u540e\u7aef\u9002\u5408\u5728\u672c\u5730\u5f00\u53d1\u8c03\u8bd5\u6d4b\u8bd5\uff0c\u751f\u4ea7\u73af\u5883\u57fa\u672c\u4e0d\u7528\u3002<\/strong><br><\/li>\n\n\n\n<li>State \u5b58\u50a8\u5728 JobManager \u7684\u5185\u5b58\u4e2d\u3002\u53d7\u9650\u4e8e JobManager \u7684\u5185\u5b58\u5927\u5c0f\u3002<br><\/li>\n\n\n\n<li>\u6bcf\u4e2a State \u9ed8\u8ba4 5MB,\u53ef\u901a\u8fc7 MemoryStateBackend \u6784\u9020\u51fd\u6570\u8c03\u6574\u3002d.\u6bcf\u4e2a Stale \u4e0d\u80fd\u8d85\u8fc7 Akka Frame \u5927\u5c0f\u3002<br><\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 FSStateBackend<\/li>\n<\/ol>\n\n\n\n<ul>\n<li>\u539f\u7406\uff1a\u8fd0\u884c\u65f6\u6240\u9700\u7684 State \u6570\u636e\u5168\u90e8\u4fdd\u5b58\u5728 TaskManager \u7684\u5185\u5b58\u4e2d\uff0c\u6267\u884c Checkpoint \u7684\u65f6\u5019\uff0c\u4f1a\u628a State \u7684\u5feb\u7167\u6570\u636e\u4fdd\u5b58\u5230\u914d\u7f6e\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\u3002TM \u662f\u5f02\u6b65\u5c06 State \u6570\u636e\u5199\u5165\u5916\u90e8\u5b58\u50a8\u3002<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-426de4db17f49a98d2ce8781a9e97e2f_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<ul>\n<li>\u9002\u7528\u573a\u666f\uff1a<\/li>\n\n\n\n<li>a.\u9002\u7528\u4e8e<strong>\u5904\u7406\u5c0f\u72b6\u6001\u3001\u77ed\u7a97\u53e3\u3001\u6216\u8005\u5c0f\u952e\u503c\u72b6\u6001\u7684\u6709\u72b6\u6001\u5904\u7406\u4efb\u52a1<\/strong>\uff0c<strong>\u4e0d\u5efa\u8bae\u5728\u5927\u72b6\u6001\u7684\u4efb\u52a1\u4e0b\u4f7f\u7528 FSStateBackend<\/strong>\u3002\u6bd4\u5982 ETL \u4efb\u52a1\uff0c\u5c0f\u65f6\u95f4\u95f4\u9694\u7684 TUMBLE \u7a97\u53e3<br>b.State \u5927\u5c0f\u4e0d\u80fd\u8d85\u8fc7 TM \u5185\u5b58\u3002<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 RocksDBStateBackend<\/li>\n<\/ol>\n\n\n\n<ul>\n<li>\u539f\u7406\uff1a\u4f7f\u7528\u5d4c\u5165\u5f0f\u7684\u672c\u5730\u6570\u636e\u5e93 RocksDB \u5c06\u6d41\u8ba1\u7b97\u6570\u636e\u72b6\u6001\u5b58\u50a8\u5728\u672c\u5730\u78c1\u76d8\u4e2d\u3002\u5728\u6267\u884c Checkpoint \u7684\u65f6\u5019\uff0c\u4f1a\u5c06\u6574\u4e2a RocksDB \u4e2d\u4fdd\u5b58\u7684 State \u6570\u636e\u5168\u91cf\u6216\u8005\u589e\u91cf\u6301\u4e45\u5316\u5230\u914d\u7f6e\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\u3002<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic2.zhimg.com\/80\/v2-f19faa25193b0c8f1cf0c815fc66a959_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<ul>\n<li>\u9002\u7528\u573a\u666f\uff1a<\/li>\n\n\n\n<li>a.\u6700\u9002\u5408\u7528\u4e8e\u5904\u7406<strong>\u5927\u72b6\u6001\u3001\u957f\u7a97\u53e3\uff0c\u6216\u5927\u952e\u503c\u72b6\u6001\u7684\u6709\u72b6\u6001\u5904\u7406\u4efb\u52a1\u3002<\/strong><\/li>\n\n\n\n<li>b.RocksDBStateBackend \u662f\u76ee\u524d<strong>\u552f\u4e00\u652f\u6301\u589e\u91cf\u68c0\u67e5\u70b9<\/strong>\u7684\u540e\u7aef\u3002<\/li>\n\n\n\n<li>c.\u589e\u91cf\u68c0\u67e5\u70b9\u975e\u5e38\u9002\u7528\u4e8e\u8d85\u5927\u72b6\u6001\u7684\u573a\u666f\u3002\u6bd4\u5982\u8ba1\u7b97 DAU \u8fd9\u79cd\u5927\u6570\u636e\u91cf\u53bb\u91cd\uff0c<strong>\u5927\u72b6\u6001\u7684\u4efb\u52a1\u90fd\u5efa\u8bae\u76f4\u63a5\u4f7f\u7528 RocksDB \u72b6\u6001\u540e\u7aef<\/strong>\u3002<\/li>\n<\/ul>\n\n\n\n<p>\u5230\u751f\u4ea7\u73af\u5883\u4e2d\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u5982\u679c\u72b6\u6001\u5f88\u5927\uff0c\u4f7f\u7528 Rocksdb\uff1b\u5982\u679c\u72b6\u6001\u4e0d\u5927\uff0c\u4f7f\u7528 Filesystem\u3002<\/li>\n\n\n\n<li>\u2b50 Rocksdb \u4f7f\u7528\u78c1\u76d8\u5b58\u50a8 State\uff0c\u6240\u4ee5\u4f1a\u6d89\u53ca\u5230\u8bbf\u95ee State \u78c1\u76d8\u5e8f\u5217\u5316\u3001\u53cd\u5e8f\u5217\u5316\uff0c\u6027\u80fd\u4f1a\u6536\u5230\u5f71\u54cd\uff0c\u800c Filesystem \u76f4\u63a5\u8bbf\u95ee\u5185\u5b58\uff0c\u5355\u7eaf\u4ece\u8bbf\u95ee\u72b6\u6001\u7684\u6027\u80fd\u6765\u8bf4 Filesystem \u8fdc\u8fdc\u597d\u4e8e Rocksdb\u3002<strong>\u751f\u4ea7\u73af\u5883\u4e2d\u5b9e\u6d4b\uff0c\u76f8\u540c\u4efb\u52a1\u4f7f\u7528 Filesystem \u6027\u80fd\u4e3a Rocksdb \u7684 n \u500d\uff0c\u56e0\u6b64\u9700\u8981\u6839\u636e\u5177\u4f53\u573a\u666f\u8bc4\u4f30\u9009\u62e9\u3002<\/strong><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"24Flink_SQL_API_State_TTL_%E7%9A%84%E8%BF%87%E6%9C%9F%E6%9C%BA%E5%88%B6%E6%98%AF_onCreateAndUpdate_%E8%BF%98%E6%98%AFonReadAndWrite\"><\/span>2.4.Flink SQL API State TTL \u7684\u8fc7\u671f\u673a\u5236\u662f onCreateAndUpdate \u8fd8\u662fonReadAndWrite<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li><strong>\u2b50 \u7ed3\u8bba\uff1aFlink SQL API State TTL \u7684\u8fc7\u671f\u673a\u5236\u76ee\u524d\u53ea\u652f\u6301 onCreateAndUpdate\uff0cDataStream API \u4e24\u4e2a\u90fd\u652f\u6301<\/strong><\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-c9f8108739f2cc9ede7513fbfa9d668c_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>3<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u5256\u6790\uff1a<\/li>\n<\/ol>\n\n\n\n<ul>\n<li>onCreateAndUpdate\uff1a\u662f\u5728\u521b\u5efa State \u548c\u66f4\u65b0 State \u65f6\u3010\u66f4\u65b0 State TTL\u3011<\/li>\n\n\n\n<li>onReadAndWrite\uff1a\u662f\u5728\u8bbf\u95ee State \u548c\u5199\u5165 State \u65f6\u3010\u66f4\u65b0 State TTL\u3011<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 \u5b9e\u9645\u8e29\u5751\u573a\u666f\uff1aFlink SQL Deduplicate \u5199\u6cd5\uff0crow_number partition by user_id order by proctime asc\uff0c\u6b64 SQL \u6700\u540e\u751f\u6210\u7684\u7b97\u5b50\u53ea\u4f1a\u5728\u7b2c\u4e00\u6761\u6570\u636e\u6765\u7684\u65f6\u5019\u66f4\u65b0 state\uff0c\u540e\u7eed\u8bbf\u95ee\u4e0d\u4f1a\u66f4\u65b0 state TTL\uff0c\u56e0\u6b64 state \u4f1a\u5728\u7528\u6237\u8bbe\u7f6e\u7684 state TTL \u65f6\u95f4\u4e4b\u540e\u8fc7\u671f\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3\"><\/span>3.\u65f6\u95f4\u7a97\u53e3<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"31watermark_%E5%88%B0%E5%BA%95%E6%98%AF%E5%B9%B2%E5%95%A5%E7%9A%84%EF%BC%9F%E5%BA%94%E7%94%A8%E5%9C%BA%E6%99%AF%EF%BC%9F\"><\/span>3.1.watermark \u5230\u5e95\u662f\u5e72\u5565\u7684\uff1f\u5e94\u7528\u573a\u666f\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u5927\u90e8\u5206\u540c\u5b66\u90fd\u53ea\u80fd\u56de\u7b54\u51fa\uff1awatermark \u662f\u7528\u4e8e\u7f13\u89e3\u65f6\u95f4\u65f6\u95f4\u7684\u4e71\u5e8f\u95ee\u9898\u7684\u3002<\/p>\n\n\n\n<p>\u6ca1\u9519\uff0c\u8fd9\u4e2a\u89c2\u70b9\u662f\u6b63\u786e\u7684\u3002\u4f46\u662f\u535a\u4e3b\u8ba4\u4e3a\u8fd9\u53ea\u662f watermark \u7b2c\u4e8c\u91cd\u8981\u7684\u4f5c\u7528\uff0c\u5176\u66f4\u91cd\u8981\u7684\u4f5c\u7528\u5728\u4e8e\u53ef\u4ee5\u6807\u8bc6\u4e00\u4e2a Flink \u4efb\u52a1\u7684\u4e8b\u4ef6&nbsp;<strong>\u65f6\u95f4\u8fdb\u5ea6<\/strong>\u3002<\/p>\n\n\n\n<p>\u600e\u4e48\u7406\u89e3&nbsp;<strong>\u65f6\u95f4\u8fdb\u5ea6<\/strong>\uff1f<\/p>\n\n\n\n<p>\u6211\u4eec\u53ef\u4ee5\u73b0\u8c61\u4e00\u4e0b\uff0c\u4e00\u4e2a\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u7684\u4efb\u52a1\uff0c\u5982\u679c\u6ca1\u6709\u4e00\u4e2a&nbsp;<strong>\u4e1c\u897f<\/strong>&nbsp;\u53bb\u6807\u8bc6\u5176\u4e8b\u4ef6\u65f6\u95f4\u7684\u8fdb\u5ea6\uff0c\u90a3\u4e48\u8fd9\u4e2a\u4e8b\u4ef6\u65f6\u95f4\u7684\u7a97\u53e3\u4e5f\u5c31\u662f\u4e0d\u77e5\u9053\u4ec0\u4e48\u65f6\u5019\u80fd\u591f\u89e6\u53d1\u4e86\uff0c\u4e5f\u5c31\u662f\u8bf4\u8fd9\u4e2a\u7a97\u53e3\u6c38\u8fdc\u4e0d\u4f1a\u89e6\u53d1\u5e76\u4e14\u8f93\u51fa\u7ed3\u679c\u3002<\/p>\n\n\n\n<p>\u6240\u4ee5\u8981\u6709\u4e00\u4e2a&nbsp;<strong>\u4e1c\u897f<\/strong>&nbsp;\u53bb\u6807\u8bc6\u5176\u4e8b\u4ef6\u65f6\u95f4\u7684\u8fdb\u5ea6\uff0c\u4ece\u800c\u8ba9\u8fd9\u4e2a\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u77e5\u9053\uff0c\u8fd9\u4e2a\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u5df2\u7ecf\u7ed3\u675f\u4e86\uff0c\u53ef\u4ee5\u89e6\u53d1\u8ba1\u7b97\u4e86\u3002\u5728 Flink \u4e2d\uff0c\u8fd9\u4e2a&nbsp;<strong>\u4e1c\u897f<\/strong>&nbsp;\u5c31\u662f&nbsp;<strong>watermark<\/strong>\u3002<\/p>\n\n\n\n<p>\u603b\u7ed3\u4e00\u4e0b\uff0c\u535a\u4e3b\u8ba4\u4e3a&nbsp;<strong>watermark<\/strong>&nbsp;\u4e3a Flink \u89e3\u51b3\u4e86\u4e24\u4e2a\u95ee\u9898\uff1a<\/p>\n\n\n\n<ol>\n<li><strong>\u2b50 \u6807\u8bc6 Flink \u4efb\u52a1\u7684\u4e8b\u4ef6\u65f6\u95f4\u8fdb\u5ea6\uff0c\u4ece\u800c\u80fd\u591f\u63a8\u52a8\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u7684\u89e6\u53d1\u3001\u8ba1\u7b97\u3002<\/strong><\/li>\n\n\n\n<li><strong>\u2b50 \u89e3\u51b3\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u7684\u4e71\u5e8f\u95ee\u9898\u3002<\/strong><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"32%E4%B8%80%E4%B8%AA_Flink_%E4%BB%BB%E5%8A%A1%E4%B8%AD%E5%8F%AF%E4%BB%A5%E6%97%A2%E6%9C%89%E4%BA%8B%E4%BB%B6%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3%EF%BC%8C%E5%8F%88%E6%9C%89%E5%A4%84%E7%90%86%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3%E5%90%97%EF%BC%9F\"><\/span>3.2.\u4e00\u4e2a Flink \u4efb\u52a1\u4e2d\u53ef\u4ee5\u65e2\u6709\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\uff0c\u53c8\u6709\u5904\u7406\u65f6\u95f4\u7a97\u53e3\u5417\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p><strong>\u7ed3\u8bba\uff1a\u4e00\u4e2a Flink \u4efb\u52a1\u53ef\u4ee5\u540c\u65f6\u6709\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\uff0c\u53c8\u6709\u5904\u7406\u65f6\u95f4\u7a97\u53e3\u3002<\/strong><\/p>\n\n\n\n<p>\u90a3\u4e48\u6709\u4e9b\u5c0f\u4f19\u4f34\u4eec\u95ee\u4e86\uff0c\u4e3a\u4ec0\u4e48\u6211\u4eec\u5e38\u89c1\u7684 Flink \u4efb\u52a1\u8981\u4e48\u8bbe\u7f6e\u4e3a\u4e8b\u4ef6\u65f6\u95f4\u8bed\u4e49\uff0c\u8981\u4e48\u8bbe\u7f6e\u4e3a\u5904\u7406\u65f6\u95f4\u8bed\u4e49\uff1f<\/p>\n\n\n\n<p>\u786e\u5b9e\uff0c\u5728\u751f\u4ea7\u73af\u5883\u4e2d\uff0c\u6211\u4eec\u7684 Flink \u4efb\u52a1\u4e00\u822c\u4e0d\u4f1a\u540c\u65f6\u62e5\u6709\u4e24\u79cd\u65f6\u95f4\u8bed\u4e49\u7684\u7a97\u53e3\u3002<\/p>\n\n\n\n<p>\u90a3\u4e48\u600e\u4e48\u89e3\u91ca\u5f00\u5934\u535a\u4e3b\u6240\u8bf4\u7684\u7ed3\u8bba\u5462\uff1f<\/p>\n\n\n\n<p>\u535a\u4e3b\u8fd9\u91cc\u4ece\u4e24\u4e2a\u89d2\u5ea6\u8fdb\u884c\u8bf4\u660e\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u6211\u4eec\u5176\u5b9e\u6ca1\u6709\u5fc5\u8981\u628a\u4e00\u4e2a Flink \u4efb\u52a1\u548c\u67d0\u79cd\u7279\u5b9a\u7684\u65f6\u95f4\u8bed\u4e49\u8fdb\u884c\u7ed1\u5b9a\u3002\u5bf9\u4e8e\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u6765\u8bf4\uff0c\u6211\u4eec\u53ea\u8981\u7ed9\u5b83 watermark\uff0c\u80fd\u8ba9 watermark \u4e00\u76f4\u5f80\u524d\u63a8\u8fdb\uff0c\u8ba9\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u80fd\u591f\u6301\u7eed\u89e6\u53d1\u8ba1\u7b97\u5c31\u884c\u3002\u5bf9\u4e8e\u5904\u7406\u65f6\u95f4\u6765\u8bf4\u66f4\u7b80\u5355\uff0c\u53ea\u8981\u7a97\u53e3\u7b97\u5b50\u6309\u7167\u672c\u5730\u65f6\u95f4\u6309\u7167\u56fa\u5b9a\u7684\u65f6\u95f4\u95f4\u9694\u8fdb\u884c\u89e6\u53d1\u5c31\u884c\u3002\u65e0\u8bba\u54ea\u79cd\u65f6\u95f4\u7a97\u53e3\uff0c\u4e3b\u8981\u6ee1\u8db3\u65f6\u95f4\u7a97\u53e3\u7684\u89e6\u53d1\u6761\u4ef6\u5c31\u884c\u3002<\/li>\n\n\n\n<li>\u2b50 Flink \u7684\u5b9e\u73b0\u4e0a\u6765\u8bf4\u4e5f\u662f\u652f\u6301\u7684\u3002Flink \u662f\u4f7f\u7528\u4e00\u4e2a\u53eb\u505a TimerService \u7684\u7ec4\u4ef6\u6765\u7ba1\u7406 timer \u7684\uff0c\u6211\u4eec\u53ef\u4ee5\u540c\u65f6\u6ce8\u518c\u4e8b\u4ef6\u65f6\u95f4\u548c\u5904\u7406\u65f6\u95f4\u7684 timer\uff0cFlink \u4f1a\u81ea\u884c\u5224\u65ad timer \u662f\u5426\u6ee1\u8db3\u89e6\u53d1\u6761\u4ef6\uff0c\u5982\u679c\u662f\uff0c\u5219\u56de\u8c03\u7a97\u53e3\u5904\u7406\u51fd\u6570\u8fdb\u884c\u8ba1\u7b97\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"33window_%E5%90%8E%E9%9D%A2%E8%B7%9F_aggregate_%E5%92%8C_process_%E7%9A%84%E4%B8%A4%E4%B8%AA%E7%AA%97%E5%8F%A3%E8%AE%A1%E7%AE%97%E7%9A%84%E5%8C%BA%E5%88%AB%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F\"><\/span>3.3.window \u540e\u9762\u8ddf aggregate \u548c process \u7684\u4e24\u4e2a\u7a97\u53e3\u8ba1\u7b97\u7684\u533a\u522b\u662f\u4ec0\u4e48\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 aggregate\uff1a\u662f\u589e\u91cf\u805a\u5408\uff0c\u6765\u4e00\u6761\u6570\u636e\u8ba1\u7b97\u5b8c\u4e86\u5b58\u50a8\u5728\u7d2f\u52a0\u5668\u4e2d\uff0c\u4e0d\u9700\u8981\u7b49\u5230\u7a97\u53e3\u89e6\u53d1\u65f6\u8ba1\u7b97\uff0c<strong>\u6027\u80fd\u8f83\u597d<\/strong>\uff1b<\/li>\n\n\n\n<li>\u2b50 process\uff1a\u5168\u91cf\u51fd\u6570\uff0c\u7f13\u5b58\u5168\u90e8\u7a97\u53e3\u5185\u7684\u6570\u636e\uff0c\u6ee1\u8db3\u7a97\u53e3\u89e6\u53d1\u6761\u4ef6\u518d\u89e6\u53d1\u8ba1\u7b97\uff0c\u540c\u65f6\u8fd8\u63d0\u4f9b\u5b9a\u65f6\u89e6\u53d1\uff0c\u7a97\u53e3\u4fe1\u606f\u7b49\u4e0a\u4e0b\u6587\u4fe1\u606f\uff1b<\/li>\n\n\n\n<li>\u2b50 \u5e94\u7528\u573a\u666f\uff1aaggregate \u4e00\u4e2a\u4e00\u4e2a\u5904\u7406\u7684\u805a\u5408\u7ed3\u679c\u5411\u540e\u4f20\u9012\u4e00\u822c\u6765\u8bf4\u90fd\u662f\u6709\u4fe1\u606f\u635f\u5931\u7684\uff0c\u800c<strong>&nbsp;process \u5219\u53ef\u4ee5\u66f4\u52a0\u5b9a\u5236\u5316\u7684\u5904\u7406<\/strong>\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4%E7%BC%96%E7%A8%8B%E6%8A%80%E5%B7%A7\"><\/span>4.\u7f16\u7a0b\u6280\u5de7<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"41%E4%B8%BA%E4%BB%80%E4%B9%88_Flink_DataStream_API_%E5%9C%A8%E5%87%BD%E6%95%B0%E5%85%A5%E5%8F%82%E6%88%96%E8%80%85%E5%87%BA%E5%8F%82%E6%9C%89%E6%B3%9B%E5%9E%8B%E6%97%B6%EF%BC%8C%E4%B8%8D%E8%83%BD%E4%BD%BF%E7%94%A8_lambda_%E8%A1%A8%E8%BE%BE%E5%BC%8F%EF%BC%9F\"><\/span>4.1.\u4e3a\u4ec0\u4e48 Flink DataStream API \u5728\u51fd\u6570\u5165\u53c2\u6216\u8005\u51fa\u53c2\u6709\u6cdb\u578b\u65f6\uff0c\u4e0d\u80fd\u4f7f\u7528 lambda \u8868\u8fbe\u5f0f\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>Flink \u7c7b\u578b\u4fe1\u606f\u7cfb\u7edf\u662f\u901a\u8fc7\u53cd\u5c04\u83b7\u53d6\u5230 Java class \u7684\u65b9\u6cd5\u7b7e\u540d\u53bb\u83b7\u53d6\u7c7b\u578b\u4fe1\u606f\u7684\u3002<\/p>\n\n\n\n<p>\u4ee5 FlatMap \u4e3a\u4f8b\uff0cFlink \u5728\u901a\u8fc7\u53cd\u5c04\u65f6\u4f1a\u68c0\u67e5\u53ca\u83b7\u53d6 FlatMap collector \u7684\u51fa\u53c2\u7c7b\u578b\u4fe1\u606f\u3002<\/p>\n\n\n\n<p>\u4f46\u662f lambda \u8868\u8fbe\u5f0f\u5199\u7684 FlatMap \u903b\u8f91\uff0c\u4f1a\u5bfc\u81f4\u53cd\u5c04\u65b9\u6cd5\u83b7\u53d6\u7c7b\u578b\u4fe1\u606f\u65f6\u3010\u76f4\u63a5\u83b7\u53d6\u4e0d\u5230\u3011collector \u7684\u51fa\u53c2\u7c7b\u578b\u53c2\u6570\uff0c\u6240\u4ee5\u624d\u4f1a\u62a5\u9519\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"42Flink_%E4%B8%BA%E4%BB%80%E4%B9%88%E5%BC%BA%E8%B0%83_function_%E5%AE%9E%E7%8E%B0%E6%97%B6%EF%BC%8C%E5%AE%9E%E4%BE%8B%E5%8C%96%E7%9A%84%E5%8F%98%E9%87%8F%E8%A6%81%E5%AE%9E%E7%8E%B0_serializable_%E6%8E%A5%E5%8F%A3%EF%BC%9F\"><\/span>4.2.Flink \u4e3a\u4ec0\u4e48\u5f3a\u8c03 function \u5b9e\u73b0\u65f6\uff0c\u5b9e\u4f8b\u5316\u7684\u53d8\u91cf\u8981\u5b9e\u73b0 serializable \u63a5\u53e3\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u5176\u5b9e\u8fd9\u4e2a\u95ee\u9898\u53ef\u4ee5\u5ef6\u4f38\u6210 3 \u4e2a\u95ee\u9898\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u4e3a\u4ec0\u4e48 Flink \u8981\u7528\u5230 Java \u5e8f\u5217\u5316\u673a\u5236\u3002\u548c Flink \u7c7b\u578b\u7cfb\u7edf\u7684\u6570\u636e\u5e8f\u5217\u5316\u673a\u5236\u7684\u7528\u9014\u6709\u5565\u533a\u522b\uff1f<\/li>\n\n\n\n<li>\u2b50 \u975e\u5b9e\u4f8b\u5316\u7684\u53d8\u91cf\u6ca1\u6709\u5b9e\u73b0 Serializable \u4e3a\u5565\u5c31\u4e0d\u62a5\u9519\uff0c\u5b9e\u4f8b\u5316\u5c31\u62a5\u9519\uff1f<\/li>\n\n\n\n<li>\u2b50 \u4e3a\u5565\u52a0 transient \u5c31\u4e0d\u62a5\u9519\uff1f<\/li>\n<\/ol>\n\n\n\n<p>\u4e0a\u9762 3 \u4e2a\u95ee\u9898\u7684\u7b54\u6848\u5982\u4e0b\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 Flink \u5199\u7684\u51fd\u6570\u5f0f\u7f16\u7a0b\u4ee3\u7801\u6216\u8005\u8bf4\u95ed\u5305\uff0c\u9700\u8981 Java \u5e8f\u5217\u5316\u4ece JobManager \u5206\u53d1\u5230 TaskManager\uff0c\u800c Flink \u7c7b\u578b\u7cfb\u7edf\u7684\u6570\u636e\u5e8f\u5217\u5316\u673a\u5236\u662f\u4e3a\u4e86\u5206\u53d1\u6570\u636e\uff0c\u4e0d\u662f\u5206\u53d1\u4ee3\u7801\uff0c\u53ef\u4ee5\u7528\u975eJava\u7684\u5e8f\u5217\u5316\u673a\u5236\uff0c\u6bd4\u5982 Kyro\u3002<\/li>\n\n\n\n<li>\u2b50 \u7f16\u8bd1\u671f\u4e0d\u505a\u5e8f\u5217\u5316\uff0c\u6240\u4ee5\u4e0d\u5b9e\u73b0 Serializable \u4e0d\u4f1a\u62a5\u9519\uff0c\u4f46\u662f\u8fd0\u884c\u671f\u4f1a\u6267\u884c\u5e8f\u5217\u5316\u52a8\u4f5c\uff0c\u6ca1\u5b9e\u73b0 Serializable \u63a5\u53e3\u7684\u5c31\u62a5\u9519\u4e86<\/li>\n\n\n\n<li>\u2b50 Flink DataStream API \u7684 Function \u4f5c\u4e3a\u95ed\u5305\u5728\u7f51\u7edc\u4f20\u8f93\uff0c\u5fc5\u987b\u91c7\u7528 Java \u5e8f\u5217\u5316\uff0c\u6240\u4ee5\u8981\u901a\u8fc7 Serializable \u63a5\u53e3\u6807\u8bb0\uff0c\u6839\u636e Java \u5e8f\u5217\u5316\u7684\u89c4\u5b9a\uff0c\u5185\u90e8\u6210\u5458\u53d8\u91cf\u8981\u4e48\u90fd\u53ef\u5e8f\u5217\u5316\uff0c\u8981\u4e48\u901a\u8fc7 transient \u5173\u952e\u5b57\u8df3\u8fc7\u5e8f\u5217\u5316\uff0c\u5426\u5219 Java \u5e8f\u5217\u5316\u7684\u65f6\u5019\u4f1a\u62a5\u9519\u3002\u9759\u6001\u53d8\u91cf\u4e0d\u53c2\u4e0e\u5e8f\u5217\u5316\uff0c\u6240\u4ee5\u4e0d\u7528\u52a0 transient\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"43Flink_%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%E5%8F%AF%E4%BB%A5%E9%80%9A%E8%BF%87%E5%93%AA%E5%87%A0%E7%A7%8D%E6%96%B9%E5%BC%8F%E8%AE%BE%E7%BD%AE%EF%BC%8C%E4%BC%98%E5%85%88%E7%BA%A7%E5%85%B3%E7%B3%BB%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F\"><\/span>4.3.Flink \u7684\u5e76\u884c\u5ea6\u53ef\u4ee5\u901a\u8fc7\u54ea\u51e0\u79cd\u65b9\u5f0f\u8bbe\u7f6e\uff0c\u4f18\u5148\u7ea7\u5173\u7cfb\u662f\u4ec0\u4e48\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 \u4ee3\u7801\u4e2d\u7b97\u5b50\u5355\u72ec\u8bbe\u7f6e<\/li>\n\n\n\n<li>\u2b50 \u4ee3\u7801\u4e2dEnv\u5168\u5c40\u8bbe\u7f6e<\/li>\n\n\n\n<li>\u2b50 \u63d0\u4ea4\u53c2\u6570<\/li>\n\n\n\n<li>\u2b50 \u9ed8\u8ba4\u914d\u7f6e\u4fe1\u606f<\/li>\n<\/ol>\n\n\n\n<p>\u4e0a\u9762\u7684 Flink \u5e76\u884c\u5ea6\u4f18\u5148\u7ea7\u4ece\u4e0a\u5f80\u4e0b\u7531\u5927\u53d8\u5c0f\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"5%E5%AE%9E%E6%88%98%E7%BB%8F%E9%AA%8C\"><\/span>5.\u5b9e\u6218\u7ecf\u9a8c<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"51Flink_SQL_%E8%AE%A1%E7%AE%97%E7%94%A8%E6%88%B7%E5%88%86%E5%B8%83\"><\/span>5.1.Flink SQL \u8ba1\u7b97\u7528\u6237\u5206\u5e03<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 \u9700\u6c42\uff1a\u4e0a\u6e38\u662f\u4e00\u4e2a kafka \u6570\u636e\u6e90\uff0c\u6570\u636e\u5185\u5bb9\u662f\u7528\u6237 QQ \u7b49\u7ea7\u53d8\u5316\u660e\u7ec6\u6570\u636e\uff08time\uff0cuid\uff0clevel\uff09\u3002\u9700\u8981\u4f60\u6c42\u51fa\u5f53\u524d\u6bcf\u4e2a\u7b49\u7ea7\u7684\u7528\u6237\u6570\u3002<br><\/li>\n\n\n\n<li>\u2b50 \u5b9e\u73b0 SQL\uff1aDeduplicate<br><\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>-- \u5982\u679c\u9700\u8981\u53ef\u4ee5\u6253\u5f00 minibatch\nselect  \n    level\n    , count(1) as uv\n    , max(time) as time\nfrom (\n    select \n        uid\n        , level\n        , time\n        , row_number() over (partition by uid order by time desc) rn \n    from source\n) tmp\nwhere rn =1 \ngroup by \n    level\n<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"52Flink_SQL_%E8%AE%A1%E7%AE%97_DAU\"><\/span>5.2.Flink SQL \u8ba1\u7b97 DAU<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 \u9700\u6c42\uff1a\u6570\u636e\u6e90\uff1a\u7528\u6237\u5fc3\u8df3\u65e5\u5fd7\uff08uid\uff0ctime\uff0ctype\uff09\u3002\u8ba1\u7b97\u5206 Android\uff0ciOS \u7684 DAU\uff0c\u6700\u665a\u4e00\u5206\u949f\u8f93\u51fa\u4e00\u6b21\u5f53\u65e5\u96f6\u70b9\u7d2f\u8ba1\u5230\u5f53\u524d\u7684\u7ed3\u679c\u3002<br><\/li>\n\n\n\n<li>\u2b50 \u5b9e\u73b0\u65b9\u5f0f 1\uff1acumulate \u7a97\u53e3<br><\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>SELECT  \n    window_start\n    , window_end\n    , platform\n    , sum(bucket_dau) as dau\nfrom (\n    SELECT\n        window_start\n        , window_end\n        , platform\n        , count(distinct uid) as bucket_dau\n    FROM TABLE(\n        CUMULATE(\n        TABLE user_log,\n        DESCRIPTOR(time),\n        INTERVAL '60' SECOND\n        , INTERVAL '1' DAY))\n    GROUP BY                                  \n        window_start\n        , window_end\n        , platform\n        , MOD(HASH_CODE(user_id), 1024)\n) tmp\nGROUP by   \n    window_start\n    , window_end\n    , platform\n<\/code><\/pre>\n\n\n\n<ul>\n<li>\u4f18\u70b9\uff1a\u5982\u679c\u662f\u66f2\u7ebf\u56fe\u7684\u9700\u6c42\uff0c\u53ef\u4ee5\u5b8c\u7f8e\u56de\u6eaf\u66f2\u7ebf\u56fe\u3002<\/li>\n\n\n\n<li>\u7f3a\u70b9\uff1a\u5927\u7a97\u53e3\u4e4b\u95f4\u5982\u679c\u6709\u6570\u636e\u4e71\u5e8f\uff0c\u6709\u4e22\u6570\u98ce\u9669\uff1b\u5e76\u4e14\u7531\u4e8e\u662f watermark \u63a8\u52a8\u4ea7\u51fa\uff0c\u6240\u4ee5\u6570\u636e\u4ea7\u51fa\u4f1a\u6709\u5ef6\u8fdf\u3002<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 \u5b9e\u73b0\u65b9\u5f0f 2\uff1aDeduplicate<\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>-- \u5982\u679c\u9700\u8981\u53ef\u4ee5\u6253\u5f00 minibatch\nselect \n    platform\n    , count(1) as dau\n    , max(time) as time\nfrom (\n    select \n        uid\n        , platform\n        , time\n        , row_number() over (partition by uid, platform, time \/ 24 \/ 3600 \/ 1000 order by time desc) rn\n    from source\n) tmp\nwhere rn = 1\ngroup by\n    platform\n<\/code><\/pre>\n\n\n\n<ul>\n<li>\u4f18\u70b9\uff1a\u8ba1\u7b97\u5feb\u3002<\/li>\n\n\n\n<li>\u7f3a\u70b9\uff1a\u4efb\u52a1\u53d1\u751f failover\uff0c\u66f2\u7ebf\u56fe\u4e0d\u80fd\u5f88\u597d\u56de\u6eaf\u3002\u6ca1\u6cd5\u652f\u6301 cube \u8ba1\u7b97\u3002<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 \u5b9e\u73b0\u65b9\u5f0f 3\uff1agroup agg<\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>-- \u5982\u679c\u9700\u8981\u53ef\u4ee5\u6253\u5f00 minibatch\nSELECT   \n    max(time) as time\n    , platform\n    , sum(bucket_dau) as dau\nfrom (\n    SELECT\n        max(time) as time\n        , platform\n        , count(distinct uid) as bucket_dau\n    FROM source\n    GROUP BY\n        platform\n        , MOD(HASH_CODE(user_id), 1024)\n) t \nGROUP by   \n    platform<\/code><\/pre>\n\n\n\n<ul>\n<li>\u4f18\u70b9\uff1a\u8ba1\u7b97\u5feb\uff0c\u652f\u6301 cube \u8ba1\u7b97\u3002<\/li>\n\n\n\n<li>\u7f3a\u70b9\uff1a\u4efb\u52a1\u53d1\u751f failover\uff0c\u66f2\u7ebf\u56fe\u4e0d\u80fd\u5f88\u597d\u56de\u6eaf\u3002<\/li>\n<\/ul>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"53%E4%BD%A0%E6%98%AF%E6%80%8E%E4%B9%88%E5%90%88%E7%90%86%E7%9A%84%E8%AF%84%E4%BC%B0_Flink_%E4%BB%BB%E5%8A%A1%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F\"><\/span>5.3.\u4f60\u662f\u600e\u4e48\u5408\u7406\u7684\u8bc4\u4f30 Flink \u4efb\u52a1\u7684\u5e76\u884c\u5ea6\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>Flink \u4efb\u52a1\u5e76\u884c\u5ea6\u5408\u7406\u884c\u4e00\u822c\u6839\u636e<strong>\u5cf0\u503c\u6d41\u91cf\u8fdb\u884c\u538b\u6d4b\u8bc4\u4f30<\/strong>\uff0c\u5e76\u4e14\u6839\u636e\u96c6\u7fa4\u8d1f\u8f7d\u60c5\u51b5<strong>\u7559\u4e00\u5b9a\u91cf\u7684 buffer \u8d44\u6e90<\/strong>\u3002<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u5982\u679c\u6570\u636e\u6e90\u5df2\u7ecf\u5b58\u5728\uff0c\u5219\u53ef\u4ee5\u76f4\u63a5\u6d88\u8d39\u8fdb\u884c\u6d4b\u8bd5<\/li>\n\n\n\n<li>\u2b50 \u5982\u679c\u6570\u636e\u6e90\u4e0d\u5b58\u5728\uff0c\u9700\u8981\u81ea\u884c\u9020\u538b\u6d4b\u6570\u636e\u8fdb\u884c\u6d4b\u8bd5<\/li>\n<\/ol>\n\n\n\n<p>\u5bf9\u4e8e\u4e00\u4e2a Flink \u4efb\u52a1\u6765\u8bf4\uff0c\u4e00\u822c\u53ef\u4ee5\u6309\u7167\u4ee5\u4e0b\u65b9\u5f0f\u8fdb\u884c\u7ec6\u7c92\u5ea6\u8bbe\u7f6e\u5e76\u884c\u5ea6\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 source \u5e76\u884c\u5ea6\u914d\u7f6e\uff1a\u4ee5 kafka \u4e3a\u4f8b\uff0csource \u7684\u5e76\u884c\u5ea6\u4e00\u822c\u8bbe\u7f6e\u4e3a kafka \u5bf9\u5e94\u7684 topic \u7684\u5206\u533a\u6570<br><\/li>\n\n\n\n<li>\u2b50 transform\uff08\u6bd4\u5982 flatmap\u3001map\u3001filter \u7b49\u7b97\u5b50\uff09\u5e76\u884c\u5ea6\u7684\u914d\u7f6e\uff1a\u8fd9\u4e9b\u7b97\u5b50\u4e00\u822c\u4e0d\u4f1a\u505a\u592a\u91cd\u7684\u64cd\u4f5c\uff0c\u5e76\u884c\u5ea6\u53ef\u4ee5\u548c source \u4fdd\u6301\u4e00\u81f4\uff0c\u4f7f\u5f97\u7b97\u5b50\u4e4b\u95f4\u53ef\u4ee5\u505a\u5230 forward \u4f20\u8f93\u6570\u636e\uff0c\u4e0d\u7ecf\u8fc7\u7f51\u7edc\u4f20\u8f93<br><\/li>\n\n\n\n<li>\u2b50 keyby \u4e4b\u540e\u7684\u5904\u7406\u7b97\u5b50\uff1a<strong>\u5efa\u8bae\u6700\u5927\u5e76\u884c\u5ea6\u4e3a\u6b64\u7b97\u5b50\u5e76\u884c\u5ea6\u7684\u6574\u6570\u500d<\/strong>\uff0c\u8fd9\u6837\u53ef\u4ee5\u4f7f\u6bcf\u4e2a\u7b97\u5b50\u4e0a\u7684 keyGroup \u662f\u76f8\u540c\u7684\uff0c\u4ece\u800c\u4f7f\u5f97\u6570\u636e\u76f8\u5bf9\u5747\u5300 shuffle \u5230\u4e0b\u6e38\u7b97\u5b50\uff0c\u5982\u4e0b\u56fe\u4e3a shuffle \u7b56\u7565<br><\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-1e65fbdab78e25e22fca54e82e48094b_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<ol>\n<li>\u2b50 sink \u5e76\u884c\u5ea6\u7684\u914d\u7f6e\uff1asink \u662f\u6570\u636e\u6d41\u5411\u4e0b\u6e38\u7684\u5730\u65b9\uff0c\u53ef\u4ee5\u6839\u636e sink \u7684\u6570\u636e\u91cf\u53ca\u4e0b\u6e38\u7684\u670d\u52a1\u6297\u538b\u80fd\u529b\u8fdb\u884c\u8bc4\u4f30\u3002\u5982\u679c sink \u662f kafka\uff0c\u53ef\u4ee5\u8bbe\u4e3a kafka \u5bf9\u5e94 topic \u7684\u5206\u533a\u6570\u3002\u6ce8\u610f sink \u5e76\u884c\u5ea6\u6700\u597d\u548c kafka partition \u6210\u500d\u6570\u5173\u7cfb\uff0c\u5426\u5219\u53ef\u80fd\u4f1a\u51fa\u73b0\u5982\u5230 kafka partition \u6570\u636e\u4e0d\u5747\u5300\u7684\u60c5\u51b5\u3002\u4f46\u662f\u5927\u591a\u6570\u60c5\u51b5\u4e0b sink \u7b97\u5b50\u5e76\u884c\u5ea6\u4e0d\u9700\u8981\u7279\u522b\u8bbe\u7f6e\uff0c\u53ea\u9700\u8981\u548c\u6574\u4e2a\u4efb\u52a1\u7684\u5e76\u884c\u5ea6\u76f8\u540c\u5c31\u884c\u3002<br><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"54%E4%BD%A0%E6%98%AF%E6%80%8E%E4%B9%88%E5%90%88%E7%90%86%E8%AF%84%E4%BC%B0%E4%BB%BB%E5%8A%A1%E6%9C%80%E5%A4%A7%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F\"><\/span>5.4.\u4f60\u662f\u600e\u4e48\u5408\u7406\u8bc4\u4f30\u4efb\u52a1\u6700\u5927\u5e76\u884c\u5ea6\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 \u524d\u63d0\uff1a\u5e76\u884c\u5ea6\u5fc5\u987b &lt;= \u6700\u5927\u5e76\u884c\u5ea6<\/li>\n\n\n\n<li>\u2b50 \u6700\u5927\u5e76\u884c\u5ea6\u7684\u4f5c\u7528\uff1a\u5408\u7406\u8bbe\u7f6e\u6700\u5927\u5e76\u884c\u5ea6\u53ef\u4ee5\u7f13\u89e3\u6570\u636e\u503e\u659c\u7684\u95ee\u9898<\/li>\n\n\n\n<li>\u2b50 \u6839\u636e\u5177\u4f53\u573a\u666f\u7684\u4e0d\u540c\uff0c\u6700\u5927\u5e76\u884c\u5ea6\u5927\u5c0f\u8bbe\u7f6e\u4e5f\u6709\u4e0d\u540c\u7684\u65b9\u5f0f\uff1a<\/li>\n<\/ol>\n\n\n\n<ul>\n<li><strong>\u5728 key \u975e\u5e38\u591a\u7684\u60c5\u51b5\u4e0b\uff0c\u6700\u5927\u5e76\u884c\u5ea6\u9002\u5408\u8bbe\u7f6e\u6bd4\u8f83\u5927\uff08\u51e0\u5343\uff09<\/strong>\uff0c\u4e0d\u5bb9\u6613\u51fa\u73b0\u6570\u636e\u503e\u659c\uff0c\u4ee5 Flink SQL \u573a\u666f\u4e3e\u4f8b\uff1arow_number = 1 partition key user_id \u7684 Deduplicate \u573a\u666f\uff08user_id \u4e00\u822c\u90fd\u975e\u5e38\u591a\uff09<\/li>\n\n\n\n<li><strong>\u5728 key \u4e0d\u662f\u5f88\u591a\u7684\u60c5\u51b5\u4e0b\uff0c\u6700\u5927\u5e76\u884c\u5ea6\u9002\u5408\u8bbe\u7f6e\u4e0d\u662f\u5f88\u5927<\/strong>\uff0c\u4e0d\u7136\u4f1a\u52a0\u91cd\u6570\u636e\u503e\u659c\uff0c\u4ee5 Flink SQL \u573a\u666f\u4e3e\u4f8b\uff1agroup by dim1,dim2 \u805a\u5408\u5e76\u4e14\u7ef4\u5ea6\u503c\u4e0d\u591a\u7684 group agg \u573a\u666f\uff08dim1\uff0cdim2 \u53ef\u4ee5\u679a\u4e3e\uff09\uff0c\u5982\u679c\u4f9d\u7136\u6709\u6570\u636e\u503e\u659c\u7684\u95ee\u9898\uff0c\u9700\u8981\u81ea\u5df1\u5148\u6253\u6563\u6570\u636e\uff0c\u7f13\u89e3\u6570\u636e\u503e\u659c<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 \u6700\u5927\u5e76\u884c\u5ea6\u7684\u4f7f\u7528\u9650\u5236\uff1a<strong>\u6700\u5927\u5e76\u884c\u5ea6\u4e00\u65e6\u8bbe\u7f6e\uff0c\u662f\u4e0d\u80fd\u968f\u610f\u53d8\u66f4\u7684<\/strong>\uff0c\u5426\u5219\u4f1a\u5bfc\u81f4\u68c0\u67e5\u70b9\u6216\u4fdd\u5b58\u70b9\u5931\u6548\uff1b\u6700\u5927\u5e76\u884c\u5ea6\u8bbe\u7f6e\u4f1a\u5f71\u54cd MapState \u72b6\u6001\u5212\u5206\u7684 KeyGroup \u6570\uff0c\u5e76\u884c\u5ea6\u4fee\u6539\u540e\u518d\u4ece\u4fdd\u5b58\u70b9\u542f\u52a8\u65f6\uff0cKeyGroup \u4f1a\u6839\u636e\u5e76\u884c\u5ea6\u7684\u8bbe\u5b9a\u8fdb\u884c\u91cd\u65b0\u5206\u5e03\u3002<\/li>\n\n\n\n<li>\u2b50 \u6700\u5927\u5e76\u884c\u5ea6\u7684\u8bbe\u7f6e\uff1a\u6700\u5927\u5e76\u884c\u5ea6\u53ef\u4ee5\u81ea\u5df1\u8bbe\u7f6e\uff0c\u4e5f\u53ef\u4ee5\u6846\u67b6\u9ed8\u8ba4\u751f\u6210\uff1b\u9ed8\u8ba4\u7684\u7b97\u6cd5\u662f\u53d6\u5f53\u524d\u7b97\u5b50\u5e76\u884c\u5ea6\u7684 1.5 \u500d\u548c 2 \u7684 7 \u6b21\u65b9\u6bd4\u8f83\uff0c\u53d6\u4e24\u8005\u4e4b\u95f4\u7684\u6700\u5927\u503c\uff0c\u7136\u540e\u7528\u4e0a\u9762\u7684\u7ed3\u679c\u548c 2 \u7684 15 \u6b21\u65b9\u6bd4\u8f83\uff0c\u53d6\u5176\u4e2d\u7684\u6700\u5c0f\u503c\u4e3a\u9ed8\u8ba4\u7684\u6700\u5927\u5e76\u884c\u5ea6\uff0c<strong>\u975e\u5e38\u4e0d\u5efa\u8bae\u81ea\u52a8\u751f\u6210\uff0c\u5efa\u8bae\u7528\u6237\u81ea\u5df1\u8bbe\u7f6e<\/strong>\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1%E5%89%8D%E8%A8%80-2\"><\/span>1.\u524d\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u672c\u6587\u4e3b\u8981\u662f\u6574\u7406\u535a\u4e3b\u6536\u96c6\u7684 Flink \u9ad8\u9891\u9762\u8bd5\u9898\u3002\u4e4b\u540e<strong>\u6bcf\u5468<\/strong>\u90fd\u4f1a\u6709\u4e00\u7bc7\u3002<\/p>\n\n\n\n<p>\u5176\u4e2d\u4e3b\u8981\u5212\u5206\u4e3a\u4e00\u4e0b 4 \u5927\u4e3b\u9898\uff0c\u9996\u5148\u662f\u524d\u4e24\u4e2a&nbsp;<strong>\u72b6\u6001\u539f\u7406<\/strong>\u3001<strong>\u65f6\u95f4\u7a97\u53e3<\/strong>&nbsp;\u662f\u7528\u4e8e\u8003\u6838\u5019\u9009\u4eba\u5bf9\u4e8e Flink \u57fa\u672c\u539f\u7406\u7684\u7406\u89e3\uff0c<strong>\u7f16\u7a0b\u6280\u5de7\u3001\u5b9e\u6218\u7ecf\u9a8c<\/strong>&nbsp;\u4e3b\u8981\u662f\u8003\u6838\u5019\u9009\u4eba\u4f7f\u7528 Flink \u7684\u7ecf\u9a8c\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic2.zhimg.com\/80\/v2-3e7852146ca46eaa3f65a0b8a58a10e1_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>1<\/p>\n\n\n\n<p>\u5185\u5bb9\u8f83\u591a\uff0c\u5e76\u4e14\u6bd4\u8f83\u8be6\u7ec6\uff0c<strong>\u5efa\u8bae\u83b7\u53d6 PDF \u8d44\u6599<\/strong>\uff0c\u516c\u4f17\u53f7\u52a0\u535a\u4e3b\u5fae\u4fe1\u83b7\u53d6\u3002\u6b22\u8fce\u5927\u5bb6\u6307\u6b63\u9519\u8bef\u4e0e\u4e0d\u8db3\u4e4b\u5904\uff0c\u4e5f\u6b22\u8fce\u5927\u5bb6\u7ee7\u7eed\u8865\u5145\u6b64\u5185\u5bb9\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2%E7%8A%B6%E6%80%81%E5%8E%9F%E7%90%86-2\"><\/span>2.\u72b6\u6001\u539f\u7406<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"21%E7%8A%B6%E6%80%81%E3%80%81%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%E3%80%81Checkpoint_%E4%B8%89%E8%80%85%E4%B9%8B%E9%97%B4%E7%9A%84%E5%8C%BA%E5%88%AB%E5%8F%8A%E5%85%B3%E7%B3%BB%EF%BC%9F-2\"><\/span>2.1.\u72b6\u6001\u3001\u72b6\u6001\u540e\u7aef\u3001Checkpoint \u4e09\u8005\u4e4b\u95f4\u7684\u533a\u522b\u53ca\u5173\u7cfb\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p><strong>\u7ed3\u8bba\uff1a\u62ff\u4e94\u4e2a\u5b57\u505a\u6bd4\u55bb\uff1a&#8221;\u94c1\u9505\u7096\u5927\u9e45&#8221;\uff0c\u94c1\u9505\u662f\u72b6\u6001\u540e\u7aef\uff0c\u5927\u9e45\u662f\u72b6\u6001\uff0cCheckpoint \u662f\u7096\u7684\u52a8\u4f5c\u3002<\/strong><\/p>\n\n\n\n<ol>\n<li>\u2b50 \u72b6\u6001\uff1a\u672c\u8d28\u6765\u8bf4\u5c31\u662f\u6570\u636e\uff0c\u5728 Flink \u4e2d\uff0c\u5176\u5b9e\u5c31\u662f Flink \u63d0\u4f9b\u7ed9\u7528\u6237\u7684\u72b6\u6001\u7f16\u7a0b\u63a5\u53e3\u3002\u6bd4\u5982 flink \u4e2d\u7684 MapState\uff0cValueState\uff0cListState\u3002<br><\/li>\n\n\n\n<li>\u2b50 \u72b6\u6001\u540e\u7aef\uff1aFlink \u63d0\u4f9b\u7684\u7528\u4e8e\u7ba1\u7406\u72b6\u6001\u7684\u7ec4\u4ef6\uff0c\u72b6\u6001\u540e\u7aef\u51b3\u5b9a\u4e86\u4ee5\u4ec0\u4e48\u6837\u6570\u636e\u7ed3\u6784\uff0c\u4ec0\u4e48\u6837\u7684\u5b58\u50a8\u65b9\u5f0f\u53bb\u5b58\u50a8\u548c\u7ba1\u7406\u6211\u4eec\u7684\u72b6\u6001\u3002Flink \u76ee\u524d\u5b98\u65b9\u63d0\u4f9b\u4e86 memory\u3001filesystem\uff0crocksdb \u4e09\u79cd\u72b6\u6001\u540e\u7aef\u6765\u5b58\u50a8\u6211\u4eec\u7684\u72b6\u6001\u3002<br><\/li>\n\n\n\n<li>\u2b50 Checkpoint\uff08\u72b6\u6001\u7ba1\u7406\uff09\uff1aFlink \u63d0\u4f9b\u7684\u7528\u4e8e\u5b9a\u65f6\u5c06\u72b6\u6001\u540e\u7aef\u4e2d\u5b58\u50a8\u7684\u72b6\u6001\u540c\u6b65\u5230\u8fdc\u7a0b\u7684\u5b58\u50a8\u7cfb\u7edf\u7684\u7ec4\u4ef6\u6216\u8005\u80fd\u529b\u3002\u4e3a\u4e86\u9632\u6b62 long run \u7684 Flink \u4efb\u52a1\u6302\u4e86\u5bfc\u81f4\u72b6\u6001\u4e22\u5931\uff0c\u4ea7\u751f\u6570\u636e\u8d28\u91cf\u95ee\u9898\uff0cFlink \u63d0\u4f9b\u4e86\u72b6\u6001\u7ba1\u7406\uff08Checkpoint\uff0cSavepoint\uff09\u7684\u80fd\u529b\u628a\u6211\u4eec\u4f7f\u7528\u7684\u72b6\u6001\u7ed9\u7ba1\u7406\u8d77\u6765\uff0c\u5b9a\u65f6\u7684\u4fdd\u5b58\u5230\u8fdc\u7a0b\u3002\u7136\u540e\u53ef\u4ee5\u5728 Flink \u4efb\u52a1 failover \u65f6\uff0c\u4ece\u8fdc\u7a0b\u628a\u72b6\u6001\u6570\u636e\u6062\u590d\u5230 Flink \u4efb\u52a1\u4e2d\uff0c\u4fdd\u969c\u6570\u636e\u8d28\u91cf\u3002<br><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"22%E6%8A%8A%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%E4%BB%8E_FileSystem_%E5%8F%98%E4%B8%BA_RocksDB_%E5%90%8E%EF%BC%8CFlink_%E4%BB%BB%E5%8A%A1%E7%8A%B6%E6%80%81%E5%AD%98%E5%82%A8%E4%BC%9A%E5%8F%91%E7%94%9F%E9%82%A3%E4%BA%9B%E5%8F%98%E5%8C%96%EF%BC%9F-2\"><\/span>2.2.\u628a\u72b6\u6001\u540e\u7aef\u4ece FileSystem \u53d8\u4e3a RocksDB \u540e\uff0cFlink \u4efb\u52a1\u72b6\u6001\u5b58\u50a8\u4f1a\u53d1\u751f\u90a3\u4e9b\u53d8\u5316\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p><strong>\u7ed3\u8bba\uff1a\u662f\u5426\u4f7f\u7528 RocksDB \u53ea\u4f1a\u5f71\u54cd Flink \u4efb\u52a1\u4e2d keyed-state \u5b58\u50a8\u7684\u65b9\u5f0f\u548c\u5730\u65b9\uff0cFlink \u4efb\u52a1\u4e2d\u7684 operator-state \u4e0d\u4f1a\u53d7\u5230\u5f71\u54cd\u3002<\/strong><\/p>\n\n\n\n<p>\u9996\u5148\u6211\u4eec\u6765\u770b\u770b\uff0cFlink \u4e2d\u7684\u72b6\u6001\u53ea\u4f1a\u5206\u4e3a\u4e24\u7c7b\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 keyed-state\uff1a\u952e\u503c\u72b6\u6001\uff0c\u5982\u5176\u540d\u5b57\uff0c\u6b64\u7c7b\u72b6\u6001\u662f\u4ee5 k-v \u7684\u5f62\u5f0f\u5b58\u50a8\uff0c\u72b6\u6001\u503c\u548c key \u7ed1\u5b9a\u3002Flink \u4e2d\u7684 keyby \u4e4b\u540e\u7d27\u8ddf\u7684\u7b97\u5b50\u7684 state \u5c31\u662f\u952e\u503c\u72b6\u6001\uff1b<\/li>\n\n\n\n<li>\u2b50 operator-state\uff1a\u7b97\u5b50\u72b6\u6001\uff0c\u975e keyed-state \u7684 state \u90fd\u662f\u7b97\u5b50\u72b6\u6001\uff0c\u975e k-v \u7ed3\u6784\uff0c\u72b6\u6001\u503c\u548c\u7b97\u5b50\u7ed1\u5b9a\uff0c\u4e0d\u548c key \u7ed1\u5b9a\u3002Flink \u4e2d\u7684 kafka source \u7b97\u5b50\u4e2d\u7528\u4e8e\u5b58\u50a8 kafka offset \u7684 state \u5c31\u662f\u7b97\u5b50\u72b6\u6001\u3002<\/li>\n<\/ol>\n\n\n\n<p>\u5982\u4e0b\u56fe\u6240\u793a\u662f 3 \u79cd\u72b6\u6001\u540e\u7aef\u548c 2 \u79cd State \u7684\u5bf9\u5e94\u5b58\u50a8\u5173\u7cfb\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic3.zhimg.com\/80\/v2-bb5636c2df2df2e8321dabc686ff88ae_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>2<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u6a2a\u5411\uff08\u884c\uff09\u6765\u770b\uff0c\u5373 Flink \u7684\u72b6\u6001\u5206\u7c7b\u3002\u5206\u4e3a Operator state-backend\u3001Keyed state-backend\uff1b<\/li>\n\n\n\n<li>\u2b50 \u7eb5\u5411\uff08\u5217\uff09\u6765\u770b\uff0c\u5373 Flink \u7684\u72b6\u6001\u540e\u7aef\u5206\u7c7b\u3002\u7528\u6237\u53ef\u4ee5\u914d\u7f6e memory\uff0cfilesystem\uff0crocksdb 3 \u4e2d\u72b6\u6001\u540e\u7aef\uff0c\u5728 Flink \u4efb\u52a1\u4e2d\u751f\u6210 MemoryStateBackend\uff0cFsStateBackend\uff0cRocksdbStateBackend\uff0c\u5176\u58f0\u660e\u4e86\u6574\u4e2a\u4efb\u52a1\u7684\u72b6\u6001\u7ba1\u7406\u540e\u7aef\u7c7b\u578b\uff1b<\/li>\n\n\n\n<li>\u2b50 \u6bcf\u4e2a\u683c\u5b50\u4e2d\u7684\u5185\u5bb9\u5c31\u662f\u7528\u6237\u5728\u914d\u7f6e xx \u72b6\u6001\u540e\u7aef\uff08\u5217\uff09\u65f6\uff0c\u7ed9\u7528\u6237\u4f7f\u7528\u7684\u72b6\u6001\uff08\u884c\uff09\u751f\u6210\u7684\u72b6\u6001\u540e\u7aef\u5b9e\u4f8b\uff0c\u751f\u6210\u7684\u8fd9\u4e2a\u5b9e\u4f8b\u5c31\u662f\u5728 Flink \u4e2d\u5b9e\u9645\u7528\u4e8e\u7ba1\u7406\u7528\u6237\u4f7f\u7528\u7684\u72b6\u6001\u7684\u7ec4\u4ef6\u3002<\/li>\n<\/ol>\n\n\n\n<p>\u56e0\u6b64\u5bf9\u5e94\u7684\u7ed3\u8bba\u5c31\u662f\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 Flink \u4efb\u52a1\u4e2d\u7684 operator-state\u3002\u65e0\u8bba\u7528\u6237\u914d\u7f6e\u54ea\u79cd\u72b6\u6001\u540e\u7aef\uff08\u65e0\u8bba\u662f memory\uff0cfilesystem\uff0crocksdb\uff09\uff0c\u90fd\u662f\u4f7f\u7528 DefaultOperatorStateBackend \u6765\u7ba1\u7406\u7684\uff0c\u72b6\u6001\u6570\u636e\u90fd\u5b58\u50a8\u5728\u5185\u5b58\u4e2d\uff0c\u505a Checkpoint \u65f6\u540c\u6b65\u5230\u8fdc\u7a0b\u6587\u4ef6\u5b58\u50a8\u4e2d\uff08\u6bd4\u5982 HDFS\uff09\u3002<\/li>\n\n\n\n<li>\u2b50 Flink \u4efb\u52a1\u4e2d\u7684 keyed-state\u3002<strong>\u7528\u6237\u5728\u914d\u7f6e rocksdb \u65f6\uff0c\u4f1a\u4f7f\u7528 RocksdbKeyedStateBackend \u53bb\u7ba1\u7406\u72b6\u6001\uff1b\u7528\u6237\u5728\u914d\u7f6e memory\uff0cfilesystem \u65f6\uff0c\u4f1a\u4f7f\u7528 HeapKeyedStateBackend \u53bb\u7ba1\u7406\u72b6\u6001\u3002\u56e0\u6b64\u5c31\u6709\u4e86\u8fd9\u4e2a\u95ee\u9898\u7684\u7ed3\u8bba\uff0c\u914d\u7f6e rocksdb \u53ea\u4f1a\u5f71\u54cd keyed-state \u5b58\u50a8\u7684\u65b9\u5f0f\u548c\u5730\u65b9\uff0coperator-state \u4e0d\u4f1a\u53d7\u5230\u5f71\u54cd\u3002<\/strong><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"23%E4%BB%80%E4%B9%88%E6%A0%B7%E7%9A%84%E4%B8%9A%E5%8A%A1%E5%9C%BA%E6%99%AF%E4%BD%A0%E4%BC%9A%E9%80%89%E6%8B%A9_filesystem%EF%BC%8C%E4%BB%80%E4%B9%88%E6%A0%B7%E7%9A%84%E4%B8%9A%E5%8A%A1%E5%9C%BA%E6%99%AF%E4%BD%A0%E4%BC%9A%E9%80%89_rocksdb_%E7%8A%B6%E6%80%81%E5%90%8E%E7%AB%AF%EF%BC%9F-2\"><\/span>2.3.\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009\u62e9 filesystem\uff0c\u4ec0\u4e48\u6837\u7684\u4e1a\u52a1\u573a\u666f\u4f60\u4f1a\u9009 rocksdb \u72b6\u6001\u540e\u7aef\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u5728\u56de\u7b54\u8fd9\u4e2a\u95ee\u9898\u524d\uff0c\u6211\u4eec\u5148\u770b\u770b\u6bcf\u79cd\u72b6\u6001\u540e\u7aef\u7684\u7279\u6027\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 MemoryStateBackend<\/li>\n<\/ol>\n\n\n\n<ul>\n<li>\u539f\u7406\uff1a\u8fd0\u884c\u65f6\u6240\u9700\u7684 State \u6570\u636e\u5168\u90e8\u4fdd\u5b58\u5728 TaskManager JVM \u5806\u4e0a\u5185\u5b58\u4e2d\uff0c\u6267\u884c Checkpoint \u7684\u65f6\u5019\uff0c\u4f1a\u628a State \u7684\u5feb\u7167\u6570\u636e\u4fdd\u5b58\u5230 JobManager \u8fdb\u7a0b \u7684\u5185\u5b58\u4e2d\u3002\u6267\u884c Savepoint \u65f6\uff0c\u53ef\u4ee5\u628a State \u5b58\u50a8\u5230\u6587\u4ef6\u7cfb\u7edf\u4e2d\u3002<br><\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic3.zhimg.com\/80\/v2-9afd973aae2e1ab14a5a6ba771fb04ce_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<ul>\n<li>\u9002\u7528\u573a\u666f\uff1a<br><\/li>\n\n\n\n<li>\u57fa\u4e8e\u5185\u5b58\u7684 StateBackend \u5728\u751f\u4ea7\u73af\u5883\u4e0b\u4e0d\u5efa\u8bae\u4f7f\u7528\uff0c\u56e0\u4e3a State \u5927\u5c0f\u8d85\u8fc7 JobManager \u5185\u5b58\u5c31 OOM \u4e86\uff0c<strong>\u6b64\u79cd\u72b6\u6001\u540e\u7aef\u9002\u5408\u5728\u672c\u5730\u5f00\u53d1\u8c03\u8bd5\u6d4b\u8bd5\uff0c\u751f\u4ea7\u73af\u5883\u57fa\u672c\u4e0d\u7528\u3002<\/strong><br><\/li>\n\n\n\n<li>State \u5b58\u50a8\u5728 JobManager \u7684\u5185\u5b58\u4e2d\u3002\u53d7\u9650\u4e8e JobManager \u7684\u5185\u5b58\u5927\u5c0f\u3002<br><\/li>\n\n\n\n<li>\u6bcf\u4e2a State \u9ed8\u8ba4 5MB,\u53ef\u901a\u8fc7 MemoryStateBackend \u6784\u9020\u51fd\u6570\u8c03\u6574\u3002d.\u6bcf\u4e2a Stale \u4e0d\u80fd\u8d85\u8fc7 Akka Frame \u5927\u5c0f\u3002<br><\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 FSStateBackend<\/li>\n<\/ol>\n\n\n\n<ul>\n<li>\u539f\u7406\uff1a\u8fd0\u884c\u65f6\u6240\u9700\u7684 State \u6570\u636e\u5168\u90e8\u4fdd\u5b58\u5728 TaskManager \u7684\u5185\u5b58\u4e2d\uff0c\u6267\u884c Checkpoint \u7684\u65f6\u5019\uff0c\u4f1a\u628a State \u7684\u5feb\u7167\u6570\u636e\u4fdd\u5b58\u5230\u914d\u7f6e\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\u3002TM \u662f\u5f02\u6b65\u5c06 State \u6570\u636e\u5199\u5165\u5916\u90e8\u5b58\u50a8\u3002<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-426de4db17f49a98d2ce8781a9e97e2f_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<ul>\n<li>\u9002\u7528\u573a\u666f\uff1a<\/li>\n\n\n\n<li>a.\u9002\u7528\u4e8e<strong>\u5904\u7406\u5c0f\u72b6\u6001\u3001\u77ed\u7a97\u53e3\u3001\u6216\u8005\u5c0f\u952e\u503c\u72b6\u6001\u7684\u6709\u72b6\u6001\u5904\u7406\u4efb\u52a1<\/strong>\uff0c<strong>\u4e0d\u5efa\u8bae\u5728\u5927\u72b6\u6001\u7684\u4efb\u52a1\u4e0b\u4f7f\u7528 FSStateBackend<\/strong>\u3002\u6bd4\u5982 ETL \u4efb\u52a1\uff0c\u5c0f\u65f6\u95f4\u95f4\u9694\u7684 TUMBLE \u7a97\u53e3<br>b.State \u5927\u5c0f\u4e0d\u80fd\u8d85\u8fc7 TM \u5185\u5b58\u3002<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 RocksDBStateBackend<\/li>\n<\/ol>\n\n\n\n<ul>\n<li>\u539f\u7406\uff1a\u4f7f\u7528\u5d4c\u5165\u5f0f\u7684\u672c\u5730\u6570\u636e\u5e93 RocksDB \u5c06\u6d41\u8ba1\u7b97\u6570\u636e\u72b6\u6001\u5b58\u50a8\u5728\u672c\u5730\u78c1\u76d8\u4e2d\u3002\u5728\u6267\u884c Checkpoint \u7684\u65f6\u5019\uff0c\u4f1a\u5c06\u6574\u4e2a RocksDB \u4e2d\u4fdd\u5b58\u7684 State \u6570\u636e\u5168\u91cf\u6216\u8005\u589e\u91cf\u6301\u4e45\u5316\u5230\u914d\u7f6e\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\u3002<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic2.zhimg.com\/80\/v2-f19faa25193b0c8f1cf0c815fc66a959_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<ul>\n<li>\u9002\u7528\u573a\u666f\uff1a<\/li>\n\n\n\n<li>a.\u6700\u9002\u5408\u7528\u4e8e\u5904\u7406<strong>\u5927\u72b6\u6001\u3001\u957f\u7a97\u53e3\uff0c\u6216\u5927\u952e\u503c\u72b6\u6001\u7684\u6709\u72b6\u6001\u5904\u7406\u4efb\u52a1\u3002<\/strong><\/li>\n\n\n\n<li>b.RocksDBStateBackend \u662f\u76ee\u524d<strong>\u552f\u4e00\u652f\u6301\u589e\u91cf\u68c0\u67e5\u70b9<\/strong>\u7684\u540e\u7aef\u3002<\/li>\n\n\n\n<li>c.\u589e\u91cf\u68c0\u67e5\u70b9\u975e\u5e38\u9002\u7528\u4e8e\u8d85\u5927\u72b6\u6001\u7684\u573a\u666f\u3002\u6bd4\u5982\u8ba1\u7b97 DAU \u8fd9\u79cd\u5927\u6570\u636e\u91cf\u53bb\u91cd\uff0c<strong>\u5927\u72b6\u6001\u7684\u4efb\u52a1\u90fd\u5efa\u8bae\u76f4\u63a5\u4f7f\u7528 RocksDB \u72b6\u6001\u540e\u7aef<\/strong>\u3002<\/li>\n<\/ul>\n\n\n\n<p>\u5230\u751f\u4ea7\u73af\u5883\u4e2d\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u5982\u679c\u72b6\u6001\u5f88\u5927\uff0c\u4f7f\u7528 Rocksdb\uff1b\u5982\u679c\u72b6\u6001\u4e0d\u5927\uff0c\u4f7f\u7528 Filesystem\u3002<\/li>\n\n\n\n<li>\u2b50 Rocksdb \u4f7f\u7528\u78c1\u76d8\u5b58\u50a8 State\uff0c\u6240\u4ee5\u4f1a\u6d89\u53ca\u5230\u8bbf\u95ee State \u78c1\u76d8\u5e8f\u5217\u5316\u3001\u53cd\u5e8f\u5217\u5316\uff0c\u6027\u80fd\u4f1a\u6536\u5230\u5f71\u54cd\uff0c\u800c Filesystem \u76f4\u63a5\u8bbf\u95ee\u5185\u5b58\uff0c\u5355\u7eaf\u4ece\u8bbf\u95ee\u72b6\u6001\u7684\u6027\u80fd\u6765\u8bf4 Filesystem \u8fdc\u8fdc\u597d\u4e8e Rocksdb\u3002<strong>\u751f\u4ea7\u73af\u5883\u4e2d\u5b9e\u6d4b\uff0c\u76f8\u540c\u4efb\u52a1\u4f7f\u7528 Filesystem \u6027\u80fd\u4e3a Rocksdb \u7684 n \u500d\uff0c\u56e0\u6b64\u9700\u8981\u6839\u636e\u5177\u4f53\u573a\u666f\u8bc4\u4f30\u9009\u62e9\u3002<\/strong><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"24Flink_SQL_API_State_TTL_%E7%9A%84%E8%BF%87%E6%9C%9F%E6%9C%BA%E5%88%B6%E6%98%AF_onCreateAndUpdate_%E8%BF%98%E6%98%AFonReadAndWrite-2\"><\/span>2.4.Flink SQL API State TTL \u7684\u8fc7\u671f\u673a\u5236\u662f onCreateAndUpdate \u8fd8\u662fonReadAndWrite<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li><strong>\u2b50 \u7ed3\u8bba\uff1aFlink SQL API State TTL \u7684\u8fc7\u671f\u673a\u5236\u76ee\u524d\u53ea\u652f\u6301 onCreateAndUpdate\uff0cDataStream API \u4e24\u4e2a\u90fd\u652f\u6301<\/strong><\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-c9f8108739f2cc9ede7513fbfa9d668c_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>3<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u5256\u6790\uff1a<\/li>\n<\/ol>\n\n\n\n<ul>\n<li>onCreateAndUpdate\uff1a\u662f\u5728\u521b\u5efa State \u548c\u66f4\u65b0 State \u65f6\u3010\u66f4\u65b0 State TTL\u3011<\/li>\n\n\n\n<li>onReadAndWrite\uff1a\u662f\u5728\u8bbf\u95ee State \u548c\u5199\u5165 State \u65f6\u3010\u66f4\u65b0 State TTL\u3011<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 \u5b9e\u9645\u8e29\u5751\u573a\u666f\uff1aFlink SQL Deduplicate \u5199\u6cd5\uff0crow_number partition by user_id order by proctime asc\uff0c\u6b64 SQL \u6700\u540e\u751f\u6210\u7684\u7b97\u5b50\u53ea\u4f1a\u5728\u7b2c\u4e00\u6761\u6570\u636e\u6765\u7684\u65f6\u5019\u66f4\u65b0 state\uff0c\u540e\u7eed\u8bbf\u95ee\u4e0d\u4f1a\u66f4\u65b0 state TTL\uff0c\u56e0\u6b64 state \u4f1a\u5728\u7528\u6237\u8bbe\u7f6e\u7684 state TTL \u65f6\u95f4\u4e4b\u540e\u8fc7\u671f\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3-2\"><\/span>3.\u65f6\u95f4\u7a97\u53e3<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"31watermark_%E5%88%B0%E5%BA%95%E6%98%AF%E5%B9%B2%E5%95%A5%E7%9A%84%EF%BC%9F%E5%BA%94%E7%94%A8%E5%9C%BA%E6%99%AF%EF%BC%9F-2\"><\/span>3.1.watermark \u5230\u5e95\u662f\u5e72\u5565\u7684\uff1f\u5e94\u7528\u573a\u666f\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u5927\u90e8\u5206\u540c\u5b66\u90fd\u53ea\u80fd\u56de\u7b54\u51fa\uff1awatermark \u662f\u7528\u4e8e\u7f13\u89e3\u65f6\u95f4\u65f6\u95f4\u7684\u4e71\u5e8f\u95ee\u9898\u7684\u3002<\/p>\n\n\n\n<p>\u6ca1\u9519\uff0c\u8fd9\u4e2a\u89c2\u70b9\u662f\u6b63\u786e\u7684\u3002\u4f46\u662f\u535a\u4e3b\u8ba4\u4e3a\u8fd9\u53ea\u662f watermark \u7b2c\u4e8c\u91cd\u8981\u7684\u4f5c\u7528\uff0c\u5176\u66f4\u91cd\u8981\u7684\u4f5c\u7528\u5728\u4e8e\u53ef\u4ee5\u6807\u8bc6\u4e00\u4e2a Flink \u4efb\u52a1\u7684\u4e8b\u4ef6&nbsp;<strong>\u65f6\u95f4\u8fdb\u5ea6<\/strong>\u3002<\/p>\n\n\n\n<p>\u600e\u4e48\u7406\u89e3&nbsp;<strong>\u65f6\u95f4\u8fdb\u5ea6<\/strong>\uff1f<\/p>\n\n\n\n<p>\u6211\u4eec\u53ef\u4ee5\u73b0\u8c61\u4e00\u4e0b\uff0c\u4e00\u4e2a\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u7684\u4efb\u52a1\uff0c\u5982\u679c\u6ca1\u6709\u4e00\u4e2a&nbsp;<strong>\u4e1c\u897f<\/strong>&nbsp;\u53bb\u6807\u8bc6\u5176\u4e8b\u4ef6\u65f6\u95f4\u7684\u8fdb\u5ea6\uff0c\u90a3\u4e48\u8fd9\u4e2a\u4e8b\u4ef6\u65f6\u95f4\u7684\u7a97\u53e3\u4e5f\u5c31\u662f\u4e0d\u77e5\u9053\u4ec0\u4e48\u65f6\u5019\u80fd\u591f\u89e6\u53d1\u4e86\uff0c\u4e5f\u5c31\u662f\u8bf4\u8fd9\u4e2a\u7a97\u53e3\u6c38\u8fdc\u4e0d\u4f1a\u89e6\u53d1\u5e76\u4e14\u8f93\u51fa\u7ed3\u679c\u3002<\/p>\n\n\n\n<p>\u6240\u4ee5\u8981\u6709\u4e00\u4e2a&nbsp;<strong>\u4e1c\u897f<\/strong>&nbsp;\u53bb\u6807\u8bc6\u5176\u4e8b\u4ef6\u65f6\u95f4\u7684\u8fdb\u5ea6\uff0c\u4ece\u800c\u8ba9\u8fd9\u4e2a\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u77e5\u9053\uff0c\u8fd9\u4e2a\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u5df2\u7ecf\u7ed3\u675f\u4e86\uff0c\u53ef\u4ee5\u89e6\u53d1\u8ba1\u7b97\u4e86\u3002\u5728 Flink \u4e2d\uff0c\u8fd9\u4e2a&nbsp;<strong>\u4e1c\u897f<\/strong>&nbsp;\u5c31\u662f&nbsp;<strong>watermark<\/strong>\u3002<\/p>\n\n\n\n<p>\u603b\u7ed3\u4e00\u4e0b\uff0c\u535a\u4e3b\u8ba4\u4e3a&nbsp;<strong>watermark<\/strong>&nbsp;\u4e3a Flink \u89e3\u51b3\u4e86\u4e24\u4e2a\u95ee\u9898\uff1a<\/p>\n\n\n\n<ol>\n<li><strong>\u2b50 \u6807\u8bc6 Flink \u4efb\u52a1\u7684\u4e8b\u4ef6\u65f6\u95f4\u8fdb\u5ea6\uff0c\u4ece\u800c\u80fd\u591f\u63a8\u52a8\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u7684\u89e6\u53d1\u3001\u8ba1\u7b97\u3002<\/strong><\/li>\n\n\n\n<li><strong>\u2b50 \u89e3\u51b3\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u7684\u4e71\u5e8f\u95ee\u9898\u3002<\/strong><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"32%E4%B8%80%E4%B8%AA_Flink_%E4%BB%BB%E5%8A%A1%E4%B8%AD%E5%8F%AF%E4%BB%A5%E6%97%A2%E6%9C%89%E4%BA%8B%E4%BB%B6%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3%EF%BC%8C%E5%8F%88%E6%9C%89%E5%A4%84%E7%90%86%E6%97%B6%E9%97%B4%E7%AA%97%E5%8F%A3%E5%90%97%EF%BC%9F-2\"><\/span>3.2.\u4e00\u4e2a Flink \u4efb\u52a1\u4e2d\u53ef\u4ee5\u65e2\u6709\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\uff0c\u53c8\u6709\u5904\u7406\u65f6\u95f4\u7a97\u53e3\u5417\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p><strong>\u7ed3\u8bba\uff1a\u4e00\u4e2a Flink \u4efb\u52a1\u53ef\u4ee5\u540c\u65f6\u6709\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\uff0c\u53c8\u6709\u5904\u7406\u65f6\u95f4\u7a97\u53e3\u3002<\/strong><\/p>\n\n\n\n<p>\u90a3\u4e48\u6709\u4e9b\u5c0f\u4f19\u4f34\u4eec\u95ee\u4e86\uff0c\u4e3a\u4ec0\u4e48\u6211\u4eec\u5e38\u89c1\u7684 Flink \u4efb\u52a1\u8981\u4e48\u8bbe\u7f6e\u4e3a\u4e8b\u4ef6\u65f6\u95f4\u8bed\u4e49\uff0c\u8981\u4e48\u8bbe\u7f6e\u4e3a\u5904\u7406\u65f6\u95f4\u8bed\u4e49\uff1f<\/p>\n\n\n\n<p>\u786e\u5b9e\uff0c\u5728\u751f\u4ea7\u73af\u5883\u4e2d\uff0c\u6211\u4eec\u7684 Flink \u4efb\u52a1\u4e00\u822c\u4e0d\u4f1a\u540c\u65f6\u62e5\u6709\u4e24\u79cd\u65f6\u95f4\u8bed\u4e49\u7684\u7a97\u53e3\u3002<\/p>\n\n\n\n<p>\u90a3\u4e48\u600e\u4e48\u89e3\u91ca\u5f00\u5934\u535a\u4e3b\u6240\u8bf4\u7684\u7ed3\u8bba\u5462\uff1f<\/p>\n\n\n\n<p>\u535a\u4e3b\u8fd9\u91cc\u4ece\u4e24\u4e2a\u89d2\u5ea6\u8fdb\u884c\u8bf4\u660e\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u6211\u4eec\u5176\u5b9e\u6ca1\u6709\u5fc5\u8981\u628a\u4e00\u4e2a Flink \u4efb\u52a1\u548c\u67d0\u79cd\u7279\u5b9a\u7684\u65f6\u95f4\u8bed\u4e49\u8fdb\u884c\u7ed1\u5b9a\u3002\u5bf9\u4e8e\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u6765\u8bf4\uff0c\u6211\u4eec\u53ea\u8981\u7ed9\u5b83 watermark\uff0c\u80fd\u8ba9 watermark \u4e00\u76f4\u5f80\u524d\u63a8\u8fdb\uff0c\u8ba9\u4e8b\u4ef6\u65f6\u95f4\u7a97\u53e3\u80fd\u591f\u6301\u7eed\u89e6\u53d1\u8ba1\u7b97\u5c31\u884c\u3002\u5bf9\u4e8e\u5904\u7406\u65f6\u95f4\u6765\u8bf4\u66f4\u7b80\u5355\uff0c\u53ea\u8981\u7a97\u53e3\u7b97\u5b50\u6309\u7167\u672c\u5730\u65f6\u95f4\u6309\u7167\u56fa\u5b9a\u7684\u65f6\u95f4\u95f4\u9694\u8fdb\u884c\u89e6\u53d1\u5c31\u884c\u3002\u65e0\u8bba\u54ea\u79cd\u65f6\u95f4\u7a97\u53e3\uff0c\u4e3b\u8981\u6ee1\u8db3\u65f6\u95f4\u7a97\u53e3\u7684\u89e6\u53d1\u6761\u4ef6\u5c31\u884c\u3002<\/li>\n\n\n\n<li>\u2b50 Flink \u7684\u5b9e\u73b0\u4e0a\u6765\u8bf4\u4e5f\u662f\u652f\u6301\u7684\u3002Flink \u662f\u4f7f\u7528\u4e00\u4e2a\u53eb\u505a TimerService \u7684\u7ec4\u4ef6\u6765\u7ba1\u7406 timer \u7684\uff0c\u6211\u4eec\u53ef\u4ee5\u540c\u65f6\u6ce8\u518c\u4e8b\u4ef6\u65f6\u95f4\u548c\u5904\u7406\u65f6\u95f4\u7684 timer\uff0cFlink \u4f1a\u81ea\u884c\u5224\u65ad timer \u662f\u5426\u6ee1\u8db3\u89e6\u53d1\u6761\u4ef6\uff0c\u5982\u679c\u662f\uff0c\u5219\u56de\u8c03\u7a97\u53e3\u5904\u7406\u51fd\u6570\u8fdb\u884c\u8ba1\u7b97\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"33window_%E5%90%8E%E9%9D%A2%E8%B7%9F_aggregate_%E5%92%8C_process_%E7%9A%84%E4%B8%A4%E4%B8%AA%E7%AA%97%E5%8F%A3%E8%AE%A1%E7%AE%97%E7%9A%84%E5%8C%BA%E5%88%AB%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F-2\"><\/span>3.3.window \u540e\u9762\u8ddf aggregate \u548c process \u7684\u4e24\u4e2a\u7a97\u53e3\u8ba1\u7b97\u7684\u533a\u522b\u662f\u4ec0\u4e48\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 aggregate\uff1a\u662f\u589e\u91cf\u805a\u5408\uff0c\u6765\u4e00\u6761\u6570\u636e\u8ba1\u7b97\u5b8c\u4e86\u5b58\u50a8\u5728\u7d2f\u52a0\u5668\u4e2d\uff0c\u4e0d\u9700\u8981\u7b49\u5230\u7a97\u53e3\u89e6\u53d1\u65f6\u8ba1\u7b97\uff0c<strong>\u6027\u80fd\u8f83\u597d<\/strong>\uff1b<\/li>\n\n\n\n<li>\u2b50 process\uff1a\u5168\u91cf\u51fd\u6570\uff0c\u7f13\u5b58\u5168\u90e8\u7a97\u53e3\u5185\u7684\u6570\u636e\uff0c\u6ee1\u8db3\u7a97\u53e3\u89e6\u53d1\u6761\u4ef6\u518d\u89e6\u53d1\u8ba1\u7b97\uff0c\u540c\u65f6\u8fd8\u63d0\u4f9b\u5b9a\u65f6\u89e6\u53d1\uff0c\u7a97\u53e3\u4fe1\u606f\u7b49\u4e0a\u4e0b\u6587\u4fe1\u606f\uff1b<\/li>\n\n\n\n<li>\u2b50 \u5e94\u7528\u573a\u666f\uff1aaggregate \u4e00\u4e2a\u4e00\u4e2a\u5904\u7406\u7684\u805a\u5408\u7ed3\u679c\u5411\u540e\u4f20\u9012\u4e00\u822c\u6765\u8bf4\u90fd\u662f\u6709\u4fe1\u606f\u635f\u5931\u7684\uff0c\u800c<strong>&nbsp;process \u5219\u53ef\u4ee5\u66f4\u52a0\u5b9a\u5236\u5316\u7684\u5904\u7406<\/strong>\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4%E7%BC%96%E7%A8%8B%E6%8A%80%E5%B7%A7-2\"><\/span>4.\u7f16\u7a0b\u6280\u5de7<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"41%E4%B8%BA%E4%BB%80%E4%B9%88_Flink_DataStream_API_%E5%9C%A8%E5%87%BD%E6%95%B0%E5%85%A5%E5%8F%82%E6%88%96%E8%80%85%E5%87%BA%E5%8F%82%E6%9C%89%E6%B3%9B%E5%9E%8B%E6%97%B6%EF%BC%8C%E4%B8%8D%E8%83%BD%E4%BD%BF%E7%94%A8_lambda_%E8%A1%A8%E8%BE%BE%E5%BC%8F%EF%BC%9F-2\"><\/span>4.1.\u4e3a\u4ec0\u4e48 Flink DataStream API \u5728\u51fd\u6570\u5165\u53c2\u6216\u8005\u51fa\u53c2\u6709\u6cdb\u578b\u65f6\uff0c\u4e0d\u80fd\u4f7f\u7528 lambda \u8868\u8fbe\u5f0f\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>Flink \u7c7b\u578b\u4fe1\u606f\u7cfb\u7edf\u662f\u901a\u8fc7\u53cd\u5c04\u83b7\u53d6\u5230 Java class \u7684\u65b9\u6cd5\u7b7e\u540d\u53bb\u83b7\u53d6\u7c7b\u578b\u4fe1\u606f\u7684\u3002<\/p>\n\n\n\n<p>\u4ee5 FlatMap \u4e3a\u4f8b\uff0cFlink \u5728\u901a\u8fc7\u53cd\u5c04\u65f6\u4f1a\u68c0\u67e5\u53ca\u83b7\u53d6 FlatMap collector \u7684\u51fa\u53c2\u7c7b\u578b\u4fe1\u606f\u3002<\/p>\n\n\n\n<p>\u4f46\u662f lambda \u8868\u8fbe\u5f0f\u5199\u7684 FlatMap \u903b\u8f91\uff0c\u4f1a\u5bfc\u81f4\u53cd\u5c04\u65b9\u6cd5\u83b7\u53d6\u7c7b\u578b\u4fe1\u606f\u65f6\u3010\u76f4\u63a5\u83b7\u53d6\u4e0d\u5230\u3011collector \u7684\u51fa\u53c2\u7c7b\u578b\u53c2\u6570\uff0c\u6240\u4ee5\u624d\u4f1a\u62a5\u9519\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"42Flink_%E4%B8%BA%E4%BB%80%E4%B9%88%E5%BC%BA%E8%B0%83_function_%E5%AE%9E%E7%8E%B0%E6%97%B6%EF%BC%8C%E5%AE%9E%E4%BE%8B%E5%8C%96%E7%9A%84%E5%8F%98%E9%87%8F%E8%A6%81%E5%AE%9E%E7%8E%B0_serializable_%E6%8E%A5%E5%8F%A3%EF%BC%9F-2\"><\/span>4.2.Flink \u4e3a\u4ec0\u4e48\u5f3a\u8c03 function \u5b9e\u73b0\u65f6\uff0c\u5b9e\u4f8b\u5316\u7684\u53d8\u91cf\u8981\u5b9e\u73b0 serializable \u63a5\u53e3\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u5176\u5b9e\u8fd9\u4e2a\u95ee\u9898\u53ef\u4ee5\u5ef6\u4f38\u6210 3 \u4e2a\u95ee\u9898\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u4e3a\u4ec0\u4e48 Flink \u8981\u7528\u5230 Java \u5e8f\u5217\u5316\u673a\u5236\u3002\u548c Flink \u7c7b\u578b\u7cfb\u7edf\u7684\u6570\u636e\u5e8f\u5217\u5316\u673a\u5236\u7684\u7528\u9014\u6709\u5565\u533a\u522b\uff1f<\/li>\n\n\n\n<li>\u2b50 \u975e\u5b9e\u4f8b\u5316\u7684\u53d8\u91cf\u6ca1\u6709\u5b9e\u73b0 Serializable \u4e3a\u5565\u5c31\u4e0d\u62a5\u9519\uff0c\u5b9e\u4f8b\u5316\u5c31\u62a5\u9519\uff1f<\/li>\n\n\n\n<li>\u2b50 \u4e3a\u5565\u52a0 transient \u5c31\u4e0d\u62a5\u9519\uff1f<\/li>\n<\/ol>\n\n\n\n<p>\u4e0a\u9762 3 \u4e2a\u95ee\u9898\u7684\u7b54\u6848\u5982\u4e0b\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 Flink \u5199\u7684\u51fd\u6570\u5f0f\u7f16\u7a0b\u4ee3\u7801\u6216\u8005\u8bf4\u95ed\u5305\uff0c\u9700\u8981 Java \u5e8f\u5217\u5316\u4ece JobManager \u5206\u53d1\u5230 TaskManager\uff0c\u800c Flink \u7c7b\u578b\u7cfb\u7edf\u7684\u6570\u636e\u5e8f\u5217\u5316\u673a\u5236\u662f\u4e3a\u4e86\u5206\u53d1\u6570\u636e\uff0c\u4e0d\u662f\u5206\u53d1\u4ee3\u7801\uff0c\u53ef\u4ee5\u7528\u975eJava\u7684\u5e8f\u5217\u5316\u673a\u5236\uff0c\u6bd4\u5982 Kyro\u3002<\/li>\n\n\n\n<li>\u2b50 \u7f16\u8bd1\u671f\u4e0d\u505a\u5e8f\u5217\u5316\uff0c\u6240\u4ee5\u4e0d\u5b9e\u73b0 Serializable \u4e0d\u4f1a\u62a5\u9519\uff0c\u4f46\u662f\u8fd0\u884c\u671f\u4f1a\u6267\u884c\u5e8f\u5217\u5316\u52a8\u4f5c\uff0c\u6ca1\u5b9e\u73b0 Serializable \u63a5\u53e3\u7684\u5c31\u62a5\u9519\u4e86<\/li>\n\n\n\n<li>\u2b50 Flink DataStream API \u7684 Function \u4f5c\u4e3a\u95ed\u5305\u5728\u7f51\u7edc\u4f20\u8f93\uff0c\u5fc5\u987b\u91c7\u7528 Java \u5e8f\u5217\u5316\uff0c\u6240\u4ee5\u8981\u901a\u8fc7 Serializable \u63a5\u53e3\u6807\u8bb0\uff0c\u6839\u636e Java \u5e8f\u5217\u5316\u7684\u89c4\u5b9a\uff0c\u5185\u90e8\u6210\u5458\u53d8\u91cf\u8981\u4e48\u90fd\u53ef\u5e8f\u5217\u5316\uff0c\u8981\u4e48\u901a\u8fc7 transient \u5173\u952e\u5b57\u8df3\u8fc7\u5e8f\u5217\u5316\uff0c\u5426\u5219 Java \u5e8f\u5217\u5316\u7684\u65f6\u5019\u4f1a\u62a5\u9519\u3002\u9759\u6001\u53d8\u91cf\u4e0d\u53c2\u4e0e\u5e8f\u5217\u5316\uff0c\u6240\u4ee5\u4e0d\u7528\u52a0 transient\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"43Flink_%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%E5%8F%AF%E4%BB%A5%E9%80%9A%E8%BF%87%E5%93%AA%E5%87%A0%E7%A7%8D%E6%96%B9%E5%BC%8F%E8%AE%BE%E7%BD%AE%EF%BC%8C%E4%BC%98%E5%85%88%E7%BA%A7%E5%85%B3%E7%B3%BB%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F-2\"><\/span>4.3.Flink \u7684\u5e76\u884c\u5ea6\u53ef\u4ee5\u901a\u8fc7\u54ea\u51e0\u79cd\u65b9\u5f0f\u8bbe\u7f6e\uff0c\u4f18\u5148\u7ea7\u5173\u7cfb\u662f\u4ec0\u4e48\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 \u4ee3\u7801\u4e2d\u7b97\u5b50\u5355\u72ec\u8bbe\u7f6e<\/li>\n\n\n\n<li>\u2b50 \u4ee3\u7801\u4e2dEnv\u5168\u5c40\u8bbe\u7f6e<\/li>\n\n\n\n<li>\u2b50 \u63d0\u4ea4\u53c2\u6570<\/li>\n\n\n\n<li>\u2b50 \u9ed8\u8ba4\u914d\u7f6e\u4fe1\u606f<\/li>\n<\/ol>\n\n\n\n<p>\u4e0a\u9762\u7684 Flink \u5e76\u884c\u5ea6\u4f18\u5148\u7ea7\u4ece\u4e0a\u5f80\u4e0b\u7531\u5927\u53d8\u5c0f\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"5%E5%AE%9E%E6%88%98%E7%BB%8F%E9%AA%8C-2\"><\/span>5.\u5b9e\u6218\u7ecf\u9a8c<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"51Flink_SQL_%E8%AE%A1%E7%AE%97%E7%94%A8%E6%88%B7%E5%88%86%E5%B8%83-2\"><\/span>5.1.Flink SQL \u8ba1\u7b97\u7528\u6237\u5206\u5e03<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 \u9700\u6c42\uff1a\u4e0a\u6e38\u662f\u4e00\u4e2a kafka \u6570\u636e\u6e90\uff0c\u6570\u636e\u5185\u5bb9\u662f\u7528\u6237 QQ \u7b49\u7ea7\u53d8\u5316\u660e\u7ec6\u6570\u636e\uff08time\uff0cuid\uff0clevel\uff09\u3002\u9700\u8981\u4f60\u6c42\u51fa\u5f53\u524d\u6bcf\u4e2a\u7b49\u7ea7\u7684\u7528\u6237\u6570\u3002<br><\/li>\n\n\n\n<li>\u2b50 \u5b9e\u73b0 SQL\uff1aDeduplicate<br><\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>-- \u5982\u679c\u9700\u8981\u53ef\u4ee5\u6253\u5f00 minibatch\nselect  \n    level\n    , count(1) as uv\n    , max(time) as time\nfrom (\n    select \n        uid\n        , level\n        , time\n        , row_number() over (partition by uid order by time desc) rn \n    from source\n) tmp\nwhere rn =1 \ngroup by \n    level\n<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"52Flink_SQL_%E8%AE%A1%E7%AE%97_DAU-2\"><\/span>5.2.Flink SQL \u8ba1\u7b97 DAU<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 \u9700\u6c42\uff1a\u6570\u636e\u6e90\uff1a\u7528\u6237\u5fc3\u8df3\u65e5\u5fd7\uff08uid\uff0ctime\uff0ctype\uff09\u3002\u8ba1\u7b97\u5206 Android\uff0ciOS \u7684 DAU\uff0c\u6700\u665a\u4e00\u5206\u949f\u8f93\u51fa\u4e00\u6b21\u5f53\u65e5\u96f6\u70b9\u7d2f\u8ba1\u5230\u5f53\u524d\u7684\u7ed3\u679c\u3002<br><\/li>\n\n\n\n<li>\u2b50 \u5b9e\u73b0\u65b9\u5f0f 1\uff1acumulate \u7a97\u53e3<br><\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>SELECT  \n    window_start\n    , window_end\n    , platform\n    , sum(bucket_dau) as dau\nfrom (\n    SELECT\n        window_start\n        , window_end\n        , platform\n        , count(distinct uid) as bucket_dau\n    FROM TABLE(\n        CUMULATE(\n        TABLE user_log,\n        DESCRIPTOR(time),\n        INTERVAL '60' SECOND\n        , INTERVAL '1' DAY))\n    GROUP BY                                  \n        window_start\n        , window_end\n        , platform\n        , MOD(HASH_CODE(user_id), 1024)\n) tmp\nGROUP by   \n    window_start\n    , window_end\n    , platform\n<\/code><\/pre>\n\n\n\n<ul>\n<li>\u4f18\u70b9\uff1a\u5982\u679c\u662f\u66f2\u7ebf\u56fe\u7684\u9700\u6c42\uff0c\u53ef\u4ee5\u5b8c\u7f8e\u56de\u6eaf\u66f2\u7ebf\u56fe\u3002<\/li>\n\n\n\n<li>\u7f3a\u70b9\uff1a\u5927\u7a97\u53e3\u4e4b\u95f4\u5982\u679c\u6709\u6570\u636e\u4e71\u5e8f\uff0c\u6709\u4e22\u6570\u98ce\u9669\uff1b\u5e76\u4e14\u7531\u4e8e\u662f watermark \u63a8\u52a8\u4ea7\u51fa\uff0c\u6240\u4ee5\u6570\u636e\u4ea7\u51fa\u4f1a\u6709\u5ef6\u8fdf\u3002<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 \u5b9e\u73b0\u65b9\u5f0f 2\uff1aDeduplicate<\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>-- \u5982\u679c\u9700\u8981\u53ef\u4ee5\u6253\u5f00 minibatch\nselect \n    platform\n    , count(1) as dau\n    , max(time) as time\nfrom (\n    select \n        uid\n        , platform\n        , time\n        , row_number() over (partition by uid, platform, time \/ 24 \/ 3600 \/ 1000 order by time desc) rn\n    from source\n) tmp\nwhere rn = 1\ngroup by\n    platform\n<\/code><\/pre>\n\n\n\n<ul>\n<li>\u4f18\u70b9\uff1a\u8ba1\u7b97\u5feb\u3002<\/li>\n\n\n\n<li>\u7f3a\u70b9\uff1a\u4efb\u52a1\u53d1\u751f failover\uff0c\u66f2\u7ebf\u56fe\u4e0d\u80fd\u5f88\u597d\u56de\u6eaf\u3002\u6ca1\u6cd5\u652f\u6301 cube \u8ba1\u7b97\u3002<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 \u5b9e\u73b0\u65b9\u5f0f 3\uff1agroup agg<\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>-- \u5982\u679c\u9700\u8981\u53ef\u4ee5\u6253\u5f00 minibatch\nSELECT   \n    max(time) as time\n    , platform\n    , sum(bucket_dau) as dau\nfrom (\n    SELECT\n        max(time) as time\n        , platform\n        , count(distinct uid) as bucket_dau\n    FROM source\n    GROUP BY\n        platform\n        , MOD(HASH_CODE(user_id), 1024)\n) t \nGROUP by   \n    platform<\/code><\/pre>\n\n\n\n<ul>\n<li>\u4f18\u70b9\uff1a\u8ba1\u7b97\u5feb\uff0c\u652f\u6301 cube \u8ba1\u7b97\u3002<\/li>\n\n\n\n<li>\u7f3a\u70b9\uff1a\u4efb\u52a1\u53d1\u751f failover\uff0c\u66f2\u7ebf\u56fe\u4e0d\u80fd\u5f88\u597d\u56de\u6eaf\u3002<\/li>\n<\/ul>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"53%E4%BD%A0%E6%98%AF%E6%80%8E%E4%B9%88%E5%90%88%E7%90%86%E7%9A%84%E8%AF%84%E4%BC%B0_Flink_%E4%BB%BB%E5%8A%A1%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F-2\"><\/span>5.3.\u4f60\u662f\u600e\u4e48\u5408\u7406\u7684\u8bc4\u4f30 Flink \u4efb\u52a1\u7684\u5e76\u884c\u5ea6\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>Flink \u4efb\u52a1\u5e76\u884c\u5ea6\u5408\u7406\u884c\u4e00\u822c\u6839\u636e<strong>\u5cf0\u503c\u6d41\u91cf\u8fdb\u884c\u538b\u6d4b\u8bc4\u4f30<\/strong>\uff0c\u5e76\u4e14\u6839\u636e\u96c6\u7fa4\u8d1f\u8f7d\u60c5\u51b5<strong>\u7559\u4e00\u5b9a\u91cf\u7684 buffer \u8d44\u6e90<\/strong>\u3002<\/p>\n\n\n\n<ol>\n<li>\u2b50 \u5982\u679c\u6570\u636e\u6e90\u5df2\u7ecf\u5b58\u5728\uff0c\u5219\u53ef\u4ee5\u76f4\u63a5\u6d88\u8d39\u8fdb\u884c\u6d4b\u8bd5<\/li>\n\n\n\n<li>\u2b50 \u5982\u679c\u6570\u636e\u6e90\u4e0d\u5b58\u5728\uff0c\u9700\u8981\u81ea\u884c\u9020\u538b\u6d4b\u6570\u636e\u8fdb\u884c\u6d4b\u8bd5<\/li>\n<\/ol>\n\n\n\n<p>\u5bf9\u4e8e\u4e00\u4e2a Flink \u4efb\u52a1\u6765\u8bf4\uff0c\u4e00\u822c\u53ef\u4ee5\u6309\u7167\u4ee5\u4e0b\u65b9\u5f0f\u8fdb\u884c\u7ec6\u7c92\u5ea6\u8bbe\u7f6e\u5e76\u884c\u5ea6\uff1a<\/p>\n\n\n\n<ol>\n<li>\u2b50 source \u5e76\u884c\u5ea6\u914d\u7f6e\uff1a\u4ee5 kafka \u4e3a\u4f8b\uff0csource \u7684\u5e76\u884c\u5ea6\u4e00\u822c\u8bbe\u7f6e\u4e3a kafka \u5bf9\u5e94\u7684 topic \u7684\u5206\u533a\u6570<br><\/li>\n\n\n\n<li>\u2b50 transform\uff08\u6bd4\u5982 flatmap\u3001map\u3001filter \u7b49\u7b97\u5b50\uff09\u5e76\u884c\u5ea6\u7684\u914d\u7f6e\uff1a\u8fd9\u4e9b\u7b97\u5b50\u4e00\u822c\u4e0d\u4f1a\u505a\u592a\u91cd\u7684\u64cd\u4f5c\uff0c\u5e76\u884c\u5ea6\u53ef\u4ee5\u548c source \u4fdd\u6301\u4e00\u81f4\uff0c\u4f7f\u5f97\u7b97\u5b50\u4e4b\u95f4\u53ef\u4ee5\u505a\u5230 forward \u4f20\u8f93\u6570\u636e\uff0c\u4e0d\u7ecf\u8fc7\u7f51\u7edc\u4f20\u8f93<br><\/li>\n\n\n\n<li>\u2b50 keyby \u4e4b\u540e\u7684\u5904\u7406\u7b97\u5b50\uff1a<strong>\u5efa\u8bae\u6700\u5927\u5e76\u884c\u5ea6\u4e3a\u6b64\u7b97\u5b50\u5e76\u884c\u5ea6\u7684\u6574\u6570\u500d<\/strong>\uff0c\u8fd9\u6837\u53ef\u4ee5\u4f7f\u6bcf\u4e2a\u7b97\u5b50\u4e0a\u7684 keyGroup \u662f\u76f8\u540c\u7684\uff0c\u4ece\u800c\u4f7f\u5f97\u6570\u636e\u76f8\u5bf9\u5747\u5300 shuffle \u5230\u4e0b\u6e38\u7b97\u5b50\uff0c\u5982\u4e0b\u56fe\u4e3a shuffle \u7b56\u7565<br><\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-1e65fbdab78e25e22fca54e82e48094b_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<ol>\n<li>\u2b50 sink \u5e76\u884c\u5ea6\u7684\u914d\u7f6e\uff1asink \u662f\u6570\u636e\u6d41\u5411\u4e0b\u6e38\u7684\u5730\u65b9\uff0c\u53ef\u4ee5\u6839\u636e sink \u7684\u6570\u636e\u91cf\u53ca\u4e0b\u6e38\u7684\u670d\u52a1\u6297\u538b\u80fd\u529b\u8fdb\u884c\u8bc4\u4f30\u3002\u5982\u679c sink \u662f kafka\uff0c\u53ef\u4ee5\u8bbe\u4e3a kafka \u5bf9\u5e94 topic \u7684\u5206\u533a\u6570\u3002\u6ce8\u610f sink \u5e76\u884c\u5ea6\u6700\u597d\u548c kafka partition \u6210\u500d\u6570\u5173\u7cfb\uff0c\u5426\u5219\u53ef\u80fd\u4f1a\u51fa\u73b0\u5982\u5230 kafka partition \u6570\u636e\u4e0d\u5747\u5300\u7684\u60c5\u51b5\u3002\u4f46\u662f\u5927\u591a\u6570\u60c5\u51b5\u4e0b sink \u7b97\u5b50\u5e76\u884c\u5ea6\u4e0d\u9700\u8981\u7279\u522b\u8bbe\u7f6e\uff0c\u53ea\u9700\u8981\u548c\u6574\u4e2a\u4efb\u52a1\u7684\u5e76\u884c\u5ea6\u76f8\u540c\u5c31\u884c\u3002<br><\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"54%E4%BD%A0%E6%98%AF%E6%80%8E%E4%B9%88%E5%90%88%E7%90%86%E8%AF%84%E4%BC%B0%E4%BB%BB%E5%8A%A1%E6%9C%80%E5%A4%A7%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F-2\"><\/span>5.4.\u4f60\u662f\u600e\u4e48\u5408\u7406\u8bc4\u4f30\u4efb\u52a1\u6700\u5927\u5e76\u884c\u5ea6\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol>\n<li>\u2b50 \u524d\u63d0\uff1a\u5e76\u884c\u5ea6\u5fc5\u987b &lt;= \u6700\u5927\u5e76\u884c\u5ea6<\/li>\n\n\n\n<li>\u2b50 \u6700\u5927\u5e76\u884c\u5ea6\u7684\u4f5c\u7528\uff1a\u5408\u7406\u8bbe\u7f6e\u6700\u5927\u5e76\u884c\u5ea6\u53ef\u4ee5\u7f13\u89e3\u6570\u636e\u503e\u659c\u7684\u95ee\u9898<\/li>\n\n\n\n<li>\u2b50 \u6839\u636e\u5177\u4f53\u573a\u666f\u7684\u4e0d\u540c\uff0c\u6700\u5927\u5e76\u884c\u5ea6\u5927\u5c0f\u8bbe\u7f6e\u4e5f\u6709\u4e0d\u540c\u7684\u65b9\u5f0f\uff1a<\/li>\n<\/ol>\n\n\n\n<ul>\n<li><strong>\u5728 key \u975e\u5e38\u591a\u7684\u60c5\u51b5\u4e0b\uff0c\u6700\u5927\u5e76\u884c\u5ea6\u9002\u5408\u8bbe\u7f6e\u6bd4\u8f83\u5927\uff08\u51e0\u5343\uff09<\/strong>\uff0c\u4e0d\u5bb9\u6613\u51fa\u73b0\u6570\u636e\u503e\u659c\uff0c\u4ee5 Flink SQL \u573a\u666f\u4e3e\u4f8b\uff1arow_number = 1 partition key user_id \u7684 Deduplicate \u573a\u666f\uff08user_id \u4e00\u822c\u90fd\u975e\u5e38\u591a\uff09<\/li>\n\n\n\n<li><strong>\u5728 key \u4e0d\u662f\u5f88\u591a\u7684\u60c5\u51b5\u4e0b\uff0c\u6700\u5927\u5e76\u884c\u5ea6\u9002\u5408\u8bbe\u7f6e\u4e0d\u662f\u5f88\u5927<\/strong>\uff0c\u4e0d\u7136\u4f1a\u52a0\u91cd\u6570\u636e\u503e\u659c\uff0c\u4ee5 Flink SQL \u573a\u666f\u4e3e\u4f8b\uff1agroup by dim1,dim2 \u805a\u5408\u5e76\u4e14\u7ef4\u5ea6\u503c\u4e0d\u591a\u7684 group agg \u573a\u666f\uff08dim1\uff0cdim2 \u53ef\u4ee5\u679a\u4e3e\uff09\uff0c\u5982\u679c\u4f9d\u7136\u6709\u6570\u636e\u503e\u659c\u7684\u95ee\u9898\uff0c\u9700\u8981\u81ea\u5df1\u5148\u6253\u6563\u6570\u636e\uff0c\u7f13\u89e3\u6570\u636e\u503e\u659c<\/li>\n<\/ul>\n\n\n\n<ol>\n<li>\u2b50 \u6700\u5927\u5e76\u884c\u5ea6\u7684\u4f7f\u7528\u9650\u5236\uff1a<strong>\u6700\u5927\u5e76\u884c\u5ea6\u4e00\u65e6\u8bbe\u7f6e\uff0c\u662f\u4e0d\u80fd\u968f\u610f\u53d8\u66f4\u7684<\/strong>\uff0c\u5426\u5219\u4f1a\u5bfc\u81f4\u68c0\u67e5\u70b9\u6216\u4fdd\u5b58\u70b9\u5931\u6548\uff1b\u6700\u5927\u5e76\u884c\u5ea6\u8bbe\u7f6e\u4f1a\u5f71\u54cd MapState \u72b6\u6001\u5212\u5206\u7684 KeyGroup \u6570\uff0c\u5e76\u884c\u5ea6\u4fee\u6539\u540e\u518d\u4ece\u4fdd\u5b58\u70b9\u542f\u52a8\u65f6\uff0cKeyGroup \u4f1a\u6839\u636e\u5e76\u884c\u5ea6\u7684\u8bbe\u5b9a\u8fdb\u884c\u91cd\u65b0\u5206\u5e03\u3002<\/li>\n\n\n\n<li>\u2b50 \u6700\u5927\u5e76\u884c\u5ea6\u7684\u8bbe\u7f6e\uff1a\u6700\u5927\u5e76\u884c\u5ea6\u53ef\u4ee5\u81ea\u5df1\u8bbe\u7f6e\uff0c\u4e5f\u53ef\u4ee5\u6846\u67b6\u9ed8\u8ba4\u751f\u6210\uff1b\u9ed8\u8ba4\u7684\u7b97\u6cd5\u662f\u53d6\u5f53\u524d\u7b97\u5b50\u5e76\u884c\u5ea6\u7684 1.5 \u500d\u548c 2 \u7684 7 \u6b21\u65b9\u6bd4\u8f83\uff0c\u53d6\u4e24\u8005\u4e4b\u95f4\u7684\u6700\u5927\u503c\uff0c\u7136\u540e\u7528\u4e0a\u9762\u7684\u7ed3\u679c\u548c 2 \u7684 15 \u6b21\u65b9\u6bd4\u8f83\uff0c\u53d6\u5176\u4e2d\u7684\u6700\u5c0f\u503c\u4e3a\u9ed8\u8ba4\u7684\u6700\u5927\u5e76\u884c\u5ea6\uff0c<strong>\u975e\u5e38\u4e0d\u5efa\u8bae\u81ea\u52a8\u751f\u6210\uff0c\u5efa\u8bae\u7528\u6237\u81ea\u5df1\u8bbe\u7f6e<\/strong>\u3002<\/li>\n<\/ol>\n\n\n\n<p>\u4e3b\u8981\u5185\u5bb9\u5982\u4e0b<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-93491b977ada49ee199cc6fe1906f014_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>\u5185\u5bb9\u8f83\u591a\uff0c\u5efa\u8bae\u83b7\u53d6PDF\u8d44\u6599\uff0c\u5173\u6ce8\u4e0b\u65b9\u516c\u4f17\u53f7\u56de\u590d\u5173\u952e\u5b57\u3010<strong>\u8d44\u6599<\/strong>\u3011\u5373\u53ef\u83b7\u53d6\u3002\u6b22\u8fce\u5927\u5bb6\u6307\u6b63\u9519\u8bef\u4e0e\u4e0d\u8db3\u4e4b\u5904\uff0c\u4e5f\u6b22\u8fce\u5927\u5bb6\u7ee7\u7eed\u8865\u5145\u6b64\u5185\u5bb9\u3002<\/p>\n\n\n\n<p>\u65b0\u589e\u7684\u5185\u5bb9\u4f1a\u91cd\u65b0\u653e\u5230\u8d44\u6599\u4e2d\uff0c\u5927\u5bb6\u53ef\u56de\u590d\u5173\u952e\u5b57<strong>\u3010\u8fdb\u7fa4\u3011<\/strong>,\u6709\u8d44\u6599\u4fee\u6539\u66f4\u65b0\u6216\u589e\u6dfb\u65b0\u5185\u5bb9\u4f1a\u9996\u53d1\u7fa4\u4e2d\u901a\u77e5\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E4%B8%80_Flink_%E6%8F%90%E4%BA%A4%E7%B3%BB%E5%88%97\"><\/span>\u4e00. Flink \u63d0\u4ea4\u7cfb\u5217<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_Flink%E6%80%8E%E4%B9%88%E6%8F%90%E4%BA%A4\"><\/span>1. Flink\u600e\u4e48\u63d0\u4ea4?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p><strong>Local\u6a21\u5f0f<\/strong><\/p>\n\n\n\n<p>JobManager \u548c TaskManager \u5171\u7528\u4e00\u4e2a JVM,\u53ea\u9700\u8981jdk\u652f\u6301\uff0c\u5355\u8282\u70b9\u8fd0\u884c\uff0c\u4e3b\u8981\u7528\u6765\u8c03\u8bd5\u3002<\/p>\n\n\n\n<p><strong>Standlone\u6a21\u5f0f<\/strong><\/p>\n\n\n\n<p>Standlone \u662fFlink\u81ea\u5e26\u7684\u4e00\u4e2a\u5206\u5e03\u5f0f\u96c6\u7fa4\uff0c\u5b83\u4e0d\u4f9d\u8d56\u5176\u4ed6\u7684\u8d44\u6e90\u8c03\u5ea6\u6846\u67b6\u3001\u4e0d\u4f9d\u8d56yarn \u7b49\u3002\u5145\u5f53Master\u89d2\u8272\u7684\u662fJobManager\u3002\u5145\u5f53Slave\/Worker\u89d2\u8272\u662fTaskManager<\/p>\n\n\n\n<p><strong>Yarn \u6a21\u5f0f<\/strong><\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_Flink%E9%9B%86%E7%BE%A4%E8%A7%84%E6%A8%A1%EF%BC%9F%E5%9C%A8Flink%E9%A1%B9%E7%9B%AE%E5%81%9A%E4%BA%86%E4%BB%80%E4%B9%88\"><\/span>2. Flink\u96c6\u7fa4\u89c4\u6a21\uff1f\u5728Flink\u9879\u76ee\u505a\u4e86\u4ec0\u4e48?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p><strong>Flink\u7fa4\u96c6\u5927\u5c0f\u65f6\u8981\u8003\u8651\u7684\u4e00\u4e9b\u65b9\u9762:<\/strong><\/p>\n\n\n\n<p>1.\u8bb0\u5f55\u6570\u548c\u6bcf\u6761\u8bb0\u5f55\u7684\u5927\u5c0f<br>\u6bcf\u79d2\u5230\u8fbe\u6d41\u5f0f\u4f20\u8f93\u6846\u67b6\u7684\u9884\u671f\u8bb0\u5f55\u6570\u4ee5\u53ca\u6bcf\u6761\u8bb0\u5f55\u7684\u5927\u5c0f\u3002\u4e0d\u540c\u7684\u8bb0\u5f55\u7c7b\u578b\u5c06\u5177\u6709\u4e0d\u540c\u7684\u5927\u5c0f\uff0c\u8fd9\u5c06\u6700\u7ec8\u5f71\u54cdFlink\u5e94\u7528\u7a0b\u5e8f\u5e73\u7a33\u8fd0\u884c\u6240\u9700\u7684\u8d44\u6e90\u3002<\/p>\n\n\n\n<p>2.\u4e0d\u540ckey\u7684\u6570\u91cf\u548c\u6bcf\u4e2a\u952e\u7684\u72b6\u6001\u5927\u5c0f\u3002<\/p>\n\n\n\n<p>3.\u72b6\u6001\u66f4\u65b0\u7684\u6570\u91cf\u548c\u72b6\u6001\u540e\u7aef\u7684\u8bbf\u95ee\u6a21\u5f0f<br>Java\u7684\u5806\u72b6\u6001\u540e\u7aef\u4e0a\u7684\u5404\u79cd\u8bbf\u95ee\u6a21\u5f0f\u53ef\u80fd\u4f1a\u663e\u7740\u5f71\u54cd\u7fa4\u96c6\u7684\u5927\u5c0f\u4ee5\u53caFlink\u4f5c\u4e1a\u6240\u9700\u7684\u8d44\u6e90\u3002<\/p>\n\n\n\n<p>4.\u7f51\u7edc\u5bb9\u91cf<br>\u7f51\u7edc\u5bb9\u91cf\u4e0d\u4ec5\u4f1a\u53d7\u5230Flink\u5e94\u7528\u7a0b\u5e8f\u672c\u8eab\u7684\u5f71\u54cd\uff0c\u8fd8\u4f1a\u53d7\u5230\u53ef\u80fd\u6b63\u5728\u4e0e\u4e4b\u4ea4\u4e92\u7684\u5916\u90e8\u670d\u52a1\uff08\u5982Kafka\u6216HDFS\uff09\u7684\u5f71\u54cd\u3002\u6b64\u7c7b\u5916\u90e8\u670d\u52a1\u53ef\u80fd\u4f1a\u5bfc\u81f4\u7f51\u7edc\u51fa\u73b0\u989d\u5916\u6d41\u91cf\u3002\u4f8b\u5982\uff0c\u542f\u7528replication \u53ef\u80fd\u4f1a\u5728\u7f51\u7edc\u7684\u6d88\u606fbrokers\u4e4b\u95f4\u521b\u5efa\u989d\u5916\u7684\u6d41\u91cf\u3002<\/p>\n\n\n\n<p>5.\u78c1\u76d8\u5e26\u5bbd\u3002<\/p>\n\n\n\n<p>6.\u673a\u5668\u6570\u91cf\u53ca\u5176\u53ef\u7528CPU\u548c\u5185\u5b58\u3002<\/p>\n\n\n\n<p><strong>Flink\u9879\u76ee\u505a\u4e86\u4ec0\u4e48\uff1f<\/strong><\/p>\n\n\n\n<p>\u5b9e\u65f6\u76d1\u63a7\uff1a<\/p>\n\n\n\n<ol>\n<li>\u7528\u6237\u884c\u4e3a\u9884\u8b66\uff0c\u670d\u52a1\u5668\u653b\u51fb\u9884\u8b66 &#8230;..<\/li>\n<\/ol>\n\n\n\n<p>\u5b9e\u65f6\u62a5\u8868\uff1a<\/p>\n\n\n\n<ol>\n<li>\u6d3b\u52a8\u76f4\u64ad\u5927\u5c4f: \u53cc11\u3001\u53cc12<\/li>\n\n\n\n<li>\u5bf9\u5916\u6570\u636e\u4ea7\u54c1\u65f6\u6548\u6027<\/li>\n\n\n\n<li>\u6570\u636e\u5316\u8fd0\u8425<\/li>\n<\/ol>\n\n\n\n<p>\u6d41\u6570\u636e\u5206\u6790\uff1a<\/p>\n\n\n\n<ol>\n<li>\u5b9e\u65f6\u8ba1\u7b97\u76f8\u5173\u6307\u6807\u53cd\u9988\u53ca\u65f6\u8c03\u6574\u51b3\u7b56<\/li>\n\n\n\n<li>\u5185\u5bb9\u6295\u653e\u3001\u65e0\u7ebf\u667a\u80fd\u63a8\u9001\u3001\u5b9e\u65f6\u4e2a\u6027\u5316\u63a8\u8350\u7b49<\/li>\n<\/ol>\n\n\n\n<p>\u5b9e\u65f6\u4ed3\u5e93\uff1a<\/p>\n\n\n\n<ol>\n<li>\u6570\u636e\u5b9e\u65f6\u6e05\u6d17\u3001\u5f52\u5e76\u3001\u7ed3\u6784\u5316<\/li>\n\n\n\n<li>\u6570\u4ed3\u7684\u8865\u5145\u548c\u4f18\u5316<\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_Flink%E6%8F%90%E4%BA%A4%E4%BD%9C%E4%B8%9A%E7%9A%84%E6%B5%81%E7%A8%8B%EF%BC%8C%E4%BB%A5%E5%8F%8A%E4%B8%8EYarn%E6%80%8E%E4%B9%88%E4%BA%A4%E4%BA%92\"><\/span>3. Flink\u63d0\u4ea4\u4f5c\u4e1a\u7684\u6d41\u7a0b\uff0c\u4ee5\u53ca\u4e0eYarn\u600e\u4e48\u4ea4\u4e92?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-0674be899e3a03840618361e5cea9d4c_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>\uff081\uff09\u63d0\u4ea4App\u4e4b\u524d\uff0c\u5148\u4e0a\u4f20Flink\u7684Jar\u5305\u548c\u914d\u7f6e\u5230HDFS\uff0c\u4ee5\u4fbfJobManager\u548cTaskManager\u5171\u4eabHDFS\u7684\u6570\u636e\u3002<\/p>\n\n\n\n<p>\uff082\uff09\u5ba2\u6237\u7aef\u5411ResourceManager\u63d0\u4ea4Job\uff0cResouceManager\u63a5\u5230\u8bf7\u6c42\u540e\uff0c\u5148\u5206\u914dcontainer\u8d44\u6e90\uff0c\u7136\u540e\u901a\u77e5NodeManager\u542f\u52a8ApplicationMaster\u3002<\/p>\n\n\n\n<p>\uff083\uff09ApplicationMaster\u4f1a\u52a0\u8f7dHDFS\u7684\u914d\u7f6e\uff0c\u542f\u52a8\u5bf9\u5e94\u7684JobManager\uff0c\u7136\u540eJobManager\u4f1a\u5206\u6790\u5f53\u524d\u7684\u4f5c\u4e1a\u56fe\uff0c\u5c06\u5b83\u8f6c\u5316\u6210\u6267\u884c\u56fe\uff08\u5305\u542b\u4e86\u6240\u6709\u53ef\u4ee5\u5e76\u53d1\u6267\u884c\u7684\u4efb\u52a1\uff09\uff0c\u4ece\u800c\u77e5\u9053\u5f53\u524d\u9700\u8981\u7684\u5177\u4f53\u8d44\u6e90\u3002<\/p>\n\n\n\n<p>\uff084\uff09\u63a5\u7740\uff0cJobManager\u4f1a\u5411ResourceManager\u7533\u8bf7\u8d44\u6e90\uff0cResouceManager\u63a5\u5230\u8bf7\u6c42\u540e\uff0c\u7ee7\u7eed\u5206\u914dcontainer\u8d44\u6e90\uff0c\u7136\u540e\u901a\u77e5ApplictaionMaster\u542f\u52a8\u66f4\u591a\u7684TaskManager\uff08\u5148\u5206\u914d\u597dcontainer\u8d44\u6e90\uff0c\u518d\u542f\u52a8TaskManager\uff09\u3002container\u5728\u542f\u52a8TaskManager\u65f6\u4e5f\u4f1a\u4eceHDFS\u52a0\u8f7d\u6570\u636e\u3002<\/p>\n\n\n\n<p>\uff085\uff09TaskManager\u542f\u52a8\u540e\uff0c\u4f1a\u5411JobManager\u53d1\u9001\u5fc3\u8df3\u5305\u3002JobManager\u5411TaskManager\u5206\u914d\u4efb\u52a1\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_Flink%E6%8F%90%E4%BA%A4Job%E7%9A%84%E6%96%B9%E5%BC%8F%E4%BB%A5%E5%8F%8A%E5%8F%82%E6%95%B0%E8%AE%BE%E7%BD%AE%EF%BC%9F\"><\/span>4. Flink\u63d0\u4ea4Job\u7684\u65b9\u5f0f\u4ee5\u53ca\u53c2\u6570\u8bbe\u7f6e\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-code\"><code>.\/bin\/flink run -t yarn-session \\\n  -Dyarn.application.id=application_XXXX_YY \\\n  .\/examples\/streaming\/TopSpeedWindowing.jar\n  \n  \n .\/bin\/flink run -t yarn-per-job \n  --detached .\/examples\/streaming\/TopSpeedWindowing.jar\n  \n  \n .\/bin\/flink run-application -t yarn-application \n .\/examples\/streaming\/TopSpeedWindowing.jar\n \n \nyn(\u5b9e\u9645) = Math.ceil(p\/ys)\nys(\u603b\u5171) = yn(\u5b9e\u9645) * ys(\u6307\u5b9a)\nys(\u4f7f\u7528) = p(\u6307\u5b9a)\n\n\nflink run \n-c,--class Flink\u5e94\u7528\u7a0b\u5e8f\u7684\u5165\u53e3\n\n-C,--classpath \u6307\u5b9a\u6240\u6709\u8282\u70b9\u90fd\u53ef\u4ee5\u8bbf\u95ee\u5230\u7684url,\u53ef\u7528\u4e8e\u591a\u4e2a\u5e94\u7528\u7a0b\u5e8f\u90fd\u9700\u8981\u7684\u5de5\u5177\u7c7b\u52a0\u8f7d\n\n-d,--detached \u662f\u5426\u4f7f\u7528\u5206\u79bb\u6a21\u5f0f\uff0c\u5c31\u662f\u63d0\u4ea4\u4efb\u52a1,cli\u662f\u5426\u9000\u51fa,\u52a0\u4e86-d\u53c2\u6570,cli\u4f1a\u9000\u51fa\n\n-n,--allowNonRestoredState \u5141\u8bb8\u8df3\u8fc7\u65e0\u6cd5\u8fd8\u539f\u7684savepoint\u3002\u6bd4\u5982\u5220\u9664\u4e86\u4ee3\u7801\u4e2d\u7684\u90e8\u5206operator\n\n-p,--parallelism \u6267\u884c\u5e76\u884c\u5ea6\n\n-s,--fromSavepoint \u4ecesavepoint\u6062\u590d\u4efb\u52a1\n\n-sae,--shutdownOnAttachedExit \u4ee5attached\u6a21\u5f0f\u63d0\u4ea4\uff0c\u5ba2\u6237\u7aef\u9000\u51fa\u7684\u65f6\u5019\u5173\u95ed\u96c6\u7fa4\n\nflink yarn-cluster \u6a21\u5f0f\n\n-d,--detached \u662f\u5426\u4f7f\u7528\u5206\u79bb\u6a21\u5f0f\n\n-m,--jobmanager \u6307\u5b9a\u63d0\u4ea4\u7684jobmanager\n\n-yat,--yarnapplicationType \u8bbe\u7f6eyarn\u5e94\u7528\u7684\u7c7b\u578b\n\n-yD \u4f7f\u7528\u7ed9\u5b9a\u5c5e\u6027\u7684\u503c\n\n-yd,--yarndetached \u4f7f\u7528yarn\u5206\u79bb\u6a21\u5f0f\n\n-yh,--yarnhelp yarn session\u7684\u5e2e\u52a9\n\n-yid,--yarnapplicationId \u6302\u5230\u6b63\u5728\u8fd0\u884c\u7684yarnsession\u4e0a\n\n-yj,--yarnjar Flink jar\u6587\u4ef6\u7684\u8def\u5f84\n\n-yjm,--yarnjobManagerMemory jobmanager\u7684\u5185\u5b58(\u5355\u4f4dM)\n\n-ynl,--yarnnodeLabel \u6307\u5b9a YARN \u5e94\u7528\u7a0b\u5e8f YARN \u8282\u70b9\u6807\u7b7e\n\n-ynm,--yarnname \u81ea\u5b9a\u4e49yarn\u5e94\u7528\u540d\u79f0\n\n-yq,--yarnquery \u663e\u793ayarn\u7684\u53ef\u7528\u8d44\u6e90\n\n-yqu,--yarnqueue \u6307\u5b9ayarn\u961f\u5217\n\n-ys,--yarnslots \u6307\u5b9a\u6bcf\u4e2ataskmanager\u7684slots\u6570\n\n-yt,--yarnship \u5728\u6307\u5b9a\u76ee\u5f55\u4e2d\u4f20\u8f93\u6587\u4ef6\n\n-ytm,--yarntaskManagerMemory \u6bcf\u4e2ataskmanager\u7684\u5185\u5b58\n\n-yz,--yarnzookeeperNamespace \u7528\u6765\u521b\u5efaha\u7684zk\u5b50\u8def\u5f84\u7684\u547d\u540d\u7a7a\u95f4\n\n-z,--zookeeperNamespace \u7528\u6765\u521b\u5efaha\u7684zk\u5b50\u8def\u5f84\u7684\u547d\u540d\u7a7a\u95f4\n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"5_Flink%E7%9A%84JobManger_%E6%9C%89%E5%A4%9A%E5%B0%91%E4%B8%AAJobManager\"><\/span>5. Flink\u7684JobManger? \u6709\u591a\u5c11\u4e2aJobManager?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"JobManger\"><\/span><strong>JobManger<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\uff081\uff09 \u63a7\u5236\u4e00\u4e2a\u5e94\u7528\u7a0b\u5e8f\u6267\u884c\u7684\u4e3b\u8fdb\u7a0b\uff0c\u4e5f\u5c31\u662f\u8bf4\uff0c\u6bcf\u4e2a\u5e94\u7528\u7a0b\u5e8f \u90fd\u4f1a\u88ab\u4e00\u4e2a\u4e0d\u540c\u7684JM\u6240\u63a7\u5236\u6267\u884c\u3002<\/p>\n\n\n\n<p>\uff082\uff09 JM\u4f1a\u5148\u63a5\u6536\u5230\u8981\u6267\u884c\u7684\u5e94\u7528\u7a0b\u5e8f\uff0c\u8fd9\u4e2a\u5e94\u7528\u7a0b\u5e8f\u4f1a\u5305\u62ec\uff1a\u4f5c\u4e1a\u56fe\uff08Job Graph\uff09\u3001\u903b\u8f91\u6570\u636e\u6d41\u56fe\uff08 ogical dataflow graph\uff09\u548c\u6253\u5305\u4e86\u6240\u6709\u7684\u7c7b\u3001\u5e93\u548c\u5176\u5b83\u8d44\u6e90\u7684JAR\u5305\u3002<\/p>\n\n\n\n<p>\uff083\uff09 JM\u4f1a\u628a Jobgraph\u8f6c\u6362\u6210\u4e00\u4e2a\u7269\u7406\u5c42\u9762\u7684 \u6570\u636e\u6d41\u56fe\uff0c\u8fd9\u4e2a\u56fe\u88ab\u53eb\u505a \u201c\u6267\u884c\u56fe\u201d(Executiongraph),\u5305\u542b\u4e86\u6240\u6709\u53ef\u4ee5\u5e76\u53d1\u6267\u884c\u7684\u4efb\u52a1\u3002Job Manager\u4f1a\u5411\u8d44\u6e90\u7ba1\u7406\u5668\uff08Resourcemanager)\u8bf7\u6c42\u6267\u884c\u4efb\u52a1\u5fc5\u8981\u7684\u8d44\u6e90\uff0c\u4e5f\u5c31\u662f \u4efb\u52a1\u7ba1\u7406\u5668(Taskmanager)\u4e0a\u7684\u63d2\u69fdslot\u3002\u4e00\u65e6\u5b83\u83b7\u53d6\u5230\u4e86\u8db3\u591f\u7684\u8d44\u6e90\uff0c\u5c31\u4f1a\u5c06\u6267\u884c\u56fe\u5206\u53d1\u5230\u771f\u6b63\u8fd0\u884c\u5b83\u4eec\u7684TM\u4e0a\u3002\u800c\u5728\u8fd0\u884c\u8fc7\u7a0b\u4e2dJM\u4f1a\u8d1f\u8d23\u6240\u6709\u9700\u8981\u4e2d\u592e\u534f\u8c03\u7684\u64cd\u4f5c\uff0c\u6bd4\u5982\u8bf4\u68c0\u67e5\u70b9(checkpoints)\u7684\u534f\u8c03\u3002<\/p>\n\n\n\n<p><strong>\u6709\u591a\u5c11\u4e2aJobManager<\/strong><\/p>\n\n\n\n<p>\u96c6\u7fa4\u9ed8\u8ba4\u53ea\u6709\u4e00\u4e2a Job Manager\u3002\u4f46\u4e3a\u4e86\u9632\u6b62\u5355\u70b9\u6545\u969c\uff0c\u6211\u4eec\u914d\u7f6e\u4e86\u9ad8\u53ef\u7528\u3002\u6211\u4eec\u516c\u53f8\u4e00\u822c\u914d\u7f6e\u4e00\u4e2a\u4e3b JobManager\uff0c\u4e24\u4e2a\u5907\u7528 JobManager\uff0c\u7136\u540e\u7ed3\u5408 ZooKeeper \u7684\u4f7f\u7528\uff0c\u6765\u8fbe\u5230\u9ad8\u53ef\u7528\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"6_JobManger_%E5%9C%A8%E9%9B%86%E7%BE%A4%E5%90%AF%E5%8A%A8%E8%BF%87%E7%A8%8B%E4%B8%AD%E8%B5%B7%E5%88%B0%E4%BB%80%E4%B9%88%E4%BD%9C%E7%94%A8%EF%BC%9F\"><\/span>6. JobManger \u5728\u96c6\u7fa4\u542f\u52a8\u8fc7\u7a0b\u4e2d\u8d77\u5230\u4ec0\u4e48\u4f5c\u7528\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>JobManager \u7684\u804c\u8d23\u4e3b\u8981\u662f\u63a5\u6536 Flink \u4f5c\u4e1a\uff0c\u8c03\u5ea6 Task\uff0c\u6536\u96c6\u4f5c\u4e1a\u72b6\u6001\u548c\u7ba1\u7406 TaskManager\u3002\u5b83\u5305\u542b\u4e00\u4e2a Actor\uff0c\u5e76\u4e14\u505a\u5982\u4e0b\u64cd\u4f5c\uff1a<\/p>\n\n\n\n<p>RegisterTaskManager: \u5b83\u7531\u60f3\u8981\u6ce8\u518c\u5230 JobManager \u7684 TaskManager \u53d1\u9001\u3002\u6ce8\u518c\u6210\u529f\u4f1a\u901a\u8fc7 AcknowledgeRegistration \u6d88\u606f\u8fdb\u884c Ack\u3002<\/p>\n\n\n\n<p>SubmitJob: \u7531\u63d0\u4ea4\u4f5c\u4e1a\u5230\u7cfb\u7edf\u7684 Client \u53d1\u9001\u3002\u63d0\u4ea4\u7684\u4fe1\u606f\u662f JobGraph \u5f62\u5f0f\u7684\u4f5c\u4e1a\u63cf\u8ff0\u4fe1\u606f\u3002<\/p>\n\n\n\n<p>CancelJob: \u8bf7\u6c42\u53d6\u6d88\u6307\u5b9a id \u7684\u4f5c\u4e1a\u3002\u6210\u529f\u4f1a\u8fd4\u56de CancellationSuccess \uff0c\u5426\u5219\u8fd4\u56deCancellationFailure\u3002<\/p>\n\n\n\n<p>UpdateTaskExecutionState: \u7531 TaskManager \u53d1\u9001\uff0c\u7528\u6765\u66f4\u65b0\u6267\u884c\u8282\u70b9(ExecutionVertex)<br>\u7684\u72b6\u6001\u3002\u6210\u529f\u5219\u8fd4\u56de true\uff0c\u5426\u5219\u8fd4\u56de false\u3002<\/p>\n\n\n\n<p>RequestNextInputSplit: TaskManager \u4e0a\u7684 Task \u8bf7\u6c42\u4e0b\u4e00\u4e2a\u8f93\u5165 split \uff0c\u6210\u529f\u5219\u8fd4\u56deNextInputSplit\uff0c\u5426\u5219\u8fd4\u56de null\u3002<\/p>\n\n\n\n<p>JobStatusChanged\uff1a\u5b83\u610f\u5473\u7740\u4f5c\u4e1a\u7684\u72b6\u6001(RUNNING, CANCELING, FINISHED,\u7b49)\u53d1\u751f\u53d8\u5316\u3002\u8fd9\u4e2a\u6d88\u606f\u7531 ExecutionGraph \u53d1\u9001\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"7_Flink%E7%9A%84TaskManager\"><\/span>7. Flink\u7684TaskManager?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-06ec82c09075f642f8c3bc33ff2dc524_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>\uff081\uff09 Flink\u4e2d\u7684\u5de5\u4f5c\u8fdb\u7a0b\u3002\u901a\u5e38\u5728 Flink\u4e2d\u4f1a\u6709\u591a\u4e2aTM\u8fd0\u884c\uff0c \u6bcf\u4e2aTM\u90fd\u5305\u542b\u4e86\u4e00\u5b9a\u6570\u91cf\u7684\u63d2\u69fdslots\u3002\u63d2\u69fd\u7684\u6570\u91cf\u9650\u5236\u4e86TM\u80fd\u591f\u6267\u884c\u7684\u4efb\u52a1\u6570\u91cf\u3002<\/p>\n\n\n\n<p>\uff082\uff09 \u542f\u52a8\u4e4b\u540e\uff0cTM\u4f1a\u5411\u8d44\u6e90\u7ba1\u7406\u5668\u6ce8\u518c\u5b83\u7684\u63d2\u69fd\uff1b\u6536\u5230\u8d44\u6e90\u7ba1\u7406\u5668\u7684\u6307\u4ee4\u540e\uff0c TM\u5c31\u4f1a\u5c06\u4e00\u4e2a\u6216\u8005\u591a\u4e2a\u63d2\u69fd\u63d0\u4f9b\u7ed9JM\u8c03\u7528\u3002TM\u5c31\u53ef\u4ee5\u5411\u63d2\u69fd\u5206\u914d\u4efb\u52a1tasks\u6765\u6267\u884c\u4e86\u3002<\/p>\n\n\n\n<p>\uff083\uff09 \u5728\u6267\u884c\u8fc7\u7a0b\u4e2d\uff0c \u4e00\u4e2aTM\u53ef\u4ee5\u8ddf\u5176\u5b83\u8fd0\u884c\u540c\u4e00\u5e94\u7528\u7a0b\u5e8f\u7684TM\u4ea4\u6362\u6570\u636e\u3002<\/p>\n\n\n\n<p>TaskManager \u76f8\u5f53\u4e8e\u6574\u4e2a\u96c6\u7fa4\u7684Slave\u8282\u70b9\uff0c\u8d1f\u8d23\u5177\u4f53\u7684\u4efb\u52a1\u6267\u884c\u548c\u5bf9\u5e94\u4efb\u52a1\u5728\u6bcf\u4e2a<br>\u8282\u70b9\u4e0a\u7684\u8d44\u6e90\u7533\u8bf7\u548c\u7ba1\u7406\u3002\u5ba2\u6237\u7aef\u901a\u8fc7\u5c06\u7f16\u5199\u597d\u7684 Flink \u5e94\u7528\u7f16\u8bd1\u6253\u5305\uff0c\u63d0\u4ea4\u5230<br>JobManager\uff0c\u7136\u540e JobManager \u4f1a\u6839\u636e\u5df2\u6ce8\u518c\u5728 JobManager \u4e2d TaskManager \u7684\u8d44\u6e90\u60c5\u51b5\uff0c\u5c06\u4efb\u52a1\u5206\u914d\u7ed9\u6709\u8d44\u6e90\u7684 TaskManager \u8282\u70b9\uff0c\u7136\u540e\u542f\u52a8\u5e76\u8fd0\u884c\u4efb\u52a1\u3002TaskManager \u4eceJobManager \u63a5\u6536\u9700\u8981\u90e8\u7f72\u7684\u4efb\u52a1\uff0c\u7136\u540e\u4f7f\u7528 Slot \u8d44\u6e90\u542f\u52a8 Task\uff0c\u5efa\u7acb\u6570\u636e\u63a5\u5165\u7684\u7f51\u7edc\u8fde\u63a5\uff0c\u63a5\u6536\u6570\u636e\u5e76\u5f00\u59cb\u6570\u636e\u5904\u7406\u3002\u540c\u65f6 TaskManager \u4e4b\u95f4\u7684\u6570\u636e\u4ea4\u4e92\u90fd\u662f\u901a\u8fc7\u6570\u636e\u6d41\u7684\u65b9\u5f0f\u8fdb\u884c\u7684\u3002\u53ef\u4ee5\u770b\u51fa\uff0cFlink\u7684\u4efb\u52a1\u8fd0\u884c\u5176\u5b9e\u662f\u91c7\u7528\u591a\u7ebf\u7a0b\u7684\u65b9\u5f0f\uff0c\u8fd9\u548c MapReduce \u591a JVM \u8fdb\u884c\u7684\u65b9\u5f0f\u6709\u5f88\u5927\u7684\u533a\u522b\uff0cFlink \u80fd\u591f\u6781\u5927\u63d0\u9ad8 CPU \u4f7f\u7528\u6548\u7387\uff0c\u5728\u591a\u4e2a\u4efb\u52a1\u548c Task \u4e4b\u95f4\u901a\u8fc7TaskSlot \u65b9\u5f0f\u5171\u4eab\u7cfb\u7edf\u8d44\u6e90\uff0c\u6bcf\u4e2a TaskManager \u4e2d\u901a\u8fc7\u7ba1\u7406\u591a\u4e2a TaskSlot \u8d44\u6e90\u6c60\u8fdb<br>\u884c\u5bf9\u8d44\u6e90\u8fdb\u884c\u6709\u6548\u7ba1\u7406\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"8_%E8%AF%B4%E4%B8%80%E4%B8%8Bslot%EF%BC%8C%E4%B8%9A%E5%8A%A1%E4%B8%AD%E4%B8%80%E4%B8%AATaskManager%E8%AE%BE%E7%BD%AE%E5%87%A0%E4%B8%AAslot%EF%BC%9F\"><\/span>8. \u8bf4\u4e00\u4e0bslot\uff0c\u4e1a\u52a1\u4e2d\u4e00\u4e2aTaskManager\u8bbe\u7f6e\u51e0\u4e2aslot\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>jobManager\uff1a\u8d1f\u8d23\u63a5\u6536Flink Client\u63d0\u4ea4\u7684Job\uff0c\u5e76\u5c06Job\u5206\u53d1\u5230TaskManager\u6267\u884c\uff0c\u4e00\u4e2aJobManager\u5305\u542b\u4e00\u4e2a\u6216\u591a\u4e2aTaskManager\u3002<\/p>\n\n\n\n<p>TaskManager\uff1a\u8d1f\u8d23\u6267\u884cClient\u63d0\u4ea4\u7684Job\u3002\u6bcf\u4e2aTaskManager\u53ef\u4ee5\u6709\u4e00\u4e2a\u6216\u591a\u4e2aslot\uff0c\u4f46slot\u7684\u4e2a\u6570\u4e0d\u80fd\u591a\u4e8ecpu-cores\u3002<\/p>\n\n\n\n<p>slot\uff1aslot\u662fFlink\u4efb\u52a1\u7684\u6700\u5c0f\u6267\u884c\u5355\u4f4d\uff0c\u5e76\u884c\u5ea6\u4e0a\u9650\u4e0d\u80fd\u5927\u4e8eslot\u7684\u6570\u91cf\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"9_Flink_%E7%9A%84%E5%B9%B6%E8%A1%8C%E5%BA%A6%EF%BC%9F\"><\/span>9. Flink \u7684\u5e76\u884c\u5ea6\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Flink \u4e2d\u7684\u4efb\u52a1\u88ab\u5206\u4e3a\u591a\u4e2a\u5e76\u884c\u4efb\u52a1\u6765\u6267\u884c\uff0c\u5176\u4e2d\u6bcf\u4e2a\u5e76\u884c\u7684\u5b9e\u4f8b\u5904\u7406\u4e00\u90e8\u5206\u6570\u636e\u3002\u8fd9\u4e9b\u5e76\u884c\u5b9e\u4f8b\u7684\u6570\u91cf\u88ab\u79f0\u4e3a\u5e76\u884c\u5ea6\u3002\u8bbe\u7f6e\u5e76\u884c\u5ea6\u4e00\u822c\u5728\u56db\u4e2a\u5c42\u9762\u8bbe\u7f6e\uff08\u4f18\u5148\u7ea7\u7531\u9ad8\u5230\u4f4e\uff09<\/p>\n\n\n\n<p>\u64cd\u4f5c\u7b97\u5b50\u5c42\u9762<\/p>\n\n\n\n<p>\u6267\u884c\u73af\u5883\u5c42\u9762<\/p>\n\n\n\n<p>\u5ba2\u6237\u7aef\u5c42\u9762<\/p>\n\n\n\n<p>\u7cfb\u7edf\u5c42\u9762<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"10_Flink_%E8%AE%A1%E7%AE%97%E8%B5%84%E6%BA%90%E7%9A%84%E8%B0%83%E5%BA%A6%E6%98%AF%E5%A6%82%E4%BD%95%E5%AE%9E%E7%8E%B0%E7%9A%84%EF%BC%9F\"><\/span>10. Flink \u8ba1\u7b97\u8d44\u6e90\u7684\u8c03\u5ea6\u662f\u5982\u4f55\u5b9e\u73b0\u7684\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>TaskManager \u4e2d\u6700\u7ec6\u7c92\u5ea6\u7684\u8d44\u6e90\u662fTaskslot\uff0c\u4ee3\u8868\u4e86\u4e00\u4e2a\u56fa\u5b9a\u5927\u5c0f\u7684\u8d44\u6e90\u5b50\u96c6\uff0c\u6bcf\u4e2aTaskManager \u4f1a\u5c06\u5176\u6240\u5360\u6709\u7684\u8d44\u6e90\u5e73\u5206\u7ed9\u5b83\u7684 slot\u3002<\/p>\n\n\n\n<p>\u901a\u8fc7\u8c03\u6574 task slot \u7684\u6570\u91cf\uff0c\u7528\u6237\u53ef\u4ee5\u5b9a\u4e49 task \u4e4b\u95f4\u662f\u5982\u4f55\u76f8\u4e92\u9694\u79bb\u7684\u3002\u6bcf\u4e2a<br>TaskManager \u6709\u4e00\u4e2a slot\uff0c\u4e5f\u5c31\u610f\u5473\u7740\u6bcf\u4e2a task \u8fd0\u884c\u5728\u72ec\u7acb\u7684 JVM \u4e2d\u3002\u6bcf\u4e2a TaskManager\u6709\u591a\u4e2a slot \u7684\u8bdd\uff0c\u4e5f\u5c31\u662f\u8bf4\u591a\u4e2a task \u8fd0\u884c\u5728\u540c\u4e00\u4e2a JVM \u4e2d\u3002\u800c\u5728\u540c\u4e00\u4e2a JVM \u8fdb\u7a0b\u4e2d\u7684task\uff0c\u53ef\u4ee5\u5171\u4eabTCP\u8fde\u63a5\uff08\u57fa\u4e8e\u591a\u8def\u590d\u7528\uff09\u548c\u5fc3\u8df3\u6d88\u606f\uff0c\u53ef\u4ee5\u51cf\u5c11\u6570\u636e\u7684\u7f51\u7edc\u4f20\u8f93\uff0c\u4e5f\u80fd\u5171\u4eab\u4e00\u4e9b\u6570\u636e\u7ed3\u6784\uff0c\u4e00\u5b9a\u7a0b\u5ea6\u4e0a\u51cf\u5c11\u4e86\u6bcf\u4e2a task \u7684\u6d88\u8017\u3002\u6bcf \u4e2a slot \u53ef\u4ee5\u63a5\u53d7\u5355\u4e2a task\uff0c\u4e5f\u53ef\u4ee5\u63a5\u53d7\u591a\u4e2a\u8fde\u7eed task \u7ec4\u6210\u7684 pipeline\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"11_Flink_%E7%9A%84_Slot_%E5%92%8C_Parallelism\"><\/span>11. Flink \u7684 Slot \u548c Parallelism<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>slot \u662f\u6307 taskmanager\u7684\u5e76\u53d1\u6267\u884c\u80fd\u529b\uff0c\u5047\u8bbe\u6211\u4eec\u5c06taskmanager.numberOfTaskSlots \u914d\u7f6e\u4e3a 3 \u90a3\u4e48\u6bcf\u4e00\u4e2a taskmanager \u4e2d\u5206\u914d 3 \u4e2a TaskSlot,3\u4e2ataskmanager\u4e00\u5171\u6709 9 \u4e2aTaskSlot\u3002<\/p>\n\n\n\n<p>parallelism \u662f\u6307 taskmanager \u5b9e\u9645\u4f7f\u7528\u7684\u5e76\u53d1\u80fd\u529b\u3002\u5047\u8bbe\u6211\u4eec\u628aparallelism.default \u8bbe\u7f6e\u4e3a 1\uff0c\u90a3\u4e48 9 \u4e2a TaskSlot \u53ea\u80fd\u7528 1 \u4e2a\uff0c\u6709 8 \u4e2a\u7a7a\u95f2\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic4.zhimg.com\/80\/v2-9bd56bdf1c9820475e99e7494265d1c3_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"12_Operator_Chains%E4%BA%86%E8%A7%A3%E5%90%97%EF%BC%9F\"><\/span>12. Operator Chains\u4e86\u89e3\u5417\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u4e3a\u4e86\u66f4\u9ad8\u6548\u5730\u5206\u5e03\u5f0f\u6267\u884c\uff0cFlink \u4f1a\u5c3d\u53ef\u80fd\u5730\u5c06 operator \u7684 subtask \u94fe\u63a5\uff08chain\uff09\u5728\u4e00\u8d77\u5f62\u6210 task\u3002\u6bcf\u4e2a task \u5728\u4e00\u4e2a\u7ebf\u7a0b\u4e2d\u6267\u884c\u3002<\/p>\n\n\n\n<p>\u5c06 operators \u94fe\u63a5\u6210 task \u662f\u975e\u5e38\u6709\u6548\u7684\u4f18\u5316\uff1a\u5b83\u80fd\u51cf\u5c11\u7ebf\u7a0b\u4e4b\u95f4\u7684\u5207\u6362\uff0c\u51cf\u5c11\u6d88\u606f\u7684\u5e8f\u5217\u5316\/\u53cd\u5e8f\u5217\u5316\uff0c\u51cf\u5c11\u6570\u636e\u5728\u7f13\u51b2\u533a\u7684\u4ea4\u6362\uff0c\u51cf\u5c11\u4e86\u5ef6\u8fdf\u7684\u540c\u65f6\u63d0\u9ad8\u6574\u4f53\u7684\u541e\u5410\u91cf\u3002\u8fd9\u5c31\u662f Operator Chains\uff08\u7b97\u5b50\u94fe\uff09\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"13_Flink_%E7%9A%84%E8%BF%90%E8%A1%8C%E5%BF%85%E9%A1%BB%E4%BE%9D%E8%B5%96_Hadoop_%E7%BB%84%E4%BB%B6%E5%90%97%EF%BC%9F\"><\/span>13. Flink \u7684\u8fd0\u884c\u5fc5\u987b\u4f9d\u8d56 Hadoop \u7ec4\u4ef6\u5417\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Flink \u53ef\u4ee5\u5b8c\u5168\u72ec\u7acb\u4e8eHadoop\uff0c\u5728\u4e0d\u4f9d\u8d56Hadoop\u7ec4\u4ef6\u4e0b\u8fd0\u884c\u3002<\/p>\n\n\n\n<p>\u4f46\u662fFlink \u96c6\u6210Yarn\u505a\u8d44\u6e90\u8c03\u5ea6\uff0c\u4e5f\u53ef\u4ee5\u8bfb\u5199 HDFS\uff0c\u6216\u8005\u5229\u7528 HDFS \u505a\u68c0\u67e5\u70b9\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"14_%E6%80%8E%E4%B9%88%E4%BF%AE%E6%94%B9%E6%AD%A3%E5%9C%A8%E8%BF%90%E8%A1%8C%E7%9A%84Flink%E7%A8%8B%E5%BA%8F%EF%BC%9F%E5%A6%82%E6%9E%9C%E6%9C%89%E6%96%B0%E7%9A%84%E5%AE%9E%E6%97%B6%E6%8C%87%E6%A0%87%E4%BD%A0%E4%BB%AC%E6%98%AF%E6%80%8E%E4%B9%88%E4%B8%8A%E7%BA%BF%E7%9A%84%EF%BC%9F\"><\/span>14. \u600e\u4e48\u4fee\u6539\u6b63\u5728\u8fd0\u884c\u7684Flink\u7a0b\u5e8f\uff1f\u5982\u679c\u6709\u65b0\u7684\u5b9e\u65f6\u6307\u6807\u4f60\u4eec\u662f\u600e\u4e48\u4e0a\u7ebf\u7684\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u6b63\u5728\u8fd0\u884c\u7684Flink\u7a0b\u5e8f\uff1a\u4fee\u6539\u4e0d\u4e86\u3002\u53ef\u52a8\u6001\u52a0\u8f7d\u914d\u7f6e\u5e7f\u64ad\u7b49\u7b49<\/p>\n\n\n\n<p>\u5b8c\u6210\u4f5c\u4e1a\u5f00\u53d1<\/p>\n\n\n\n<p>\u4f5c\u4e1a\u8c03\u8bd5\uff0c\u5e76\u4e14\u901a\u8fc7\u8bed\u6cd5\u68c0\u67e5\u540e<\/p>\n\n\n\n<p>\u4e0a\u7ebf\u4f5c\u4e1a\uff0c\u5373\u53ef\u5c06\u6570\u636e\u53d1\u5e03\u81f3\u751f\u4ea7\u73af\u5883\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E4%BA%8C_%E7%8A%B6%E6%80%81%E7%BC%96%E7%A8%8B%E7%B3%BB%E5%88%97\"><\/span>\u4e8c. \u72b6\u6001\u7f16\u7a0b\u7cfb\u5217<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_%E8%AF%B4%E4%B8%80%E4%B8%8B%E7%8A%B6%E6%80%81%E7%BC%96%E7%A8%8B%EF%BC%88operator_state%EF%BC%8Ckeyed_state%EF%BC%89\"><\/span>1. \u8bf4\u4e00\u4e0b\u72b6\u6001\u7f16\u7a0b\uff08operator state\uff0ckeyed state\uff09<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u6211\u4eec\u77e5\u9053Flink\u4e0a\u7684\u805a\u5408\u548c\u7a97\u53e3\u64cd\u4f5c\uff0c\u4e00\u822c\u90fd\u662f\u57fa\u4e8eKeyedStream\u7684\uff0c<br>\u6570\u636e\u4f1a\u6309\u7167key\u7684\u54c8\u5e0c\u503c\u8fdb\u884c\u5206\u533a\uff0c\u805a\u5408\u5904\u7406\u7684\u7ed3\u679c\u4e5f\u5e94\u8be5\u662f\u53ea\u5bf9\u5f53\u524dkey\u6709\u6548\u3002\u7136\u800c\u540c\u4e00\u4e2a\u5206\u533a\uff08\u4e5f\u5c31\u662fslot\uff09\u4e0a\u6267\u884c\u7684\u4efb\u52a1\u5b9e\u4f8b\uff0c\u53ef\u80fd\u4f1a\u5305\u542b\u591a\u4e2akey\u7684\u6570\u636e\uff0c<br>\u5b83\u4eec\u540c\u65f6\u8bbf\u95ee\u548c\u66f4\u6539\u672c\u5730\u53d8\u91cf\uff0c\u5c31\u4f1a\u5bfc\u81f4\u8ba1\u7b97\u7ed3\u679c\u9519\u8bef\u3002\u6240\u4ee5\u8fd9\u65f6\u72b6\u6001\u5e76\u4e0d\u662f\u5355\u7eaf\u7684\u672c\u5730\u53d8\u91cf\u3002<\/p>\n\n\n\n<p>\u5bb9\u9519\u6027\uff0c\u4e5f\u5c31\u662f\u6545\u969c\u540e\u7684\u6062\u590d\u3002\u72b6\u6001\u53ea\u4fdd\u5b58\u5728\u5185\u5b58\u4e2d\u663e\u7136\u662f\u4e0d\u591f\u7a33\u5b9a\u7684\uff0c\u6211\u4eec\u9700\u8981\u5c06\u5b83\u6301\u4e45\u5316\u4fdd\u5b58\uff0c<br>\u505a\u4e00\u4e2a\u5907\u4efd\uff1b\u5728\u53d1\u751f\u6545\u969c\u540e\u53ef\u4ee5\u4ece\u8fd9\u4e2a\u5907\u4efd\u4e2d\u6062\u590d\u72b6\u6001\u3002<\/p>\n\n\n\n<p>\u5904\u7406\u7684\u6570\u636e\u91cf\u589e\u5927\u65f6\uff0c\u6211\u4eec\u5e94\u8be5\u76f8\u5e94\u5730\u5bf9\u8ba1\u7b97\u8d44\u6e90\u6269\u5bb9\uff0c<br>\u8c03\u5927\u5e76\u884c\u5ea6\u3002\u8fd9\u65f6\u5c31\u6d89\u53ca\u5230\u4e86\u72b6\u6001\u7684\u91cd\u7ec4\u8c03\u6574\u3002<\/p>\n\n\n\n<p><strong>\u72b6\u6001\u7684\u5206\u7c7b\uff1a<\/strong><\/p>\n\n\n\n<ol>\n<li>\u6258\u7ba1\u72b6\u6001\uff08Managed State\uff09\u548c\u539f\u59cb\u72b6\u6001\uff08Raw State\uff09<\/li>\n<\/ol>\n\n\n\n<p>\u6258\u7ba1\u72b6\u6001\u662f\u7531Flink\u7684\u8fd0\u884c\u65f6\uff08Runtime\uff09\u6765\u6258\u7ba1\u7684\uff1b\u5728\u914d\u7f6e\u5bb9\u9519\u673a\u5236\u540e\uff0c\u72b6\u6001\u4f1a\u81ea\u52a8\u6301\u4e45\u5316\u4fdd\u5b58\uff0c<br>\u5e76\u5728\u53d1\u751f\u6545\u969c\u65f6\u81ea\u52a8\u6062\u590d\u3002\u5f53\u5e94\u7528\u53d1\u751f\u6a2a\u5411\u6269\u5c55\u65f6\uff0c\u72b6\u6001\u4e5f\u4f1a\u81ea\u52a8\u5730\u91cd\u7ec4\u5206\u914d\u5230\u6240\u6709\u7684\u5b50\u4efb\u52a1\u5b9e\u4f8b\u4e0a\u3002<\/p>\n\n\n\n<p>\u5bf9\u4e8e\u5177\u4f53\u7684\u72b6\u6001\u5185\u5bb9\uff0cFlink\u4e5f\u63d0\u4f9b\u4e86\u503c\u72b6\u6001\uff08ValueState\uff09\u3001\u5217\u8868\u72b6\u6001\uff08ListState\uff09\u3001\u6620\u5c04\u72b6\u6001\uff08MapState\uff09\u3001<br>\u805a\u5408\u72b6\u6001\uff08AggregateState\uff09\u7b49\u591a\u79cd\u7ed3\u6784\uff0c\u5185\u90e8\u652f\u6301\u5404\u79cd\u6570\u636e\u7c7b\u578b\u3002<\/p>\n\n\n\n<p>\u805a\u5408\u3001\u7a97\u53e3\u7b49\u7b97\u5b50\u4e2d\u5185\u7f6e\u7684\u72b6\u6001\uff0c\u5c31\u90fd\u662f\u6258\u7ba1\u72b6\u6001\u3002<\/p>\n\n\n\n<p>\u6211\u4eec\u4e5f\u53ef\u4ee5\u5728\u5bcc\u51fd\u6570\u7c7b\uff08RichFunction\uff09\u4e2d\u901a\u8fc7\u4e0a\u4e0b\u6587\u6765\u81ea\u5b9a\u4e49\u72b6\u6001\uff0c\u8fd9\u4e9b\u4e5f\u90fd\u662f\u6258\u7ba1\u72b6\u6001\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E7%AE%97%E5%AD%90%E7%8A%B6%E6%80%81%EF%BC%88Operator_State%EF%BC%89%E5%92%8C%E6%8C%89%E9%94%AE%E5%88%86%E5%8C%BA%E7%8A%B6%E6%80%81\"><\/span>\u7b97\u5b50\u72b6\u6001\uff08Operator State\uff09\u548c\u6309\u952e\u5206\u533a\u72b6\u6001<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p><strong>1. \u6309\u952e\u5206\u533a\u72b6\u6001<\/strong><\/p>\n\n\n\n<p>\u6309\u952e\u5206\u533a\u72b6\u6001\uff08Keyed State\uff09\u987e\u540d\u601d\u4e49\uff0c\u662f\u4efb\u52a1\u6309\u7167\u952e\uff08key\uff09\u6765\u8bbf\u95ee\u548c\u7ef4\u62a4\u7684\u72b6\u6001\u3002\u5b83\u7684\u7279\u70b9\u975e\u5e38\u9c9c\u660e\uff0c\u5c31\u662f\u4ee5key\u4e3a\u4f5c\u7528\u8303\u56f4\u8fdb\u884c\u9694\u79bb\u3002Keyed State\u7c7b\u4f3c\u4e8e\u4e00\u4e2a\u5206\u5e03\u5f0f\u7684\u6620\u5c04\uff08map\uff09\u6570\u636e\u7ed3\u6784\uff0c\u6240\u6709\u7684\u72b6\u6001\u4f1a\u6839\u636ekey\u4fdd\u5b58\u6210\u952e\u503c\u5bf9\uff08key-value\uff09\u7684\u5f62\u5f0f\u3002\u8fd9\u6837\u5f53\u4e00\u6761\u6570\u636e\u5230\u6765\u65f6\uff0c\u4efb\u52a1\u5c31\u4f1a\u81ea\u52a8\u5c06\u72b6\u6001\u7684\u8bbf\u95ee\u8303\u56f4\u9650\u5b9a\u4e3a\u5f53\u524d\u6570\u636e\u7684key\uff0c\u4ecemap\u5b58\u50a8\u4e2d\u8bfb\u53d6\u51fa\u5bf9\u5e94\u7684\u72b6\u6001\u503c\u3002\u6240\u4ee5\u5177\u6709\u76f8\u540ckey\u7684\u6240\u6709\u6570\u636e\u90fd\u4f1a\u5230\u8bbf\u95ee\u76f8\u540c\u7684\u72b6\u6001\uff0c\u800c\u4e0d\u540ckey\u7684\u72b6\u6001\u4e4b\u95f4\u662f\u5f7c\u6b64\u9694\u79bb\u7684\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-f907faa8dc9f5e630779a80231316ac8_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p><strong>\u503c\u72b6\u6001\uff08ValueState\uff09:<\/strong><\/p>\n\n\n\n<p>\u72b6\u6001\u4e2d\u53ea\u4fdd\u5b58\u4e00\u4e2a\u201c\u503c\u201d\uff08value\uff09\u3002<\/p>\n\n\n\n<p><strong>\u5217\u8868\u72b6\u6001\uff08ListState\uff09:<\/strong><\/p>\n\n\n\n<p>\u4ee5\u5217\u8868\uff08List\uff09\u7684\u5f62\u5f0f\u7ec4\u7ec7\u8d77\u6765\u3002\u5728ListState\u63a5\u53e3\u4e2d\u540c\u6837\u6709\u4e00\u4e2a\u7c7b\u578b\u53c2\u6570T\uff0c\u8868\u793a\u5217\u8868\u4e2d\u6570\u636e\u7684\u7c7b\u578b\u3002ListState\u4e5f\u63d0\u4f9b\u4e86\u4e00\u7cfb\u5217\u7684\u65b9\u6cd5\u6765\u64cd\u4f5c\u72b6\u6001\uff0c\u4f7f\u7528\u65b9\u5f0f\u4e0e\u4e00\u822c\u7684List\u975e\u5e38\u76f8\u4f3c\u3002<\/p>\n\n\n\n<p><strong>\u6620\u5c04\u72b6\u6001\uff08MapState\uff09:<\/strong><\/p>\n\n\n\n<p>\u628a\u4e00\u4e9b\u952e\u503c\u5bf9\uff08key-value\uff09\u4f5c\u4e3a\u72b6\u6001\u6574\u4f53\u4fdd\u5b58\u8d77\u6765\uff0c\u53ef\u4ee5\u8ba4\u4e3a\u5c31\u662f\u4e00\u7ec4key-value\u6620\u5c04\u7684\u5217\u8868\u3002<\/p>\n\n\n\n<p><strong>\u5f52\u7ea6\u72b6\u6001\uff08ReducingState\uff09:<\/strong><\/p>\n\n\n\n<p>\u7c7b\u4f3c\u4e8e\u503c\u72b6\u6001\uff08Value\uff09\uff0c\u4e0d\u8fc7\u9700\u8981\u5bf9\u6dfb\u52a0\u8fdb\u6765\u7684\u6240\u6709\u6570\u636e\u8fdb\u884c\u5f52\u7ea6\uff0c\u5c06\u5f52\u7ea6\u805a\u5408\u4e4b\u540e\u7684\u503c\u4f5c\u4e3a\u72b6\u6001\u4fdd\u5b58\u4e0b\u6765\u3002<\/p>\n\n\n\n<p><strong>\u805a\u5408\u72b6\u6001\uff08AggregatingState\uff09:<\/strong><\/p>\n\n\n\n<p>\u4e0e\u5f52\u7ea6\u72b6\u6001\u975e\u5e38\u7c7b\u4f3c\uff0c\u805a\u5408\u72b6\u6001\u4e5f\u662f\u4e00\u4e2a\u503c\uff0c\u7528\u6765\u4fdd\u5b58\u6dfb\u52a0\u8fdb\u6765\u7684\u6240\u6709\u6570\u636e\u7684\u805a\u5408\u7ed3\u679c\u3002<\/p>\n\n\n\n<p><strong>2. \u7b97\u5b50\u72b6\u6001<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic3.zhimg.com\/80\/v2-aec822e7a3241455563640016404608e_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>\u7b97\u5b50\u72b6\u6001\uff08Operator State\uff09\u5c31\u662f\u4e00\u4e2a\u7b97\u5b50\u5e76\u884c\u5b9e\u4f8b\u4e0a\u5b9a\u4e49\u7684\u72b6\u6001\uff0c\u4f5c\u7528\u8303\u56f4\u88ab\u9650\u5b9a\u4e3a\u5f53\u524d\u7b97\u5b50\u4efb\u52a1\u3002\u7b97\u5b50\u72b6\u6001\u8ddf\u6570\u636e\u7684key\u65e0\u5173\uff0c\u6240\u4ee5\u4e0d\u540ckey\u7684\u6570\u636e\u53ea\u8981\u88ab\u5206\u53d1\u5230\u540c\u4e00\u4e2a\u5e76\u884c\u5b50\u4efb\u52a1\uff0c<br>\u5c31\u4f1a\u8bbf\u95ee\u5230\u540c\u4e00\u4e2aOperator State\u3002<\/p>\n\n\n\n<p>\u7b97\u5b50\u72b6\u6001\u7684\u5b9e\u9645\u5e94\u7528\u573a\u666f\u4e0d\u5982Keyed State\u591a\uff0c\u4e00\u822c\u7528\u5728Source\u6216Sink\u7b49\u4e0e\u5916\u90e8\u7cfb\u7edf\u8fde\u63a5\u7684\u7b97\u5b50\u4e0a\uff0c<br>\u6216\u8005\u5b8c\u5168\u6ca1\u6709key\u5b9a\u4e49\u7684\u573a\u666f\u3002\u6bd4\u5982Flink\u7684Kafka\u8fde\u63a5\u5668\u4e2d\uff0c\u5c31\u7528\u5230\u4e86\u7b97\u5b50\u72b6\u6001\u3002<\/p>\n\n\n\n<p>\u5f53\u7b97\u5b50\u7684\u5e76\u884c\u5ea6\u53d1\u751f\u53d8\u5316\u65f6\uff0c\u7b97\u5b50\u72b6\u6001\u4e5f\u652f\u6301\u5728\u5e76\u884c\u7684\u7b97\u5b50\u4efb\u52a1\u5b9e\u4f8b\u4e4b\u95f4\u505a\u91cd\u7ec4\u5206\u914d\u3002\u6839\u636e\u72b6\u6001\u7684\u7c7b\u578b\u4e0d\u540c\uff0c\u91cd\u7ec4\u5206\u914d\u7684\u65b9\u6848\u4e5f\u4f1a\u4e0d\u540c\u3002<\/p>\n\n\n\n<p>\u72b6\u6001\u7c7b\u578b:ListState\u3001UnionListState\u548cBroadcastState\u3002<\/p>\n\n\n\n<p><strong>ListState<\/strong><\/p>\n\n\n\n<p>Keyed State\u4e2d\u7684\u5217\u8868\u72b6\u6001\u7684\u533a\u522b\u662f\uff1a\u5728\u7b97\u5b50\u72b6\u6001\u7684\u4e0a\u4e0b\u6587\u4e2d\uff0c\u4e0d\u4f1a\u6309\u952e\uff08key\uff09\u5206\u522b\u5904\u7406\u72b6\u6001\uff0c<br>\u6240\u4ee5\u6bcf\u4e00\u4e2a\u5e76\u884c\u5b50\u4efb\u52a1\u4e0a\u53ea\u4f1a\u4fdd\u7559\u4e00\u4e2a\u201c\u5217\u8868\u201d\uff08list\uff09\uff0c\u4e5f\u5c31\u662f\u5f53\u524d\u5e76\u884c\u5b50\u4efb\u52a1\u4e0a\u6240\u6709\u72b6\u6001\u9879\u7684\u96c6\u5408\u3002<\/p>\n\n\n\n<p>\u5217\u8868\u4e2d\u7684\u72b6\u6001\u9879\u5c31\u662f\u53ef\u4ee5\u91cd\u65b0\u5206\u914d\u7684\u6700\u7ec6\u7c92\u5ea6\uff0c\u5f7c\u6b64\u4e4b\u95f4\u5b8c\u5168\u72ec\u7acb\u3002\u5f53\u7b97\u5b50\u5e76\u884c\u5ea6\u8fdb\u884c\u7f29\u653e\u8c03\u6574\u65f6\uff0c\u7b97\u5b50\u7684\u5217\u8868\u72b6\u6001\u4e2d\u7684\u6240\u6709\u5143\u7d20\u9879\u4f1a\u88ab\u7edf\u4e00\u6536\u96c6\u8d77\u6765\uff0c\u76f8\u5f53\u4e8e\u628a\u591a\u4e2a\u5206\u533a\u7684\u5217\u8868\u5408\u5e76\u6210\u4e86\u4e00\u4e2a\u201c\u5927\u5217\u8868\u201d\uff0c<br>\u7136\u540e\u518d\u5747\u5300\u5730\u5206\u914d\u7ed9\u6240\u6709\u5e76\u884c\u4efb\u52a1\u3002\u8fd9\u79cd\u201c\u5747\u5300\u5206\u914d\u201d\u7684\u5177\u4f53\u65b9\u6cd5\u5c31\u662f\u201c\u8f6e\u8be2\u201d\uff08round-robin\uff09\uff0c\u4e0e\u4e4b\u524d\u4ecb\u7ecd\u7684rebanlance\u6570\u636e\u4f20\u8f93\u65b9\u5f0f\u7c7b\u4f3c\uff0c<br>\u662f\u901a\u8fc7\u9010\u4e00\u201c\u53d1\u724c\u201d\u7684\u65b9\u5f0f\u5c06\u72b6\u6001\u9879\u5e73\u5747\u5206\u914d\u7684\u3002\u8fd9\u79cd\u65b9\u5f0f\u4e5f\u53eb\u4f5c\u201c\u5e73\u5747\u5206\u5272\u91cd\u7ec4\u201d\uff08even-split redistribution\uff09\u3002<\/p>\n\n\n\n<p>\u7b97\u5b50\u72b6\u6001\u4e2d\u4e0d\u4f1a\u5b58\u5728\u201c\u952e\u7ec4\u201d\uff08key group\uff09\u8fd9\u6837\u7684\u7ed3\u6784\uff0c\u6240\u4ee5\u4e3a\u4e86\u65b9\u4fbf\u91cd\u7ec4\u5206\u914d\uff0c\u5c31\u628a\u5b83\u76f4\u63a5\u5b9a\u4e49\u6210\u4e86\u201c\u5217\u8868\u201d\uff08list\uff09\u3002\u8fd9\u4e5f\u5c31\u89e3\u91ca\u4e86\uff0c\u4e3a\u4ec0\u4e48\u7b97\u5b50\u72b6\u6001\u4e2d\u6ca1\u6709\u6700\u7b80\u5355\u7684\u503c\u72b6\u6001\uff08ValueState\uff09\u3002<\/p>\n\n\n\n<p><strong>\u8054\u5408\u5217\u8868\u72b6\u6001\uff08UnionListState\uff09<\/strong><\/p>\n\n\n\n<p>\u4e0eListState\u7c7b\u4f3c\uff0c\u8054\u5408\u5217\u8868\u72b6\u6001\u4e5f\u4f1a\u5c06\u72b6\u6001\u8868\u793a\u4e3a\u4e00\u4e2a\u5217\u8868\u3002\u5b83\u4e0e\u5e38\u89c4\u5217\u8868\u72b6\u6001\u7684\u533a\u522b\u5728\u4e8e\uff0c\u7b97\u5b50\u5e76\u884c\u5ea6\u8fdb\u884c\u7f29\u653e\u8c03\u6574\u65f6\u5bf9\u4e8e\u72b6\u6001\u7684\u5206\u914d\u65b9\u5f0f\u4e0d\u540c\u3002UnionListState\u7684\u91cd\u70b9\u5c31\u5728\u4e8e\u201c\u8054\u5408\u201d\uff08union\uff09\u3002\u5728\u5e76\u884c\u5ea6\u8c03\u6574\u65f6\uff0c\u5e38\u89c4\u5217\u8868\u72b6\u6001\u662f\u8f6e\u8be2\u5206\u914d\u72b6\u6001\u9879\uff0c\u800c\u8054\u5408\u5217\u8868\u72b6\u6001\u7684\u7b97\u5b50\u5219\u4f1a\u76f4\u63a5\u5e7f\u64ad\u72b6\u6001\u7684\u5b8c\u6574\u5217\u8868\u3002\u8fd9\u6837\uff0c\u5e76\u884c\u5ea6\u7f29\u653e\u4e4b\u540e\u7684\u5e76\u884c\u5b50\u4efb\u52a1\u5c31\u83b7\u53d6\u5230\u4e86\u8054\u5408\u540e\u5b8c\u6574\u7684\u201c\u5927\u5217\u8868\u201d\uff0c\u53ef\u4ee5\u81ea\u884c\u9009\u62e9\u8981\u4f7f\u7528\u7684\u72b6\u6001\u9879\u548c\u8981\u4e22\u5f03\u7684\u72b6\u6001\u9879\u3002\u8fd9\u79cd\u5206\u914d\u4e5f\u53eb\u4f5c\u201c\u8054\u5408\u91cd\u7ec4\u201d\uff08union redistribution\uff09\u3002\u5982\u679c\u5217\u8868\u4e2d\u72b6\u6001\u9879\u6570\u91cf\u592a\u591a\uff0c\u4e3a\u8d44\u6e90\u548c\u6548\u7387\u8003\u8651\u4e00\u822c\u4e0d\u5efa\u8bae\u4f7f\u7528\u8054\u5408\u91cd\u7ec4\u7684\u65b9\u5f0f\u3002<\/p>\n\n\n\n<p><strong>\u5e7f\u64ad\u72b6\u6001\uff08BroadcastState\uff09<\/strong><\/p>\n\n\n\n<p>\u6709\u65f6\u6211\u4eec\u5e0c\u671b\u7b97\u5b50\u5e76\u884c\u5b50\u4efb\u52a1\u90fd\u4fdd\u6301\u540c\u4e00\u4efd\u201c\u5168\u5c40\u201d\u72b6\u6001\uff0c\u7528\u6765\u505a\u7edf\u4e00\u7684\u914d\u7f6e\u548c\u89c4\u5219\u8bbe\u5b9a\u3002<\/p>\n\n\n\n<p>\u8fd9\u65f6\u6240\u6709\u5206\u533a\u7684\u6240\u6709\u6570\u636e\u90fd\u4f1a\u8bbf\u95ee\u5230\u540c\u4e00\u4e2a\u72b6\u6001\uff0c\u72b6\u6001\u5c31\u50cf\u88ab\u201c\u5e7f\u64ad\u201d\u5230\u6240\u6709\u5206\u533a\u4e00\u6837\uff0c\u8fd9\u79cd\u7279\u6b8a\u7684\u7b97\u5b50\u72b6\u6001\uff0c\u5c31\u53eb\u4f5c\u5e7f\u64ad\u72b6\u6001\uff08BroadcastState\uff09\u3002\u56e0\u4e3a\u5e7f\u64ad\u72b6\u6001\u5728\u6bcf\u4e2a\u5e76\u884c\u5b50\u4efb\u52a1\u4e0a\u7684\u5b9e\u4f8b\u90fd\u4e00\u6837\uff0c\u6240\u4ee5\u5728\u5e76\u884c\u5ea6\u8c03\u6574\u7684\u65f6\u5019\u5c31\u6bd4\u8f83\u7b80\u5355\uff0c\u53ea\u8981\u590d\u5236\u4e00\u4efd\u5230\u65b0\u7684\u5e76\u884c\u4efb\u52a1\u5c31\u53ef\u4ee5\u5b9e\u73b0\u6269\u5c55\uff1b\u800c\u5bf9\u4e8e\u5e76\u884c\u5ea6\u7f29\u5c0f\u7684\u60c5\u51b5\uff0c<br>\u53ef\u4ee5\u5c06\u591a\u4f59\u7684\u5e76\u884c\u5b50\u4efb\u52a1\u8fde\u540c\u72b6\u6001\u76f4\u63a5\u780d\u6389\u2014\u2014\u56e0\u4e3a\u72b6\u6001\u90fd\u662f\u590d\u5236\u51fa\u6765\u7684\uff0c\u5e76\u4e0d\u4f1a\u4e22\u5931\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_10%E4%B8%AAint%E4%BB%A5%E6%95%B0%E7%BB%84%E7%9A%84%E5%BD%A2%E5%BC%8F%E4%BF%9D%E5%AD%98%E4%BF%9D%E5%AD%98%E5%9C%A8%E4%BB%80%E4%B9%88%E7%8A%B6%E6%80%81%E5%A5%BDVlaueState%E8%BF%98%E6%98%AFListState%E5%AD%98%E5%9C%A8%E5%93%AA%E4%B8%AA%E7%9A%84%E6%80%A7%E8%83%BD%E6%AF%94%E8%BE%83%E5%A5%BD\"><\/span>2. 10\u4e2aint\u4ee5\u6570\u7ec4\u7684\u5f62\u5f0f\u4fdd\u5b58,\u4fdd\u5b58\u5728\u4ec0\u4e48\u72b6\u6001\u597d?VlaueState\u8fd8\u662fListState?\u5b58\u5728\u54ea\u4e2a\u7684\u6027\u80fd\u6bd4\u8f83\u597d?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>ValueState[Array[Int]] update\u5f62\u5f0f\u3002<\/p>\n\n\n\n<p>ListState[Int]: add\u5f62\u5f0f\u6dfb\u52a0\u3002<\/p>\n\n\n\n<p>\u5bf9\u4e8e\u64cd\u63a7\u6765\u8bf4 ListState \u65b9\u4fbf\u53d6\u503c\u4e0e\u66f4\u6539\u3002<\/p>\n\n\n\n<p>\u6309\u952e\u5206\u533a\u72b6\u6001\uff08Keyed State\uff09\u9009\u62e9ValueState ListState\u3002<\/p>\n\n\n\n<p>\u7b97\u5b50\u72b6\u6001\uff08Operator State\uff09\u9009\u62e9ListState\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_%E4%BD%BF%E7%94%A8MapStage%EF%BC%8Cgroup_by_id_%E5%A6%82%E4%BD%95%E8%AE%BE%E8%AE%A1%EF%BC%9Fid%E4%B8%8D%E6%94%BE%E5%9C%A8key%E8%A1%8C%E4%B8%8D%E8%A1%8C%EF%BC%9F\"><\/span>3. \u4f7f\u7528MapStage\uff0cgroup by id \u5982\u4f55\u8bbe\u8ba1\uff1fid\u4e0d\u653e\u5728key\u884c\u4e0d\u884c\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>MapState \u662f KeyedState\uff0c\u4e5f\u5c31\u662f keyBy \u540e\u624d\u80fd\u4f7f\u7528MapState\u3002\u6240\u4ee5 State \u4e2d\u80af\u5b9a\u8981\u4fdd\u5b58 key\u3002<\/p>\n\n\n\n<p>group by id\u3002\u5047\u8bbeid\u6709id1\u3001id2\u8fd9\u4e24\u4e2a\u503c\uff0cid1\u3001id2\u5c31\u662fkey\u3002<\/p>\n\n\n\n<p>id \u7684\u8bbe\u8ba1\u662f\u7ed3\u5408\u4e1a\u52a1\u573a\u666f\uff0c\u5c31\u662f\u628a\u540c\u7c7b\u6570\u636e\u6216\u8005\u540c\u903b\u8f91\u6570\u636e\u653e\u5230\u4e00\u8d77\u8ba1\u7b97\u5904\u7406\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_Flink%E6%98%AF%E5%A6%82%E4%BD%95%E7%AE%A1%E7%90%86kafka%E7%9A%84offset%EF%BC%8C%E4%BD%BF%E7%94%A8%E4%BB%80%E4%B9%88%E7%B1%BB%E5%9E%8B%E7%9A%84%E7%8A%B6%E6%80%81%E4%BF%9D%E5%AD%98offset%EF%BC%9F\"><\/span>4. Flink\u662f\u5982\u4f55\u7ba1\u7406kafka\u7684offset\uff0c\u4f7f\u7528\u4ec0\u4e48\u7c7b\u578b\u7684\u72b6\u6001\u4fdd\u5b58offset\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>checkpoint\u662fFlink\u7684\u5185\u90e8\u673a\u5236\uff0c\u53ef\u4ee5\u4ece\u6545\u969c\u4e2d\u6062\u590d\u3002\u901a\u4fd7\u7684\u7406\u89e3\u662fcheckpoint\u662fFlink\u5e94\u7528\u7a0b\u5e8f\u72b6\u6001\u7684\u4e00\u81f4\u6027\u526f\u672c\uff0c\u5305\u62ec\u8f93\u5165\u7684\u8bfb\u53d6\u4f4d\u7f6e(offset)\u3002\u5982\u679c\u53d1\u751f\u6545\u969c\uff0cFlink\u5c06\u901a\u8fc7\u4ececheckpoint\u52a0\u8f7d\u72b6\u6001\u540e\u7aef\u5e76\u4ece\u6062\u590d\u7684\u8bfb\u53d6\u4f4d\u7f6e\u7ee7\u7eed\u6062\u590d\u5e94\u7528\u7a0b\u5e8f\uff0c\u53ef\u4ee5\u505a\u5230\u6240\u8c13\u7684\u65ad\u70b9\u7eed\u4f20\u3002<\/p>\n\n\n\n<p>checkpoint\u4f7fFlink\u5177\u6709\u5bb9\u9519\u80fd\u529b\uff0c\u5e76\u786e\u4fdd\u5728\u53d1\u751f\u6545\u969c\u65f6\u5177\u6709\u5bb9\u9519\u7684\u80fd\u529b\u3002\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u5b9a\u671f\u89e6\u53d1\u68c0\u67e5\u70b9\u3002<\/p>\n\n\n\n<p>Flink\u4e2d\u7684Kafka\u6d88\u8d39\u8005\u5c06Flink\u7684\u68c0\u67e5\u70b9\u673a\u5236\u4e0e\u6709\u72b6\u6001\u8fd0\u7b97\u7b26\u96c6\u6210\u5728\u4e00\u8d77\uff0c<br>\u5176\u72b6\u6001\u662f\u6240\u6709Kafka\u5206\u533a\u4e2d\u7684\u8bfb\u53d6\u504f\u79fb\u91cf\u3002\u89e6\u53d1checkpoint\u65f6\uff0c<br>\u6bcf\u4e2a\u5206\u533a\u7684\u504f\u79fb\u91cf\u90fd\u5b58\u50a8\u5728checkpoint\u4e2d\u3002Flink\u7684checkpoint\u673a\u5236\u786e\u4fdd\u6240\u6709\u64cd\u4f5c\u5458\u4efb\u52a1\u7684\u5b58\u50a8\u72b6\u6001\u662f\u4e00\u81f4\u7684\u3002<\/p>\n\n\n\n<p>\u5373\u5b83\u4eec\u57fa\u4e8e\u76f8\u540c\u7684\u8f93\u5165\u6570\u636e\u3002\u5f53\u6240\u6709\u64cd\u4f5c\u5458\u4efb\u52a1\u6210\u529f\u5b58\u50a8\u5176\u72b6\u6001\u65f6\uff0c\u68c0\u67e5\u70b9\u5b8c\u6210\u3002\u56e0\u6b64\uff0c\u5f53\u4ece\u6f5c\u5728\u7684\u7cfb\u7edf\u6545\u969c\u91cd\u65b0\u542f\u52a8\u65f6\uff0c\u7cfb\u7edf\u63d0\u4f9b\u4e00\u6b21\u6027\u72b6\u6001\u66f4\u65b0\u4fdd\u8bc1\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>private transient ListState&lt;Tuple2&lt;KafkaTopicPartition, Long&gt;&gt; unionOffsetStates;\n\n\npublic final void snapshotState(FunctionSnapshotContext context) throws Exception {\n        if (!running) {\n            LOG.debug(\"snapshotState() called on closed source\");\n        } else {\n            unionOffsetStates.clear();\n\n            final AbstractFetcher&lt;?, ?&gt; fetcher = this.kafkaFetcher;\n            if (fetcher == null) {\n                \/\/ the fetcher has not yet been initialized, which means we need to return the\n                \/\/ originally restored offsets or the assigned partitions\n                for (Map.Entry&lt;KafkaTopicPartition, Long&gt; subscribedPartition :\n                        subscribedPartitionsToStartOffsets.entrySet()) {\n                    unionOffsetStates.add(\n                            Tuple2.of(\n                                    subscribedPartition.getKey(), subscribedPartition.getValue()));\n                }\n\n                if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {\n                    \/\/ the map cannot be asynchronously updated, because only one checkpoint call\n                    \/\/ can happen\n                    \/\/ on this function at a time: either snapshotState() or\n                    \/\/ notifyCheckpointComplete()\n                    pendingOffsetsToCommit.put(context.getCheckpointId(), restoredState);\n                }\n            } else {\n                HashMap&lt;KafkaTopicPartition, Long&gt; currentOffsets = fetcher.snapshotCurrentState();\n\n                if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {\n                    \/\/ the map cannot be asynchronously updated, because only one checkpoint call\n                    \/\/ can happen\n                    \/\/ on this function at a time: either snapshotState() or\n                    \/\/ notifyCheckpointComplete()\n                    pendingOffsetsToCommit.put(context.getCheckpointId(), currentOffsets);\n                }\n\n                for (Map.Entry&lt;KafkaTopicPartition, Long&gt; kafkaTopicPartitionLongEntry :\n                        currentOffsets.entrySet()) {\n                    unionOffsetStates.add(\n                            Tuple2.of(\n                                    kafkaTopicPartitionLongEntry.getKey(),\n                                    kafkaTopicPartitionLongEntry.getValue()));\n                }\n            }\n\n            if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {\n                \/\/ truncate the map of pending offsets to commit, to prevent infinite growth\n                while (pendingOffsetsToCommit.size() &gt; MAX_NUM_PENDING_CHECKPOINTS) {\n                    pendingOffsetsToCommit.remove(0);\n                }\n            }\n        }\n    }\n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"5_%E4%B8%80%E4%B8%AA%E7%AA%97%E5%8F%A3%EF%BC%8C%E7%8E%B0%E5%9C%A8%E5%8F%AA%E5%8F%96%E7%AC%AC%E4%B8%80%E5%B8%A7%E5%92%8C%E6%9C%80%E5%90%8E%E4%B8%80%E5%B8%A7%EF%BC%8C%E6%80%8E%E4%B9%88%E5%81%9A%EF%BC%9F\"><\/span>5. \u4e00\u4e2a\u7a97\u53e3\uff0c\u73b0\u5728\u53ea\u53d6\u7b2c\u4e00\u5e27\u548c\u6700\u540e\u4e00\u5e27\uff0c\u600e\u4e48\u505a\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u5982\u679c\u9700\u8981\u8bbf\u95ee\u7a97\u53e3\u4e2d\u7684\u6700\u540e\u4e00\u4e2a\u5143\u7d20\uff0c<br>\u5219\u53ef\u80fd\u5e94\u8be5\u53ea\u4f7f\u7528WindowFunction\u5e76\u5728apply\u65b9\u6cd5\u4e2d\u83b7\u53d6\u53ef\u8fed\u4ee3\u8f93\u5165\uff08input.last\uff09\u7684\u7b2c\u4e00\u4e2a\u548c\u6700\u540e\u4e00\u4e2a\u5143\u7d20\u3002<\/p>\n\n\n\n<p>getstart getend \u83b7\u53d6\u65f6\u95f4\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E4%B8%89_%E5%8F%8D%E5%8E%8B%E4%B8%8E%E9%97%AE%E9%A2%98%E7%B3%BB%E5%88%97\"><\/span>\u4e09. \u53cd\u538b\u4e0e\u95ee\u9898\u7cfb\u5217<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_Flink%E7%94%A8%E4%BB%80%E4%B9%88%E7%9B%91%E6%8E%A7%EF%BC%8C%E7%9B%91%E6%8E%A7%E4%BB%80%E4%B9%88%EF%BC%9F%E5%A6%82%E4%BD%95%E6%9C%89%E6%95%88%E5%A4%84%E7%90%86%E6%95%B0%E6%8D%AE%E7%A7%AF%E5%8E%8B%EF%BC%9F\"><\/span>1. Flink\u7528\u4ec0\u4e48\u76d1\u63a7\uff0c\u76d1\u63a7\u4ec0\u4e48\uff1f\u5982\u4f55\u6709\u6548\u5904\u7406\u6570\u636e\u79ef\u538b\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u6211\u4eec\u76d1\u63a7\u4e86Flink\u7684\u4efb\u52a1\u662f\u5426\u505c\u6b62\uff0c\u7684Kafka\u7684LAG\uff0c\u6211\u4eec\u4f1a\u8fdb\u884c\u5b9e\u65f6\u6570\u636e\u5bf9\u8d26<\/p>\n\n\n\n<p><strong>1. Flink Web UI<\/strong><\/p>\n\n\n\n<p>\u6bd4\u5982\u53cd\u538b\uff1a\u901a\u8fc7Thread.getStackTrace() \u91c7\u96c6\u5728 TaskManager \u4e0a\u6b63\u5728\u8fd0\u884c\u7684\u6240\u6709\u7ebf\u7a0b\uff0c<br>\u6536\u96c6\u5728\u7f13\u51b2\u533a\u8bf7\u6c42\u4e2d\u963b\u585e\u7684\u7ebf\u7a0b\u6570\uff08\u610f\u5473\u7740\u4e0b\u6e38\u963b\u585e\uff09\uff0c\u5e76\u8ba1\u7b97\u7f13\u51b2\u533a\u963b\u585e\u7ebf\u7a0b\u6570\u4e0e\u603b\u7ebf\u7a0b\u6570\u7684\u6bd4\u503c rate\u3002\u5176\u4e2d\uff0crate &lt; 0.1 \u4e3a OK\uff0c0.1 &lt;= rate &lt;= 0.5 \u4e3a LOW\uff0crate &gt; 0.5 \u4e3a HIGH\u3002<\/p>\n\n\n\n<p><strong>2. Prometheus&amp;Grafana\u76d1\u63a7<\/strong><\/p>\n\n\n\n<p><strong>\u4f5c\u4e1a\u7684\u53ef\u7528\u6027<\/strong>\uff0c\u5982 uptime (\u4f5c\u4e1a\u6301\u7eed\u8fd0\u884c\u7684\u65f6\u95f4)\u3001fullRestarts (\u4f5c\u4e1a\u91cd\u542f\u7684\u6b21\u6570)<\/p>\n\n\n\n<p><strong>\u4f5c\u4e1a\u7684\u6d41\u91cf<\/strong>\uff0c\u53ef\u4ee5\u901a\u8fc7 numRecordsIn\u3001numBytesInLocal \u7b49\u76f8\u5173\u6307\u6807\u6765\u5173\u6ce8\u4f5c\u4e1a\u6bcf\u5929\u5904\u7406\u7684\u6d88\u606f\u6570\u76ee\u53ca\u9ad8\u5cf0\u65f6\u95f4\u6bb5\u7684\u6d41\u91cf\uff0c<br>\u901a\u8fc7\u5173\u6ce8\u8fd9\u4e9b\u6307\u6807\u53ef\u4ee5\u89c2\u5bdf\u4f5c\u4e1a\u7684\u6d41\u91cf\u8868\u73b0\u662f\u5426\u6b63\u5e38\u3002<\/p>\n\n\n\n<p><strong>CPU<\/strong>\uff08\u5982\uff1aCPU.Load\uff09\u3001\u5185\u5b58\uff08\u5982\uff1aHeap.Used \uff09\u3001GC ( \u5982\uff1aGarbageCollector.Count\u3001GarbageCollector.Time )\u53ca\u7f51\u7edc ( inputQueueLength\u3001outputQueueLength) \u76f8\u5173\u6307\u6807\uff0c\u8fd9\u4e9b\u6307\u6807\u4e00\u822c\u662f\u7528\u6765\u6392\u67e5\u4f5c\u4e1a\u7684\u6545\u969c\u4fe1\u606f\u3002<\/p>\n\n\n\n<p><strong>checkpoint \u76f8\u5173\u4fe1\u606f<\/strong>\uff0c\u4f8b\u5982\u6700\u8fd1\u5b8c\u6210\u7684 checkpoint \u7684\u65f6\u957f\uff08 lastCheckpointDuration \uff09\u3001<br>\u6700\u8fd1\u5b8c\u6210\u7684 checkpoint \u7684\u5927\u5c0f\uff08 lastCheckpointSize \uff09\u3001<br>\u4f5c\u4e1a\u5931\u8d25\u540e\u6062\u590d\u7684\u80fd\u529b\uff08 lastCheckpointRestoreTimestamp \uff09\u3001<br>\u6210\u529f\u548c\u5931\u8d25\u7684 checkpoint \u6570\u76ee\uff08 numberOfCompletedCheckpoints\u3001numberOfFailedCheckpoints \uff09<br>\u4ee5\u53ca\u5728 Exactly once \u6a21\u5f0f\u4e0b barrier \u5bf9\u9f50\u65f6\u95f4\uff08 checkpointAlignmentTime \uff09<\/p>\n\n\n\n<p><strong>connector \u7684\u6307\u6807<\/strong>\uff0c\u4f8b\u5982\u5e38\u7528\u7684 Kafka connector \uff0cKafka \u81ea\u8eab\u63d0\u4f9b\u4e86\u4e00\u4e9b\u6307\u6807\uff0c\u53ef\u4ee5\u5e2e\u52a9\u6211\u4eec\u4e86\u89e3\u5230\u4f5c\u4e1a\u6700\u65b0\u6d88\u8d39\u7684\u6d88\u606f\u7684\u72b6\u51b5\u3001<br>\u4f5c\u4e1a\u662f\u5426\u6709\u5ef6\u8fdf\u7b49<\/p>\n\n\n\n<p><strong>\u5176\u4ed6\u81ea\u5b9a\u4e49\u6307\u6807<\/strong>\uff1a\u8d85\u65f6\u4e22\u5f03\u7684\u6570\u636e\u91cf\uff0c filter\u8fc7\u6ee4\u7684\u6570\u636e\u91cf\/\u5360\u6bd4 \u5904\u7406\u5931\u8d25\u7684\u6570\u636e\uff0c\u7b49\u7b49<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E8%83%8C%E5%8E%8B\"><\/span>\u80cc\u538b<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u80cc\u538b\u4ea7\u751f\u7684\u539f\u56e0\uff1a\u4e0b\u6e38\u6d88\u8d39\u7684\u901f\u5ea6\u8ddf\u4e0d\u4e0a\u4e0a\u6e38\u751f\u4ea7\u6570\u636e\u7684\u901f\u5ea6\uff0c\u53ef\u80fd\u51fa\u73b0\u7684\u539f\u56e0\u5982\u4e0b\uff1a<\/p>\n\n\n\n<p>\uff081\uff09\u8282\u70b9\u6709\u6027\u80fd\u74f6\u9888\uff0c\u53ef\u80fd\u662f\u8be5\u8282\u70b9\u6240\u5728\u7684\u673a\u5668\u6709\u7f51\u7edc\u3001\u78c1\u76d8\u7b49\u7b49\u6545\u969c\uff0c\u673a\u5668\u7684\u7f51\u7edc\u5ef6\u8fdf\u548c\u78c1\u76d8\u4e0d\u8db3\u3001\u9891\u7e41GC\u3001\u6570\u636e\u70ed\u70b9\u7b49\u539f\u56e0\u3002<\/p>\n\n\n\n<p>\uff082\uff09\u6570\u636e\u6e90\u751f\u4ea7\u6570\u636e\u7684\u901f\u5ea6\u8fc7\u5feb\uff0c\u8ba1\u7b97\u6846\u67b6\u5904\u7406\u4e0d\u53ca\u65f6\u3002\u6bd4\u5982\u6d88\u606f\u4e2d\u95f4\u4ef6kafka\uff0c\u751f\u4ea7\u8005\u751f\u4ea7\u6570\u636e\u8fc7\u5feb\uff0c\u4e0b\u6e38flink\u6d88\u8d39\u8ba1\u7b97\u4e0d\u53ca\u65f6\u3002<\/p>\n\n\n\n<p>\uff083\uff09flink\u7b97\u5b50\u95f4\u5e76\u884c\u5ea6\u4e0d\u540c\uff0c\u4e0b\u6e38\u7b97\u5b50\u76f8\u6bd4\u4e0a\u6e38\u7b97\u5b50\u8fc7\u5c0f\u3002<\/p>\n\n\n\n<p>\u80cc\u538b\u5bfc\u81f4\u7684\u5f71\u54cd<br>\u9996\u5148\uff0c\u80cc\u538b\u4e0d\u4f1a\u76f4\u63a5\u5bfc\u81f4\u7cfb\u7edf\u7684\u5d29\u76d8\uff0c\u53ea\u662f\u5904\u5728\u4e00\u4e2a\u4e0d\u5065\u5eb7\u7684\u8fd0\u884c\u72b6\u6001\u3002<\/p>\n\n\n\n<p>\uff081\uff09\u80cc\u538b\u4f1a\u5bfc\u81f4\u6d41\u5904\u7406\u4f5c\u4e1a\u6570\u636e\u5ef6\u8fdf\u7684\u589e\u52a0\u3002<\/p>\n\n\n\n<p>\uff082\uff09\u5f71\u54cd\u5230Checkpoint\uff0c\u5bfc\u81f4\u5931\u8d25\uff0c\u5bfc\u81f4\u72b6\u6001\u6570\u636e\u4fdd\u5b58\u4e0d\u4e86\uff0c\u5982\u679c\u4e0a\u6e38\u662fkafka\u6570\u636e\u6e90\uff0c\u5728\u4e00\u81f4\u6027\u7684\u8981\u6c42\u4e0b\uff0c\u53ef\u80fd\u4f1a\u5bfc\u81f4offset\u7684\u63d0\u4ea4\u4e0d\u4e0a\u3002<\/p>\n\n\n\n<p>\u539f\u7406: \u7531\u4e8eFlink\u7684Checkpoint\u673a\u5236\u9700\u8981\u8fdb\u884cBarrier\u5bf9\u9f50\uff0c\u5982\u679c\u6b64\u65f6\u67d0\u4e2aTask\u51fa\u73b0\u4e86\u80cc\u538b\uff0cBarrier\u6d41\u52a8\u7684\u901f\u5ea6\u5c31\u4f1a\u53d8\u6162\uff0c\u5bfc\u81f4Checkpoint\u6574\u4f53\u65f6\u95f4\u53d8\u957f\uff0c\u5982\u679c\u80cc\u538b\u5f88\u4e25\u91cd\uff0c\u8fd8\u6709\u53ef\u80fd\u5bfc\u81f4Checkpoint\u8d85\u65f6\u5931\u8d25\u3002<\/p>\n\n\n\n<p>\uff083\uff09\u5f71\u54cdstate\u7684\u5927\u5c0f\uff0c\u8fd8\u662f\u56e0\u4e3acheckpoint barrier\u5bf9\u9f50\u8981\u6c42\u3002\u5bfc\u81f4state\u53d8\u5927\u3002<\/p>\n\n\n\n<p><strong>\u539f\u7406\uff1a<\/strong>\u63a5\u53d7\u5230\u8f83\u5feb\u7684\u8f93\u5165\u7ba1\u9053\u7684barrier\u540e\uff0c\u5b83\u540e\u9762\u6570\u636e\u4f1a\u88ab\u7f13\u5b58\u8d77\u6765\u4f46\u4e0d\u5904\u7406\uff0c\u76f4\u5230\u8f83\u6162\u7684\u8f93\u5165\u7ba1\u9053\u7684barrier\u4e5f\u5230\u8fbe\u3002\u8fd9\u4e9b\u88ab\u7f13\u5b58\u7684\u6570\u636e\u4f1a\u88ab\u653e\u5230state \u91cc\u9762\uff0c\u5bfc\u81f4state\u53d8\u5927\u3002<\/p>\n\n\n\n<p>Flink\u4e0d\u9700\u8981\u4e00\u4e2a\u7279\u6b8a\u7684\u673a\u5236\u6765\u5904\u7406\u80cc\u538b\uff0c\u56e0\u4e3aFlink\u4e2d\u7684\u6570\u636e\u4f20\u8f93\u76f8\u5f53\u4e8e\u5df2\u7ecf\u63d0\u4f9b\u4e86\u5e94\u5bf9\u80cc\u538b\u7684\u673a\u5236\u3002\u6240\u4ee5\u53ea\u6709\u4ece\u4ee3\u7801\u4e0a\u4e0e\u8d44\u6e90\u4e0a\u53bb\u505a\u4e00\u4e9b\u8c03\u6574\u3002<\/p>\n\n\n\n<p>\uff081\uff09\u80cc\u538b\u90e8\u5206\u539f\u56e0\u53ef\u80fd\u662f\u7531\u4e8e\u6570\u636e\u503e\u659c\u9020\u6210\u7684\uff0c\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7 Web UI \u5404\u4e2a SubTask \u7684 \u6307\u6807\u503c\u6765\u786e\u8ba4\u3002Checkpoint detail \u91cc\u4e0d\u540c SubTask \u7684 State size \u4e5f\u662f\u4e00\u4e2a\u5206\u6790\u6570\u636e\u503e\u659c\u7684\u6709\u7528\u6307\u6807\u3002\u89e3\u51b3\u65b9\u5f0f\u628a\u6570\u636e\u5206\u7ec4\u7684 key \u9884\u805a\u5408\u6765\u6d88\u9664\u6570\u636e\u503e\u659c\u3002<\/p>\n\n\n\n<p>\uff082\uff09\u4ee3\u7801\u7684\u6267\u884c\u6548\u7387\u95ee\u9898\uff0c\u963b\u585e\u6216\u8005\u6027\u80fd\u95ee\u9898\u3002<\/p>\n\n\n\n<p>\uff083\uff09TaskManager \u7684\u5185\u5b58\u5927\u5c0f\u5bfc\u81f4\u80cc\u538b\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_%E9%81%87%E5%88%B0Flink%E4%B8%8D%E5%A4%AA%E8%83%BD%E8%A7%A3%E5%86%B3%E7%9A%84%E9%97%AE%E9%A2%98%EF%BC%9F%EF%BC%88PVUV%E6%94%BE%E5%86%85%E5%AD%98%EF%BC%8COOM%E4%BA%86%EF%BC%8C%E5%90%8E%E9%9D%A2%E9%85%8D%E5%90%88redis%E4%BB%A5%E5%8F%8A%E5%B8%83%E9%9A%86%E8%BF%87%E6%BB%A4%E5%99%A8%EF%BC%89\"><\/span>2. \u9047\u5230Flink\u4e0d\u592a\u80fd\u89e3\u51b3\u7684\u95ee\u9898\uff1f\uff08PV,UV\u653e\u5185\u5b58\uff0cOOM\u4e86\uff0c\u540e\u9762\u914d\u5408redis\u4ee5\u53ca\u5e03\u9686\u8fc7\u6ee4\u5668\uff09<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u6709\u65f6\u5019\u9700\u8981\u6c42uv \uff0c\u5185\u5b58\u6216\u8005\u72b6\u6001\u4e2d\u5b58\u8fc7\u591a\u6570\u636e\uff0c\u5bfc\u81f4\u538b\u529b\u5de8\u5927\uff0c<br>\u8fd9\u4e2a\u65f6\u5019\u53ef\u4ee5\u7ed3\u5408Redis \u6216\u8005 \u5e03\u9686\u8fc7\u6ee4\u5668\u6765\u53bb\u91cd\u3002<\/p>\n\n\n\n<p>\u6ce8\u610f\uff1a\u5e03\u9686\u8fc7\u6ee4\u5668\u5b58\u5728\u975e\u5e38\u5c0f\u7684\u8bef\u5224\u51e0\u7387\uff0c\u4e0d\u80fd\u5224\u65ad\u67d0\u4e2a\u5143\u7d20\u4e00\u5b9a\u767e\u5206\u4e4b\u767e\u5b58\u5728\uff0c\u6240\u4ee5\u53ea\u80fd\u7528\u5728\u5141\u8bb8\u6709\u5c11\u91cf\u8bef\u5224\u7684\u573a\u666f\uff0c\u4e0d\u80fd\u7528\u5728\u9700\u8981100%\u7cbe\u786e\u5224\u65ad\u5b58\u5728\u7684\u573a\u666f\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_%E4%BD%BF%E7%94%A8flink%E7%BB%9F%E8%AE%A1%E8%AE%A2%E5%8D%95%E8%A1%A8%E7%9A%84GMV%EF%BC%8C%E5%A6%82%E6%9E%9Cmysql%E4%B8%AD%E7%9A%84%E6%95%B0%E6%8D%AE%E5%87%BA%E7%8E%B0%E9%94%99%E8%AF%AF%EF%BC%8C%E4%B9%8B%E5%90%8E%E5%9C%A8mysql%E4%B8%AD%E5%81%9A%E6%95%B0%E6%8D%AE%E7%9A%84%E4%BF%AE%E6%94%B9%E6%93%8D%E4%BD%9C%EF%BC%8C%E9%82%A3%E4%B9%88flink%E7%A8%8B%E5%BA%8F%E5%A6%82%E4%BD%95%E4%BF%9D%E8%AF%81GMV%E7%9A%84%E6%AD%A3%E7%A1%AE%E6%80%A7%EF%BC%8C%E4%BD%A0%E4%BB%AC%E6%98%AF%E5%A6%82%E4%BD%95%E8%A7%A3%E5%86%B3%EF%BC%9F\"><\/span>3. \u4f7f\u7528flink\u7edf\u8ba1\u8ba2\u5355\u8868\u7684GMV\uff0c\u5982\u679cmysql\u4e2d\u7684\u6570\u636e\u51fa\u73b0\u9519\u8bef\uff0c\u4e4b\u540e\u5728mysql\u4e2d\u505a\u6570\u636e\u7684\u4fee\u6539\u64cd\u4f5c\uff0c\u90a3\u4e48flink\u7a0b\u5e8f\u5982\u4f55\u4fdd\u8bc1GMV\u7684\u6b63\u786e\u6027\uff0c\u4f60\u4eec\u662f\u5982\u4f55\u89e3\u51b3\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u4e00\u822c\u6709\u79bb\u7ebfJob\u6765\u6062\u590d\u548c\u5b8c\u5584\u5b9e\u65f6\u6570\u636e\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%9B%9B_Spark_%E4%B8%8E_Flink_%E5%AF%B9%E6%AF%94\"><\/span>\u56db. Spark \u4e0e Flink \u5bf9\u6bd4<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_Spark%E4%B8%8EFlink%E5%8C%BA%E5%88%AB\"><\/span>1. Spark\u4e0eFlink\u533a\u522b<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<ol>\n<li>\u67b6\u6784\u6a21\u578b\uff1aSpark Streaming \u5728\u8fd0\u884c\u65f6\u7684\u4e3b\u8981\u89d2\u8272\u5305\u62ec\uff1aMaster\u3001Worker\u3001Driver\u3001Executor\uff0cFlink \u5728\u8fd0\u884c\u65f6\u4e3b\u8981\u5305\u542b\uff1aJobmanager\u3001Taskmanager\u548cSlot\u3002<br><\/li>\n\n\n\n<li>\u4efb\u52a1\u8c03\u5ea6\uff1aSpark Streaming \u8fde\u7eed\u4e0d\u65ad\u7684\u751f\u6210\u5fae\u5c0f\u7684\u6570\u636e\u6279\u6b21\uff0c\u6784\u5efa\u6709\u5411\u65e0\u73af\u56feDAG\uff0cSpark Streaming \u4f1a\u4f9d\u6b21\u521b\u5efa DStreamGraph\u3001JobGenerator\u3001JobScheduler\u3002Flink \u6839\u636e\u7528\u6237\u63d0\u4ea4\u7684\u4ee3\u7801\u751f\u6210 StreamGraph\uff0c\u7ecf\u8fc7\u4f18\u5316\u751f\u6210 JobGraph\uff0c\u7136\u540e\u63d0\u4ea4\u7ed9 JobManager\u8fdb\u884c\u5904\u7406\uff0cJobManager \u4f1a\u6839\u636e JobGraph \u751f\u6210 ExecutionGraph\uff0cExecutionGraph \u662f Flink \u8c03\u5ea6\u6700\u6838\u5fc3\u7684\u6570\u636e\u7ed3\u6784\uff0cJobManager \u6839\u636e ExecutionGraph \u5bf9 Job \u8fdb\u884c\u8c03\u5ea6\u3002<br><\/li>\n\n\n\n<li>\u65f6\u95f4\u673a\u5236\uff1aSpark Streaming \u652f\u6301\u7684\u65f6\u95f4\u673a\u5236\u6709\u9650\uff0c\u53ea\u652f\u6301\u5904\u7406\u65f6\u95f4\u3002Flink \u652f\u6301\u4e86\u6d41\u5904\u7406\u7a0b\u5e8f\u5728\u65f6\u95f4\u4e0a\u7684\u4e09\u4e2a\u5b9a\u4e49\uff1a\u5904\u7406\u65f6\u95f4\u3001\u4e8b\u4ef6\u65f6\u95f4\u3001\u6ce8\u5165\u65f6\u95f4\u3002\u540c\u65f6\u4e5f\u652f\u6301 watermark \u673a\u5236\u6765\u5904\u7406\u6ede\u540e\u6570\u636e\u3002<br><\/li>\n\n\n\n<li>\u5bb9\u9519\u673a\u5236\uff1a\u5bf9\u4e8e Spark Streaming \u4efb\u52a1\uff0c\u6211\u4eec\u53ef\u4ee5\u8bbe\u7f6e checkpoint\uff0c\u7136\u540e\u5047\u5982\u53d1\u751f\u6545\u969c\u5e76\u91cd\u542f\uff0c\u6211\u4eec\u53ef\u4ee5\u4ece\u4e0a\u6b21 checkpoint \u4e4b\u5904\u6062\u590d\uff0c\u4f46\u662f\u8fd9\u4e2a\u884c\u4e3a\u53ea\u80fd\u4f7f\u5f97\u6570\u636e\u4e0d\u4e22\u5931\uff0c\u53ef\u80fd\u4f1a\u91cd\u590d\u5904\u7406\uff0c\u4e0d\u80fd\u505a\u5230\u6070\u597d\u4e00\u6b21\u5904\u7406\u8bed\u4e49\u3002Flink \u5219\u4f7f\u7528\u4e24\u9636\u6bb5\u63d0\u4ea4\u534f\u8bae\u6765\u89e3\u51b3\u8fd9\u4e2a\u95ee\u9898\u3002<br><\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_Flink%E7%9A%84key_By%E5%92%8CSpark%E7%9A%84_group_by%E6%9C%89%E4%BB%80%E4%B9%88%E5%8C%BA%E5%88%AB\"><\/span>2. Flink\u7684key By\u548cSpark\u7684 group by\u6709\u4ec0\u4e48\u533a\u522b?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>keyBy\u7b97\u5b50\u5c06DataStream\u8f6c\u6362\u6210\u4e00\u4e2aKeyedStream\u3002KeyedStream\u662f\u4e00\u79cd\u7279\u6b8a\u7684DataStream\uff0c<br>\u4e8b\u5b9e\u4e0a\uff0cKeyedStream\u7ee7\u627f\u4e86DataStream\uff0cDataStream\u7684\u5404\u5143\u7d20\u968f\u673a\u5206\u5e03\u5728\u5404Task Slot\u4e2d\uff0c<br>KeyedStream\u7684\u5404\u5143\u7d20\u6309\u7167Key\u5206\u7ec4\uff0c\u5206\u914d\u5230\u5404Task Slot\u4e2d\u3002<\/p>\n\n\n\n<p>groupBy\u6309\u7167\u4f20\u5165\u51fd\u6570\u7684\u8fd4\u56de\u503c\u8fdb\u884c\u5206\u7ec4\u3002\u5c06\u76f8\u540c\u7684key\u5bf9\u5e94\u7684\u503c\u653e\u5165\u4e00\u4e2a\u8fed\u4ee3\u5668\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_%E4%B8%BA%E4%BB%80%E4%B9%88%E8%A6%81%E7%94%A8Flink%E6%9B%BF%E4%BB%A3SparkStreaming%EF%BC%9F\"><\/span>3. \u4e3a\u4ec0\u4e48\u8981\u7528Flink\u66ff\u4ee3SparkStreaming\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Spark\u548cFlink\u90fd\u5177\u6709\u6d41\u548c\u6279\u5904\u7406\u80fd\u529b\uff0c\u4f46\u662f\u4ed6\u4eec\u7684\u505a\u6cd5\u662f\u76f8\u53cd\u7684\u3002Spark Streaming\u662f\u628a\u6d41\u8f6c\u5316\u6210\u4e00\u4e2a\u4e2a\u5c0f\u7684\u6279\u6765\u5904\u7406\uff0c\u8fd9\u79cd\u65b9\u6848\u7684\u4e00\u4e2a\u95ee\u9898\u662f\u6211\u4eec\u9700\u8981\u7684\u5ef6\u8fdf\u8d8a\u4f4e\uff0c<br>\u989d\u5916\u5f00\u9500\u5360\u7684\u6bd4\u4f8b\u5c31\u4f1a\u8d8a\u5927\uff0c\u8fd9\u5bfc\u81f4\u4e86Spark Streaming\u5f88\u96be\u505a\u5230\u79d2\u7ea7\u751a\u81f3\u4e9a\u79d2\u7ea7\u7684\u5ef6\u8fdf\u3002Flink\u662f\u628a\u6279\u5f53\u4f5c\u4e00\u79cd\u6709\u9650\u7684\u6d41\uff0c\u8fd9\u79cd\u505a\u6cd5\u7684\u4e00\u4e2a\u7279\u70b9\u662f\u5728\u6d41\u548c\u6279\u5171\u4eab\u5927\u90e8\u5206\u4ee3\u7801\u7684\u540c\u65f6\u8fd8\u80fd\u591f\u4fdd\u7559\u6279\u5904\u7406\u7279\u6709\u7684\u4e00\u7cfb\u5217\u7684\u4f18\u5316\u3002<\/p>\n\n\n\n<p>\u540c\u65f6\uff0cFlink\u76f8\u6bd4\u4e8eSpark\u800c\u8a00\u8fd8\u6709\u8bf8\u591a\u660e\u663e\u4f18\u52bf\uff1a<\/p>\n\n\n\n<p>\u652f\u6301\u9ad8\u6548\u5bb9\u9519\u7684\u72b6\u6001\u7ba1\u7406\uff0c\u4fdd\u8bc1\u5728\u4efb\u4f55\u65f6\u95f4\u90fd\u80fd\u8ba1\u7b97\u51fa\u6b63\u786e\u7684\u7ed3\u679c\uff1b\u540c\u65f6\u652f\u6301\u9ad8\u541e\u5410\u3001\u4f4e\u5ef6\u8fdf\u3001\u9ad8\u6027\u80fd\u7684\u5206\u5e03\u5f0f\u6d41\u5f0f\u6570\u636e\u5904\u7406\u6846\u67b6\uff1b\u652f\u6301\u4e8b\u4ef6\u65f6\u95f4\uff08Event Time\uff09\u6982\u5ff5\uff0c\u4e8b\u4ef6\u5373\u4f7f\u65e0\u5e8f\u5230\u8fbe\u751a\u81f3\u5ef6\u8fdf\u5230\u8fbe\uff0c\u6570\u636e\u6d41\u90fd\u80fd\u591f\u8ba1\u7b97\u51fa\u7cbe\u786e\u7684\u7ed3\u679c\uff1b\u8f7b\u91cf\u7ea7\u5206\u5e03\u5f0f\u5feb\u7167\uff08Snapshot\uff09\u5b9e\u73b0\u7684\u5bb9\u9519\uff0c\u80fd\u5c06\u8ba1\u7b97\u8fc7\u7a0b\u5206\u5e03\u5230\u5355\u53f0\u5e76\u884c\u8282\u70b9\u4e0a\u8fdb\u884c\u5904\u7406\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E4%BA%94_Checkpoint%E7%B3%BB%E5%88%97\"><\/span>\u4e94. Checkpoint\u7cfb\u5217<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_Flink_checkpoint%E7%9A%84%E5%AE%9E%E7%8E%B0%E5%8E%9F%E7%90%86\"><\/span>1. Flink checkpoint\u7684\u5b9e\u73b0\u539f\u7406<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Flink\u5728\u65b0\u7248\u672c\u4e0a\u6539\u9020\u4e4b\u524d\u65b9\u5f0f\uff1a\u6682\u505c\u5e94\u7528, \u7136\u540e\u5f00\u59cb\u505a\u68c0\u67e5\u70b9, \u518d\u91cd\u65b0\u6062\u590d\u5e94\u7528<br>\u8fd9\u79cd\u65b9\u5f0f\u6548\u7387\u4f4e\uff0c\u6240\u4ee5Flink\u6539\u9020\u5206\u5e03\u5f0f\u5feb\u7167\u7b97\uff0c\u5f02\u6b65 barrier \u5feb\u7167\u3002<\/p>\n\n\n\n<p>\u6bcf\u4e2a\u9700\u8981checkpoint\u7684\u5e94\u7528\u5728\u542f\u52a8\u65f6\uff0cFlink\u7684JobManager\u4e3a\u5176\u521b\u5efa\u4e00\u4e2a CheckpointCoordinator\uff0c<br>CheckpointCoordinator\u5168\u6743\u8d1f\u8d23\u672c\u5e94\u7528\u7684\u5feb\u7167\u5236\u4f5c\u3002<\/p>\n\n\n\n<p>\u5982\u679c\u63a5\u7740\u95ee Barrier \u662f\u5565\uff1f\u591a\u4e2abarrier\u88ab\u63d2\u5165\u5230\u6570\u636e\u6d41\u4e2d, \u7136\u540e\u4f5c\u4e3a\u6570\u636e\u6d41\u7684\u4e00\u90e8\u5206\u968f\u7740\u6570\u636e\u6d41\u52a8(\u6709\u70b9\u7c7b\u4f3c\u4e8eWatermark).\u8fd9\u4e9bbarrier\u4e0d\u4f1a\u8de8\u8d8a\u6d41\u4e2d\u7684\u6570\u636e.<br>\u6bcf\u4e2abarrier\u4f1a\u628a\u6570\u636e\u6d41\u5206\u6210\u4e24\u90e8\u5206: \u4e00\u90e8\u5206\u6570\u636e\u8fdb\u5165\u5f53\u524d\u7684\u5feb\u7167 , \u53e6\u4e00\u90e8\u5206\u6570\u636e\u8fdb\u5165\u4e0b\u4e00\u4e2a\u5feb\u7167 . \u6bcf\u4e2abarrier\u643a\u5e26\u7740\u5feb\u7167\u7684id. barrier \u4e0d\u4f1a\u6682\u505c\u6570\u636e\u7684\u6d41\u52a8, \u6240\u4ee5\u975e\u5e38\u8f7b\u91cf\u7ea7. \u5728\u6d41\u4e2d, \u540c\u4e00\u65f6\u95f4\u53ef\u4ee5\u6709\u6765\u6e90\u4e8e\u591a\u4e2a\u4e0d\u540c\u5feb\u7167\u7684\u591a\u4e2abarrier, \u8fd9\u4e2a\u610f\u5473\u7740\u53ef\u4ee5\u5e76\u53d1\u7684\u51fa\u73b0\u4e0d\u540c\u7684\u5feb\u7167\u3002<\/p>\n\n\n\n<p><strong>Flink\u7684\u68c0\u67e5<\/strong>\u70b9\u5236\u4f5c\u8fc7\u7a0b<\/p>\n\n\n\n<ol>\n<li>Checkpoint Coordinator \u5411\u6240\u6709 source \u8282\u70b9 trigger Checkpoint. \u7136\u540eSource Task\u4f1a\u5728\u6570\u636e\u6d41\u4e2d\u5b89\u63d2CheckPoint barrier<\/li>\n<\/ol>\n\n\n\n<p>Job Manager \u5bf9\u6bcf\u4e00\u4e2ajob\u90fd\u4f1a\u4ea7\u751f\u4e00\u4e2aCheckpoint Coordinator<br>\u5411\u6240\u6709 source \u8282\u70b9 \u89e6\u53d1 trigger Checkpoint\u8282\u70b9\uff0c \u5e76\u884c\u5ea6\u662f\u51e0\uff0c\u5c31\u4f1a\u89e6\u53d1\u591a\u5c11\u4e2a\u3002source \u4f1a\u5411\u6d41\u4e2d\u89e6\u53d1Barrier\uff0c\u63a5\u6536\u5230Barrier\u7684\u8282\u70b9\u5c31\u4f1a\u4fdd\u5b58\u5feb\u7167\uff08\u5305\u62ecsource\uff09\u3002<\/p>\n\n\n\n<ol>\n<li>source \u8282\u70b9\u5411\u4e0b\u6e38\u5e7f\u64ad barrier\uff0c\u8fd9\u4e2a barrier \u5c31\u662f\u5b9e\u73b0 Chandy-Lamport \u5206\u5e03\u5f0f\u5feb\u7167\u7b97\u6cd5\u7684\u6838\u5fc3\uff0c<br>\u4e0b\u6e38\u7684 task \u53ea\u6709\u6536\u5230\u6240\u6709\u8fdb\u6765\u7684 barrier \u624d\u4f1a\u6267\u884c\u76f8\u5e94\u7684 Checkpoint(barrier\u5bf9\u9f50\uff09<br>\u6ce8\u610f\uff1a\u65b0\u7248\u672c\u6709\u4e00\u79cd\u65b0\u975e\u5bf9\u9f50barrier<\/li>\n<\/ol>\n\n\n\n<p>source\u5148\u6536\u5230barrier\uff0c\u7136\u540e\u5f80\u540e\u4f20\u9012\uff0c\u82e5\u662f\u591a\u5e76\u884c\u5ea6\uff0c\u76f8\u5f53\u4e8e\u591a\u7ec4\u63a5\u529b\u8d5b\u8dd1\u6bd4\u8d5b\uff0c\u6240\u4ee5\u987a\u5e8f\u662f\u4e0d\u4e00\u81f4\u7684\uff0c\u5e76\u4e0d\u662f\u540c\u6b65\u3002<\/p>\n\n\n\n<ol>\n<li>\u4e0b\u6e38\u7684 sink \u8282\u70b9\u6536\u96c6\u9f50\u4e0a\u6e38\u4e24\u4e2a input \u7684 barrier \u4e4b\u540e\uff0c\u4f1a\u6267\u884c\u672c\u5730\u5feb\u7167\u3002<br><\/li>\n\n\n\n<li>\u540c\u6837\u7684\uff0csink \u8282\u70b9\u5728\u5b8c\u6210\u81ea\u5df1\u7684 Checkpoint \u4e4b\u540e\uff0c\u4f1a\u5c06 state handle \u8fd4\u56de\u901a\u77e5 Coordinator\u3002<br><\/li>\n\n\n\n<li>\u6700\u540e\uff0c\u5f53 Checkpoint coordinator \u6536\u96c6\u9f50\u6240\u6709 task \u7684 state handle\uff0c\u5c31\u8ba4\u4e3a\u8fd9\u4e00\u6b21\u7684 Checkpoint \u5168\u5c40\u5b8c\u6210\u4e86\uff0c\u5411\u6301\u4e45\u5316\u5b58\u50a8\u4e2d\u518d\u5907\u4efd\u4e00\u4e2a Checkpoint meta \u6587\u4ef6\u3002<br><\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_Checkpoint_%E5%AD%98%E5%82%A8\"><\/span>2. Checkpoint \u5b58\u50a8<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p><strong>MemoryStateBackend<\/strong><\/p>\n\n\n\n<p>\u5185\u5b58\u7ea7\u7684\u72b6\u6001\u540e\u7aef\uff0c\u4f1a\u5c06\u952e\u63a7\u72b6\u6001\u4f5c\u4e3a\u5185\u5b58\u4e2d\u7684\u5bf9\u8c61\u8fdb\u884c\u7ba1\u7406\uff0c\u5c06\u5b83\u4eec\u5b58\u50a8<br>\u5728 TaskManager \u7684 JVM \u5806\u4e0a\uff1b\u800c\u5c06 checkpoint \u5b58\u50a8\u5728 JobManager \u7684\u5185\u5b58\u4e2d\u3002<\/p>\n\n\n\n<p><strong>FsStateBackend<\/strong><\/p>\n\n\n\n<p>\u5c06 checkpoint \u5b58\u5230\u8fdc\u7a0b\u7684\u6301\u4e45\u5316\u6587\u4ef6\u7cfb\u7edf\uff08FileSystem\uff09\u4e0a\u3002\u800c\u5bf9\u4e8e\u672c\u5730\u72b6\u6001\uff0c<br>\u8ddf MemoryStateBackend \u4e00\u6837\uff0c\u4e5f\u4f1a\u5b58\u5728 TaskManager \u7684 JVM \u5806\u4e0a\u3002<\/p>\n\n\n\n<p><strong>RocksDBStateBackend<\/strong><\/p>\n\n\n\n<p>\u5c06\u6240\u6709\u72b6\u6001\u5e8f\u5217\u5316\u540e\uff0c\u5b58\u5165\u672c\u5730\u7684 RocksDB \u4e2d\u5b58\u50a8\u3002<\/p>\n\n\n\n<p>\u8ffd\u95ee\uff1a\u4e86\u89e3RocksDB\u4e0eRocksDBStateBackend\uff1f<\/p>\n\n\n\n<p>RocksDB\u662f\u4e00\u4e2a\u7528\u4e8e\u5feb\u901f\u5b58\u50a8\u7684\u53ef\u5d4c\u5165\u6301\u4e45\u5316\u952e\u503c\u5b58\u50a8\u3002\u5b83\u901a\u8fc7Java Native\u63a5\u53e3\uff08JNI\uff09\u4e0eFlink\u8fdb\u884c\u4ea4\u4e92\u3002Flink\u4f5c\u4e1a\u8fd0\u884c\u65f6\uff0cRocksDB\u4f1a\u88ab\u5185\u5d4c\u5230TaskManager\u8fdb\u7a0b\u4e2d\u3002<\/p>\n\n\n\n<p>RocksDBStateBackend\u8fd8\u652f\u6301\u4f5c\u4e3a\u6027\u80fd\u8c03\u4f18\u9009\u9879\u7684\u589e\u91cfcheckpoint\u3002\u589e\u91cfcheckpoint\u4ec5\u5b58\u50a8\u4e0a\u6b21checkpoint\u4e4b\u540e\u53d1\u751f\u7684\u6539\u53d8\u3002\u4e0e\u6267\u884c\u5b8c\u6574\u5feb\u7167\u76f8\u6bd4\uff0c\u8fd9\u5927\u5927\u51cf\u5c11\u4e86checkpoint\u7684\u65f6\u95f4\u3002<\/p>\n\n\n\n<p><strong>\u573a\u666f<\/strong><\/p>\n\n\n\n<p>1. \u4f5c\u4e1a\u7684\u72b6\u6001\u5927\u5c0f\u5927\u4e8e\u5730\u5185\u5b58(\u5982\u8f83\u957f\u7684\u7a97\u53e3\uff0c\u8f83\u5927\u7684Keyed\u72b6\u6001)\u3002<\/p>\n\n\n\n<p>2. \u4f5c\u4e1a\u9700\u8981\u4f7f\u7528\u589e\u91cfcheckpoint\uff0c\u4ee5\u51cf\u5c11checkpoint\u7684\u65f6\u95f4\u3002<\/p>\n\n\n\n<p>3. \u4f5c\u4e1a\u9700\u8981\u4fdd\u8bc1\u53ef\u9884\u6d4b\u7684\u5ef6\u8fdf\uff0c\u4e0d\u53d7JVM\u5783\u573e\u56de\u6536\u7684\u5f71\u54cd\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_Flink%E7%9A%84checkpoint%E6%9C%BA%E5%88%B6%E4%BB%A5%E5%8F%8A%E7%B2%BE%E5%87%86%E4%B8%80%E6%AC%A1%E6%80%A7%E6%B6%88%E8%B4%B9%E5%A6%82%E4%BD%95%E5%AE%9E%E7%8E%B0%EF%BC%9F\"><\/span>4. Flink\u7684checkpoint\u673a\u5236\u4ee5\u53ca\u7cbe\u51c6\u4e00\u6b21\u6027\u6d88\u8d39\u5982\u4f55\u5b9e\u73b0\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>1. souce\uff1a\u4f7f\u7528\u6267\u884cExactlyOnce\u7684\u6570\u636e\u6e90\uff0c\u6bd4\u5982kafka\u7b49<\/p>\n\n\n\n<p>2. \u5185\u90e8\u4f7f\u7528FlinkKafakConsumer\uff0c\u5e76\u5f00\u542fCheckPoint\uff0c\u504f\u79fb\u91cf\u4f1a\u4fdd\u5b58\u5230StateBackend\u4e2d\uff0c\u5e76\u4e14\u9ed8\u8ba4\u4f1a\u5c06\u504f\u79fb\u91cf\u5199\u5165\u5230topic\u4e2d\u53bb\uff0c\u5373_consumer_offsets Flink\u8bbe\u7f6eCheckepointingModel.EXACTLY_ONCE<\/p>\n\n\n\n<p>3. sink\uff1a\u5b58\u50a8\u7cfb\u7edf\u652f\u6301\u8986\u76d6\u4e5f\u5373\u5e42\u7b49\u6027\uff1a\u5982Redis,Hbase,ES\u7b49<br>\u5b58\u50a8\u7cfb\u7edf\u4e0d\u652f\u6301\u8986\uff1a\u9700\u8981\u652f\u6301\u4e8b\u52a1(\u9884\u5199\u5f0f\u65e5\u5fd7\u6216\u8005\u4e24\u9636\u6bb5\u63d0\u4ea4),\u4e24\u9636\u6bb5\u63d0\u4ea4\u53ef\u53c2\u8003Flink\u96c6\u6210\u7684kafka sink\u7684\u5b9e\u73b0<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_%E7%B2%BE%E7%A1%AE%E4%B8%80%E6%AC%A1%EF%BC%8C%E8%87%B3%E5%B0%91%E4%B8%80%E6%AC%A1%E5%AF%B9checkpoint%E6%9C%89%E4%BB%80%E4%B9%88%E5%BD%B1%E5%93%8D%EF%BC%9F\"><\/span>3. \u7cbe\u786e\u4e00\u6b21\uff0c\u81f3\u5c11\u4e00\u6b21\u5bf9checkpoint\u6709\u4ec0\u4e48\u5f71\u54cd\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u7cbe\u786e\u4e00\u6b21:<br>\u5728\u591a\u5e76\u884c\u5ea6\u4e0b, \u5982\u679c\u8981\u5b9e\u73b0\u4e25\u683c\u4e00\u6b21, \u5219\u8981\u6267\u884cbarrier\u5bf9\u9f50.<br>\u5f53 job graph \u4e2d\u7684\u6bcf\u4e2a operator \u63a5\u6536\u5230 barriers \u65f6\uff0c\u5b83\u5c31\u4f1a\u8bb0\u5f55\u4e0b\u5176\u72b6\u6001\u3002\u6709\u4e14\u53ea\u6709\u4e00\u6b21\uff0c\u6d88\u606f\u4e0d\u4e22\u5931\u4e0d\u91cd\u590d\uff0c\u4e14\u53ea\u6d88\u8d39\u4e00\u6b21<\/p>\n\n\n\n<p>\u81f3\u5c11\u4e00\u6b21\u8bed\u4e49:<br>barrier\u4e0d\u5bf9\u9f50,\u4f1a\u91cd\u590d\u6d88\u8d39\u3002\u5982\u679c\u4e0d\u5bf9\u9f50\uff0c\u90a3\u4e48\u5728 chk-100 \u5feb\u7167\u4e4b\u524d\uff0c\u5df2\u7ecf\u5904\u7406\u4e86\u4e00\u4e9b chk-100 \u5bf9\u5e94\u7684 offset \u4e4b\u540e\u7684\u6570\u636e\uff0c<br>\u5f53\u7a0b\u5e8f\u4ece chk-100 \u6062\u590d\u4efb\u52a1\u65f6\uff0cchk-100 \u5bf9\u5e94\u7684 offset \u4e4b\u540e\u7684\u6570\u636e\u8fd8\u4f1a\u88ab\u5904\u7406\u4e00\u6b21\uff0c<br>\u6240\u4ee5\u5c31\u51fa\u73b0\u4e86\u91cd\u590d\u6d88\u8d39\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_Savapoint%E4%BA%86%E8%A7%A3%E5%A4%9A%E5%B0%91%EF%BC%9F\"><\/span>4. Savapoint\u4e86\u89e3\u591a\u5c11\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>savepoint\u662f\u201c\u901a\u8fc7checkpoint\u673a\u5236\u201d\u521b\u5efa\u7684\uff0c\u6240\u4ee5savepoint\u672c\u8d28\u4e0a\u662f\u7279\u6b8a\u7684checkpoint<\/p>\n\n\n\n<p>savepoint\u7684\u4fa7\u91cd\u70b9\u662f\u201c\u7ef4\u62a4\u201d\uff0c\u5373Flink\u4f5c\u4e1a\u9700\u8981\u5728\u4eba\u5de5\u5e72\u9884\u4e0b\u624b\u52a8\u91cd\u542f\u3001\u5347\u7ea7\u3001\u8fc1\u79fb\u6216A\/B\u6d4b\u8bd5\u65f6\uff0c\u5148\u5c06\u72b6\u6001\u6574\u4f53\u5199\u5165\u53ef\u9760\u5b58\u50a8\uff0c\u7ef4\u62a4\u5b8c\u6bd5\u4e4b\u540e\u518d\u4ecesavepoint\u6062\u590d\u73b0\u573a<\/p>\n\n\n\n<p>savepoint\u5219\u4ee5\u4e8c\u8fdb\u5236\u5f62\u5f0f\u5b58\u50a8\u6240\u6709\u72b6\u6001\u6570\u636e\u548c\u5143\u6570\u636e<\/p>\n\n\n\n<p>avepoint\u5e76\u4e0d\u4f1a\u8fde\u7eed\u81ea\u52a8\u89e6\u53d1\uff0c\u6240\u4ee5savepoint\u6ca1\u6709\u5fc5\u8981\u652f\u6301\u589e\u91cf<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"5_Flink_checkpoint%E7%9A%84%E8%B6%85%E6%97%B6%E9%97%AE%E9%A2%98_%E5%A6%82%E4%BD%95%E8%A7%A3%E5%86%B3\"><\/span>5. Flink checkpoint\u7684\u8d85\u65f6\u95ee\u9898 \u5982\u4f55\u89e3\u51b3?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>1\u3001\u662f\u5426\u7f51\u7edc\u95ee\u9898<\/p>\n\n\n\n<p>2\u3001\u662f\u5426\u662fbarrir\u95ee\u9898<\/p>\n\n\n\n<p>3\u3001\u67e5\u770bwebui\uff0c\u662f\u5426\u6709\u6570\u636e\u503e\u659c<\/p>\n\n\n\n<p>4\u3001\u6709\u6570\u636e\u503e\u659c\u7684\u8bdd\uff0c\u90a3\u4e48\u89e3\u51b3\u6570\u636e\u503e\u659c\u540e\uff0c\u4f1a\u6709\u6539\u5584\uff0c<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"6_%E4%BD%9C%E4%B8%9A%E6%8C%82%E6%8E%89%E4%BA%86%EF%BC%8C%E6%81%A2%E5%A4%8D%E4%B8%8A%E4%B8%80%E4%B8%AACheckpoint%EF%BC%8C%E7%94%A8%E4%BB%80%E4%B9%88%E5%91%BD%E4%BB%A4%EF%BC%9F\"><\/span>6. \u4f5c\u4e1a\u6302\u6389\u4e86\uff0c\u6062\u590d\u4e0a\u4e00\u4e2aCheckpoint\uff0c\u7528\u4ec0\u4e48\u547d\u4ee4\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-code\"><code>-s hdfs:\/\/192.168.0.1:8020\/flink\/checkpoint\/...\n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"7_%E4%BB%80%E4%B9%88%E6%98%AFFlink%E7%9A%84%E9%9D%9Ebarrier%E5%AF%B9%E9%BD%90%EF%BC%8C%E5%A6%82%E4%BD%95%E5%AE%9E%E7%8E%B0%EF%BC%9F\"><\/span>7. \u4ec0\u4e48\u662fFlink\u7684\u975ebarrier\u5bf9\u9f50\uff0c\u5982\u4f55\u5b9e\u73b0\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u975e\u5bf9\u9f50\u603b\u4f53\u6d41\u7a0b\uff1a\u5728\u63a5\u53d7\u4e0a\u6e38\u591a\u4e2a\u8f93\u5165\u60c5\u51b5\u4e0b\uff0c\u6bcf\u4e00\u4e2a\u6279\u6b21\u7684checkpoint\u4e0d\u4f1a\u53d1\u751f\u6570\u636e\u7f13\u5b58\uff0c<br>\u4f1a\u76f4\u63a5\u4ea4\u7ed9\u4e0b\u6e38\u53bb\u5904\u7406\uff0ccheckpoint\u4fe1\u606f\u4f1a\u88ab\u7f13\u5b58\u5728\u4e00\u4e2aCheckpointBarrierCount\u7c7b\u578b\u7684\u961f\u5217\u4e2d\uff0c<br>CheckpointBarrierCount\u6807\u8bc6\u4e86\u4e00\u6b21checkpoint\u4e0e\u5176channel\u8f93\u5165checkpointBarrier\u4e2a\u6570\uff0c<br>\u5f53checkpointBarrier\u4e2a\u6570\u4e0echannel\u4e2a\u6570\u76f8\u540c\u5219\u4f1a\u89e6\u53d1checkpoint\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%85%AD_%E7%AA%97%E5%8F%A3_%E4%B8%8E_Watermark%E7%B3%BB%E5%88%97\"><\/span>\u516d. \u7a97\u53e3 \u4e0e Watermark\u7cfb\u5217<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_Flink%E6%97%B6%E9%97%B4%E8%AF%AD%E4%B9%89\"><\/span>1. Flink\u65f6\u95f4\u8bed\u4e49<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u4e8b\u4ef6\u65f6\u95f4Event Time<\/p>\n\n\n\n<p>\u5904\u7406\u65f6\u95f4Process Time<\/p>\n\n\n\n<p>\u8fdb\u5165\u65f6\u95f4Ingestion Time<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_%E4%BB%80%E4%B9%88%E6%98%AFWatermark%E5%8F%8A%E4%B8%BB%E8%A6%81%E4%BD%9C%E7%94%A8%EF%BC%9F%E4%BB%80%E4%B9%88%E6%97%B6%E5%80%99%E5%8E%BB%E8%A7%A6%E5%8F%91%E8%AE%A1%E7%AE%97\"><\/span>2. \u4ec0\u4e48\u662fWatermark\u53ca\u4e3b\u8981\u4f5c\u7528\uff1f\u4ec0\u4e48\u65f6\u5019\u53bb\u89e6\u53d1\u8ba1\u7b97?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Watermark \u662f\u4e00\u79cd\u8861\u91cf Event Time \u8fdb\u5c55\u7684\u673a\u5236\uff0c\u7528\u4e8e\u5904\u7406\u4e71\u5e8f\u4e8b\u4ef6\u7684\uff0c\u5355\u8c03\u9012\u589e\u7684\u65f6\u95f4\u6233\uff1b\u6570\u636e\u6d41\u4e2d\u7684 Watermark \u7528\u4e8e\u8868\u793a timestamp \u5c0f\u4e8e Watermark \u7684\u6570\u636e\uff0c\u90fd\u5df2\u7ecf\u5230\u8fbe\uff1b<\/p>\n\n\n\n<p>watermark(\u6c34\u4f4d\u7ebf\uff0c\u5305\u542b\u5ef6\u8fdf) &gt; \u7a97\u53e3\u7ed3\u675f\u65f6\u95f4<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_%E6%B6%88%E6%81%AF%E8%B6%85%E8%BF%87watermark%E7%9A%84%E6%97%B6%E9%97%B4%E4%BC%9A%E4%B8%A2%E5%A4%B1%E6%95%B0%E6%8D%AE%E5%90%97\"><\/span>3. \u6d88\u606f\u8d85\u8fc7watermark\u7684\u65f6\u95f4\u4f1a\u4e22\u5931\u6570\u636e\u5417?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>allowedLateness\u4e5f\u662fFlink\u5904\u7406\u4e71\u5e8f\u4e8b\u4ef6\u7684\u4e00\u4e2a\u7279\u522b\u91cd\u8981\u7684\u7279\u6027,\u9ed8\u8ba4\u60c5\u51b5\u4e0b,<br>\u5f53wartermark\u901a\u8fc7window\u540e,\u518d\u8fdb\u6765\u7684\u6570\u636e,\u4e5f\u5c31\u662f\u8fdf\u5230\u6216\u8005\u665a\u5230\u7684\u6570\u636e\u5c31\u4f1a\u522b\u4e22\u5f03\u6389\u4e86,<br>\u4f46\u662f\u6709\u7684\u65f6\u5019\u6211\u4eec\u5e0c\u671b\u5728\u4e00\u4e2a\u53ef\u4ee5\u63a5\u53d7\u7684\u8303\u56f4\u5185,<br>\u8fdf\u5230\u7684\u6570\u636e,\u4e5f\u53ef\u4ee5\u88ab\u5904\u7406\u6216\u8005\u8ba1\u7b97,\u8fd9\u5c31\u662fallowedLateness\u4ea7\u751f\u7684\u539f\u56e0\u4e86<\/p>\n\n\n\n<p>\u8fdf\u5230\u7684\u5143\u7d20\u4e5f\u53ef\u4ee5\u4f7f\u7528\u4fa7\u8f93\u51fa(side output)\u7279\u6027\u88ab\u91cd\u5b9a\u5411\u5230\u53e6\u5916\u7684\u4e00\u6761\u6d41\u4e2d\u53bb\u3002\u8fdf\u5230\u5143\u7d20\u6240\u7ec4\u6210\u7684\u4fa7\u8f93\u51fa\u6d41\u53ef\u4ee5\u7ee7\u7eed\u5904\u7406\u6216\u8005sink\u5230\u6301\u4e45\u5316\u8bbe\u65bd\u4e2d\u53bb\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_%E5%BC%80%E7%AA%97%E5%87%BD%E6%95%B0%E6%9C%89%E5%93%AA%E4%BA%9B%EF%BC%9F\"><\/span>4. \u5f00\u7a97\u51fd\u6570\u6709\u54ea\u4e9b\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<pre class=\"wp-block-code\"><code>FlinkSQL\n\u7a97\u53e3\uff1a\nTUMBLE(TABLE data, DESCRIPTOR(timecol), size)\n\nHOP(TABLE data, DESCRIPTOR(timecol), slide, size &#91;, offset ])\n\nSESSION(&lt;time-attr&gt;, &lt;gap-interval&gt;)\n&lt;gap-interval&gt;: INTERVAL 'string' timeUnit\n\n\u7d2f\u79ef\u7a97\u53e3\u51fd\u6570\uff1aCUMULATE(TABLE data, DESCRIPTOR(timecol), step, size)\n\n\u7a97\u53e3\u5206\u7ec4\u805a\u5408GROUPING SETS\n\nover\u51fd\u6570\n\nCUBE\n\n\nFlink DataStream\n\u589e\u91cf\u805a\u5408\u548c\u5168\u91cf\u805a\u5408\n\u589e\u91cf\u805a\u5408: \u7a97\u53e3\u4e0d\u7ef4\u62a4\u539f\u59cb\u6570\u636e\uff0c\u53ea\u7ef4\u62a4\u4e2d\u95f4\u7ed3\u679c\uff0c\u6bcf\u6b21\u57fa\u4e8e\u4e2d\u95f4\u7ed3\u679c\u548c\u589e\u91cf\u6570\u636e\u8fdb\u884c\u805a\u5408\u3002\n\u5982: ReduceFunction\u3001AggregateFunction\n\n\u5168\u91cf\u805a\u5408: \u7a97\u53e3\u9700\u8981\u7ef4\u62a4\u5168\u90e8\u539f\u59cb\u6570\u636e\uff0c\u7a97\u53e3\u89e6\u53d1\u8fdb\u884c\u5168\u91cf\u805a\u5408\u3002\u5982:ProcessWindowFunction\n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"5_%E6%B2%A1%E6%9C%89%E6%95%B0%E6%8D%AE%E6%B5%81%E7%9A%84%E6%97%B6%E5%80%99%E7%AA%97%E5%8F%A3%E8%BF%99%E4%B8%AA%E7%AA%97%E5%8F%A3%E5%AD%98%E5%9C%A8%E5%90%97\"><\/span>5. (\u6ca1\u6709\u6570\u636e\u6d41\u7684\u65f6\u5019)\u7a97\u53e3,\u8fd9\u4e2a\u7a97\u53e3\u5b58\u5728\u5417?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u6ca1\u6709\u6570\u636e\uff0c\u7a97\u53e3\u4e0d\u4ea7\u751f<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"6_1%E5%B0%8F%E6%97%B6%E7%9A%84%E6%BB%9A%E5%8A%A8%E7%AA%97%E5%8F%A3%E4%B8%80%E5%B0%8F%E6%97%B6%E5%A4%84%E7%90%86%E4%B8%80%E6%AC%A1%E7%9A%84%E5%8E%8B%E5%8A%9B%E6%AF%94%E8%BE%83%E5%A4%A7%E6%83%B3%E8%AE%A9%E4%BB%965%E5%88%86%E9%92%9F%E5%A4%84%E7%90%86%E4%B8%80%E6%AC%A1%E6%80%8E%E4%B9%88%E5%8A%9E\"><\/span>6. 1\u5c0f\u65f6\u7684\u6eda\u52a8\u7a97\u53e3,\u4e00\u5c0f\u65f6\u5904\u7406\u4e00\u6b21\u7684\u538b\u529b\u6bd4\u8f83\u5927,\u60f3\u8ba9\u4ed65\u5206\u949f\u5904\u7406\u4e00\u6b21.\u600e\u4e48\u529e?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u81ea\u5b9a\u4e49\u89e6\u53d1\u5668\uff0c4\u4e2a\u65b9\u6cd5\uff0c\u4e00\u4e2aClose\u4e09\u4e2a\u7528\u4e8e\u63a7\u5236\u8ba1\u7b97\u548c\u8f93\u51fa<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E4%B8%83_Join%E7%B3%BB%E5%88%97\"><\/span>\u4e03. Join\u7cfb\u5217<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_Flink%E7%9A%84%E5%8F%8C%E6%B5%81join%E7%9A%84%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86\"><\/span>1. Flink\u7684\u53cc\u6d41join\u7684\u5e95\u5c42\u539f\u7406?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>union\uff1aunion \u652f\u6301\u53cc\u6d41 Join\uff0c\u4e5f\u652f\u6301\u591a\u6d41 Join\u3002\u591a\u4e2a\u6d41\u7c7b\u578b\u5fc5\u987b\u4e00\u81f4\uff1b<\/p>\n\n\n\n<p>connector\uff1aconnector \u652f\u6301\u53cc\u6d41 Join\uff0c\u4e24\u4e2a\u6d41\u7684\u7c7b\u578b\u53ef\u4ee5\u4e0d\u4e00\u81f4\uff1b<\/p>\n\n\n\n<p>join\uff1a\u8be5\u65b9\u6cd5\u53ea\u652f\u6301 inner join\uff0c\u5373\uff1a\u76f8\u540c\u7a97\u53e3\u4e0b\uff0c\u4e24\u4e2a\u6d41\u4e2d\uff0cKey\u90fd\u5b58\u5728\u4e14\u76f8\u540c\u65f6\u624d\u4f1a\u5173\u8054\u6210\u529f\uff1b<\/p>\n\n\n\n<p>coGroup\uff1a\u540c\u6837\u80fd\u591f\u5b9e\u73b0\u53cc\u6d41 Join\u3002\u5373\uff1a\u5c06\u540c\u4e00 Window \u7a97\u53e3\u5185\u7684\u4e24\u4e2aDataStream \u8054\u5408\u8d77\u6765\uff0c\u4e24\u4e2a\u6d41\u6309\u7167 Key \u6765\u8fdb\u884c\u5173\u8054\uff0c\u5e76\u901a\u8fc7 apply()\u65b9\u6cd5 new CoGroupFunction() \u7684\u5f62\u5f0f\uff0c\u91cd\u5199 join() \u65b9\u6cd5\u8fdb\u884c\u903b\u8f91\u5904\u7406\u3002<\/p>\n\n\n\n<p>intervalJoin\uff1aInterval Join \u6ca1\u6709 Window \u7a97\u53e3\u7684\u6982\u5ff5\uff0c\u76f4\u63a5\u7528\u65f6\u95f4\u6233\u4f5c\u4e3a\u5173\u8054\u7684\u6761\u4ef6\uff0c\u66f4\u5177\u8868\u8fbe\u529b\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%9F%BA%E4%BA%8EConnect%E7%9A%84%E5%8F%8C%E6%B5%81JOIN%E5%AE%9E%E7%8E%B0%E6%9C%BA%E5%88%B6\"><\/span>\u57fa\u4e8eConnect\u7684\u53cc\u6d41JOIN\u5b9e\u73b0\u673a\u5236<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u5bf9\u4e24\u4e2aDataStream\u6267\u884cconnect\u64cd\u4f5c\uff0c\u5c06\u5176\u8f6c\u5316\u4e3aConnectedStreams,<br>\u751f\u6210\u7684Streams\u53ef\u4ee5\u8c03\u7528\u4e0d\u540c\u65b9\u6cd5\u5728\u4e24\u4e2a\u5b9e\u65f6\u6d41\u4e0a\u6267\u884c\uff0c\u4e14\u53cc\u6d41\u4e4b\u95f4\u53ef\u4ee5\u5171\u4eab\u72b6\u6001\u3002<\/p>\n\n\n\n<p>orderStream.connect(orderDetailStream)<br>.keyBy(&#8220;orderId&#8221;, &#8220;orderId&#8221;)<br>.process(new orderProcessFunc());<\/p>\n\n\n\n<p>\u603b\u4f53\u601d\u60f3\uff1a\u57fa\u4e8e\u6570\u636e\u65f6\u95f4\u5b9e\u73b0\u8ba2\u5355\u6570\u636e\u53ca\u8ba2\u5355\u660e\u7ec6\u6570\u636e\u7684\u5173\u8054\uff0c\u8d85\u65f6\u6216\u8005\u7f3a\u5931\u5219\u7531\u4fa7\u8f93\u51fa\u6d41\u8f93\u51fa\u3002<\/p>\n\n\n\n<p>\u5728connect\u4e2d\u9488\u5bf9\u8ba2\u5355\u6d41\u548c\u8ba2\u5355\u660e\u7ec6\u6d41\uff0c\u5148\u521b\u5efa\u5b9a\u65f6\u5668\u5e76\u4fdd\u5b58state\u72b6\u6001\uff0c\u5904\u4e8e\u7a97\u53e3\u5185\u5c31\u8fdb\u884cjoin, \u5426\u5219\u8fdb\u5165\u4fa7\u8f93\u51fa\u6d41\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%9F%BA%E4%BA%8EWindow_Join%E7%9A%84%E5%8F%8C%E6%B5%81JOIN%E5%AE%9E%E7%8E%B0%E6%9C%BA%E5%88%B6\"><\/span>\u57fa\u4e8eWindow Join\u7684\u53cc\u6d41JOIN\u5b9e\u73b0\u673a\u5236<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u5c06\u4e24\u6761\u5b9e\u65f6\u6d41\u4e2d\u5143\u7d20\u5206\u914d\u5230\u540c\u4e00\u4e2a\u65f6\u95f4\u7a97\u53e3\u4e2d\u5b8c\u6210Join\u3002\u5e95\u5c42\u539f\u7406: \u4e24\u6761\u5b9e\u65f6\u6d41\u6570\u636e\u7f13\u5b58\u5728Window State\u4e2d\uff0c\u5f53\u7a97\u53e3\u89e6\u53d1\u8ba1\u7b97\u65f6\uff0c\u6267\u884cjoin\u64cd\u4f5c\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>orderStream.join(orderDetailStream)\n    .where(r =&gt; r._1)  \/\/\u8ba2\u5355id\n    .equalTo(r =&gt; r._2) \/\/\u8ba2\u5355id\n    .window(TumblingProcessTimeWindows.of(\n          Time.seconds(60)))\n    .apply {(r1, r2) =&gt; r1 + \" : \" + r2}\n    \n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"coGroup%E7%AE%97%E5%AD%90\"><\/span>coGroup\u7b97\u5b50<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>coGroup\u7b97\u5b50\u4e5f\u662f\u57fa\u4e8ewindow\u7a97\u53e3\u673a\u5236\uff0c\u4e0d\u8fc7coGroup\u7b97\u5b50\u6bd4Join\u7b97\u5b50\u66f4\u52a0\u7075\u6d3b\uff0c\u53ef\u4ee5\u6309\u7167\u7528\u6237\u6307\u5b9a\u7684\u903b\u8f91\u5339\u914d\u5de6\u6d41\u6216\u53f3\u6d41\u6570\u636e\u5e76\u8f93\u51fa\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>orderDetailStream\n  .coGroup(orderStream)\n  .where(r -&gt; r.getOrderId())\n  .equalTo(r -&gt; r.getOrderId())\n  .window(TumblingProcessingTimeWindows.of(Time.seconds(60)))\n  .apply(new CoGroupFunction&lt;OrderDetail, Order, Tuple2&lt;String, Long&gt;&gt;() {\n    @Override\n    public void coGroup(Iterable&lt;OrderDetail&gt; orderDetailRecords, Iterable&lt;Order&gt; orderRecords, Collector&lt;Tuple2&lt;String, Long&gt;&gt; collector)  {\n      for (OrderDetail orderDetaill : orderDetailRecords) {\n        boolean flag = false;\n        for (Order orderRecord : orderRecords) {\n          \/\/ \u53f3\u6d41\u4e2d\u6709\u5bf9\u5e94\u7684\u8bb0\u5f55\n          collector.collect(new Tuple2&lt;&gt;(orderDetailRecords.getGoods_name(), orderDetailRecords.getGoods_price()));\n          flag = true;\n        }\n        if (!flag) {\n          \/\/ \u53f3\u6d41\u4e2d\u6ca1\u6709\u5bf9\u5e94\u7684\u8bb0\u5f55\n          collector.collect(new Tuple2&lt;&gt;(orderDetailRecords.getGoods_name(), null));\n        }\n      }\n    }\n  })\n  .print();\n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_A%E8%A1%A8left_join_B%E8%A1%A8\"><\/span>2. A\u8868left join B\u8868<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>(1\uff09A\u8868\u6570\u636e\u6765\u4e86\uff0cB\u6ca1\u6765<\/p>\n\n\n\n<p>(2\uff09A\u8868\u6570\u636e\u6765\u4e86\uff0cB\u5728\u89c4\u5b9a\u65f6\u95f4\u5185\u5230<\/p>\n\n\n\n<p>(3)A\u8868\u6570\u636e\u6765\u4e86\uff0cB\u5728\u89c4\u5b9a\u65f6\u95f4\u540e\u9762\u5230<\/p>\n\n\n\n<p>\u600e\u4e48\u5904\u7406\uff1f<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>left join\u4e0eright join\u7531\u4e8eFlink\u5b98\u65b9\u5e76\u6ca1\u6709\u7ed9\u51fa\u660e\u786e\u7684\u65b9\u6848\uff0c\u65e0\u6cd5\u901a\u8fc7join\u6765\u5b9e\u73b0\uff0c\u4f46\u662f\u53ef\u4ee5\u7528coGroup\n\npublic static class LeftJoin implements CoGroupFunction&lt;Tuple3&lt;String, String, Long&gt;, Tuple3&lt;String, String,\n            Long&gt;, Tuple5&lt;String,\n            String, String, Long, Long&gt;&gt; {\n\n        \/\/ \u5c06key\u76f8\u540c\uff0c\u5e76\u4e14\u5728\u540c\u4e00\u7a97\u53e3\u7684\u6570\u636e\u53d6\u51fa\u6765\n        @Override\n        public void coGroup(Iterable&lt;Tuple3&lt;String, String, Long&gt;&gt; first, Iterable&lt;Tuple3&lt;String, String, Long&gt;&gt; second,\n                            Collector&lt;Tuple5&lt;String, String, String, Long, Long&gt;&gt; out) throws Exception {\n\n            for (Tuple3&lt;String, String, Long&gt; leftElem : first) {\n                boolean hadElements = false;\n\n                \/\/\u5982\u679c\u5de6\u8fb9\u7684\u6d41join\u4e0a\u4e86\u53f3\u8fb9\u7684\u6d41rightElements\u5c31\u4e0d\u4e3a\u7a7a\uff0c\u5c31\u4f1a\u8d70\u4e0b\u9762\u7684\u589e\u5f3afor\u5faa\u73af\n                for (Tuple3&lt;String, String, Long&gt; rightElem : second) {\n                    \/\/\u5c06join\u4e0a\u7684\u6570\u636e\u8f93\u51fa\n                    out.collect(new Tuple5&lt;&gt;(leftElem.f0, leftElem.f1, rightElem.f1, leftElem.f2,\n                            rightElem.f2));\n                    hadElements = true;\n                }\n\n                if (!hadElements) {\n                    \/\/\u6ca1join\u4e0a\uff0c\u7ed9\u53f3\u8fb9\u7684\u6570\u636e\u8d4b\u7a7a\u503c\n                    out.collect(new Tuple5&lt;&gt;(leftElem.f0, leftElem.f1, \"null\", leftElem.f2, -1L));\n                }\n            }\n        }\n    }\n    \n    \n    \u5bf9\u4e8e\u6b63\u5e38\u6570\u636e\uff0c\u76f4\u63a5join\n    \n    \u5bf9\u4e8e\u5176\u4ed6\u60c5\u51b5\uff0c\u770b\u4e1a\u52a1\u9700\u6c42\u5427\uff0c\u4e0d\u9700\u8981\u7acb\u9a6c\u820d\u5f03\u7684\uff0c\u53ef\u7528\u72b6\u6001\u6216\u8005\u7b2c\u4e09\u65b9\u5b58\u50a8\u6765\u7b49\u5f85\n    \n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_Flink_%E7%BB%B4%E8%A1%A8%E5%85%B3%E8%81%94%E6%80%8E%E4%B9%88%E5%81%9A%E7%9A%84%EF%BC%9F\"><\/span>3. Flink \u7ef4\u8868\u5173\u8054\u600e\u4e48\u505a\u7684\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>1\u3001async io<\/p>\n\n\n\n<p>2\u3001broadcast<\/p>\n\n\n\n<p>3\u3001async io + cache<\/p>\n\n\n\n<p>4\u3001open\u65b9\u6cd5\u4e2d\u8bfb\u53d6\uff0c\u7136\u540e\u5b9a\u65f6\u7ebf\u7a0b\u5237\u65b0\uff0c\u7f13\u5b58\u66f4\u65b0\u662f\u5148\u5220\u9664\uff0c\u4e4b\u540e\u518d\u6765\u4e00\u6761\u4e4b\u540e\u518d\u8d1f\u8d23\u5199\u5165\u7f13\u5b58<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%85%AB_%E5%85%B6%E4%BB%96\"><\/span>\u516b. \u5176\u4ed6<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_process%E7%94%A8%E7%9A%84%E7%A7%8D%E7%B1%BB%EF%BC%9F\"><\/span>1. process\u7528\u7684\u79cd\u7c7b\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>ProcessFunction\u7528\u6765\u6784\u5efa\u4e8b\u4ef6\u9a71\u52a8\u7684\u5e94\u7528\u4ee5\u53ca\u5b9e\u73b0\u81ea\u5b9a\u4e49\u7684\u4e1a\u52a1\u903b\u8f91(\u4f7f\u7528\u4e4b\u524d\u7684window\u51fd\u6570\u548c\u8f6c\u6362\u7b97\u5b50\u65e0\u6cd5\u5b9e\u73b0)\u3002\u4f8b\u5982\uff0cFlink SQL\u5c31\u662f\u4f7f\u7528Process Function\u5b9e\u73b0\u7684\u3002<\/p>\n\n\n\n<p>ProcessFunction\u662f\u4e00\u4e2a\u4f4e\u7ea7\u7684\u6d41\u5904\u7406\u64cd\u4f5c\uff0c\u5141\u8bb8\u8bbf\u95ee\u6240\u6709(\u975e\u5faa\u73af)\u6d41\u5e94\u7528\u7a0b\u5e8f\u7684\u57fa\u672c\u6784\u4ef6\uff1a<\/p>\n\n\n\n<p>events\uff1a\u6570\u636e\u6d41\u4e2d\u7684\u5143\u7d20<br>state\uff1a\u72b6\u6001\uff0c\u7528\u4e8e\u5bb9\u9519\u548c\u4e00\u81f4\u6027\uff0c\u4ec5\u7528\u4e8ekeyed stream<br>timers\uff1a\u5b9a\u65f6\u5668\uff0c\u652f\u6301\u4e8b\u4ef6\u65f6\u95f4\u548c\u5904\u7406\u65f6\u95f4\uff0c\u4ec5\u7528\u4e8ekeyed stream<br>Flink\u63d0\u4f9b\u4e868\u4e2aProcess Function\uff1a<\/p>\n\n\n\n<p>ProcessFunction\uff1adataStream<br>KeyedProcessFunction\uff1a\u7528\u4e8eKeyedStream\uff0ckeyBy\u4e4b\u540e\u7684\u6d41\u5904\u7406<\/p>\n\n\n\n<p>CoProcessFunction\uff1a\u7528\u4e8econnect\u8fde\u63a5\u7684\u6d41<\/p>\n\n\n\n<p>ProcessJoinFunction\uff1a\u7528\u4e8ejoin\u6d41\u64cd\u4f5c<\/p>\n\n\n\n<p>BroadcastProcessFunction\uff1a\u7528\u4e8e\u5e7f\u64ad<\/p>\n\n\n\n<p>KeyedBroadcastProcessFunction\uff1akeyBy\u4e4b\u540e\u7684\u5e7f\u64ad<\/p>\n\n\n\n<p>ProcessWindowFunction\uff1a\u7a97\u53e3\u589e\u91cf\u805a\u5408<\/p>\n\n\n\n<p>ProcessAllWindowFunction\uff1a\u5168\u7a97\u53e3\u805a\u5408<\/p>\n\n\n\n<p>\u53ef\u4ee5\u5c06ProcessFunction\u770b\u4f5c\u662f\u4e00\u4e2a\u5177\u6709key state\u548c\u5b9a\u65f6\u5668(timer)\u8bbf\u95ee\u6743\u7684FlatMapFunction\u3002\u5bf9\u4e8e\u5728\u8f93\u5165\u6d41\u4e2d\u63a5\u6536\u5230\u7684\u6bcf\u4e00\u4e2a\u4e8b\u4ef6\uff0c\u6b64\u51fd\u6570\u5c31\u4f1a\u88ab\u8c03\u7528\u4ee5\u5904\u7406\u8be5\u4e8b\u4ef6\u3002<\/p>\n\n\n\n<p>\u5bf9\u4e8e\u5bb9\u9519\u72b6\u6001\uff0cProcessFunction \u53ef\u4ee5\u901a\u8fc7 RuntimeContext \u8bbf\u95eeFlink\u7684keyed state\uff0c\u8fd9\u4e0e\u5176\u4ed6\u6709\u72b6\u6001\u51fd\u6570\u8bbf\u95eekeyed state\u7684\u65b9\u5f0f\u7c7b\u4f3c\u3002<\/p>\n\n\n\n<p>\u5b9a\u65f6\u5668\u53ef\u8ba9\u5e94\u7528\u7a0b\u5e8f\u5bf9\u5728\u5904\u7406\u65f6\u95f4\u548c\u4e8b\u4ef6\u65f6\u95f4\u4e2d\u7684\u53d8\u5316\u8fdb\u884c\u54cd\u5e94\u3002\u6bcf\u6b21\u8c03\u7528 processElement(&#8230;)\u51fd\u6570\u65f6\u90fd\u53ef\u4ee5\u83b7\u5f97\u4e00\u4e2aContext\u5bf9\u8c61\uff0c\u901a\u8fc7\u8be5\u5bf9\u8c61\u53ef\u4ee5\u8bbf\u95ee\u5143\u7d20\u7684\u4e8b\u4ef6\u65f6\u95f4\uff08event time\uff09\u65f6\u95f4\u6233\u4ee5\u53ca TimerService\u3002\u53ef\u4ee5\u4f7f\u7528TimerService\u4e3a\u5c06\u6765\u7684\u4e8b\u4ef6\u65f6\u95f4\/\u5904\u7406\u65f6\u95f4\u5b9e\u4f8b\u6ce8\u518c\u56de\u8c03\u3002\u5bf9\u4e8e\u4e8b\u4ef6\u65f6\u95f4\u8ba1\u65f6\u5668\uff0c\u5f53\u5f53\u524d\u6c34\u5370\u88ab\u63d0\u5347\u5230\u6216\u8d85\u8fc7\u8ba1\u65f6\u5668\u7684\u65f6\u95f4\u6233\u65f6\uff0c\u5c06\u8c03\u7528onTimer(\u2026)\u65b9\u6cd5\uff0c\u800c\u5bf9\u4e8e\u5904\u7406\u65f6\u95f4\u8ba1\u65f6\u5668\uff0c\u5f53\u6302\u949f\u65f6\u95f4\u8fbe\u5230\u6307\u5b9a\u65f6\u95f4\u65f6\uff0c\u5c06\u8c03\u7528onTimer(\u2026)\u65b9\u6cd5\u3002\u5728\u8c03\u7528\u671f\u95f4\uff0c\u6240\u6709\u72b6\u6001\u7684\u8303\u56f4\u518d\u6b21\u9650\u5b9a\u4e3a\u521b\u5efa\u5b9a\u65f6\u5668\u6240\u7528\u7684key\uff0c\u4ece\u800c\u5141\u8bb8\u5b9a\u65f6\u5668\u64cd\u4f5ckeyed state\u3002<\/p>\n\n\n\n<p>\u5982\u679c\u60f3\u8981\u5728\u6d41\u5904\u7406\u8fc7\u7a0b\u4e2d\u8bbf\u95eekeyed state\u548c\u5b9a\u65f6\u5668\uff0c\u5c31\u5fc5\u987b\u5728\u4e00\u4e2akeyed stream\u4e0a\u5e94\u7528ProcessFunction\u51fd\u6570\uff0c\u4ee3\u7801\u5982\u4e0b\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>stream.keyBy(...).process(new MyProcessFunction())\n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_Flink%E7%9A%84%E5%86%85%E5%AD%98%E7%AE%A1%E7%90%86%EF%BC%9F\"><\/span>2. Flink\u7684\u5185\u5b58\u7ba1\u7406\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Flink\u5e76\u4e0d\u662f\u5c06\u5927\u91cf\u5bf9\u8c61\u5b58\u5728\u5806\u4e0a\uff0c\u800c\u662f\u5c06\u5bf9\u8c61\u90fd\u5e8f\u5217\u5316\u5230\u4e00\u4e2a\u9884\u5206\u914d\u7684\u5185\u5b58\u5757\u4e0a\u3002\u6b64\u5916\uff0cFlink \u5927\u91cf\u7684\u4f7f\u7528\u4e86\u5806\u5916\u5185\u5b58\u3002\u5982\u679c\u9700\u8981\u5904\u7406\u7684\u6570\u636e\u8d85\u51fa\u4e86\u5185\u5b58\u9650\u5236\uff0c\u5219\u4f1a\u5c06\u90e8\u5206\u6570\u636e \u5b58\u50a8\u5230\u786c\u76d8\u4e0a\u3002Flink \u4e3a\u4e86\u76f4\u63a5\u64cd\u4f5c\u4e8c\u8fdb\u5236\u6570\u636e\u5b9e\u73b0\u4e86\u81ea\u5df1\u7684\u5e8f\u5217\u5316\u6846\u67b6\u3002\u7406\u8bba\u4e0a Flink \u7684\u3002\u5185\u5b58\u7ba1\u7406\u5206\u4e3a\u4e09\u90e8\u5206\uff1a<\/p>\n\n\n\n<p><strong>Network Buffers<\/strong><\/p>\n\n\n\n<p>\u8fd9\u4e2a\u662f\u5728 TaskManager\u542f\u52a8\u7684\u65f6\u5019\u5206\u914d\u7684\uff0c\u8fd9\u662f\u4e00\u7ec4\u7528\u4e8e\u7f13\u5b58\u7f51\u7edc\u6570\u636e\u7684\u5185\u5b58\uff0c\u6bcf\u4e2a\u5757\u662f32K\uff0c\u9ed8\u8ba4\u5206\u914d2048\u4e2a\uff0c\u53ef\u4ee5\u901a\u8fc7\u201ctaskmanager.network.numberOfBuffers\u201d \u4fee\u6539\u3002<\/p>\n\n\n\n<p><strong>Memory Manage pool<\/strong><\/p>\n\n\n\n<p>\u5927\u91cf\u7684 Memory Segment \u5757\uff0c\u7528\u4e8e\u8fd0\u884c\u65f6\u7684\u7b97\u6cd5\uff08Sort\/Join\/Shuffle \u7b49\uff09\uff0c\u8fd9\u90e8\u5206\u542f\u52a8\u7684\u65f6\u5019\u5c31\u4f1a\u5206\u914d\u3002<\/p>\n\n\n\n<p><strong>User Code<\/strong><\/p>\n\n\n\n<p>\u8fd9\u90e8\u5206\u662f\u9664\u4e86 Memory Manager \u4e4b\u5916\u7684\u5185\u5b58\u7528\u4e8e User code \u548c TaskManager \u672c\u8eab\u7684\u6570\u636e\u7ed3\u6784\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"JobManager_%E5%86%85%E5%AD%98%E6%A8%A1%E5%9E%8B\"><\/span>JobManager \u5185\u5b58\u6a21\u578b<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic3.zhimg.com\/80\/v2-400d0e47bb1218e53c9d2bae2681a95a_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>\u914d\u7f6eJobManager\u7684\u603b\u8fdb\u7a0b\u5185\u5b58<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>#The heap size for the JobManager JVM\n\njobmanager.heap.size:1024m\n\n#Flink1.11\u7248\u672c\u53ca\u4ee5\u540e\n\n#JobManager\u603b\u8fdb\u7a0b\u5185\u5b58\njobmanager.memory.process.size:4096m\n\n# \u4f5c\u4e1a\u7ba1\u7406\u5668\u7684 JVM \u5806\u5185\u5b58\u5927\u5c0f\njobmanager.memory.heap.size\uff1a2048m\n\n# \u4f5c\u4e1a\u7ba1\u7406\u5668\u7684\u5806\u5916\u5185\u5b58\u5927\u5c0f\u3002\u6b64\u9009\u9879\u6db5\u76d6\u6240\u6709\u5806\u5916\u5185\u5b58\u4f7f\u7528\u3002\njobmanager.memory.off-heap.size\uff1a1536m\n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"TaskManager_%E5%86%85%E5%AD%98%E6%A8%A1%E5%9E%8B\"><\/span>TaskManager \u5185\u5b58\u6a21\u578b<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/pic1.zhimg.com\/80\/v2-26028a0f2ebe2cdb414be48e4aacd904_720w.webp\" alt=\"\"\/><\/figure>\n\n\n\n<p>TaskManager \u5185\u5b58\u6a21\u578b\u4e00\u5171\u5305\u542b 3\u5927\u90e8\u5206\uff0c\u5206\u522b\u4e3a\u603b\u4f53\u5185\u5b58\u3001JVM Heap \u5806\u4e0a\u5185\u5b58\u3001Off-Heap \u5806\u5916\u5185\u5b58\u7b49\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%80%BB%E4%BD%93%E5%86%85%E5%AD%98\"><\/span>\u603b\u4f53\u5185\u5b58<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<ol>\n<li>Total Process Memory\uff1aFlink Java \u5e94\u7528\u7a0b\u5e8f\uff08\u5305\u62ec\u7528\u6237\u4ee3\u7801\uff09\u548c JVM \u8fd0\u884c\u6574\u4e2a\u8fdb\u7a0b\u6240\u6d88\u8017\u7684\u603b\u5185\u5b58\u3002<\/li>\n<\/ol>\n\n\n\n<p>\u603b\u8fdb\u7a0b\u5185\u5b58(Total Process Memory) = Flink \u603b\u5185\u5b58 + JVM \u5143\u7a7a\u95f4 + JVM \u6267\u884c\u5f00\u9500\u3002<\/p>\n\n\n\n<ol>\n<li>Total Flink Memory\uff1a\u4ec5 Flink Java \u5e94\u7528\u7a0b\u5e8f\u6d88\u8017\u7684\u5185\u5b58\uff0c\u5305\u62ec\u7528\u6237\u4ee3\u7801\uff0c\u4f46\u4e0d\u5305\u62ec JVM \u4e3a\u5176\u8fd0\u884c\u800c\u5206\u914d\u7684\u5185\u5b58\u3002<\/li>\n<\/ol>\n\n\n\n<p>Flink \u603b\u5185\u5b58 = Framework\u5806\u5185\u5916 + task \u5806\u5185\u5916 + network + managed Memory<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"JVM_Heap_JVM_%E5%A0%86%E4%B8%8A%E5%86%85%E5%AD%98\"><\/span>JVM Heap (JVM \u5806\u4e0a\u5185\u5b58)<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<ol>\n<li>Framework Heap \uff1a\u6846\u67b6\u5806\u5185\u5b58<\/li>\n\n\n\n<li>Task Heap : \u4efb\u52a1\u5806\u5185\u5b58<br>\u5982\u679c\u5185\u5b58\u5927\u5c0f\u6ca1\u6709\u6307\u5b9a\uff0c\u5b83\u5c06\u88ab\u63a8\u5bfc\u51fa\u4e3a\u603b Flink \u5185\u5b58\u51cf\u53bb\u6846\u67b6\u5806\u5185\u5b58\u3001\u6846\u67b6\u5806\u5916\u5185\u5b58\u3001\u4efb\u52a1\u5806\u5916\u5185\u5b58\u3001\u6258\u7ba1\u5185\u5b58\u548c\u7f51\u7edc\u5185\u5b58\u3002<\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Off-Heap_MempryJVM_%E5%A0%86%E5%A4%96%E5%86%85%E5%AD%98\"><\/span>Off-Heap Mempry(JVM \u5806\u5916\u5185\u5b58)<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<ol>\n<li>Managed memory: \u6258\u7ba1\u5185\u5b58<\/li>\n<\/ol>\n\n\n\n<p>\u7531 Flink \u7ba1\u7406\u7684\u539f\u751f\u6258\u7ba1\u5185\u5b58\uff0c\u4fdd\u7559\u7528\u4e8e\u6392\u5e8f\u3001\u54c8\u5e0c\u8868\u3001\u4e2d\u95f4\u7ed3\u679c\u7f13\u5b58\u548c RocksDB \u72b6\u6001\u540e\u7aef\u3002<\/p>\n\n\n\n<ol>\n<li>DirectMemory\uff1aJVM \u76f4\u63a5\u5185\u5b58<\/li>\n<\/ol>\n\n\n\n<p>Framework Off-Heap Memory\uff1aFlink \u6846\u67b6\u5806\u5916\u5185\u5b58\u3002<\/p>\n\n\n\n<p>Task Off-Heap \uff1aTask \u5806\u5916\u5185\u5b58\u3002<\/p>\n\n\n\n<p>Network Memory\uff1a\u7f51\u7edc\u5185\u5b58\u3002<\/p>\n\n\n\n<ol>\n<li>JVM metaspace\uff1aJVM \u5143\u7a7a\u95f4\u3002\u9ed8\u8ba4\u4e3a256MB\u3002<\/li>\n\n\n\n<li>JVM Overhead \uff1aJVM\u6267\u884c\u5f00\u9500<\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_Flink%E7%9A%84%E5%BA%8F%E5%88%97%E5%8C%96%E6%9C%BA%E5%88%B6%EF%BC%9F\"><\/span>3. Flink\u7684\u5e8f\u5217\u5316\u673a\u5236\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u5f53\u4e24\u4e2a\u8fdb\u7a0b\u5728\u8fdb\u884c\u8fdc\u7a0b\u901a\u4fe1\u65f6\uff0c\u5f7c\u6b64\u53ef\u4ee5\u53d1\u9001\u5404\u79cd\u7c7b\u578b\u7684\u6570\u636e\u3002\u65e0\u8bba\u662f\u4f55\u79cd\u7c7b\u578b\u7684\u6570\u636e\uff0c\u90fd\u4f1a\u4ee5\u4e8c\u8fdb\u5236\u5e8f\u5217\u7684\u5f62\u5f0f\u5728\u7f51\u7edc\u4e0a\u4f20\u9001\u3002\u53d1\u9001\u65b9\u9700\u8981\u628a\u8fd9\u4e2aJava\u5bf9\u8c61\u8f6c\u6362\u4e3a\u5b57\u8282\u5e8f\u5217\uff0c\u624d\u80fd\u5728\u7f51\u7edc\u4e0a\u4f20\u9001\uff1b\u63a5\u6536\u65b9\u5219\u9700\u8981\u628a\u5b57\u8282\u5e8f\u5217\u518d\u6062\u590d\u4e3aJava\u5bf9\u8c61\u3002\u628aJava\u5bf9\u8c61\u8f6c\u6362\u4e3a\u5b57\u8282\u5e8f\u5217\u7684\u8fc7\u7a0b\u79f0\u4e3a\u5bf9\u8c61\u7684\u5e8f\u5217\u5316\u3002<\/p>\n\n\n\n<p>\u628a\u5b57\u8282\u5e8f\u5217\u6062\u590d\u4e3aJava\u5bf9\u8c61\u7684\u8fc7\u7a0b\u79f0\u4e3a\u5bf9\u8c61\u7684\u53cd\u5e8f\u5217\u5316\u3002<\/p>\n\n\n\n<p>Apache Flink \u6452\u5f03\u4e86 Java \u539f\u751f\u7684\u5e8f\u5217\u5316\u65b9\u6cd5\uff0c\u4ee5\u72ec\u7279\u7684\u65b9\u5f0f\u5904\u7406<br>\u6570\u636e\u7c7b\u578b\u548c\u5e8f\u5217\u5316\uff0c\u5305\u542b\u81ea\u5df1\u7684\u7c7b\u578b\u63cf\u8ff0\u7b26\uff0c\u6cdb\u578b\u7c7b\u578b\u63d0\u53d6\u548c\u7c7b\u578b\u5e8f\u5217\u5316\u6846\u67b6\u3002TypeInformation \u662f\u6240\u6709\u7c7b\u578b\u63cf\u8ff0\u7b26\u7684\u57fa\u7c7b\u3002\u5b83\u63ed\u793a\u4e86\u8be5\u7c7b\u578b\u7684\u4e00\u4e9b\u57fa\u672c\u5c5e\u6027\uff0c\u5e76\u4e14\u53ef\u4ee5\u751f\u6210\u5e8f\u5217\u5316\u5668\u3002TypeInformation \u652f\u6301\u4ee5\u4e0b\u51e0\u79cd\u7c7b\u578b\uff1a<\/p>\n\n\n\n<p>BasicTypeInfo: \u4efb\u610f Java \u57fa\u672c\u7c7b\u578b\u6216 String \u7c7b\u578b<\/p>\n\n\n\n<p>BasicArrayTypeInfo: \u4efb\u610f Java \u57fa\u672c\u7c7b\u578b\u6570\u7ec4\u6216 String \u6570\u7ec4<\/p>\n\n\n\n<p>WritableTypeInfo: \u4efb\u610f Hadoop Writable \u63a5\u53e3\u7684\u5b9e\u73b0\u7c7b<\/p>\n\n\n\n<p>TupleTypeInfo: \u4efb\u610f\u7684 Flink Tuple \u7c7b\u578b\u3002<\/p>\n\n\n\n<p>CaseClassTypeInfo: \u4efb\u610f\u7684 Scala CaseClass\u3002<\/p>\n\n\n\n<p>PojoTypeInfo: \u4efb\u610f\u7684 POJO (Java or Scala)\uff0c\u4f8b\u5982\uff0cJava \u5bf9\u8c61\u7684\u6240\u6709\u6210\u5458\u53d8\u91cf\uff0c\u8981\u4e48\u662fpublic \u4fee\u9970\u7b26\u5b9a\u4e49\uff0c\u8981\u4e48\u6709 getter\/setter \u65b9\u6cd5\u3002<\/p>\n\n\n\n<p>GenericTypeInfo: \u4efb\u610f\u65e0\u6cd5\u5339\u914d\u4e4b\u524d\u51e0\u79cd\u7c7b\u578b\u7684\u7c7b<\/p>\n\n\n\n<p>\u9488\u5bf9\u524d\u516d\u79cd\u7c7b\u578b\u6570\u636e\u96c6\uff0cFlink\u7686\u53ef\u4ee5\u81ea\u52a8\u751f\u6210\u5bf9\u5e94\u7684TypeSerializer\uff0c\u80fd\u975e\u5e38\u9ad8\u6548\u5730\u5bf9\u6570\u636e\u96c6\u8fdb\u884c\u5e8f\u5217\u5316\u548c\u53cd\u5e8f\u5217\u5316\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_Rich_Functions%E4%B8%8EFunctions%E5%8C%BA%E5%88%AB\"><\/span>4. Rich Functions\u4e0eFunctions\u533a\u522b?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Rich\u7248\u672c\u3002\u5b83\u4e0e\u5e38\u89c4\u51fd\u6570\u7684\u4e0d\u540c\u5728\u4e8e\uff0c\u53ef\u4ee5\u83b7\u53d6\u8fd0\u884c\u73af\u5883\u7684\u4e0a\u4e0b\u6587\uff0c\u5e76\u62e5\u6709\u4e00\u4e9b\u751f\u547d\u5468\u671f\u65b9\u6cd5\uff0c<br>\u6240\u4ee5\u53ef\u4ee5\u5b9e\u73b0\u66f4\u590d\u6742\u7684\u529f\u80fd\u3002\u4e5f\u6709\u610f\u5473\u7740\u63d0\u4f9b\u4e86\u66f4\u591a\u7684\uff0c\u66f4\u4e30\u5bcc\u7684\u529f\u80fd\u3002\u9ed8\u8ba4\u751f\u547d\u5468\u671f\u65b9\u6cd5\uff1a1.\u521d\u59cb\u5316\u65b9\u6cd5open(), \u5728\u6bcf\u4e2a\u5e76\u884c\u5ea6\u4e0a\u53ea\u4f1a\u88ab\u8c03\u7528\u4e00\u6b21, \u800c\u4e14\u5148\u88ab\u8c03\u7528\uff1b<\/p>\n\n\n\n<p>2.\u6700\u540e\u4e00\u4e2a\u65b9\u6cd5close(), \u505a\u4e00\u4e9b\u6e05\u7406\u5de5\u4f5c, \u5728\u6bcf\u4e2a\u5e76\u884c\u5ea6\u4e0a\u53ea\u8c03\u7528\u4e00\u6b21, \u800c\u4e14\u662f\u6700\u540e\u88ab\u8c03\u7528\uff0c<br>\u4f46\u8bfb\u6587\u4ef6\u65f6\u5728\u6bcf\u4e2a\u5e76\u884c\u5ea6\u4e0a\u8c03\u7528\u4e24\u6b21\uff1b<\/p>\n\n\n\n<p>3.getRuntimeContext()\u65b9\u6cd5\u63d0\u4f9b\u4e86\u51fd\u6570\u7684RuntimeContext\u7684\u4e00\u4e9b\u4fe1\u606f\uff0c<br>\u53ef\u4ee5\u901a\u8fc7\u6539\u65b9\u6cd5\u83b7\u53d6\u5230\u51fd\u6570\u6267\u884c\u5e76\u884c\u5ea6\uff0c\u4efb\u52a1\u540d\u79f0\uff0cstate\u72b6\u6001\u7b49\u4fe1\u606f<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"5_Flink_%E5%88%86%E5%8C%BA%E7%AD%96%E7%95%A5%EF%BC%9F\"><\/span>5. Flink \u5206\u533a\u7b56\u7565\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"6_Flink_%E4%B8%AD%E7%9A%84%E5%88%86%E5%B8%83%E5%BC%8F%E7%BC%93%E5%AD%98\"><\/span>6. Flink \u4e2d\u7684\u5206\u5e03\u5f0f\u7f13\u5b58<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Flink\u8bfb\u53d6\u6587\u4ef6\uff0c\u628a\u6570\u636e\u653e\u5728 taskmanager \u8282\u70b9\u4e2d\uff0c\u9632\u6b62 task \u91cd\u590d\u62c9\u53d6\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>env.registerCachedFile(\"hdfs:\/\/\/\", \"hdfsFile\")\n<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"7_Flink%E9%87%8C%E9%9D%A2%E5%BC%82%E6%AD%A5IO%E4%BB%A3%E7%A0%81%E5%85%B7%E4%BD%93%E6%80%8E%E4%B9%88%E5%86%99%E7%9A%84\"><\/span>7. Flink\u91cc\u9762\u5f02\u6b65IO\u4ee3\u7801\u5177\u4f53\u600e\u4e48\u5199\u7684?<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Flink source\u6536\u5230\u4e00\u6761\u6570\u636e\u5c31\u4f1a\u8fdb\u884c\u5904\u7406\uff0c\u5982\u679c\u9700\u8981\u901a\u8fc7\u8fd9\u6761\u6570\u636e\u5173\u8054\u5916\u90e8\u6570\u636e\u6e90\uff0c<br>\u4f8b\u5982mysql\uff0c\u5728\u53d1\u51fa\u67e5\u8be2\u8bf7\u6c42\u540e\uff0c\u540c\u6b65IO\u7684\u65b9\u5f0f\u662f\u4f1a\u7b49\u5f85\u67e5\u8be2\u7ed3\u679c\u518d\u5904\u7406\u4e0b\u4e00\u6761\u6570\u636e\u7684\u67e5\u8be2\uff0c<br>\u4e5f\u5c31\u662f\u6bcf\u4e00\u6761\u6570\u636e\u90fd\u8981\u7b49\u5f85\u4e0a\u4e00\u4e2a\u67e5\u8be2\u7ed3\u675f\u3002\u800c\u5f02\u6b65IO\u662f\u6307\u6570\u636e\u6765\u4e86\u4ee5\u540e\u53d1\u51fa\u67e5\u8be2\u8bf7\u6c42\uff0c<br>\u5148\u4e0d\u7b49\u67e5\u8be2\u7ed3\u679c\uff0c\u76f4\u63a5\u7ee7\u7eed\u53d1\u9001\u4e0b\u4e00\u6761\u7684\u67e5\u8be2\u8bf7\u6c42\uff0c\u5bf9\u4e8e\u67e5\u8be2\u7ed3\u679c\u662f\u5f02\u6b65\u8fd4\u56de\u7684\uff0c<br>\u8fd4\u56de\u7ed3\u679c\u4e4b\u540e\u518d\u8fdb\u5165\u4e0b\u4e00\u4e2a\u7b97\u5b50\u7684\u8ba1\u7b97\u3002<\/p>\n\n\n\n<ol>\n<li>\u5b9a\u4e49\u4e00\u4e2aAsyncFunction\u7528\u4e8e\u5b9e\u73b0\u8bf7\u6c42\u7684\u5206\u53d1<br><\/li>\n\n\n\n<li>\u5b9a\u4e49\u4e00\u4e2acallback\u56de\u8c03\u51fd\u6570\uff0c\u8be5\u51fd\u6570\u7528\u4e8e\u53d6\u51fa\u5f02\u6b65\u8bf7\u6c42\u7684\u8fd4\u56de\u7ed3\u679c\uff0c\u5e76\u5c06\u8fd4\u56de\u7684\u7ed3\u679c\u4f20\u9012\u7ed9ResultFuture .<br><\/li>\n\n\n\n<li>\u5bf9DataStream\u4f7f\u7528Async\u64cd\u4f5c\u3002<br><\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code\"><code>class AsyncDatabaseRequest extends AsyncFunction&#91;String, (String, String)] {\n\n    \/** \u53ef\u4ee5\u5f02\u6b65\u8bf7\u6c42\u7684\u7279\u5b9a\u6570\u636e\u5e93\u7684\u5ba2\u6237\u7aef *\/\n    lazy val client: DatabaseClient = new DatabaseClient(host, post, credentials)\n\n    \/** future \u7684\u56de\u8c03\u7684\u6267\u884c\u4e0a\u4e0b\u6587\uff08\u5f53\u524d\u7ebf\u7a0b\uff09 *\/\n    implicit lazy val executor: ExecutionContext = ExecutionContext.fromExecutor(Executors.directExecutor())\n\n    override def asyncInvoke(str: String, asyncCollector: AsyncCollector&#91;(String, String)]): Unit = {\n\n        \/\/ \u53d1\u8d77\u4e00\u4e2a\u5f02\u6b65\u8bf7\u6c42\uff0c\u8fd4\u56de\u7ed3\u679c\u7684 future\n        val resultFuture: Future&#91;String] = client.query(str)\n\n        \/\/ \u8bbe\u7f6e\u8bf7\u6c42\u5b8c\u6210\u65f6\u7684\u56de\u8c03: \u5c06\u7ed3\u679c\u4f20\u9012\u7ed9 collector\n        resultFuture.onSuccess {\n            case result: String =&gt; asyncCollector.collect(Iterable((str, result)));\n        }\n    }\n}\n\/\/ \u521b\u5efa\u4e00\u4e2a\u539f\u59cb\u7684\u6d41\nval stream: DataStream&#91;String] = ...\n\n\/\/ \u6dfb\u52a0\u4e00\u4e2a async I\/O \nval resultStream: DataStream&#91;(String, String)] =\n    AsyncDataStream.(un)orderedWait(\n      stream, new AsyncDatabaseRequest(),\n      500, TimeUnit.MILLISECONDS, \/\/ \u8d85\u65f6\u65f6\u95f4\n      120)  \/\/ \u8fdb\u884c\u4e2d\u7684\u5f02\u6b65\u8bf7\u6c42\u7684\u6700\u5927\u6570\u91cf\n<\/code><\/pre>\n\n\n\n<p><strong>Capacity<\/strong><\/p>\n\n\n\n<p>\u8be5\u53c2\u6570\u7528\u4e8e\u5b9a\u4e49\u540c\u65f6\u6700\u591a\u4f1a\u6709\u591a\u5c11\u4e2a\u5f02\u6b65\u8bf7\u6c42\u5728\u540c\u65f6\u5904\u7406\u3002\u8be5\u53c2\u6570\u53ef\u4ee5\u9650\u5236\u5e76\u53d1\u8bf7\u6c42\u6570\u91cf\uff0c\u8fd9\u6837\u7684\u8bdd\u4e0d\u4f1a\u79ef\u538b\u975e\u5e38\u7684\u672a\u5904\u7406\u8bf7\u6c42\u3002<\/p>\n\n\n\n<p><strong>Timeout<\/strong><\/p>\n\n\n\n<p>\u53ef\u4ee5\u8bbe\u7f6eTimeout\u53c2\u6570\u6765\u91ca\u653e\u6302\u6389\u6216\u8005\u5931\u8d25\u7684\u8bf7\u6c42\u6240\u5360\u7528\u7684\u8d44\u6e90\uff0c \u63d0\u9ad8\u8d44\u6e90\u5229\u7528\u7387<\/p>\n\n\n\n<p><strong>\u8d85\u65f6\u5904\u7406<\/strong><\/p>\n\n\n\n<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c \u5f53\u4e00\u4e2a\u5f02\u6b65\u8bf7\u6c42\u591a\u6b21\u8d85\u65f6\u65f6\uff0c\u7a0b\u5e8f\u4f1a\u629b\u51fa\u4e00\u4e2a\u5f02\u5e38\u5e76\u91cd\u542fjob \u3002\u4f46\u662f\u4e00\u822c\u6765\u8bf4\u6211\u4eec\u4f1a\u91cd\u5199AsyncFunction\u4e2d\u7684timeout\u65b9\u6cd5\u6765\u81ea\u5b9a\u4e49\u8d85\u65f6\u4e4b\u540e\u7684\u5904\u7406\u65b9\u5f0f.<\/p>\n\n\n\n<p><strong>\u7ed3\u679c\u7684\u987a\u5e8f<\/strong><\/p>\n\n\n\n<p>AsyncDataStream\u5305\u542b\u4e24\u79cd\u8f93\u51fa\u6a21\u5f0f: \u6709\u5e8f\u548c\u65e0\u5e8f\uff0c\u5206\u522b\u5bf9\u5e94\u9759\u6001\u65b9\u6cd5 orderedWait\u4e0eunorderedWait\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"8_Flink%E7%9A%84%E9%87%8D%E5%90%AF%E7%AD%96%E7%95%A5\"><\/span>8. Flink\u7684\u91cd\u542f\u7b56\u7565<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p><strong>\u56fa\u5b9a\u5ef6\u8fdf\u91cd\u542f\u7b56\u7565<\/strong><\/p>\n\n\n\n<p>\u56fa\u5b9a\u5ef6\u8fdf\u91cd\u542f\u7b56\u7565\u662f\u5c1d\u8bd5\u7ed9\u5b9a\u6b21\u6570\u91cd\u65b0\u542f\u52a8\u4f5c\u4e1a\u3002\u5982\u679c\u8d85\u8fc7\u6700\u5927\u5c1d\u8bd5\u6b21\u6570\uff0c\u5219\u4f5c\u4e1a\u5931\u8d25\u3002\u5728\u4e24\u6b21\u8fde\u7eed\u91cd\u542f\u5c1d\u8bd5\u4e4b\u95f4\uff0c\u4f1a\u6709\u4e00\u4e2a\u56fa\u5b9a\u7684\u5ef6\u8fdf\u7b49\u5f85\u65f6\u95f4\u3002<\/p>\n\n\n\n<p><strong>\u6545\u969c\u7387\u91cd\u542f\u7b56\u7565<\/strong><\/p>\n\n\n\n<p>\u6545\u969c\u7387\u91cd\u542f\u7b56\u7565\u5728\u6545\u969c\u540e\u91cd\u65b0\u4f5c\u4e1a\uff0c\u5f53\u8bbe\u7f6e\u7684\u6545\u969c\u7387\uff08failure rate\uff09\u8d85\u8fc7\u6bcf\u4e2a\u65f6\u95f4\u95f4\u9694\u7684\u6545\u969c\u65f6\uff0c\u4f5c\u4e1a\u6700\u7ec8\u5931\u8d25\u3002\u5728\u4e24\u6b21\u8fde\u7eed\u91cd\u542f\u5c1d\u8bd5\u4e4b\u95f4\uff0c\u91cd\u542f\u7b56\u7565\u5ef6\u8fdf\u7b49\u5f85\u4e00\u6bb5\u65f6\u95f4\u3002<\/p>\n\n\n\n<p><strong>\u65e0\u91cd\u542f\u7b56\u7565<\/strong><\/p>\n\n\n\n<p>\u4f5c\u4e1a\u76f4\u63a5\u5931\u8d25\uff0c\u4e0d\u5c1d\u8bd5\u91cd\u542f\u3002<\/p>\n\n\n\n<p><strong>\u540e\u5907\u91cd\u542f\u7b56\u7565<\/strong><\/p>\n\n\n\n<p>\u4f7f\u7528\u7fa4\u96c6\u5b9a\u4e49\u7684\u91cd\u65b0\u542f\u52a8\u7b56\u7565\u3002\u8fd9\u5bf9\u4e8e\u542f\u7528\u68c0\u67e5\u70b9\u7684\u6d41\u5f0f\u4f20\u8f93\u7a0b\u5e8f\u5f88\u6709\u5e2e\u52a9\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u5982\u679c\u6ca1\u6709\u5b9a\u4e49\u5176\u4ed6\u91cd\u542f\u7b56\u7565\uff0c\u5219\u9009\u62e9\u56fa\u5b9a\u5ef6\u8fdf\u91cd\u542f\u7b56\u7565\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"9_Flink_%E6%95%B0%E6%8D%AE%E5%80%BE%E6%96%9C\"><\/span>9. Flink \u6570\u636e\u503e\u659c<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Flink \u4efb\u52a1\u51fa\u73b0\u6570\u636e\u503e\u659c\u7684\u76f4\u89c2\u8868\u73b0\u662f\u4efb\u52a1\u8282\u70b9\u9891\u7e41\u51fa\u73b0\u53cd\u538b\u3002\u90e8\u5206\u8282\u70b9\u51fa\u73b0 OOM \u5f02\u5e38\uff0c\u662f\u56e0\u4e3a\u5927\u91cf\u7684\u6570\u636e\u96c6\u4e2d\u5728\u67d0\u4e2a\u8282\u70b9\u4e0a\uff0c\u5bfc\u81f4\u8be5\u8282\u70b9\u5185\u5b58\u88ab\u7206\uff0c\u4efb\u52a1\u5931\u8d25\u91cd\u542f\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%96%B9%E6%A1%88\"><\/span>\u65b9\u6848<span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p><strong>\u6570\u636e\u6e90 source \u6d88\u8d39\u4e0d\u5747\u5300<\/strong><\/p>\n\n\n\n<p>\u901a\u8fc7\u8c03\u6574Flink\u5e76\u884c\u5ea6\uff0c\u89e3\u51b3\u6570\u636e\u6e90\u6d88\u8d39\u4e0d\u5747\u5300\u6216\u8005\u6570\u636e\u6e90\u53cd\u538b\u7684\u60c5\u51b5\u3002\u6211\u4eec\u5e38\u5e38\u4f8b\u5982kafka\u6570\u636e\u6e90\uff0c\u8c03\u6574\u5e76\u884c\u5ea6\u7684\u539f\u5219\uff1aSource\u5e76\u884c\u5ea6\u4e0e kafka\u5206\u533a\u6570\u662f\u4e00\u6837\u7684\uff0c\u6216\u8005 kafka \u5206\u533a\u6570\u662fKafkaSource \u5e76\u53d1\u5ea6\u7684\u6574\u6570\u500d\u3002\u5efa\u8bae\u662f\u5e76\u884c\u5ea6\u7b49\u4e8e\u5206\u533a\u6570\u3002<\/p>\n\n\n\n<p><strong>key \u5206\u5e03\u4e0d\u5747\u5300<\/strong><\/p>\n\n\n\n<p>\u4e0a\u6e38\u6570\u636e\u5206\u5e03\u4e0d\u5747\u5300\uff0c\u4f7f\u7528keyBy\u6765\u6253\u6563\u6570\u636e\u7684\u65f6\u5019\u51fa\u73b0\u503e\u659c\u3002\u901a\u8fc7\u6dfb\u52a0\u968f\u673a\u524d\u7f00\uff0c\u6253\u6563 key \u7684\u5206\u5e03\uff0c\u4f7f\u5f97\u6570\u636e\u4e0d\u4f1a\u96c6\u4e2d\u5728\u51e0\u4e2a Subtask\u3002<\/p>\n\n\n\n<p><strong>\u4e24\u9636\u6bb5\u805a\u5408\u89e3\u51b3 KeyBy\uff08\u52a0\u76d0\u5c40\u90e8\u805a\u5408+\u53bb\u76d0\u5168\u5c40\u805a\u5408\uff09<\/strong><\/p>\n\n\n\n<p>\u9884\u805a\u5408\uff1a\u52a0\u76d0\u5c40\u90e8\u805a\u5408\uff0c\u5728\u539f\u6765\u7684 key \u4e0a\u52a0\u968f\u673a\u7684\u524d\u7f00\u6216\u8005\u540e\u7f00\u3002<\/p>\n\n\n\n<p>\u805a\u5408\uff1a\u53bb\u76d0\u5168\u5c40\u805a\u5408\uff0c\u5220\u9664\u9884\u805a\u5408\u6dfb\u52a0\u7684\u524d\u7f00\u6216\u8005\u540e\u7f00\uff0c\u7136\u540e\u8fdb\u884c\u805a\u5408\u7edf\u8ba1\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>1.\u524d\u8a00 \u672c\u6587\u4e3b\u8981\u662f\u6574\u7406\u535a\u4e3b\u6536\u96c6\u7684 Flink \u9ad8\u9891\u9762\u8bd5\u9898\u3002\u4e4b\u540e\u6bcf\u5468\u90fd\u4f1a\u6709\u4e00\u7bc7\u3002 \u5176\u4e2d\u4e3b\u8981\u5212\u5206\u4e3a\u4e00\u4e0b&hellip; <a href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%b8%b8%e8%a7%81%e7%9a%84%e9%97%ae%e7%ad%94\/\" class=\"more-link read-more\" rel=\"bookmark\">\u7ee7\u7eed\u9605\u8bfb <span class=\"screen-reader-text\">\u3010\u8fd0\u7ef4\u5de5\u5177\u3011flink sql \u57fa\u7840\u5b9e\u8df5 &#8211; \u5e38\u89c1\u7684\u7591\u95ee\u70b9<\/span><i class=\"fa fa-arrow-right\"><\/i><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[1],"tags":[25],"views":617,"_links":{"self":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1295"}],"collection":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/comments?post=1295"}],"version-history":[{"count":5,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1295\/revisions"}],"predecessor-version":[{"id":1301,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1295\/revisions\/1301"}],"wp:attachment":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/media?parent=1295"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/categories?post=1295"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/tags?post=1295"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}