{"id":1281,"date":"2023-03-19T06:41:59","date_gmt":"2023-03-18T22:41:59","guid":{"rendered":"http:\/\/viplao.com\/?p=1281"},"modified":"2023-03-19T07:42:46","modified_gmt":"2023-03-18T23:42:46","slug":"%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client","status":"publish","type":"post","link":"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/","title":{"rendered":"\u3010\u8fd0\u7ef4\u5de5\u5177\u3011Flink SQL \u57fa\u7840\u5b9e\u8df5 &#8211; SQL Client"},"content":{"rendered":"\n<figure class=\"wp-block-embed is-type-rich is-provider-\u5d4c\u5165\u5904\u7406\u7a0b\u5e8f wp-block-embed-\u5d4c\u5165\u5904\u7406\u7a0b\u5e8f\"><div class=\"wp-block-embed__wrapper\">\n<div style=\"width: 640px;\" class=\"wp-video\"><!--[if lt IE 9]><script>document.createElement('video');<\/script><![endif]-->\n<video class=\"wp-video-shortcode\" id=\"video-1281-1\" width=\"640\" height=\"360\" preload=\"metadata\" controls=\"controls\"><source type=\"video\/mp4\" src=\"https:\/\/cloud.video.taobao.com\/play\/u\/null\/p\/1\/e\/6\/t\/1\/d\/ud\/372169841042.mp4?_=1\" \/><a href=\"https:\/\/cloud.video.taobao.com\/play\/u\/null\/p\/1\/e\/6\/t\/1\/d\/ud\/372169841042.mp4\">https:\/\/cloud.video.taobao.com\/play\/u\/null\/p\/1\/e\/6\/t\/1\/d\/ud\/372169841042.mp4<\/a><\/video><\/div>\n<\/div><\/figure>\n\n\n\n<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_71 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u6587\u7ae0\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 eztoc-toggle-hide-by-default' ><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E5%89%8D%E8%A8%80\" title=\"\u524d\u8a00\">\u524d\u8a00<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E5%A6%82%E4%BD%95%E4%BD%BF%E7%94%A8\" title=\"\u5982\u4f55\u4f7f\u7528\">\u5982\u4f55\u4f7f\u7528<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E5%9C%BA%E6%99%AF%E7%A4%BA%E4%BE%8B\" title=\"\u573a\u666f\u793a\u4f8b\">\u573a\u666f\u793a\u4f8b<\/a><ul class='ez-toc-list-level-2' ><li class='ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E5%87%86%E5%A4%87%E6%95%B0%E6%8D%AE\" title=\"\u51c6\u5907\u6570\u636e\">\u51c6\u5907\u6570\u636e<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E5%AE%89%E8%A3%85Flink%E5%92%8C%E4%BE%9D%E8%B5%96\" title=\"\u5b89\u88c5Flink\u548c\u4f9d\u8d56\">\u5b89\u88c5Flink\u548c\u4f9d\u8d56<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E5%90%AF%E5%8A%A8Flink_SQL_client\" title=\"\u542f\u52a8Flink SQL client\">\u542f\u52a8Flink SQL client<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E6%89%A7%E8%A1%8CSQL%E6%8F%90%E4%BA%A4%E4%BD%9C%E4%B8%9A\" title=\"\u6267\u884cSQL\u63d0\u4ea4\u4f5c\u4e1a\">\u6267\u884cSQL\u63d0\u4ea4\u4f5c\u4e1a<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E6%95%B0%E6%8D%AE%E7%B1%BB%E5%9E%8B\" title=\"\u6570\u636e\u7c7b\u578b\">\u6570\u636e\u7c7b\u578b<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E5%B8%B8%E8%A7%81%E9%94%99%E8%AF%AF\" title=\"\u5e38\u89c1\u9519\u8bef\">\u5e38\u89c1\u9519\u8bef<\/a><ul class='ez-toc-list-level-2' ><li class='ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E6%89%BE%E4%B8%8D%E5%88%B0Hadoop%E7%9B%B8%E5%85%B3%E4%BE%9D%E8%B5%96\" title=\"\u627e\u4e0d\u5230Hadoop\u76f8\u5173\u4f9d\u8d56\">\u627e\u4e0d\u5230Hadoop\u76f8\u5173\u4f9d\u8d56<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#SQL_client_%E6%8F%90%E4%BA%A4%E4%BB%BB%E5%8A%A1%E7%9A%84%E6%97%B6%E5%80%99%E8%BF%9E%E6%8E%A5%E6%8B%92%E7%BB%9D\" title=\"SQL client \u63d0\u4ea4\u4efb\u52a1\u7684\u65f6\u5019\u8fde\u63a5\u62d2\u7edd\">SQL client \u63d0\u4ea4\u4efb\u52a1\u7684\u65f6\u5019\u8fde\u63a5\u62d2\u7edd<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/#%E8%A1%A5%E5%85%85%E5%86%85%E5%AE%B9\" title=\"\u8865\u5145\u5185\u5bb9\">\u8865\u5145\u5185\u5bb9<\/a><\/li><\/ul><\/nav><\/div>\n<h1 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%89%8D%E8%A8%80\"><\/span>\u524d\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h1>\n\n\n\n<p>Flink\u63d0\u4f9b\u4e86SQL client\uff0c\u6709\u4e86\u5b83\u6211\u4eec\u53ef\u4ee5\u50cfHive\u7684beeline\u4e00\u6837\u76f4\u63a5\u5728\u63a7\u5236\u53f0\u7f16\u5199SQL\u5e76\u63d0\u4ea4\u4f5c\u4e1a\u3002\u5b8c\u5168\u4e0d\u7528\u4f7f\u7528Java\/Scala\u7f16\u5199Flink\u4f5c\u4e1a\u6253\u5305\u4e3ajar\u63d0\u4ea4\u3002\u6beb\u65e0\u7591\u95eeSQL client\u4f7f\u7528\u8d77\u6765\u66f4\u4e3a\u7b80\u5355\u7075\u6d3b\u3002<\/p>\n\n\n\n<h1 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%A6%82%E4%BD%95%E4%BD%BF%E7%94%A8\"><\/span>\u5982\u4f55\u4f7f\u7528<span class=\"ez-toc-section-end\"><\/span><\/h1>\n\n\n\n<p>Flink SQL client\u652f\u6301\u8fd0\u884c\u5728standalone\u96c6\u7fa4\u548cYarn\u96c6\u7fa4\u4e0a\u3002\u63d0\u4ea4\u4efb\u52a1\u7684\u547d\u4ee4\u6709\u6240\u4e0d\u540c\u3002<\/p>\n\n\n\n<p>SQL client\u8fd0\u884c\u4e8estandalone\u6a21\u5f0f\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>.\/sql-client.sh embedded\n<\/code><\/pre>\n\n\n\n<blockquote class=\"wp-block-quote\">\n<p>\u6ce8\u610f\u3002\u5982\u679c\u4f7f\u7528standalone\u6a21\u5f0f\u8fd0\u884c\uff0c\u9700\u8981\u4e8b\u5148\u542f\u52a8\u4e00\u4e2aFlink standalone\u96c6\u7fa4\u3002\u65b9\u6cd5\u5982\u4e0b<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>.\/start-cluster.sh\n<\/code><\/pre>\n<\/blockquote>\n\n\n\n<p>\u8fd0\u884c\u5230Yarn\u96c6\u7fa4\u4e0a\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>.\/sql-client.sh embedded -s yarn-session\n<\/code><\/pre>\n\n\n\n<p>\u5927\u5bb6\u53ef\u80fd\u4f1a\u95ee\uff1aSQL slient\u600e\u4e48\u77e5\u9053\u63d0\u4ea4\u5230\u54ea\u4e2ayarn app\u4e0a\uff1f<\/p>\n\n\n\n<p>\u6307\u5b9a<code>yarn-session<\/code>\u65b9\u5f0f\u542f\u52a8\u7684\u65f6\u5019\u4f1a\u770b\u5230\u5982\u4e0b\u65e5\u5fd7\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                &#091;] - Found Yarn properties file under \/tmp\/.yarn-properties-root.\n<\/code><\/pre>\n\n\n\n<p>Flink\u6bcf\u6b21\u542f\u52a8<code>yarn-session<\/code>\uff0c\u90fd\u4f1a\u521b\u5efa\u4e00\u4e2a<code>\/tmp\/.yarn-properties-root<\/code>\u6587\u4ef6(root\u4e3a\u7528\u6237\u540d\uff0c\u6e90\u7801\u4f4d\u4e8e<code>FlinkYarnSessionCli<\/code>)\u3002\u5185\u5bb9\u5982\u4e0b\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>dynamicPropertiesString=\napplicationID=application_xxxxxxxxxxxxx_xxxx\n<\/code><\/pre>\n\n\n\n<p>\u8bb0\u5f55\u4e86\u6700\u8fd1\u4e00\u6b21\u63d0\u4ea4\u7684yarn session\u5bf9\u5e94\u7684application ID\u3002<\/p>\n\n\n\n<p>SQL client\u5982\u679c\u4f7f\u7528yarn session\u6a21\u5f0f\u542f\u52a8\uff0c\u4f1a\u67e5\u627e<code>\/tmp\/.yarn-properties-{\u7528\u6237\u540d}<\/code>\u6587\u4ef6\u4e2d\u6307\u5b9a\u7684application id\uff0c\u5c06SQL\u63d0\u4ea4\u5230\u8fd9\u4e2ayarn session\u4e0a\u8fd0\u884c\u3002\u6240\u4ee5\u8bf4\uff0c\u6211\u4eec\u5728\u4f7f\u7528Yarn\u65b9\u5f0f\u7684\u65f6\u5019\u9700\u8981\u989d\u5916\u6ce8\u610f\uff0c\u542f\u52a8Yarn session\u548cSQL client\u5fc5\u987b\u4f7f\u7528\u76f8\u540c\u7684\u7528\u6237\u3002<\/p>\n\n\n\n<h1 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%9C%BA%E6%99%AF%E7%A4%BA%E4%BE%8B\"><\/span>\u573a\u666f\u793a\u4f8b<span class=\"ez-toc-section-end\"><\/span><\/h1>\n\n\n\n<p>\u5728\u672c\u8282\u6211\u4eec\u4f7f\u7528Flink sql client\uff0c\u5b8c\u6210\u4e00\u4e2a\u7b80\u5355\u7684\u573a\u666f\uff1a\u4eceKafka\u4e2d\u8bfb\u53d6CSV\u6570\u636e\u5c06\u5176\u5199\u5165MySQL\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%87%86%E5%A4%87%E6%95%B0%E6%8D%AE\"><\/span>\u51c6\u5907\u6570\u636e<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u6211\u4eec\u63d0\u524d\u5c06\u51c6\u5907\u597d\u7684\u6d4b\u8bd5\u6570\u636e\u704c\u5165Kafka topic\u4e2d\uff0c\u4f8b\u5982<code>fludesc<\/code>\u8fd9\u4e2atopic\u3002\u4f7f\u7528\u5982\u4e0b\u65b9\u5f0f\u67e5\u770bKafka topic\u4e2d\u7684\u6570\u636e\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>.\/kafka-console-consumer.sh --topic fludesc --bootstrap-server broker1:9092,broker2:9092,broker3:9092 --from-beginning\n<\/code><\/pre>\n\n\n\n<p>\u90e8\u5206\u6837\u4f8b\u6570\u636e\u5982\u4e0b\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>511653962048,Zomfq,53,woman,532120,534.61,313020,cart,15926130785,UyxghCpKMD@huawei.com,2019-08-03\n751653962048,Qvtil,27,man,532120,655.7,313023,cart,13257423096,cJfbNhRYow@163.com,2019-08-05\n121653962048,Spdwh,35,woman,480071,97.35,313018,cart,18825789463,LkVYmpcWXC@qq.com,2019-08-05\n871653962048,Fdhpc,18,man,650012,439.40,313012,cart,15059872140,sfzuPWvNEe@qq.com,2019-08-06\n841653962048,Iqoyh,51,woman,152121,705.6,313012,buy,13646513897,jISbcYdxZO@126.com,2019-08-04\n761653962048,Xgzhy,29,woman,480071,329.60,313013,cart,15069315824,NtTDRlAdeZ@qq.com,2019-08-04\n<\/code><\/pre>\n\n\n\n<p>\u63a5\u4e0b\u6765\u6211\u4eec\u9700\u8981\u5b89\u88c5Flink\u548c\u76f8\u5173\u4f9d\u8d56\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%AE%89%E8%A3%85Flink%E5%92%8C%E4%BE%9D%E8%B5%96\"><\/span>\u5b89\u88c5Flink\u548c\u4f9d\u8d56<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u4ece\u5b98\u7f51\u4e0b\u8f7dFlink\u89e3\u538b\u5230\u96c6\u7fa4\u67d0\u8282\u70b9\u4efb\u610f\u76ee\u5f55\uff08\u4ee5Flink 1.13.2\u7248\u672c\u4e3a\u4f8b\uff09\u3002\u7136\u540e\u5206\u6790\u573a\u666f\u4e2d\u9700\u8981\u7528\u5230\u54ea\u4e9b\u4f9d\u8d56\u3002<br>\u9879\u76ee\u4e2d\u7528\u5230\u4e86CSV\u6587\u4ef6\u683c\u5f0f\uff0c\u8bfb\u53d6Kafka\u548c\u5199\u5165MySQL\u3002\u6211\u4eec\u9700\u8981\u627e\u5230\u4e0b\u5217\u4f9d\u8d56\uff1a<\/p>\n\n\n\n<ul>\n<li>flink-csv-1.13.2.jar: CSV\u683c\u5f0f\u652f\u6301<\/li>\n\n\n\n<li>flink-connector-jdbc_2.11-1.13.2.jar: \u8bfb\u5199JDBC\u652f\u6301<\/li>\n\n\n\n<li>flink-connector-kafka_2.11-1.13.2.jar: \u8bfb\u5199Kafka\u652f\u6301<\/li>\n\n\n\n<li>kafka-clients-2.4.1.jar: Kafka\u5ba2\u6237\u7aef<\/li>\n\n\n\n<li>mysql-connector-java-8.0.29.jar: MySQL JDBC\u9a71\u52a8<\/li>\n<\/ul>\n\n\n\n<p>\u5c06\u5176\u653e\u5165Flink\u5b89\u88c5\u4f4d\u7f6e\u7684lib\u76ee\u5f55\u4e0b\u3002<\/p>\n\n\n\n<p>\u5230\u8fd9\u91ccFlink\u73af\u5883\u6574\u7406\u5b8c\u6bd5\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%90%AF%E5%8A%A8Flink_SQL_client\"><\/span>\u542f\u52a8Flink SQL client<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u9996\u5148\u542f\u52a8Flink yarn-session\u3002\u5f00\u542f\u4e00\u4e2ashell\uff0c\u6267\u884c\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>export HADOOP_CLASSPATH=`hadoop classpath`\n# \u5207\u6362\u5230\u5177\u6709\u8bbf\u95eeHDFS\u548c\u63d0\u4ea4Yarn\u4efb\u52a1\u6743\u9650\u7684\u7528\u6237\nsu hdfs\ncd ${FLINK_HOME}\/bin\n.\/yarn-session.sh -s 1 -jm 1024 -tm 2048\n<\/code><\/pre>\n\n\n\n<blockquote class=\"wp-block-quote\">\n<p>\u542f\u52a8yarn-session.sh\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u6dfb\u52a0-d\u53c2\u6570\u4f7f\u7528detached\u6a21\u5f0f\uff0c\u8fd9\u6837yarn session\u542f\u52a8\u5b8c\u6bd5\u4e4b\u540eshell\u5c31\u53ef\u4ee5\u5173\u95ed\u3002<\/p>\n<\/blockquote>\n\n\n\n<p>\u7136\u540e\u542f\u52a8Flink SQL client\u3002<\/p>\n\n\n\n<p>\u518d\u5f00\u542f\u4e00\u4e2ashell\uff0c\u6267\u884c\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>export HADOOP_CLASSPATH=`hadoop classpath`\n# \u5207\u6362\u5230\u5177\u6709\u8bbf\u95eeHDFS\u548c\u63d0\u4ea4Yarn\u4efb\u52a1\u6743\u9650\u7684\u7528\u6237\nsu hdfs\ncd ${FLINK_HOME}\/bin\n.\/sql-client.sh embedded -s yarn-session\n<\/code><\/pre>\n\n\n\n<p>\u770b\u5230Flink SQL Client\u7684logo\u8bf4\u660e\u542f\u52a8\u6210\u529f\uff0c\u53ef\u4ee5\u7f16\u5199SQL\u63d0\u4ea4\u4f5c\u4e1a\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%89%A7%E8%A1%8CSQL%E6%8F%90%E4%BA%A4%E4%BD%9C%E4%B8%9A\"><\/span>\u6267\u884cSQL\u63d0\u4ea4\u4f5c\u4e1a<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u9996\u5148\u5728MySQL\u4e2d\u521b\u5efa\u9700\u8981\u63a5\u6536Kafka\u6570\u636e\u7684\u8868\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>create database demo_db character set utf8mb4;\nuse demo_db;\ncreate table fludesc (\n    id varchar(32),\n    use_rname varchar(32),\n    age int,\n    gender varchar(32),\n    goods_no varchar(32),\n    goods_price Float,\n    store_id int,\n    shopping_type varchar(32),\n    tel varchar(32),\n    email varchar(32),\n    shopping_date date\n);\n<\/code><\/pre>\n\n\n\n<blockquote class=\"wp-block-quote\">\n<p>\u6ce8\uff1a\u4e0a\u9762SQL\u7684Schema\u9700\u8981\u548cKafka CSV\u6570\u636e\u5bf9\u5e94\u3002<\/p>\n<\/blockquote>\n\n\n\n<p>\u7136\u540e\u6211\u4eec\u5728Flink SQL client\u4e2d\u6267\u884c\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>create table kafka_source (\n    id STRING,\n    use_rname STRING,\n    age integer,\n    gender STRING,\n    goods_no STRING,\n    goods_price Float,\n    store_id integer,\n    shopping_type STRING,\n    tel STRING,\n    email STRING,\n    shopping_date Date\n) with (\n    'connector' = 'kafka',\n    'properties.bootstrap.servers' = 'broker1:9092,broker2:9092,broker3:9092',\n    'topic' = 'fludesc',\n    'scan.startup.mode' = 'earliest-offset',\n    'format' = 'csv',\n    'csv.ignore-parse-errors' = 'true'\n);\n\nCREATE TABLE mysql_sink (\n    id STRING,\n    use_rname STRING,\n    age integer,\n    gender STRING,\n    goods_no STRING,\n    goods_price Float,\n    store_id integer,\n    shopping_type STRING,\n    tel STRING,\n    email STRING,\n    shopping_date Date\n) WITH (\n   'connector' = 'jdbc',\n   'url' = 'jdbc:mysql:\/\/mysql_server:3306\/demo_db',\n   'table-name' = 'fludesc',\n   'username' = 'root',\n   'password' = 'password'\n);\n\ninsert into mysql_sink select * from kafka_source;\n<\/code><\/pre>\n\n\n\n<p>\u53c2\u6570\u89e3\u6790\uff1a<\/p>\n\n\n\n<ul>\n<li>properties.bootstrap.servers\uff1aKafka broker\u5730\u5740\u548c\u7aef\u53e3\u53f7\u3002<\/li>\n\n\n\n<li>topic: \u6570\u636e\u6e90Kafka topic \u540d\u79f0\u3002<\/li>\n\n\n\n<li>scan.startup.mode: \u4f8b\u5b50\u4e2d\u4e3a\u4eceKafka topic\u4e2d\u4ece\u5934\u5f00\u59cb\u6d88\u8d39\u3002<\/li>\n\n\n\n<li>format: \u4f7f\u7528\u7684\u6570\u636e\u683c\u5f0f\uff0c\u6837\u4f8b\u4e2d\u4e3aCSV\u3002<\/li>\n\n\n\n<li>url: \u8fde\u63a5MySQL\u7684JDBC\u8fde\u63a5\u4e32\u3002<\/li>\n\n\n\n<li>table-name: MySQL\u8868\u540d\u3002<\/li>\n\n\n\n<li>username: MySQL\u7528\u6237\u540d\u3002<\/li>\n\n\n\n<li>password: MySQL\u5bc6\u7801\u3002<\/li>\n\n\n\n<li>csv.ignore-parse-errors: \u5ffd\u7565\u89e3\u6790\u9519\u8bef\u3002\u542f\u7528\u6b64\u914d\u7f6e\u53ef\u4ee5\u5bb9\u5fcd\u810f\u6570\u636e\u3002<\/li>\n<\/ul>\n\n\n\n<p>\u5982\u679c\u5168\u8fc7\u7a0b\u6ca1\u6709\u95ee\u9898\uff0c\u53ef\u8fdb\u5165MySQL\u63a7\u5236\u53f0\u6267\u884c\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>use demo_db;\nselect * from fludesc;\n<\/code><\/pre>\n\n\n\n<p>\u53ef\u4ee5\u67e5\u770b\u5230Kafka\u4e2d\u7684\u6570\u636e\u5df2\u5199\u5165\u5230MySQL\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%95%B0%E6%8D%AE%E7%B1%BB%E5%9E%8B\"><\/span>\u6570\u636e\u7c7b\u578b<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>Flink SQL\u548cMySQL\u6570\u636e\u7c7b\u578b\u5728SQL\u4e2d\u5199\u6cd5\u662f\u4e0d\u4e00\u6837\u7684\u3002\u5177\u4f53\u5199\u6cd5\u548c\u5bf9\u7167\u5173\u7cfb\u53ef\u53c2\u8003\uff1a<\/p>\n\n\n\n<p>Flink SQL\u6570\u636e\u7c7b\u578b\uff1a<a href=\"https:\/\/links.jianshu.com\/go?to=https%3A%2F%2Fnightlies.apache.org%2Fflink%2Fflink-docs-release-1.15%2Fzh%2Fdocs%2Fdev%2Ftable%2Ftypes%2F\" target=\"_blank\" rel=\"noreferrer noopener\">Data Types | Apache Flink<\/a><\/p>\n\n\n\n<p><\/p>\n\n\n\n<h1 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%B8%B8%E8%A7%81%E9%94%99%E8%AF%AF\"><\/span>\u5e38\u89c1\u9519\u8bef<span class=\"ez-toc-section-end\"><\/span><\/h1>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%89%BE%E4%B8%8D%E5%88%B0Hadoop%E7%9B%B8%E5%85%B3%E4%BE%9D%E8%B5%96\"><\/span>\u627e\u4e0d\u5230Hadoop\u76f8\u5173\u4f9d\u8d56<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u542f\u52a8yarn-session\u7684\u65f6\u5019\u51fa\u73b0\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>Setting HADOOP_CONF_DIR=\/etc\/hadoop\/conf because no HADOOP_CONF_DIR or HADOOP_CLASSPATH was set.\nSetting HBASE_CONF_DIR=\/etc\/hbase\/conf because no HBASE_CONF_DIR was set.\nError: A JNI error has occurred, please check your installation and try again\nException in thread \"main\" java.lang.NoClassDefFoundError: org\/apache\/hadoop\/yarn\/exceptions\/YarnException\n        at java.lang.Class.getDeclaredMethods0(Native Method)\n        at java.lang.Class.privateGetDeclaredMethods(Class.java:2701)\n        at java.lang.Class.privateGetMethodRecursive(Class.java:3048)\n        at java.lang.Class.getMethod0(Class.java:3018)\n        at java.lang.Class.getMethod(Class.java:1784)\n        at sun.launcher.LauncherHelper.validateMainClass(LauncherHelper.java:544)\n        at sun.launcher.LauncherHelper.checkAndLoadMain(LauncherHelper.java:526)\nCaused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.exceptions.YarnException\n        at java.net.URLClassLoader.findClass(URLClassLoader.java:381)\n        at java.lang.ClassLoader.loadClass(ClassLoader.java:424)\n        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)\n        at java.lang.ClassLoader.loadClass(ClassLoader.java:357)\n        ... 7 more\n<\/code><\/pre>\n\n\n\n<p>\u8bf4\u660eFlink\u627e\u4e0d\u5230HADOOP\u7684\u76f8\u5173\u914d\u7f6e\u548c\u4f9d\u8d56\u3002\u9700\u8981\u63d0\u524d\u6267\u884c\uff1a<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>export HADOOP_CLASSPATH=`hadoop classpath`\n<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"SQL_client_%E6%8F%90%E4%BA%A4%E4%BB%BB%E5%8A%A1%E7%9A%84%E6%97%B6%E5%80%99%E8%BF%9E%E6%8E%A5%E6%8B%92%E7%BB%9D\"><\/span>SQL client \u63d0\u4ea4\u4efb\u52a1\u7684\u65f6\u5019\u8fde\u63a5\u62d2\u7edd<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p><em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>&#091;INFO] SUbmitting SQL update statement to the cluster...\n&#091;ERROR] Could not execute SQL statement. Reason:\njava.net.ConnectException: Connection refused\n<\/code><\/pre>\n\n\n\n<p>\u8fd9\u4e2a\u95ee\u9898\u62a5\u9519\u6bd4\u8f83\u7b3c\u7edf\u3002\u771f\u5b9e\u7684\u539f\u56e0\u662fsql-client\u65e0\u6cd5\u8fde\u63a5\u5230Flink\u96c6\u7fa4\u7684job manager\u3002<\/p>\n\n\n\n<p>\u5982\u679c\u4f7f\u7528standalone\u6a21\u5f0f\uff0c\u9700\u8981\u6267\u884c<code>.\/start-cluster.sh<\/code>\u542f\u52a8\u4e00\u4e2astandalone\u96c6\u7fa4\u3002<\/p>\n\n\n\n<p>\u5982\u679c\u4f7f\u7528Yarn session\u6a21\u5f0f\uff0c\u5219\u9700\u8981\uff1a<\/p>\n\n\n\n<ul>\n<li>\u542f\u52a8sql client\u4e4b\u524d\u9700\u8981export HADOOP_CLASSPATH\u73af\u5883\u53d8\u91cf\u3002<\/li>\n\n\n\n<li>\u63d0\u4ea4yarn session\u548c\u542f\u52a8sql client\u9700\u8981\u4f7f\u7528\u540c\u4e00\u4e2a\u7528\u6237\uff0c\u5426\u5219\u4f1a\u627e\u4e0d\u5230yarn session\u5bf9\u5e94\u7684application id\u3002<\/li>\n\n\n\n<li>\u786e\u4fdd\u5f53\u524d\u673a\u5668\u7684Yarn\u5ba2\u6237\u7aef\u914d\u7f6e\u65e0\u95ee\u9898\u3002\u53ef\u901a\u8fc7\u6267\u884cyarn\u547d\u4ee4\u662f\u5426\u80fd\u6b63\u5e38\u8fd4\u56de\u96c6\u7fa4\u4fe1\u606f\u786e\u8ba4\u3002<\/li>\n<\/ul>\n\n\n\n<h1 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E8%A1%A5%E5%85%85%E5%86%85%E5%AE%B9\"><\/span>\u8865\u5145\u5185\u5bb9<span class=\"ez-toc-section-end\"><\/span><\/h1>\n\n\n\n<p>Flink SQL client\u652f\u6301\u591a\u79cd\u683c\u5f0f\u5c55\u793a\u67e5\u8be2\u51fa\u7684\u6570\u636e\u3002\u5728\u6267\u884c\u67e5\u8be2\u4e4b\u524d\uff0c\u53ef\u4ee5\u901a\u8fc7<code>SET<\/code>\u8bed\u53e5\u6307\u5b9a\u6570\u636e\u67e5\u770b\u683c\u5f0f\u3002<em><\/em><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># \u5728\u4e13\u95e8\u7684\u754c\u9762\u5c55\u793a\uff0c\u4f7f\u7528\u5206\u9875table\u683c\u5f0f\u3002\u53ef\u6309\u7167\u754c\u9762\u4e0b\u65b9\u8bf4\u660e\uff0c\u4f7f\u7528\u5feb\u6377\u952e\u524d\u540e\u7ffb\u9875\u548c\u9000\u51fa\u5230SQL\u547d\u4ee4\u884c\nSET sql-client.execution.result-mode = table;\n\n# changelog\u683c\u5f0f\u5c55\u793a\uff0c\u53ef\u5c55\u793a\u6570\u636e\u589e(I)\u5220(D)\u6539(U)\nSET sql-client.execution.result-mode = changelog;\n\n# \u63a5\u8fd1\u4f20\u7edf\u6570\u636e\u5e93\u7684\u5c55\u793a\u65b9\u5f0f\uff0c\u4e0d\u4f7f\u7528\u4e13\u95e8\u754c\u9762\nSET sql-client.execution.result-mode = tableau;<\/code><\/pre>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u524d\u8a00 Flink\u63d0\u4f9b\u4e86SQL client\uff0c\u6709\u4e86\u5b83\u6211\u4eec\u53ef\u4ee5\u50cfHive\u7684beeline\u4e00\u6837\u76f4\u63a5\u5728\u63a7\u5236\u53f0&hellip; <a href=\"http:\/\/viplao.com\/index.php\/2023\/03\/19\/%e3%80%90%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7%e3%80%91flink-sql-%e5%9f%ba%e7%a1%80%e5%ae%9e%e8%b7%b5-sql-client\/\" class=\"more-link read-more\" rel=\"bookmark\">\u7ee7\u7eed\u9605\u8bfb <span class=\"screen-reader-text\">\u3010\u8fd0\u7ef4\u5de5\u5177\u3011Flink SQL \u57fa\u7840\u5b9e\u8df5 &#8211; SQL Client<\/span><i class=\"fa fa-arrow-right\"><\/i><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[1],"tags":[25],"views":1274,"_links":{"self":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1281"}],"collection":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/comments?post=1281"}],"version-history":[{"count":2,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1281\/revisions"}],"predecessor-version":[{"id":1283,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1281\/revisions\/1283"}],"wp:attachment":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/media?parent=1281"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/categories?post=1281"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/tags?post=1281"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}