{"id":1066,"date":"2022-10-30T15:39:57","date_gmt":"2022-10-30T07:39:57","guid":{"rendered":"http:\/\/viplao.com\/?p=1066"},"modified":"2022-10-30T21:54:19","modified_gmt":"2022-10-30T13:54:19","slug":"%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7-hadoop%e9%ab%98%e5%8f%af%e7%94%a8%e7%9a%84%e6%9e%b6%e6%9e%84%e7%9f%a5%e8%af%86","status":"publish","type":"post","link":"http:\/\/viplao.com\/index.php\/2022\/10\/30\/%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7-hadoop%e9%ab%98%e5%8f%af%e7%94%a8%e7%9a%84%e6%9e%b6%e6%9e%84%e7%9f%a5%e8%af%86\/","title":{"rendered":"\u8fd0\u7ef4\u5de5\u5177 &#8211; Hadoop\u9ad8\u53ef\u7528\u7684\u67b6\u6784\u77e5\u8bc6"},"content":{"rendered":"\n<p><\/p>\n\n\n\n<p>hadoop 2.0 <\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" loading=\"lazy\" width=\"704\" height=\"400\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-63.png\" alt=\"\" class=\"wp-image-1075\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-63.png 704w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-63-300x170.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-63-528x300.png 528w\" sizes=\"(max-width: 704px) 100vw, 704px\" \/><\/figure>\n\n\n\n<p>\u5bf9\u4e8eHadoop\u9ad8\u53ef\u7528\u67b6\u6784\u8282\u70b9\u4ecb\u7ecd<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" loading=\"lazy\" width=\"651\" height=\"411\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-57.png\" alt=\"\" class=\"wp-image-1067\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-57.png 651w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-57-300x189.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-57-475x300.png 475w\" sizes=\"(max-width: 651px) 100vw, 651px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" loading=\"lazy\" width=\"817\" height=\"519\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-62.png\" alt=\"\" class=\"wp-image-1073\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-62.png 817w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-62-300x191.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-62-768x488.png 768w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-62-472x300.png 472w\" sizes=\"(max-width: 817px) 100vw, 817px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" loading=\"lazy\" width=\"1024\" height=\"623\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-58-1024x623.png\" alt=\"\" class=\"wp-image-1068\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-58-1024x623.png 1024w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-58-300x183.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-58-768x467.png 768w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-58-1536x935.png 1536w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-58-2048x1247.png 2048w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-58-493x300.png 493w\" sizes=\"(max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<p>NN\u670d\u52a1\u5668\u4f1a\u51fa\u73b0\u8111\u88c2(brain-split)\u60c5\u51b5<br>\u4ec0\u4e48\u662f\u8111\u88c2<br>\u5728hadoop2.x\u7248\u672c\u4e2d\uff0c\u5982\u679c\u5b58\u5728\u4e24\u4e2aNameNode\u8282\u70b9\u540c\u65f6\u670d\u52a1\uff0c\u8fd9\u79cd\u60c5\u51b5\u79f0\u4e4b\u4e3a\u201c\u8111\u88c2\u201d<\/p>\n\n\n\n<p>\u4e3a\u4ec0\u4e48\u4f1a\u51fa\u73b0\u8111\u88c2<br>\u8111\u88c2\u51fa\u73b0\u539f\u56e0\u4e00\u822c\u53d1\u751f\u5728\u4e3b\u5907NamoNode\u5207\u6362\uff0c\u7531\u4e8e\u7f51\u7edc\u5ef6\u8fdf\u3001\u8bbe\u5907\u6545\u969c\u7b49\uff0c\u5907\u7528\u7684StandbyNameNode\u3010\u5907\u7528\u8282\u70b9\u3011\u8ba4\u4e3aActiveNameNode\u3010\u4e3b\u8282\u70b9\u3011\u5931\u6548\uff0c\u6b64\u65f6StandbyNameNode\u4f1a\u8f6c\u6362\u4e3a\u6d3b\u8dc3\u72b6\u6001\u3010\u4e3b\u8282\u70b9\u3011\uff0c\u8fd9\u65f6\u5019\u5982\u679c\u539f\u6765\u7684\u4e3b\u8282\u70b9\u81ea\u5df1\u6062\u590d\u6b63\u5e38\uff0c\u8fd9\u65f6\u5019\u4f1a\u51fa\u73b0\u4e24\u4e2a\u4e3b\u8282\u70b9\u540c\u65f6\u5de5\u4f5c\u7684\u60c5\u51b5<\/p>\n\n\n\n<p>\u8111\u88c2\u7684\u573a\u666f<br>NameNode \u53ef\u80fd\u4f1a\u51fa\u73b0\u8fd9\u79cd\u60c5\u51b5\uff0cNameNode \u5728\u5783\u573e\u56de\u6536\uff08GC\uff09\u65f6\uff0c\u53ef\u80fd\u4f1a\u5728\u957f\u65f6\u95f4\u5185\u6574\u4e2a\u7cfb\u7edf\u65e0\u54cd\u5e94<br>zkfc\u5ba2\u6237\u7aef\u4e5f\u5c31\u65e0\u6cd5\u5411 zk \u5199\u5165\u5fc3\u8df3\u4fe1\u606f,\u8fd9\u6837\u7684\u8bdd\u53ef\u80fd\u4f1a\u5bfc\u81f4\u4e34\u65f6\u8282\u70b9\u6389\u7ebf,\u5907 NameNode\u4f1a\u5207\u6362\u5230 Active \u72b6\u6001<br>\u8fd9\u79cd\u60c5\u51b5\u53ef\u80fd\u4f1a\u5bfc\u81f4\u6574\u4e2a\u96c6\u7fa4\u4f1a\u6709\u540c\u65f6\u6709\u4e24\u4e2aActive NameNode<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" loading=\"lazy\" width=\"1024\" height=\"667\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-59-1024x667.png\" alt=\"\" class=\"wp-image-1069\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-59-1024x667.png 1024w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-59-300x196.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-59-768x501.png 768w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-59-1536x1001.png 1536w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-59-2048x1335.png 2048w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-59-460x300.png 460w\" sizes=\"(max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<p>\u89e3\u51b3\u65b9\u6848<br>\u5728\u4e24\u4e2aNameNode\u8282\u70b9\u670d\u52a1\u5668\u4e2d\uff0c\u90fd\u4f1a\u6709\u4e00\u4e2a\u8fdb\u7a0b\u3010ZKFC\u3011\u76d1\u542c\u5f53\u524dNN\u670d\u52a1\u5668\u7684\u5065\u5eb7\u72b6\u51b5<br>\u5728\u96c6\u7fa4\u542f\u52a8\u540e\uff0c\u4f7f\u7528\u7b2c\u4e09\u65b9\u7ec4\u4ef6\u3010\u8fd9\u91cc\u4f7f\u7528Zookeeper\u3011\u5e2e\u52a9\u9009\u4e3e\u4e3b\u8282\u70b9\u3010ANN\u3011\uff0c\u7ade\u9009\u5b8cANN\u540e\uff0c\u4f1a\u5728Zookerper\u96c6\u7fa4\u4e2d\u521b\u5efa\u9009\u4e3e\u6210\u529f\u540eANN\u4fe1\u606f\uff0c\u4e00\u4e2azookeeper\u4e34\u65f6\u8282\u70b9\uff08\u4e3b\u8282\u70b9\u670d\u52a1\u5668\u6302\u6389\uff0c\u4e34\u65f6\u8282\u70b9\u5c31\u4e0d\u5b58\u5728\uff09\uff0c\u548c\u4e00\u4e2a\u6b63zookerper\u6b63\u5e38\u8282\u70b9\uff08\u5728\u6b63\u5e38\u5173\u95edANN\uff0c\u6b63\u5e38\u8282\u70b9\u4e5f\u4f1a\u5173\u95ed\uff0c\u5982\u679c\u662f\u5f02\u5e38\u5173\u95edANN\uff0c\u6b63\u5e38\u8282\u70b9\u5b58\u5728\uff09<br>\u5207\u6362\u4e3b\u5907\u8282\u70b9\u65b9\u5f0f<br>3.1. \u5f53\u5907\u7528\u8282\u70b9\u3010SNN\u3011\u53d1\u73b0zookerper\u4e2d\uff0cANN\u7684\u4e34\u65f6\u8282\u70b9\u548c\u6b63\u5e38\u8282\u70b9\u90fd\u4e0d\u5b58\u5728\uff0c\u81ea\u5df1\u53ef\u4ee5\u653e\u5fc3\u5207\u6362\u4e3a\u65b0\u7684ANN\u4e3b\u8282\u70b9<br>3.2. \u5f53SNN\u53d1\u73b0Zookerper\u4e2d\uff0c\u539f\u6765ANN\u7684\u4e34\u65f6\u8282\u70b9\u4e0d\u5b58\u5728\uff0c\u6b63\u5e38\u8282\u70b9\u5b58\u5728\u7684\u65f6\u5019\uff0c\u5982\u679c\u76f4\u63a5\u5207\u6362\u81ea\u5df1\u4e3a\u65b0\u7684ANN\u53ef\u80fd\u51fa\u73b0\u8111\u88c2<br>\u9488\u5bf9\u8fd9\u4e24\u79cd\u60c5\u51b5\uff0c\u600e\u4e48\u5207\u6362\u4e3b\u5907\u670d\u52a1\u5668<br>4.1 \u9996\u5148\u5c31\u4f7f\u7528RPC\u8fdc\u7a0b\u8c03\u7528ANN\u670d\u52a1\u5668\u7684ActiveBreadCrumb\u65b9\u6cd5\uff0c\u5c1d\u8bd5\u5c06\u539f\u6765\u7684ANN\u5207\u6362\u4e3aSNN<br>4.2. \u5982\u679cRPC\u8fdc\u7a0b\u8c03\u7528\u5931\u8d25\uff0c\u4f1a\u6267\u884c\u9884\u5b9a\u4e49\u7684\u9694\u79bb\u63aa\u65bd<br>\u9694\u79bb\u63aa\u65bd<br>5.1. sshfence: \u901a\u8fc7SSH\uff08\u767b\u9646\u4fe1\u606f\u5728zookerper\u8282\u70b9\u4e2d\u5b58\u653e\uff0c\u53ef\u4ee5\u83b7\u53d6\u5230\uff09\u767b\u9646\u5230\u76ee\u6807\u673a\u5668\u4e0a\uff0c\u6267\u884c\u547d\u4ee4fuser\u5c06\u5bf9\u5e94\u8fdb\u57ce\u6740\u6b7b<br>5.2. shellfence\uff1a\u6267\u884c\u7528\u6237\u81ea\u5b9a\u4e49\u7684shell\u811a\u672c\u8bb2\u5bf9\u5e94\u7684\u8fdb\u57ce\u9694\u79bb<br>\u5904\u7406\u5b8c\u6bd5\u539f\u6765\u7684ANN\u670d\u52a1\u5668\u540e\uff0cSNN\u5c31\u4f1a\u8c03\u7528becomoActive\u6210\u4e3a\u4e3b\u8282\u70b9\uff0c\u5bf9\u5916\u63d0\u4f9b\u670d\u52a1<br>\u65b0\u7684ANN\u670d\u52a1\u5668\u4e5f\u4f1a\u5728zookeeper\u4e2d\u521b\u5efa\u81ea\u5df1\u7684\u4e34\u65f6\u8282\u70b9\u548c\u6b63\u5e38\u8282\u70b9<\/p>\n\n\n\n<p>\u5982\u679c\u53d1\u751f\u8111\u88c2\u7684\uff0c\u9632\u6b62\u591a\u6761\u6307\u4ee4\u7684\u60c5\u51b5<br>\u5728\u6bcf\u6b21\u9009\u4e3e\u4e4b\u540e\uff0cANN\u90fd\u4f1a\u4ea7\u751f\u4e00\u4e2a\u65b0\u7684\u5e8f\u5217\u53f7\u3010\u53ef\u4ee5\u7406\u89e3\u4e3a\u81ea\u589eid\u3011\u7136\u540e\u8fd9\u4e2a\u5e8f\u5217\u53f7\u4f1a\u53d1\u9001\u7ed9DN<br>\u5728\u6bcf\u6b21NN\u670d\u52a1\u5668\u53d1\u9001\u6307\u4ee4\u7684\u65f6\u5019\uff0c\u5982\u679c\u53d1\u751f\u8111\u88c2\uff0c\u591a\u4e2aNN\u670d\u52a1\u5668\u53d1\u9001\u6307\u4ee4\uff0cDN\u670d\u52a1\u5668\u4e00\u6700\u65b0\u7684\u6307\u4ee4\u4e3a\u51c6<\/p>\n\n\n\n<p>\u8054\u90a6\u673a\u5236<br>\u4ec0\u4e48\u662f\u8054\u90a6\u673a\u5236<br>\u53ef\u4ee5\u540c\u65f6\u8ba9\u591a\u4e2aNameNode\u53c2\u4e0e\u6570\u636e\u7684\u7ba1\u7406\u3010\u8fd9\u662f\u6c34\u5e73\u62d3\u5c55\u3011<\/p>\n\n\n\n<p>\u4e3a\u4ec0\u4e48\u4f7f\u7528\u8054\u90a6\u673a\u5236<br>== \u5355\u4e2aNN\u8282\u70b9\u4f7f\u7528\u5b58\u5728\u5c40\u9650\u6027==<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" loading=\"lazy\" width=\"1024\" height=\"564\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-60-1024x564.png\" alt=\"\" class=\"wp-image-1070\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-60-1024x564.png 1024w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-60-300x165.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-60-768x423.png 768w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-60-1536x847.png 1536w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-60-544x300.png 544w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-60.png 1651w\" sizes=\"(max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<p>Namespace\uff08\u547d\u540d\u7a7a\u95f4\uff09\u7684\u9650\u5236<\/p>\n\n\n\n<ul><li>NameNode\u6240\u80fd\u5b58\u50a8\u7684\u5bf9\u8c61\uff08\u6587\u4ef6+\u5757\uff09\u6570\u76ee\u53d7\u5230NameNode\u6240\u5728JVM\u7684heap size\u7684\u9650\u5236\u3002<\/li><li>50G\u7684heap\u80fd\u591f\u5b58\u50a820\u4ebf\uff08200million\uff09\u4e2a\u5bf9\u8c61\uff0c\u8fd920\u4ebf\u4e2a\u5bf9\u8c61\u652f\u63014000\u4e2aDataNode\uff0c<br>12PB\u7684\u5b58\u50a8<\/li><li>DataNode\u4ece4T\u589e\u957f\u523036T\uff0c\u96c6\u7fa4\u7684\u5c3a\u5bf8\u589e\u957f\u52308000\u4e2aDataNode\u3002\u5b58\u50a8\u7684\u9700\u6c42\u4ece12PB\u589e\u957f\u5230\u5927\u4e8e100PB\u3002<br>\u6027\u80fd\u7684\u74f6\u9888<\/li><li>\u6574\u4e2aHDFS\u6587\u4ef6\u7cfb\u7edf\u7684\u541e\u5410\u91cf\u53d7\u9650\u4e8e\u5355\u4e2aNamenode\u7684\u541e\u5410\u91cf<br>\u9694\u79bb\u95ee\u9898<\/li><li>HDFS\u4e0a\u7684\u4e00\u4e2a\u5b9e\u9a8c\u7a0b\u5e8f\u5c31\u5f88\u6709\u53ef\u80fd\u5f71\u54cd\u6574\u4e2aHDFS\u4e0a\u8fd0\u884c\u7684\u7a0b\u5e8f<br>\u96c6\u7fa4\u7684\u53ef\u7528\u6027<\/li><li>Namenode\u7684\u5b95\u673a\u65e0\u7591\u4f1a\u5bfc\u81f4\u6574\u4e2a\u96c6\u7fa4\u4e0d\u53ef\u7528\u3002<br>Namespace\u548cBlock Management\u7684\u7d27\u5bc6\u8026\u5408<br>\u7eb5\u5411\u6269\u5c55\u76ee\u524d\u7684Namenode\u4e0d\u53ef\u884c<\/li><li>\u5c06Namenode\u7684Heap\u7a7a\u95f4\u6269\u5927\u5230512GB\u542f\u52a8\u82b1\u8d39\u7684\u65f6\u95f4\u592a\u957f<\/li><li>Namenode\u5728Full GC\u65f6\uff0c\u5982\u679c\u53d1\u751f\u9519\u8bef\u5c06\u4f1a\u5bfc\u81f4\u6574\u4e2a\u96c6\u7fa4\u5b95\u673a<br>\u8054\u90a6\u673a\u5236\u67b6\u6784<br>1\uff1a\u6bcf\u4e2aNameNode\u3010\u8fd9\u91cc\u4f7f\u7528\u9ad8\u53ef\u7528\u67b6\u6784\uff0c\u5b58\u5728\u4e3b\u5907\u8282\u70b9\u3011\u7ba1\u7406\u6307\u5b9a\u7684\u6587\u4ef6\u5185\u5bb9<br>2\uff1a\u6bcf\u4e2aNameNode\u90fd\u7ba1\u7406\u7ef4\u62a4\u4e00\u4e2aNameSpace(\u547d\u540d\u7a7a\u95f4\u6c60)<br>3\uff1a\u6240\u6709\u7684DN\u670d\u52a1\u5668\u90fd\u90fd\u5171\u4eab\u7ed9\u5168\u90e8NameSpace<\/li><\/ul>\n\n\n\n<p>\u5757\u6c60Block Pool<\/p>\n\n\n\n<ul><li>Block pool(\u5757\u6c60)\u5c31\u662f\u5c5e\u4e8e\u5355\u4e2a\u547d\u540d\u7a7a\u95f4\u7684\u4e00\u7ec4block(\u5757)\u7ba1\u7406\u533a\u57df<\/li><li>\u6bcf\u4e00\u4e2adatanode\u4e3a\u6240\u6709\u7684block pool\u5b58\u50a8<\/li><li>Datanode\u662f\u4e00\u4e2a\u7269\u7406\u6982\u5ff5\uff0c\u800cblock pool\u662f\u4e00\u4e2a\u91cd\u65b0\u5c06block\u5212\u5206\u7684\u903b\u8f91\u6982\u5ff5<\/li><li>\u4e00\u4e2aNamenode\u5931\u6548\u4e0d\u4f1a\u5f71\u54cd\u5176\u4e0b\u7684datanode\u4e3a\u5176\u4ed6Namenode\u7684\u670d\u52a1<\/li><li>datanode\u4e0eNamenode\u5efa\u7acb\u8054\u7cfb\u5e76\u5f00\u59cb\u4f1a\u8bdd\u540e\u81ea\u52a8\u5efa\u7acbBlock pool<br>Namespace Volume\uff08\u547d\u540d\u7a7a\u95f4\u5377\uff09<\/li><li>\u4e00\u4e2aNamespace\u548c\u5b83\u7684Block Pool\u5408\u5728\u4e00\u8d77\u79f0\u4f5cNamespace Volume<\/li><li>Namespace Volume\u662f\u4e00\u4e2a\u72ec\u7acb\u5b8c\u6574\u7684\u7ba1\u7406\u5355\u5143\u3002\u5f53\u4e00\u4e2aNamenode\/Namespace\u88ab\u5220\u9664\uff0c\u4e0e\u4e4b\u76f8\u5bf9\u5e94\u7684Block Pool\u4e5f\u4e5f\u88ab\u5220\u9664\u3002<br>\u901a\u8fc7\u591a\u4e2anamenode\/namespace\u628a\u5143\u6570\u636e\u7684\u5b58\u50a8\u548c\u7ba1\u7406\u5206\u6563\u5230\u591a\u4e2a\u8282\u70b9\u4e2d<\/li><li>\u964d\u4f4e\u5355\u4e2aNN\u8282\u70b9\u6570\u636e\u538b\u529b,\u8ba1\u7b97\u538b\u529b<br>namenode\/namespace\u53ef\u4ee5\u901a\u8fc7\u589e\u52a0\u673a\u5668\u6765\u8fdb\u884c\u6c34\u5e73\u6269\u5c55<\/li><li>\u53ef\u4ee5\u8ba9\u66f4\u591a\u7684\u8282\u70b9\u53c2\u4e0e\u5230\u8fd0\u7b97<\/li><li>namespace\u547d\u540d\u7a7a\u95f4\uff0c\u901a\u8fc7\u8fd9\u79cd\u65b9\u5f0f\u786e\u5b9a\u8981\u5904\u7406\u6570\u636e\u7684\u8def\u5f84<br>\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7namenode\u548cnamespace\u7ec4\u5408\u4f7f\u7528<\/li><li>\u6240\u6709\u7684nn\u5171\u4eabdn<\/li><li>\u4f46\u662f\u6bcf\u4e00\u4e2anamespace\u4f1a\u5355\u72ec\u7ba1\u7406\u81ea\u5df1\u7684\u5757<\/li><li>\u4f1a\u521b\u5efa\u4e00\u4e2a\u7ba1\u7406\u5757\u7684\u673a\u5236\uff1ablocks pool<br><\/li><\/ul>\n\n\n\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" loading=\"lazy\" width=\"1024\" height=\"615\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-61-1024x615.png\" alt=\"\" class=\"wp-image-1071\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-61-1024x615.png 1024w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-61-300x180.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-61-768x461.png 768w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-61-500x300.png 500w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-61.png 1243w\" sizes=\"(max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<p><\/p>\n\n\n\n<p>\u968f\u7740Hadoop\u751f\u6001\u7684\u53d1\u5c55\uff0c\u5f00\u6e90\u793e\u533a\u51fa\u73b0\u4e86\u591a\u79cd\u591a\u6837\u7684\u6280\u672f\u7ec4\u4ef6\u3002\u6709\u7528\u6765\u6784\u5efa\u6570\u636e\u4ed3\u5e93\u7684Hive\uff0c\u4e5f\u6709\u57fa\u4e8e\u5185\u5b58\u7684\u8ba1\u7b97\u6846\u67b6\u00a0Spark\uff0c\u8fd8\u6709\u6211\u4eec\u4e4b\u524d\u4ecb\u7ecd\u8fc7\u7684NoSQL\u6570\u636e\u5e93\u00a0HBase\u7b49\u3002<br>\u8fd9\u4e9b\u6280\u672f\u7ec4\u4ef6\u7684\u51fa\u73b0\uff0c\u6781\u5927\u5730\u4e30\u5bcc\u4e86\u5927\u6570\u636e\u7684\u751f\u6001\u4f53\u7cfb\uff0c\u4f46\u540c\u65f6\u4e5f\u5f15\u51fa\u4e86\u4e00\u4e9b\u65b0\u7684\u95ee\u9898\u3002\u4f5c\u4e3a\u4e00\u4e2a\u5927\u6570\u636e\u5e95\u5c42\u652f\u6491\u5e73\u53f0\uff0c\u540c\u65f6\u90e8\u7f72Hive\u3001HBase\u548cSpark\u7b49\u591a\u79cd\u6280\u672f\u7ec4\u4ef6\u662f\u4e00\u4ef6\u5341\u5206\u5e73\u5e38\u7684\u4e8b\u60c5\u3002\u8fd9\u4e9b\u4e3a\u5927\u6570\u636e\u573a\u666f\u8bbe\u8ba1\u7684\u6280\u672f\u7ec4\u4ef6\u53ef\u4ee5\u8bf4\u4e2a\u4e2a\u90fd\u662f\u6d88\u8017\u8d44\u6e90\u7684\u5927\u6237\uff0c\u8fd9\u4e9b\u8d44\u6e90\u5305\u62ec\u670d\u52a1\u5668\u7684CPU\u548c\u5185\u5b58\u3002<br>\u901a\u5e38\u8fd9\u4e9b\u6280\u672f\u7ec4\u4ef6\u90fd\u6709\u4e00\u5957\u81ea\u5df1\u7684\u8d44\u6e90\u8c03\u5ea6\u7cfb\u7edf\u7528\u6765\u7ba1\u7406\u4efb\u52a1\u7684\u8d44\u6e90\u5206\u914d\uff0c\u4f46\u5f53\u5b83\u4eec\u540c\u65f6\u90e8\u7f72\u5728\u4e00\u8d77\u7684\u65f6\u5019\u5c31\u51fa\u95ee\u9898\u4e86\u3002\u8fd9\u65f6\u4f1a\u6709\u4e24\u79cd\u60c5\u51b5\u4ea7\u751f\uff0c\u7b2c\u4e00\u79cd\u60c5\u51b5\u662f\u67d0\u4e9b\u7ec4\u4ef6\u53ef\u80fd\u7533\u8bf7\u4e0d\u5230\u670d\u52a1\u5668\u8d44\u6e90\u3002<br>\u6bd4\u5982\u4e00\u53f0\u62e5\u670932G\u5185\u5b58\u7684\u670d\u52a1\u5668\u540c\u65f6\u90e8\u7f72\u4e86HBase\u548cSpark\uff0cHBase\u7684RegionServer\u542f\u52a8\u65f6\u5360\u7528\u4e8620GB\u5185\u5b58\uff0c\u8fd9\u65f6Spark\u5f00\u59cb\u6267\u884c\u67d0\u4e2a\u4efb\u52a1\u4e5f\u9700\u8981\u4f7f\u752820GB\u5185\u5b58\uff0c\u4f46\u8fd9\u65f6\u53d1\u73b0\u6ca1\u6709\u8db3\u591f\u7684\u5185\u5b58\u8d44\u6e90\u4f7f\u7528\u4e86\u3002\u56e0\u4e3a\u4ece\u6bcf\u4e2a\u7ec4\u4ef6\u72ec\u7acb\u7684\u89c6\u89d2\u6765\u770b\u4ed6\u4eec\u90fd\u8ba4\u4e3a\u81ea\u5df1\u80fd\u4f7f\u7528100%\u7684\u670d\u52a1\u5668\u8d44\u6e90\uff0c\u4f46\u670d\u52a1\u5668\u8d44\u6e90\u7684\u603b\u91cf\u5c31\u90a3\u4e48\u591a\uff0c\u4e0d\u53ef\u80fd\u540c\u65f6\u6ee1\u8db3\u6240\u6709\u7ec4\u4ef6\u7684\u9700\u6c42\u3002<br>\u7b2c\u4e8c\u79cd\u662f\u53ef\u80fd\u4f1a\u51fa\u73b0\u8d44\u6e90\u5206\u914d\u4e0d\u5408\u7406\u7684\u60c5\u51b5\uff0c\u5bfc\u81f4\u6574\u4f53\u8d44\u6e90\u4f7f\u7528\u7387\u504f\u4f4e\u3002\u6211\u4eec\u540c\u6837\u7528\u521a\u624d\u7684\u573a\u666f\u4e3e\u4f8b\uff0cSpark\u542f\u52a8\u4e86\u4e00\u4e2a\u4efb\u52a1\u7533\u8bf7\u4f7f\u752830GB\u7684\u5185\u5b58\uff0c\u4f46\u662f\u5b9e\u9645\u4e0a\u5b83\u7684\u7a0b\u5e8f\u903b\u8f91\u5e76\u4e0d\u9700\u8981\u4f7f\u7528\u8fd9\u4e48\u591a\u8d44\u6e90\u3002\u8fd9\u5c31\u51fa\u73b0\u4e86\u4e00\u79cdHBase\u6ca1\u6709\u8d44\u6e90\u4ec0\u4e48\u4e8b\u60c5\u4e5f\u505a\u4e0d\u4e86\uff0c\u4f46Spark\u5360\u7528\u4e86\u8d44\u6e90\u5374\u6ca1\u6709\u4e8b\u60c5\u53ef\u505a\u7684\u5c40\u9762\u3002<br>\u4e3a\u4e86\u89e3\u51b3\u7c7b\u4f3c\u7684\u95ee\u9898\uff0c\u6211\u4eec\u9700\u8981\u4e00\u79cd\u901a\u7528\u7684\u8d44\u6e90\u8c03\u5ea6\u6846\u67b6\uff0c\u5bf9\u6574\u4e2a\u96c6\u7fa4\u7684\u8d44\u6e90\u8fdb\u884c\u7edf\u7b79\u7ba1\u7406\u3002<br>YARN\u5c31\u662f\u4e00\u6b3e\u4f18\u79c0\u7684\u96c6\u7fa4\u8d44\u6e90\u8c03\u5ea6\u6846\u67b6\u3002YARN\u662fYet Another Resource Negotiator\u7684\u7f29\u5199\uff0c\u5b83\u662fHadoop\u7684\u7b2c\u4e8c\u4ee3\u96c6\u7fa4\u8d44\u6e90\u8c03\u5ea6\u6846\u67b6\u3002<br>\u89e3\u51b3\u4e86Hadoop\u7b2c\u4e00\u4ee3\u96c6\u7fa4\u8d44\u6e90\u8c03\u5ea6\u6846\u67b6\u4e0a\u53ef\u9760\u6027\u5dee\u3001\u6269\u5c55\u6027\u5dee\u7b49\u4e00\u7cfb\u5217\u95ee\u9898\uff0c\u540c\u65f6YARN\u4eceMapReduce\u4e2d\u5b8c\u5168\u72ec\u7acb\u51fa\u6765\uff0c\u4ece\u4e13\u95e8\u652f\u6491MapReduce\u4efb\u52a1\u8c03\u5ea6\u5347\u7ea7\u6210\u4e3a\u4e86\u4e00\u4e2a\u652f\u6301\u591a\u79cd\u5e94\u7528\u7c7b\u578b\u7684\u901a\u7528\u96c6\u7fa4\u8d44\u6e90\u8c03\u5ea6\u6846\u67b6\u3002<br>\u9664\u4e86MapReduce\u4e4b\u5916\uff0cSpark\u3001Hive\u7b49\u4e00\u7cfb\u5217\u670d\u52a1\u90fd\u53ef\u4ee5\u4f5c\u4e3a\u5e94\u7528\u8fd0\u884c\u5728YARN\u4e4b\u4e0a\uff0c\u7edf\u4e00\u4f7f\u7528YARN\u4e3a\u6574\u4e2a\u96c6\u7fa4\u8d44\u6e90\u8fdb\u884c\u5b8f\u89c2\u7684\u8c03\u5ea6\u4e0e\u5206\u914d<br><\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" loading=\"lazy\" width=\"1024\" height=\"394\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-66-1024x394.png\" alt=\"\" class=\"wp-image-1083\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-66-1024x394.png 1024w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-66-300x116.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-66-768x296.png 768w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-66-779x300.png 779w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-66.png 1080w\" sizes=\"(max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<p>02 \u8d44\u6e90\u6a21\u578b\u548cContainer<br>YARN\u5c06\u670d\u52a1\u5668\u8d44\u6e90\u8fdb\u884c\u4e86\u62bd\u8c61\u5c01\u88c5\uff0c\u5b83\u4f7f\u7528Container\u5bf9\u8c61\u4ee3\u8868\u7533\u8bf7\u8d44\u6e90\u7684\u57fa\u672c\u5355\u5143\u3002<br>\u8fd9\u4e9b\u8d44\u6e90\u5305\u62ec\u8d44\u6e90\u540d\u79f0\uff08\u670d\u52a1\u5668\u540d\u79f0\u3001\u673a\u67b6\u7b49\uff09\u3001\u5185\u5b58\u548cCPU\uff0cYARN\u901a\u8fc7Container\u673a\u5236\u5c06\u670d\u52a1\u5668\u8d44\u6e90\u8fdb\u884c\u4e86\u9694\u79bb\u3002\u6bcf\u4e2a\u5e94\u7528\u90fd\u53ef\u4ee5\u901a\u8fc7ApplicationMaster\u5411ResourceManager\u7533\u8bf7\u8d44\u6e90\uff0c\u5f53ApplicationMaster\u5411ResourceManager\u7533\u8bf7\u8d44\u6e90\u65f6\uff0cResourceManager\u8fd4\u56de\u7684\u8d44\u6e90\u4f7f\u7528Container\u7684\u4e2a\u6570\u6765\u8868\u793a\uff0c\u6bd4\u5982\u4e00\u4e2aSpark\u8ba1\u7b97\u4efb\u52a1\u9700\u89815\u4e2aContainer\u8d44\u6e90\u3002<\/p>\n\n\n\n<p>03 ResourceManager<br>ResourceManager\u662f\u4e00\u4e2a\u5168\u5c40\u7684\u8d44\u6e90\u7ba1\u7406\u5668\uff0c\u8d1f\u8d23\u6574\u4e2a\u7cfb\u7edf\u7684\u8d44\u6e90\u7ba1\u7406\u548c\u5206\u914d\u4ee5\u4fdd\u8bc1\u6574\u4e2a\u96c6\u7fa4\u7684\u9ad8\u6548\u8fd0\u884c\u3002\u5b83\u4f1a\u6839\u636e\u5bb9\u91cf\u3001\u961f\u5217\u7b49\u9650\u5236\u6761\u4ef6\uff08\u5982\u6bcf\u4e2a\u961f\u5217\u5206\u914d\u4e00\u5b9a\u7684\u8d44\u6e90\uff0c\u6700\u591a\u6267\u884c\u4e00\u5b9a\u6570\u91cf\u7684\u4f5c\u4e1a\u7b49\uff09\uff0c\u5c06\u7cfb\u7edf\u4e2d\u7684\u8d44\u6e90\u5206\u914d\u7ed9\u5404\u4e2a\u6b63\u5728\u8fd0\u884c\u7684\u5e94\u7528\u7a0b\u5e8f\u3002<br>ResourceManager\u53ea\u8d1f\u8d23\u6839\u636e\u5404\u4e2a\u5e94\u7528\u7a0b\u5e8f\u7684\u8d44\u6e90\u8bf7\u6c42\u8fdb\u884c\u8d44\u6e90\u5206\u914d\uff0c\u4e0d\u53c2\u4e0e\u4efb\u4f55\u4e0e\u5177\u4f53\u5e94\u7528\u7a0b\u5e8f\u76f8\u5173\u7684\u5de5\u4f5c\uff0c\u6bd4\u5982\u4e0d\u8d1f\u8d23\u76d1\u63a7\u6216\u8005\u8ddf\u8e2a\u5e94\u7528\u7684\u6267\u884c\u72b6\u6001\u7b49\uff0c\u4e5f\u4e0d\u8d1f\u8d23\u91cd\u65b0\u542f\u52a8\u56e0\u5e94\u7528\u6267\u884c\u5931\u8d25\u6216\u8005\u786c\u4ef6\u6545\u969c\u800c\u4ea7\u751f\u7684\u5931\u8d25\u4efb\u52a1\uff0c\u8fd9\u4e9b\u5747\u4ea4\u7531\u5e94\u7528\u7a0b\u5e8f\u76f8\u5173\u7684ApplicationMaster\u5b8c\u6210\u3002\u8d44\u6e90\u5206\u914d\u5355\u4f4d\u7528\u7684\u662f\u6211\u4eec\u521a\u624d\u4ecb\u7ecd\u8fc7\u7684Container\u5bf9\u8c61\u3002<br>\u6b64\u5916\uff0cResourceManager\u8fd8\u652f\u6301\u4e00\u4e2a\u53ef\u63d2\u62d4\u7684\u8c03\u5ea6\u5668\u63d2\u4ef6\u6765\u652f\u6301\u591a\u79cd\u8d44\u6e90\u8c03\u5ea6\u7b56\u7565\uff0c\u6bd4\u5982\u4f7f\u7528\u516c\u5e73\u8c03\u5ea6\u6216\u662f\u5bb9\u91cf\u8c03\u5ea6\u3002<\/p>\n\n\n\n<p>04 ApplicationMaster<br>\u6bcf\u4e00\u4e2a\u60f3\u8981\u8fd0\u884c\u5728YARN\u4e0a\u7684\u5e94\u7528\u90fd\u5fc5\u987b\u6709\u4e00\u4e2a\u76f8\u5e94\u7684ApplicationMaster\u5b9e\u73b0\uff0c\u5e94\u7528\u5c06\u5185\u90e8\u7684\u4efb\u52a1\u8c03\u5ea6\u903b\u8f91\u548c\u76d1\u63a7\u90fd\u4ea4\u7531\u5b83\u4eec\u81ea\u5df1\u7684ApplicationMaster\u5b9e\u73b0\u7c7b\u6765\u5904\u7406\u3002<br>ApplicationMaster\u662fYARN\u7684\u4e00\u4e2a\u521b\u65b0\u8bbe\u8ba1\uff0cYARN\u901a\u8fc7\u8fd9\u79cd\u673a\u5236\u5c06\u81ea\u5df1\u6253\u9020\u6210\u4e86\u4e00\u4e2a\u6269\u5c55\u6027\u6781\u5f3a\u7684\u901a\u7528\u8d44\u6e90\u8c03\u5ea6\u6846\u67b6\uff0c\u56e0\u4e3a\u5b83\u5141\u8bb8\u7528\u6237\u5f00\u53d1\u81ea\u5df1\u7684ApplicationMaster\u5b9e\u73b0\u3002<br>ApplicationMaster\u8fdb\u7a0b\u5728\u8fd0\u884c\u7684\u8fc7\u7a0b\u4e2d\u4e3b\u8981\u8d1f\u8d23\u4e0eResourceManager\u8fdb\u884c\u901a\u4fe1\uff0c\u4ee5\u7533\u8bf7\u6267\u884c\u4efb\u52a1\u65f6\u6240\u9700\u8981\u7684\u8d44\u6e90\uff0c\u5728\u7533\u8bf7\u5230\u8d44\u6e90\u4e4b\u540e\u518d\u8fdb\u4e00\u6b65\u6267\u884c\u81ea\u8eab\u5185\u90e8\u7684\u8c03\u5ea6\u4efb\u52a1\u3002\u540c\u65f6ApplicationMaster\u4e5f\u8d1f\u8d23\u76d1\u63a7\u81ea\u5df1\u8fd0\u884c\u7684\u5185\u90e8\u4efb\u52a1\u72b6\u6001\uff0c\u5728\u4efb\u52a1\u5931\u8d25\u7684\u65f6\u5019\u91cd\u65b0\u4e3a\u4efb\u52a1\u7533\u8bf7\u76f8\u5e94\u8d44\u6e90\u5e76\u91cd\u542f\u4efb\u52a1\u3002<br>ApplicationMaster\u901a\u5e38\u4f5c\u4e3a\u4e00\u4e2a\u5e94\u7528\u7684\u4e3b\u8fdb\u7a0b\uff0c\u4e3b\u8981\u7528\u6765\u626e\u6f14\u62c6\u5206\u5b50\u4efb\u52a1\u3001\u6c47\u603b\u7ed3\u679c\u6570\u636e\u8fd9\u7c7b\u7684\u603b\u4f53\u8c03\u5ea6\uff0c\u6bd4\u5982Spark\u7684Driver\u8fdb\u7a0b\u3002\u800c\u771f\u6b63\u7684\u6267\u884c\u7a0b\u5e8f\u4e1a\u52a1\u903b\u8f91\u7684\u8fdb\u7a0b\u662f\u5728NodeManager\u8fdb\u7a0b\u4e0a\u6267\u884c\u7684\u3002<\/p>\n\n\n\n<p>05 NodeManager<br>NodeManager\u662f\u6bcf\u4e2a\u670d\u52a1\u5668\u8282\u70b9\u4e0a\u8d44\u6e90\u7ba1\u7406\u5668\uff0c\u8d1f\u8d23\u7ba1\u7406\u81ea\u5df1\u6240\u5904\u670d\u52a1\u5668Containers\u7684\u6574\u4e2a\u751f\u547d\u5468\u671f\u3002<br>\u5728YARN\u4e0a\u8fd0\u884c\u7684\u5e94\u7528\u6700\u7ec8\u7684\u903b\u8f91\u6267\u884c\u7a0b\u5e8f\uff08\u6bd4\u5982Spark\u7684task\u3001MapReduce\u7684job\uff09\u90fd\u4f1a\u5728NodeManager\u7684Container\u4e2d\u8fd0\u884c\uff0c\u53ef\u4ee5\u8bf4NodeManager\u662fYARN\u8ba1\u7b97\u8282\u70b9\u7684\u4ee3\u7406\uff0c\u56e0\u4e3aResourceManager\u53ea\u4f1a\u5c06\u4efb\u52a1\u5206\u914d\u5230\u542f\u52a8\u4e86NodeManager\u8fdb\u7a0b\u7684\u670d\u52a1\u5668\u3002<br>\u5f53NodeManager\u8fdb\u7a0b\u542f\u52a8\u7684\u65f6\u5019\u5b83\u4f1a\u5411ResourceManager\u8fdb\u884c\u6ce8\u518c\uff0c\u5e76\u5b9a\u65f6\u6c47\u62a5\u81ea\u5df1\u6240\u5728\u670d\u52a1\u5668\u7684\u8d44\u6e90\u4f7f\u7528\u60c5\u51b5\u548cContainer\u8fd0\u884c\u72b6\u6001\uff0c\u540c\u65f6\u5b83\u4e5f\u63a5\u53d7\u5e76\u5904\u7406\u6765\u81eaApplicationMaster\u7684Container\u542f\u52a8\u548c\u505c\u6b62\u7b49\u5404\u79cd\u8bf7\u6c42\u3002<\/p>\n\n\n\n<p>06 \u5355\u4e00\u96c6\u7fa4\u67b6\u6784<br>\u901a\u8fc7\u4e0a\u9762\u7684\u4ecb\u7ecd\u6211\u4eec\u4e0d\u96be\u53d1\u73b0\uff0cResourceManager\u3001NodeManager\u548cContainer\u7ec4\u4ef6\u90fd\u4e0d\u5173\u5fc3\u5177\u4f53\u7684\u5e94\u7528\u7a0b\u5e8f\u6216\u4efb\u52a1\u7684\u7c7b\u578b\uff0c\u53ea\u6709ApplicationMaster\u624d\u662f\u5e94\u7528\u7c7b\u578b\u76f8\u5173\u7684\u3002<br>YARN\u901a\u8fc7\u4f7f\u7528\u5f00\u653eApplicationMaster\u7684\u96c6\u6210\u65b9\u5f0f\uff0c\u5141\u8bb8\u7b2c\u4e09\u65b9\u5e94\u7528\u6846\u67b6\u4fbf\u6377\u7684\u548cYARN\u8fdb\u884c\u96c6\u6210\u3002\u8fd9\u624d\u6709\u4e86\u50cfMapReduce On YARN\u3001Storm On YARN\u3001Spark On YARN\u548cTez On YARN\u7b49\u4f17\u591a\u7b2c\u4e09\u65b9\u5e94\u7528\u96c6\u6210\u65b9\u6848\u7684\u51fa\u73b0\u3002<br>\u901a\u8fc7\u8fd9\u79cd\u8d44\u6e90\u5171\u4eab\u7684\u5355\u4e00\u96c6\u7fa4\u67b6\u6784\uff0c\u6211\u4eec\u5728\u4f01\u4e1a\u5185\u90e8\u53ef\u4ee5\u5b9e\u73b0\u670d\u52a1\u5668\u8d44\u6e90\u771f\u6b63\u7684\u5171\u4eab\u4f7f\u7528\uff0c\u4ee5\u8fbe\u5230\u964d\u4f4e\u6280\u672f\u96c6\u6210\u6210\u672c\u548c\u589e\u5f3a\u8d44\u6e90\u6574\u4f53\u5229\u7528\u7387\u7684\u76ee\u7684\u3002<\/p>\n\n\n\n<p>07 \u5de5\u4f5c\u6d41\u7a0b<br>\u63a5\u4e0b\u6765\u6211\u4eec\u7b80\u5355\u770b\u4e00\u4e0bYARN\u7684\u6574\u4e2a\u5de5\u4f5c\u8fc7\u7a0b\uff0c\u5982\u56fe2-13\u6240\u793a\u3002<\/p>\n\n\n\n<p>\u7528\u6237\u5411YARN\u4e2d\u63d0\u4ea4\u5e94\u7528\u7a0b\u5e8f\u3002<br>ResourceManager\u4e3a\u8be5\u5e94\u7528\u7a0b\u627e\u5230\u4e00\u4e2a\u53ef\u7528\u7684NodeManager\u5e76\u5206\u914d\u7b2c\u4e00\u4e2aContainer\uff0c\u7136\u540e\u5728\u8fd9\u4e2aContainer\u4e2d\u542f\u52a8\u5e94\u7528\u7a0b\u5e8f\u7684ApplicationMaster\u3002<br>ApplicationMaster\u5411ResourceManager\u8fdb\u884c\u6ce8\u518c\uff0c\u8fd9\u6837\u7528\u6237\u5c31\u53ef\u4ee5\u901a\u8fc7ResourceManager\u67e5\u770b\u5e94\u7528\u7a0b\u5e8f\u7684\u8fd0\u884c\u72b6\u6001\u5e76\u5bf9\u4efb\u52a1\u8fdb\u884c\u76d1\u63a7\u3002<br>ApplicationMaster\u91c7\u7528\u8f6e\u8be2\u7684\u65b9\u5f0f\u901a\u8fc7RPC\u534f\u8bae\u5411ResourceManager\u7533\u8bf7\u548c\u9886\u53d6\u8d44\u6e90\u3002<br>ApplicationMaster\u7533\u8bf7\u5230\u8d44\u6e90\u540e\u4e0e\u5bf9\u5e94\u7684NodeManager\u901a\u4fe1\uff0c\u8981\u6c42\u5b83\u542f\u52a8Container\u5e76\u4e3a\u4efb\u52a1\u8bbe\u7f6e\u597d\u8fd0\u884c\u73af\u5883\u3002<br>\u5e94\u7528\u7a0b\u5e8f\u7684\u4efb\u52a1\u5f00\u59cb\u5728\u542f\u52a8\u7684Container\u4e2d\u8fd0\u884c\uff0c\u5404\u4e2a\u4efb\u52a1\u5411ApplicationMaster\u6c47\u62a5\u81ea\u5df1\u7684\u72b6\u6001\u548c\u8fdb\u5ea6\uff0c\u4ee5\u4fbfApplicationMaster\u968f\u65f6\u638c\u63e1\u5404\u4e2a\u4efb\u52a1\u7684\u8fd0\u884c\u72b6\u6001\uff0c\u4ece\u800c\u53ef\u4ee5\u5728\u4efb\u52a1\u5931\u8d25\u65f6\u91cd\u65b0\u542f\u52a8\u4efb\u52a1\u3002<br>\u5e94\u7528\u5728\u8fd0\u884c\u7684\u8fc7\u7a0b\u4e2d\uff0c\u5ba2\u6237\u7aef\u901a\u8fc7\u8f6e\u8be2\u7684\u65b9\u5f0f\u4e3b\u52a8\u4e0eApplicationMaster\u901a\u4fe1\u4ee5\u83b7\u5f97\u5e94\u7528\u7684\u8fd0\u884c\u72b6\u6001\u3001\u6267\u884c\u8fdb\u5ea6\u7b49\u4fe1\u606f\u3002<br>\u5e94\u7528\u7a0b\u5e8f\u8fd0\u884c\u5b8c\u6210\u540e\uff0cApplicationMaster\u5411ResourceManager\u6ce8\u9500\u5e76\u5173\u95ed\u81ea\u5df1\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" loading=\"lazy\" width=\"923\" height=\"660\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-64.png\" alt=\"\" class=\"wp-image-1077\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-64.png 923w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-64-300x215.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-64-768x549.png 768w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-64-420x300.png 420w\" sizes=\"(max-width: 923px) 100vw, 923px\" \/><\/figure>\n\n\n\n<p>\u25b2\u56fe2-13 YARN\u7684\u5de5\u4f5c\u6d41\u7a0b<\/p>\n\n\n\n<p>08 \u4f7f\u7528\u573a\u666f<br>\u57fa\u4e8eYARN\u6269\u5c55\u6027\u5f3a\u3001\u53ef\u9760\u6027\u5f3a\u3001\u652f\u6301\u591a\u7528\u6237\u548c\u652f\u6301\u591a\u5e94\u7528\u7684\u7279\u70b9\uff0c\u5b83\u975e\u5e38\u9002\u5408\u4e8e\u652f\u6491\u4f01\u4e1a\u5185\u90e8\u6784\u5efa\u7edf\u4e00\u7684\u8d44\u6e90\u5171\u4eab\u578b\u5927\u6570\u636e\u5e73\u53f0\u3002\u501f\u52a9YARN\u6211\u4eec\u53ef\u4ee5\u771f\u6b63\u5b9e\u73b0\u901a\u8fc7\u4e00\u5957\u8d44\u6e90\u8c03\u5ea6\u7cfb\u7edf\u96c6\u6210\u6240\u6709\u5e94\u7528\u7ec4\u4ef6\u7684\u5355\u4e00\u5927\u96c6\u7fa4\u67b6\u6784\u3002<\/p>\n\n\n\n<ol><li>Spark\u4efb\u52a1\u8c03\u5ea6<br>&nbsp;Spark\u662f\u4e00\u6b3e\u5206\u5e03\u5f0f\u5185\u5b58\u8ba1\u7b97\u6846\u67b6\uff0cSpark\u53ef\u4ee5\u5c06\u81ea\u8eab\u7684\u4efb\u52a1\u8c03\u5ea6\u90e8\u5206\u59d4\u6258YARN\u8fdb\u884c\u7ba1\u7406\uff0c\u4ece\u800c\u5b9e\u73b0\u96c6\u7fa4\u8d44\u6e90\u9ad8\u6548\u6574\u5408\u4e0e\u5229\u7528\u3002<\/li><li>MapReduce\u4efb\u52a1\u8c03\u5ea6<\/li><\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><strong>\u540c\u6837\u7684\uff0cMapReduce\u4efb\u52a1\u7684\u6574\u4e2a\u751f\u547d\u5468\u671f\u90fd\u53ef\u4ee5\u501f\u52a9YARN\u8fdb\u884c\u7ba1\u7406\uff0c\u5305\u62ec\u4efb\u52a1\u7684\u5206\u914d\u3001\u8d44\u6e90\u7684\u8c03\u5ea6\uff0c\u7b49\u7b49\u3002<\/strong><\/h2>\n\n\n\n<p>HBase\u5206\u5e03\u5f0f\u67b6\u6784\u5904\u7406\u5927\u6570\u636e\u91cf<\/p>\n\n\n\n<p><a href=\"https:\/\/blog.csdn.net\/zih58888888\/article\/details\/124734494\">https:\/\/blog.csdn.net\/zih58888888\/article\/details\/124734494<\/a><\/p>\n\n\n\n<p>HBase<\/p>\n\n\n\n<p>\u4ec0\u4e48\u662fBigTable\uff1f: \u628a\u6240\u6709\u7684\u6570\u636e\u4fdd\u5b58\u5230\u4e00\u5f20\u8868\u4e2d\uff0c\u91c7\u7528\u5197\u4f59 &#8212;&gt; \u597d\u5904\uff1a\u63d0\u9ad8\u6548\u7387<\/p>\n\n\n\n<p>1\u3001\u56e0\u4e3a\u6709\u4e86bigtable\u7684\u601d\u60f3\uff1aNoSQL\uff1aHBase\u6570\u636e\u5e93<br>2\u3001HBase\u57fa\u4e8eHadoop\u7684HDFS\u7684<br>3\u3001\u63cf\u8ff0HBase\u7684\u8868\u7ed3\u6784<br>\u6838\u5fc3\u601d\u60f3\u662f\uff1a\u5229\u7528\u7a7a\u95f4\u6362\u6548\u7387<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" loading=\"lazy\" width=\"1014\" height=\"486\" src=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-65.png\" alt=\"\" class=\"wp-image-1082\" srcset=\"http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-65.png 1014w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-65-300x144.png 300w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-65-768x368.png 768w, http:\/\/viplao.com\/wp-content\/uploads\/2022\/10\/image-65-626x300.png 626w\" sizes=\"(max-width: 1014px) 100vw, 1014px\" \/><\/figure>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>hadoop 2.0 \u5bf9\u4e8eHadoop\u9ad8\u53ef\u7528\u67b6\u6784\u8282\u70b9\u4ecb\u7ecd NN\u670d\u52a1\u5668\u4f1a\u51fa\u73b0\u8111\u88c2(brain-spli&hellip; <a href=\"http:\/\/viplao.com\/index.php\/2022\/10\/30\/%e8%bf%90%e7%bb%b4%e5%b7%a5%e5%85%b7-hadoop%e9%ab%98%e5%8f%af%e7%94%a8%e7%9a%84%e6%9e%b6%e6%9e%84%e7%9f%a5%e8%af%86\/\" class=\"more-link read-more\" rel=\"bookmark\">\u7ee7\u7eed\u9605\u8bfb <span class=\"screen-reader-text\">\u8fd0\u7ef4\u5de5\u5177 &#8211; Hadoop\u9ad8\u53ef\u7528\u7684\u67b6\u6784\u77e5\u8bc6<\/span><i class=\"fa fa-arrow-right\"><\/i><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[1],"tags":[],"views":1063,"_links":{"self":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1066"}],"collection":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/comments?post=1066"}],"version-history":[{"count":7,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1066\/revisions"}],"predecessor-version":[{"id":1084,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/posts\/1066\/revisions\/1084"}],"wp:attachment":[{"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/media?parent=1066"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/categories?post=1066"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/viplao.com\/index.php\/wp-json\/wp\/v2\/tags?post=1066"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}