{"id":6638,"date":"2023-10-26T15:35:00","date_gmt":"2023-10-26T07:35:00","guid":{"rendered":"https:\/\/ai-stack.ai\/ring-attention-processes-millions-words"},"modified":"2024-11-14T16:04:05","modified_gmt":"2024-11-14T08:04:05","slug":"ring-attention-processes-millions-words","status":"publish","type":"post","link":"https:\/\/ai-stack.orca-biz.tw\/ja\/ring-attention-processes-millions-words","title":{"rendered":"\u6570\u767e\u4e07\u6587\u5b57\u306e\u30c7\u30fc\u30bf\u3092\u51e6\u7406\u3067\u304d\u308bAI\u30e2\u30c7\u30eb\uff1aRing Attention"},"content":{"rendered":"\n<p>\u4eba\u5de5\u77e5\u80fd\uff08AI\uff09\u306e\u4e16\u754c\u3067\u306f\u3001\u30c7\u30fc\u30bf\u304c\u3059\u3079\u3066\u306e\u57fa\u76e4\u3067\u3059\u3002\u3057\u304b\u3057\u3001\u30c7\u30fc\u30bf\u306e\u898f\u6a21\u306b\u3064\u3044\u3066\u8a71\u3059\u3068\u304d\u3001\u5f93\u6765\u306eAI\u30e2\u30c7\u30eb\u306f\u3057\u3070\u3057\u3070\u30dc\u30c8\u30eb\u30cd\u30c3\u30af\u306b\u76f4\u9762\u3057\u307e\u3059\u3002\u6570\u767e\u4e07\u6587\u5b57\u3092\u542b\u3080\u672c\u304c\u3042\u308b\u306e\u306b\u3001AI\u30e2\u30c7\u30eb\u304c\u6570\u5343\u6587\u5b57\u3057\u304b\u8aad\u3081\u306a\u3044\u3068\u3057\u305f\u3089\u3001\u305d\u308c\u306f\u3069\u308c\u307b\u3069\u5236\u9650\u3055\u308c\u3066\u3044\u308b\u3067\u3057\u3087\u3046\u304b\uff01\u3057\u304b\u3057\u4eca\u3001\u300cRing Attention\u300d\u3068\u547c\u3070\u308c\u308b\u65b0\u3057\u3044\u6280\u8853\u306b\u3088\u308a\u3001\u3059\u3079\u3066\u304c\u4e00\u5909\u3057\u307e\u3057\u305f\u3002UC Berkeley\u3068Google DeepMind\u306e\u7814\u7a76\u8005\u306b\u3088\u3063\u3066\u958b\u767a\u3055\u308c\u305f\u3053\u306e\u6280\u8853\u306f\u3001\u30e1\u30e2\u30ea\u5236\u9650\u306e\u554f\u984c\u3092\u89e3\u6c7a\u3059\u308b\u3060\u3051\u3067\u306a\u304f\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u51e6\u7406\u306b\u304a\u3051\u308bAI\u30e2\u30c7\u30eb\u306e\u80fd\u529b\u3092\u5927\u5e45\u306b\u5411\u4e0a\u3055\u305b\u307e\u3057\u305f\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"-\">\u5f93\u6765\u306e\u30c8\u30e9\u30f3\u30b9\u30d5\u30a9\u30fc\u30de\u30fc\u306e\u30e1\u30e2\u30ea\u5236\u9650<\/h2>\n\n\n\n<p>\u30c8\u30e9\u30f3\u30b9\u30d5\u30a9\u30fc\u30de\u30fc\u306f\u767b\u5834\u4ee5\u6765\u3001\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\uff08NLP\uff09\u3068\u6a5f\u68b0\u5b66\u7fd2\uff08ML\uff09\u306e\u5206\u91ce\u3067\u5927\u304d\u306a\u5f79\u5272\u3092\u679c\u305f\u3057\u3066\u304d\u307e\u3057\u305f\u3002\u3057\u304b\u3057\u3001\u3053\u306e\u67b6\u69cb\u306b\u306f\u660e\u3089\u304b\u306a\u6b20\u70b9\u304c\u3042\u308a\u307e\u3059\uff1a\u9577\u3044\u9806\u5e8f\u30c7\u30fc\u30bf\u3092\u51e6\u7406\u3059\u308b\u969b\u306b\u30e1\u30e2\u30ea\u5236\u9650\u306b\u76f4\u9762\u3059\u308b\u3053\u3068\u3067\u3059\u3002\u3053\u308c\u306f\u4e3b\u306b\u3001\u30c8\u30e9\u30f3\u30b9\u30d5\u30a9\u30fc\u30de\u30fc\u304c\u300c\u81ea\u5df1\u6ce8\u610f\u300d\uff08self-attention\uff09\u3068\u3044\u3046\u30e1\u30ab\u30cb\u30ba\u30e0\u3092\u4f7f\u7528\u3057\u3066\u3044\u308b\u305f\u3081\u3067\u3001\u3053\u308c\u306f\u975e\u5e38\u306b\u30e1\u30e2\u30ea\u96c6\u7d04\u7684\u306a\u30d7\u30ed\u30bb\u30b9\u3067\u3059\u3002\u5f93\u6765\u3001\u3053\u306e\u5236\u9650\u306b\u3088\u308a\u30c8\u30e9\u30f3\u30b9\u30d5\u30a9\u30fc\u30de\u30fc\u306f\u30b3\u30f3\u30c6\u30ad\u30b9\u30c8\u9577\u3092\u62e1\u5f35\u3059\u308b\u3053\u3068\u304c\u96e3\u3057\u304f\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306e\u51e6\u7406\u80fd\u529b\u304c\u5236\u9650\u3055\u308c\u3066\u3044\u307e\u3057\u305f\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"ring-attention-\">Ring Attention\uff1a\u753b\u671f\u7684\u306a\u89e3\u6c7a\u7b56<\/h2>\n\n\n\n<p>\u3053\u306e\u554f\u984c\u3092\u89e3\u6c7a\u3059\u308b\u305f\u3081\u306b\u3001UC Berkeley\u306e\u7814\u7a76\u8005\u306f\u300cRing Attention\u300d\u3068\u547c\u3070\u308c\u308b\u65b0\u3057\u3044\u65b9\u6cd5\u3092\u958b\u767a\u3057\u307e\u3057\u305f\u3002\u3053\u306e\u65b9\u6cd5\u306e\u6838\u5fc3\u7684\u306a\u30a2\u30a4\u30c7\u30a2\u306f\u3001\u8a08\u7b97\u30d7\u30ed\u30bb\u30b9\u3092\u8907\u6570\u306e\u30c7\u30d0\u30a4\u30b9\u9593\u3067\u30d6\u30ed\u30c3\u30af\u5f62\u5f0f\u3067\u5206\u6563\u3055\u305b\u308b\u3053\u3068\u3067\u3059\u3002\u3053\u308c\u306b\u3088\u308a\u3001\u5404\u30c7\u30d0\u30a4\u30b9\u306f\u30c7\u30fc\u30bf\u306e\u4e00\u90e8\u5206\u306e\u307f\u3092\u51e6\u7406\u3059\u308c\u3070\u3088\u304f\u3001\u30e1\u30e2\u30ea\u8981\u6c42\u3092\u5927\u5e45\u306b\u6e1b\u5c11\u3055\u305b\u307e\u3059\u3002<\/p>\n\n\n\n<p>\u5177\u4f53\u7684\u306b\u306f\u3001Ring Attention\u306f\u30ea\u30f3\u30b0\u72b6\u306e\u69cb\u9020\u3092\u63a1\u7528\u3057\u3001\u30ad\u30fc\u30d0\u30ea\u30e5\u30fc\uff08key-value\uff09\u30d6\u30ed\u30c3\u30af\u3092\u4e00\u3064\u306e\u30c7\u30d0\u30a4\u30b9\u304b\u3089\u5225\u306e\u30c7\u30d0\u30a4\u30b9\u306b\u9001\u4fe1\u3057\u307e\u3059\u3002\u3053\u306e\u30d6\u30ed\u30c3\u30af\u5358\u4f4d\u306e\u6ce8\u610f\uff08blockwise attention\uff09\u3068\u30d5\u30a3\u30fc\u30c9\u30d5\u30a9\u30ef\u30fc\u30c9\u6f14\u7b97\uff08feedforward operations\uff09\u306b\u3088\u308a\u3001\u5404\u5165\u529b\u30d6\u30ed\u30c3\u30af\u304c\u7279\u5b9a\u306e\u64cd\u4f5c\u3092\u6301\u3064\u3053\u3068\u304c\u3067\u304d\u3001\u52b9\u7387\u7684\u306a\u8a08\u7b97\u3092\u5b9f\u73fe\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"-\">\u5b9f\u969b\u306e\u5fdc\u7528\u3068\u5c06\u6765\u306e\u5c55\u671b<\/h2>\n\n\n\n<p>\u3053\u306e\u65b0\u3057\u3044\u65b9\u6cd5\u306f\u3001\u30e1\u30e2\u30ea\u5236\u9650\u3092\u514b\u670d\u3059\u308b\u3060\u3051\u3067\u306a\u304f\u3001AI\u30e2\u30c7\u30eb\u304c\u3053\u308c\u307e\u3067\u3088\u308a\u3082\u306f\u308b\u304b\u306b\u9577\u3044\u9806\u5e8f\u3092\u51e6\u7406\u3067\u304d\u308b\u3088\u3046\u306b\u306a\u308a\u307e\u3057\u305f\u3002\u7814\u7a76\u5831\u544a\u306b\u3088\u308b\u3068\u3001Ring Attention\u306f\u4ee5\u524d\u306e\u30e1\u30e2\u30ea\u52b9\u7387\u30e2\u30c7\u30eb\u3068\u6bd4\u3079\u3066\u6700\u5927500\u500d\u9577\u3044\u9806\u5e8f\u3092\u51e6\u7406\u3067\u304d\u307e\u3059\u3002\u3053\u308c\u306f\u3001\u73fe\u5728\u306eAI\u30e2\u30c7\u30eb\u304c\u6570\u767e\u4e07\u6587\u5b57\u306e\u30c7\u30fc\u30bf\u91cf\u3092\u5bb9\u6613\u306b\u51e6\u7406\u3067\u304d\u308b\u3053\u3068\u3092\u610f\u5473\u3057\u3001\u5927\u898f\u6a21\u306a\u30d3\u30c7\u30aa\u3001\u97f3\u58f0\u3001\u8a00\u8a9e\u30e2\u30c7\u30eb\u306b\u3068\u3063\u3066\u5927\u304d\u306a\u30d6\u30ec\u30fc\u30af\u30b9\u30eb\u30fc\u3067\u3059\u3002<\/p>\n\n\n\n<p>\u3053\u306e\u6280\u8853\u306e\u6f5c\u5728\u7684\u306a\u5fdc\u7528\u7bc4\u56f2\u306f\u975e\u5e38\u306b\u5e83\u304f\u3001\u5927\u898f\u6a21\u306a\u30d3\u30c7\u30aa\u8a00\u8a9e\u30e2\u30c7\u30eb\u304b\u3089\u907a\u4f1d\u5b50\u914d\u5217\u306a\u3069\u306e\u79d1\u5b66\u30c7\u30fc\u30bf\u307e\u3067\u53ca\u3073\u307e\u3059\u3002\u3055\u3089\u306b\u3001\u3053\u306e\u7814\u7a76\u306f\u5c06\u6765\u306e\u6700\u5927\u9806\u5e8f\u9577\u3068\u6700\u5927\u8a08\u7b97\u6027\u80fd\u306e\u63a2\u7d22\u306b\u65b0\u305f\u306a\u53ef\u80fd\u6027\u3092\u63d0\u4f9b\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"ring-attention-\">Ring Attention\u306e\u6280\u8853\u5b9f\u73fe\u65b9\u6cd5<\/h2>\n\n\n\n<p>Ring Attention\u3092\u5b9f\u73fe\u3059\u308b\u305f\u3081\u306e\u9375\u306f\u3001\u8a08\u7b97\u30d7\u30ed\u30bb\u30b9\u3092\u8907\u6570\u306e\u30c7\u30d0\u30a4\u30b9\u306b\u52b9\u679c\u7684\u306b\u5206\u6563\u3055\u305b\u308b\u3053\u3068\u3067\u3059\u3002\u4ee5\u4e0b\u306b\u3044\u304f\u3064\u304b\u306e\u5b9f\u7528\u7684\u306a\u30b9\u30c6\u30c3\u30d7\u3092\u793a\u3057\u307e\u3059\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u30d6\u30ed\u30c3\u30af\u5206\u5272\uff08Block Partitioning\uff09\uff1a\u307e\u305a\u3001\u5168\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u3092\u8907\u6570\u306e\u5c0f\u30d6\u30ed\u30c3\u30af\u306b\u5206\u5272\u3057\u307e\u3059\u3002<\/li>\n\n\n\n<li>\u30ea\u30f3\u30b0\u69cb\u9020\u8a2d\u8a08\uff08Ring Structure Design\uff09\uff1a\u3059\u3079\u3066\u306e\u30c7\u30d0\u30a4\u30b9\u304c\u30ea\u30f3\u30b0\u72b6\u306e\u69cb\u9020\u3067\u914d\u7f6e\u3055\u308c\u308b\u3088\u3046\u306b\u3057\u307e\u3059\u3002<\/li>\n\n\n\n<li>\u30ad\u30fc\u30d0\u30ea\u30e5\u30fc\u30d6\u30ed\u30c3\u30af\u8ee2\u9001\uff08Key-Value Block Transfer\uff09\uff1a\u8a08\u7b97\u3092\u884c\u3044\u306a\u304c\u3089\u3001\u30ad\u30fc\u30d0\u30ea\u30e5\u30fc\u30d6\u30ed\u30c3\u30af\u3092\u4e00\u3064\u306e\u30c7\u30d0\u30a4\u30b9\u304b\u3089\u5225\u306e\u30c7\u30d0\u30a4\u30b9\u306b\u9001\u4fe1\u3057\u307e\u3059\u3002<\/li>\n\n\n\n<li>\u30d6\u30ed\u30c3\u30af\u5358\u4f4d\u306e\u6ce8\u610f\u3068\u30d5\u30a3\u30fc\u30c9\u30d5\u30a9\u30ef\u30fc\u30c9\u6f14\u7b97\uff08Blockwise Attention and Feedforward Operations\uff09\uff1a\u5404\u30c7\u30d0\u30a4\u30b9\u304c\u62c5\u5f53\u3059\u308b\u30c7\u30fc\u30bf\u30d6\u30ed\u30c3\u30af\u306b\u5bfe\u3057\u3066\u6ce8\u610f\u3068\u30d5\u30a3\u30fc\u30c9\u30d5\u30a9\u30ef\u30fc\u30c9\u6f14\u7b97\u3092\u884c\u3044\u307e\u3059\u3002<\/li>\n<\/ul>\n\n\n\n<p>\u3053\u306e\u3088\u3046\u306b\u3057\u3066\u3001\u5404\u30c7\u30d0\u30a4\u30b9\u306f\u8a08\u7b97\u306e\u4e00\u90e8\u5206\u306e\u307f\u3092\u62c5\u5f53\u3059\u308c\u3070\u3088\u304f\u3001\u5168\u4f53\u306e\u30e1\u30e2\u30ea\u8981\u6c42\u3092\u5927\u5e45\u306b\u6e1b\u5c11\u3055\u305b\u307e\u3059\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"-\">\u7d50\u8ad6\uff1a\u30e1\u30e2\u30ea\u5236\u9650\u3092\u6253\u7834\u3057\u3001\u7121\u9650\u306e\u53ef\u80fd\u6027\u3092\u958b\u304f<\/h2>\n\n\n\n<p>Ring Attention\u306e\u767b\u5834\u306f\u3001\u9593\u9055\u3044\u306a\u304fAI\u5206\u91ce\u306e\u91cd\u5927\u306a\u30d6\u30ec\u30fc\u30af\u30b9\u30eb\u30fc\u3067\u3059\u3002\u3053\u308c\u306f\u9577\u5e74\u7814\u7a76\u8005\u3092\u60a9\u307e\u305b\u3066\u304d\u305f\u30e1\u30e2\u30ea\u554f\u984c\u3092\u89e3\u6c7a\u3059\u308b\u3060\u3051\u3067\u306a\u304f\u3001AI\u30e2\u30c7\u30eb\u304c\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u3092\u51e6\u7406\u3059\u308b\u5168\u304f\u65b0\u3057\u3044\u53ef\u80fd\u6027\u3092\u63d0\u4f9b\u3057\u307e\u3059\u3002\u3053\u308c\u304b\u3089\u306f\u3001\u6570\u767e\u4e07\u6587\u5b57\u306e\u30c7\u30fc\u30bf\u91cf\u306f\u3082\u306f\u3084\u554f\u984c\u3067\u306f\u306a\u304f\u3001AI\u306e\u5fdc\u7528\u7bc4\u56f2\u3082\u3053\u308c\u306b\u3088\u308a\u5927\u304d\u304f\u62e1\u5927\u3059\u308b\u3067\u3057\u3087\u3046\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>UC\u30d0\u30fc\u30af\u30ec\u30fc\u3068Google DeepMind\u306e\u7814\u7a76\u30c1\u30fc\u30e0\u306f\u3001\u65b0\u6280\u8853\u300c\u30ea\u30f3\u30b0\u30a2\u30c6\u30f3\u30b7\u30e7\u30f3\u300d\u3092\u958b\u767a\u3057\u3001\u5927\u898f\u6a21\u30c7\u30fc\u30bf\u51e6\u7406\u306b\u304a\u3051\u308b\u5f93\u6765\u306eAI\u30e2\u30c7\u30eb\u306e\u30e1\u30e2\u30ea\u5236\u9650\u3092\u7a81\u7834\u3057\u307e\u3057\u305f\u3002\u3053\u306e\u6280\u8853\u306f\u3001\u8a08\u7b97\u30d7\u30ed\u30bb\u30b9\u3092\u8907\u6570\u306e\u30c7\u30d0\u30a4\u30b9\u306b\u5206\u6563\u3055\u305b\u308b\u3053\u3068\u3067\u3001AI\u30e2\u30c7\u30eb\u304c\u9577\u3044\u30b7\u30fc\u30b1\u30f3\u30b9\u30c7\u30fc\u30bf\u3092\u51e6\u7406\u3059\u308b\u80fd\u529b\u3092\u52b9\u679c\u7684\u306b\u5411\u4e0a\u3055\u305b\u3001AI\u306e\u69d8\u3005\u306a\u5206\u91ce\u3067\u306e\u5fdc\u7528\u306b\u65b0\u305f\u306a\u53ef\u80fd\u6027\u3092\u958b\u62d3\u3057\u307e\u3057\u305f\u3002<\/p>\n","protected":false},"author":253372381,"featured_media":5310,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"jetpack_post_was_ever_published":false,"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[96987672,96987620],"tags":[96987615,96987686],"class_list":["post-6638","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-ai-focus","category-featured-articles","tag-ai","tag-ai-ja"],"blocksy_meta":[],"acf":[],"jetpack_featured_media_url":"https:\/\/i0.wp.com\/ai-stack.orca-biz.tw\/wp-content\/uploads\/2024\/03\/Featured-Article-05.jpg?fit=824%2C464&quality=100&ssl=1","jetpack_shortlink":"https:\/\/wp.me\/ph344V-1J4","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/posts\/6638","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/users\/253372381"}],"replies":[{"embeddable":true,"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/comments?post=6638"}],"version-history":[{"count":0,"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/posts\/6638\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/media\/5310"}],"wp:attachment":[{"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/media?parent=6638"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/categories?post=6638"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/ai-stack.orca-biz.tw\/ja\/wp-json\/wp\/v2\/tags?post=6638"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}