{"id":1854,"date":"2020-10-09T12:46:49","date_gmt":"2020-10-09T12:46:49","guid":{"rendered":"https:\/\/kukan-joho.plus\/?p=1854"},"modified":"2024-04-30T08:19:41","modified_gmt":"2024-04-30T08:19:41","slug":"prophetnet","status":"publish","type":"post","link":"https:\/\/club.informatix.co.jp\/?p=1854","title":{"rendered":"\u4e8b\u524d\u5b66\u7fd2\u306e\u6539\u5584\u3067\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306e\u7cbe\u5ea6\u5411\u4e0a\uff5cTransformer\u30d9\u30fc\u30b9\u306e\u30e2\u30c7\u30eb\u300cProphetNet\u300d"},"content":{"rendered":"<p>\u3053\u3093\u306b\u3061\u306f\u3001<a href=\"https:\/\/www.informatix.co.jp\/\" target=\"_blank\" rel=\"noopener noreferrer\">\u30a4\u30f3\u30d5\u30a9\u30de\u30c6\u30a3\u30af\u30b9<\/a>\u3067\u6a5f\u68b0\u5b66\u7fd2\u696d\u52d9\u3092\u62c5\u5f53\u3057\u3066\u3044\u308b\u5927\u6a4b\u3067\u3059\u3002<\/p>\n<p>\u4eca\u56de\u306f\u30c7\u30a3\u30fc\u30d7\u30e9\u30fc\u30cb\u30f3\u30b0\u3092\u7528\u3044\u305f\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306e\u8ad6\u6587\u3001ProphetNet [1] \u3092\u7d39\u4ecb\u3057\u305f\u3044\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n<p>\u65e2\u5b58\u306e\u69cb\u9020\u306b\u5bfe\u3059\u308b\u4fee\u6b63\u3092\u6700\u5c0f\u9650\u306b\u3057\u3001\u3061\u3087\u3063\u3068\u3057\u305f\u767a\u60f3\u3067\u4e8b\u524d\u5b66\u7fd2\u306e\u65b9\u6cd5\u3092\u6539\u5584\u3057\u3066\u7cbe\u5ea6\u3092\u5411\u4e0a\u3055\u305b\u305f\u3053\u3068\u304c\u66f8\u304b\u308c\u3066\u3044\u308b\u8208\u5473\u6df1\u3044\u8ad6\u6587\u3067\u3059\u3002<\/p>\n\n<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_84 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u76ee\u6b21<\/p>\n<label for=\"ez-toc-cssicon-toggle-item-6a2394a95c852\" class=\"ez-toc-cssicon-toggle-label\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/label><input type=\"checkbox\"  id=\"ez-toc-cssicon-toggle-item-6a2394a95c852\" checked aria-label=\"Toggle\" \/><nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E8%87%AA%E7%84%B6%E8%A8%80%E8%AA%9E%E5%87%A6%E7%90%86%E3%81%A8%E8%BB%A2%E7%A7%BB%E5%AD%A6%E7%BF%92\" >\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u3068\u8ee2\u79fb\u5b66\u7fd2<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#ProphetNet%E6%A6%82%E8%A6%81\" >ProphetNet\u6982\u8981<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#ProphetNet%E8%A9%B3%E7%B4%B0\" >ProphetNet\u8a73\u7d30<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E3%83%8D%E3%83%83%E3%83%88%E3%83%AF%E3%83%BC%E3%82%AF%E3%81%AE%E5%85%A5%E5%87%BA%E5%8A%9B\" >\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u306e\u5165\u51fa\u529b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E3%82%A8%E3%83%B3%E3%82%B3%E3%83%BC%E3%83%80\" >\u30a8\u30f3\u30b3\u30fc\u30c0<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E3%83%87%E3%82%B3%E3%83%BC%E3%83%80\" >\u30c7\u30b3\u30fc\u30c0<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E7%9B%AE%E7%9A%84%E9%96%A2%E6%95%B0\" >\u76ee\u7684\u95a2\u6570<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E3%83%87%E3%82%B3%E3%83%BC%E3%83%80%E3%81%AE%E6%B3%A8%E6%84%8F%E6%A9%9F%E6%A7%8B\" >\u30c7\u30b3\u30fc\u30c0\u306e\u6ce8\u610f\u6a5f\u69cb<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E3%83%87%E3%82%B3%E3%83%BC%E3%83%80%E3%81%B8%E3%81%AE%E5%85%A5%E5%8A%9B\" >\u30c7\u30b3\u30fc\u30c0\u3078\u306e\u5165\u529b<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E7%B2%BE%E5%BA%A6\" >\u7cbe\u5ea6<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E8%A6%81%E7%B4%84\" >\u8981\u7d04<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E6%96%87%E7%AB%A0%E7%94%9F%E6%88%90\" >\u6587\u7ae0\u751f\u6210<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-13\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E3%83%87%E3%83%BC%E3%82%BF%E3%82%B5%E3%82%A4%E3%82%BA%E3%81%A8%E7%B2%BE%E5%BA%A6\" >\u30c7\u30fc\u30bf\u30b5\u30a4\u30ba\u3068\u7cbe\u5ea6<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-14\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E4%BA%88%E6%B8%AC%E5%8D%98%E8%AA%9E%E6%95%B0%E3%81%AE%E9%81%95%E3%81%84\" >\u4e88\u6e2c\u5358\u8a9e\u6570\u306e\u9055\u3044<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-15\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E3%81%8A%E3%82%8F%E3%82%8A%E3%81%AB\" >\u304a\u308f\u308a\u306b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-16\" href=\"https:\/\/club.informatix.co.jp\/?p=1854\/#%E5%8F%82%E8%80%83%E6%96%87%E7%8C%AE\" >\u53c2\u8003\u6587\u732e<\/a><\/li><\/ul><\/nav><\/div>\n<h2><span class=\"ez-toc-section\" id=\"%E8%87%AA%E7%84%B6%E8%A8%80%E8%AA%9E%E5%87%A6%E7%90%86%E3%81%A8%E8%BB%A2%E7%A7%BB%E5%AD%A6%E7%BF%92\"><\/span>\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u3068\u8ee2\u79fb\u5b66\u7fd2<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p><a href=\"https:\/\/club.informatix.co.jp\/?p=922\" target=\"_blank\" rel=\"noopener noreferrer\">\u81ea\u7136\u8a00\u8a9e\u51e6\u7406<\/a>\u3068\u306f\u3001\u6a5f\u68b0\u306b\u65e5\u672c\u8a9e\u3084\u82f1\u8a9e\u7b49\u306e\u8a00\u8a9e\u3092\u7406\u89e3\u30fb\u5224\u5225\u3055\u305b\u305f\u308a\u751f\u6210\u3055\u305b\u305f\u308a\u3059\u308b\u51e6\u7406\u306e\u3053\u3068\u3067\u3059\u3002\u4f8b\u3048\u3070\u8ff7\u60d1\u30e1\u30fc\u30eb\u306e\u5224\u5225\u3001\u6587\u7ae0\u306e\u81ea\u52d5\u8981\u7d04\u3001\u65e5\u82f1\u7ffb\u8a33\u306a\u3069\u304c\u3053\u308c\u306b\u3042\u305f\u308a\u307e\u3059\u3002<\/p>\n<p>\u3053\u308c\u3089\u3092\u6a5f\u68b0\u5b66\u7fd2\u3067\u5b9f\u73fe\u3057\u3088\u3046\u3068\u3057\u305f\u3068\u304d\u306b\u554f\u984c\u306b\u306a\u308b\u306e\u304c\u3001\u6b63\u89e3\u4ed8\u304d\u5b66\u7fd2\u30c7\u30fc\u30bf\u306e\u4f5c\u6210\u3067\u3059\u3002<\/p>\n<p>\u5341\u5206\u306a\u7cbe\u5ea6\u3092\u51fa\u3059\u306b\u306f\u76f8\u5fdc\u306e\u91cf\u3092\u7528\u610f\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u3001\u591a\u304f\u306e\u5834\u5408\u4eba\u9593\u304c\u624b\u4f5c\u696d\u3067\u4f5c\u6210\u3059\u308b\u305f\u3081\u3001\u304b\u306a\u308a\u306e\u6642\u9593\u304c\u304b\u304b\u308a\u307e\u3059\u3002\u5148\u306e\u8981\u7d04\u306e\u4f8b\u3060\u3068\u3001\u30cb\u30e5\u30fc\u30b9\u8a18\u4e8b\u7b49\u306e\u6587\u7ae0\u306b\u5bfe\u3057\u3066\u305d\u306e\u8981\u7d04\u6587\u3092\u4f5c\u6210\u3059\u308b\u4f5c\u696d\u3067\u3059\u3002<\/p>\n<p>\u3053\u306e\u554f\u984c\u3092\u8efd\u6e1b\u3059\u308b\u305f\u3081\u3001\u6a5f\u68b0\u5b66\u7fd2\u3067\u306f\u300c\u8ee2\u79fb\u5b66\u7fd2\u300d\u3068\u3044\u3046\u624b\u6cd5\u304c\u3088\u304f\u7528\u3044\u3089\u308c\u307e\u3059\u3002\u3053\u306e\u624b\u6cd5\u3067\u306f<strong>\u2460\u4e8b\u524d\u5b66\u7fd2<\/strong>\u3068<strong>\u2461\u30d5\u30a1\u30a4\u30f3\u30c1\u30e5\u30fc\u30cb\u30f3\u30b0<\/strong>\u306e2\u6bb5\u968e\u3067\u5b66\u7fd2\u3055\u305b\u307e\u3059\u3002<\/p>\n<p>\u307e\u305a\u300c\u2460\u4e8b\u524d\u5b66\u7fd2\u300d\u3067\u3001Web\u7b49\u306b\u5927\u91cf\u306b\u5b58\u5728\u3057\u3066\u3044\u308b\u30c6\u30ad\u30b9\u30c8\u30c7\u30fc\u30bf\u3092\u7528\u3044\u3066\u6587\u6cd5\u3084\u5358\u8a9e\u306e\u610f\u5473\u7b49\u3092\u5b66\u7fd2\u3055\u305b\u307e\u3059\u3002\u3053\u306e\u5b66\u7fd2\u3067\u306f\u4eba\u9593\u304c\u6b63\u89e3\u30c7\u30fc\u30bf\u3092\u4f5c\u6210\u3059\u308b\u5fc5\u8981\u304c\u306a\u304f\u3001\u30c7\u30fc\u30bf\u3092\u53ce\u96c6\u3057\u3066\u304f\u308b\u3060\u3051\u3067\u3088\u3044\u306e\u3067\u5b89\u4fa1\u3067\u3059\u3002<\/p>\n<p>\u305d\u306e\u5f8c\u300c\u2461\u30d5\u30a1\u30a4\u30f3\u30c1\u30e5\u30fc\u30cb\u30f3\u30b0\u300d\u3067\u3001\u4eba\u9593\u304c\u7528\u610f\u3057\u305f\u30c7\u30fc\u30bf\u3092\u7528\u3044\u3066\u5b66\u7fd2\u3055\u305b\u307e\u3059\u3002\u65e2\u306b\u8a00\u8a9e\u306e\u57fa\u672c\u7684\u306a\u77e5\u8b58\u3092\u5099\u3048\u305f\u72b6\u614b\u3067\u5b66\u7fd2\u3059\u308b\u306e\u3067\u30010\u304b\u3089\u5b66\u7fd2\u3059\u308b\u306e\u306b\u6bd4\u3079\u3066\u5c11\u91cf\u306e\u30c7\u30fc\u30bf\u3067\u6e08\u307f\u307e\u3059\u3002\u2460\u3067\u5b66\u7fd2\u3057\u305f\u3082\u306e\u3092\u2461\u306e\u5b66\u7fd2\u306b\u8ee2\u7528\u30fb\u8ee2\u79fb\u3059\u308b\u306e\u3067<strong>\u8ee2\u79fb\u5b66\u7fd2<\/strong>\u3068\u547c\u3070\u308c\u307e\u3059\u3002<\/p>\n<p>\u4e8b\u524d\u5b66\u7fd2\u306e\u3055\u305b\u65b9\u306b\u3082\u8272\u3005\u306a\u3082\u306e\u304c\u63d0\u6848\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u3053\u306e\u90e8\u5206\u3092\u5de5\u592b\u3057\u305f\u3089\u6587\u7ae0\u8981\u7d04\u3084\u6587\u7ae0\u751f\u6210\u306e\u7cbe\u5ea6\u304c\u5411\u4e0a\u3057\u305f\u3068\u3044\u3046\u306e\u304c\u4eca\u56de\u7d39\u4ecb\u3059\u308b\u300cProphetNet\u300d\u3067\u3059\u3002<\/p>\n<p>\u203b\u672c\u8a18\u4e8b\u306f<a href=\"https:\/\/club.informatix.co.jp\/?p=1265\">\u524d\u56de\u8a18\u4e8b<\/a>\u540c\u69d8\u3001Transformer [2] \u306e\u77e5\u8b58\u304c\u3042\u308b\u7a0b\u5ea6\u3042\u308b\u3053\u3068\u3092\u524d\u63d0\u3068\u3057\u3066\u66f8\u304b\u308c\u3066\u3044\u307e\u3059\u3002Transformer\u306b\u3064\u3044\u3066\u3082\u3001\u3044\u305a\u308c\u89e3\u8aac\u3057\u305f\u3044\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"ProphetNet%E6%A6%82%E8%A6%81\"><\/span>ProphetNet\u6982\u8981<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u5148\u8ff0\u3057\u305f\u901a\u308a\u3001\u4e8b\u524d\u5b66\u7fd2\u306b\u306f\u3044\u308d\u3044\u308d\u306a\u3082\u306e\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n<p>\u4ee3\u8868\u7684\u306a\u3082\u306e\u3068\u3057\u3066\u3001\u6587\u7ae0\u3092\u9014\u4e2d\u307e\u3067\u4e0e\u3048\u3066\u6b21\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3055\u305b\u308b\u3082\u306e\uff08Language Model\uff09\u3084\u3001\u6587\u7ae0\u306e\u4e2d\u304b\u3089\u30e9\u30f3\u30c0\u30e0\u306b\u5358\u8a9e\u3092\u96a0\u3057\u866b\u98df\u3044\u72b6\u614b\u306b\u3057\u3066\u305d\u306e\u90e8\u5206\u3092\u4e88\u6e2c\u3055\u305b\u308b\u3082\u306e\uff08Denoise\uff09\u7b49\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n<p>\u4f8b\u3048\u3070\u300c\u543e\u8f29\u306f\u732b\u3067\u3042\u308b\u300d\u3068\u3044\u3046\u6587\u7ae0\u3092\u8003\u3048\u307e\u3059\u3002<\/p>\n<p>\u3053\u306e\u3068\u304d\u300c\u543e\u8f29\u306f\u300d\u307e\u3067\u3092\u4e0e\u3048\u3066\u300c\u732b\u300d\u3092\u4e88\u6e2c\u3055\u305b\u308b\u306e\u304cLanguage Model\u3067\u3001\u300c\u543e\u8f29\u306f\u00d7\u3067\u3042\u308b\u300d\u3092\u4e0e\u3048\u3066\u00d7\u306e\u90e8\u5206\u3092\u300c\u732b\u300d\u3068\u4e88\u6e2c\u3055\u305b\u308b\u306e\u304cDenoise\u3067\u3059\u3002<\/p>\n<p>\u4eca\u56de\u7d39\u4ecb\u3059\u308bProphetNet\u306fLanguage Model\u306b\u985e\u3059\u308b\u4e8b\u524d\u5b66\u7fd2\u3092\u884c\u3044\u307e\u3059\u3002<\/p>\n<p>\u5177\u4f53\u7684\u306b\u306f\u3001\u6587\u7ae0\u3092\u9014\u4e2d\u307e\u3067\u4e0e\u3048\u305f\u3068\u304d\u306b\u6b21\u306e\u5358\u8a9e\u306e\u307f\u3092\u4e88\u6e2c\u3059\u308b\u306e\u3067\u306f\u306a\u304f\\(N\\)\u500b\u5148\u307e\u3067\u4e88\u6e2c\u3055\u305b\u307e\u3059\u3002<\/p>\n<p>\u3064\u307e\u308a\\(N=2\\)\u306e\u5834\u5408\u3001\u5148\u307b\u3069\u306e\u4f8b\u3067\u306f\u300c\u543e\u8f29\u306f\u300d\u3092\u4e0e\u3048\u305f\u3068\u304d\u306b\u300c\u732b\u3067\u300d\u307e\u3067\u3092\u4e88\u6e2c\u3055\u305b\u308b\u3053\u3068\u306b\u5bfe\u5fdc\u3057\u307e\u3059\u3002<\/p>\n<p>\u56f3\u793a\u3059\u308b\u3068\u56f31\u3067\u3059\u3002<\/p>\n<p>\u56f3\u4e0b\u65b9\u306e\\(X_0\\)\uff5e\\(X_5\\)\u304c\u30e2\u30c7\u30eb\u3078\u306e\u5165\u529b\u3067\u3001\u56f3\u4e0a\u65b9\u306e\\(X_1\\)\uff5e\\(X_7\\)\u304c\u30e2\u30c7\u30eb\u306e\u51fa\u529b\u3067\u3059\u3002<\/p>\n<p>Language Model\u306f\u5404\u5358\u8a9e\u306b\u5bfe\u3057\u3066\u4e00\u3064\u306e\u6b21\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3057\u3066\u3044\u307e\u3059\u304c\u3001Bigram ProphetNet\u306f\u6b21\u3068\u305d\u306e\u6b21\u306e2\u3064\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3057\u307e\u3059\u3002<\/p>\n<p><img decoding=\"async\" class=\" wp-image-1857 aligncenter\" src=\"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020-p\/09\/p1-300x125.png\" alt=\"\" width=\"403\" height=\"168\" \/><\/p>\n<p style=\"text-align: center;\">\u56f31\uff1a\u53c2\u8003\u6587\u732e[1]\u3088\u308a\u629c\u7c8b<\/p>\n<p>\u3053\u306e\u3088\u3046\u306a\u624b\u6cd5\u304c\u63d0\u6848\u3055\u308c\u305f\u7d4c\u7def\u306b\u306f\u3001Language Model\u306e\u5834\u5408\u3001\u76f4\u524d\u306e\u5358\u8a9e\u306b\u5f37\u304f\u5f71\u97ff\u3055\u308c\u3066\u3057\u307e\u3044\u6587\u7ae0\u5168\u4f53\u3068\u3057\u3066\u306e\u610f\u5473\u3092\u6349\u3048\u304d\u308c\u3066\u3044\u306a\u3044\u3068\u6307\u6458\u3055\u308c\u3066\u3044\u305f\u3068\u3044\u3046\u3053\u3068\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n<p>\u4e88\u6e2c\u3059\u308b\u7bc4\u56f2\u3092\u5927\u304d\u304f\u3057\u3066\u3001\u3088\u308a\u6587\u7ae0\u5168\u4f53\u3092\u898b\u6e21\u305b\u308b\u3088\u3046\u306b\u5de5\u592b\u3057\u305f\u306e\u304c\u4eca\u56de\u63d0\u6848\u3055\u308c\u305f\u624b\u6cd5\u3067\u3059\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"ProphetNet%E8%A9%B3%E7%B4%B0\"><\/span>ProphetNet\u8a73\u7d30<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u305d\u308c\u3067\u306f\u3069\u306e\u3088\u3046\u306b\\(N\\)\u500b\u5148\u307e\u3067\u4e88\u6e2c\u3055\u305b\u308b\u306e\u3067\u3057\u3087\u3046\u304b\uff1f<\/p>\n<p>ProphetNet\u306fTransformer\u3092\u30d9\u30fc\u30b9\u306b\u4f5c\u3089\u308c\u305f\u30a8\u30f3\u30b3\u30fc\u30c0\u30fb\u30c7\u30b3\u30fc\u30c0\u30e2\u30c7\u30eb\u3067\u3059\u3002\u307e\u305a\u57fa\u672c\u7684\u306a\u69cb\u6210\u3092\u56f3\u793a\u3059\u308b\u3068\u56f32\u306e\u3088\u3046\u306b\u306a\u308a\u307e\u3059\u3002<\/p>\n<p><img decoding=\"async\" class=\"aligncenter wp-image-1859 size-full\" src=\"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020-p\/09\/p2-e1601361016939.png\" alt=\"\" width=\"500\" height=\"356\" \/><\/p>\n<p style=\"text-align: center;\">\u56f32\uff1a\u53c2\u8003\u6587\u732e[1]\u3088\u308a\u629c\u7c8b<\/p>\n<p>\u9806\u3092\u8ffd\u3063\u3066\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E3%83%8D%E3%83%83%E3%83%88%E3%83%AF%E3%83%BC%E3%82%AF%E3%81%AE%E5%85%A5%E5%87%BA%E5%8A%9B\"><\/span>\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u306e\u5165\u51fa\u529b<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u56f32\u5de6\u4e0b\u304c\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u306e\u5165\u529b\u3067\u3059\u3002\u4e8b\u524d\u5b66\u7fd2\u306e\u969b\u306f\u3001\u5358\u8a9e\u5217\u3092\u4e00\u5b9a\u78ba\u7387\u3067\u866b\u98df\u3044\u5f0f\u306b\u96a0\u3057\u305f\u3082\u306e\u3092\u5165\u529b\u3068\u3057\u307e\u3059\u3002<\/p>\n<p>\u56f32\u53f3\u4e0a\u304c\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u306e\u51fa\u529b\u3067\u3059\u3002\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u306f\u3001\u5165\u529b\u6587\u7ae0\u3092\u3082\u3068\u306b\u96a0\u3055\u308c\u305f\u5358\u8a9e\u304b\u3089\\(n\\)\u500b\u5148\u307e\u3067\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3057\u307e\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E3%82%A8%E3%83%B3%E3%82%B3%E3%83%BC%E3%83%80\"><\/span>\u30a8\u30f3\u30b3\u30fc\u30c0<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u56f32\u306e\u5de6\u4e2d\u306f\u30a8\u30f3\u30b3\u30fc\u30c0\u306b\u3042\u305f\u308a\u307e\u3059\u3002<\/p>\n<p>\u30a8\u30f3\u30b3\u30fc\u30c0\u90e8\u5206\u306f\u3001\u30aa\u30ea\u30b8\u30ca\u30eb\u306eTrasnformer\u306e\u3082\u306e\u3068\u5909\u66f4\u3042\u308a\u307e\u305b\u3093\u3002\u5165\u529b\u5358\u8a9e\u5217\\(x_1,\\dots ,x_M\\)\u306b\u5bfe\u3057\u3066\u51fa\u529b\u3092\\(H_{enc}\\)\u3068\u66f8\u304f\u3053\u3068\u306b\u3059\u308b\u3068\u3001\u30a8\u30f3\u30b3\u30fc\u30c0\u90e8\u5206\u306f<\/p>\n<p style=\"text-align: center;\">\\(H_{enc}=Encoder(x_1,\\dots ,x_M)\\)<\/p>\n<p>\u3068\u62bd\u8c61\u7684\u306b\u66f8\u3051\u307e\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E3%83%87%E3%82%B3%E3%83%BC%E3%83%80\"><\/span>\u30c7\u30b3\u30fc\u30c0<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u56f32\u306e\u53f3\u4e2d\u306f\u30c7\u30b3\u30fc\u30c0\u306b\u3042\u305f\u308a\u307e\u3059\u3002<\/p>\n<p>Transformer\u306e\u30c7\u30b3\u30fc\u30c0\u306f\u3001\u30a8\u30f3\u30b3\u30fc\u30c0\u306e\u51fa\u529b\u3068\\(t-1\\)\u756a\u76ee\u307e\u3067\u306e\u4e88\u6e2c\u3092\u7528\u3044\u3066\\(t\\)\u756a\u76ee\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3057\u307e\u3059\u304c\u3001\u305d\u306e\u4e88\u6e2c\u78ba\u7387\\(p(y_t|y_{&lt; t},x)\\)\u306f\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u66f8\u3051\u307e\u3059\u3002<\/p>\n<p style=\"text-align: center;\">\\(p(y_t|y_{&lt; t},x) =Decoder(y_{&lt; t},H_{enc})\\)<\/p>\n<p>ProphetNet\u30c7\u30b3\u30fc\u30c0\u90e8\u5206\u306f\u3001\u3053\u306e\u90e8\u5206\u3092\u6b21\u306e\u5358\u8a9e\u3060\u3051\u3067\u306a\u304f\\(n\\)\u500b\u5148\u307e\u3067\u4e88\u6e2c\u3059\u308b\u3088\u3046\u3001\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u5909\u66f4\u3057\u307e\u3059\u3002<\/p>\n<p style=\"text-align: center;\">\\(p(y_t|y_{&lt; t},x),\\dots ,p(y_{t+n-1}|y_{&lt; t},x) =Decoder(y_{&lt; t},H_{enc})\\)<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E7%9B%AE%E7%9A%84%E9%96%A2%E6%95%B0\"><\/span>\u76ee\u7684\u95a2\u6570<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>Language Model\u3067\u306f\u30b9\u30c6\u30c3\u30d7\u306b\u3064\u304d1\u5358\u8a9e\u3057\u304b\u4e88\u6e2c\u3057\u307e\u305b\u3093\u3067\u3057\u305f\u304c\u3001ProphetNet\u3067\u306f\u8907\u6570\u5358\u8a9e\u3092\u4e88\u6e2c\u3057\u307e\u3059\u3002\u305d\u306e\u305f\u3081\u3001\u305d\u308c\u3089\u306e\u51fa\u529b\u306b\u5bfe\u3057\u3066\u76ee\u7684\u95a2\u6570\u3092\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u62e1\u5f35\u3057\u307e\u3059\u3002<\/p>\n<p style=\"text-align: center;\">\\(\\mathcal{L}=-\\sum^{N-1}_{n=0}\\alpha_{n}\\left(\\sum^{T-n}_{t=1}\\log p_{\\theta}(y_{t+n}|y_{&lt; t},x)\\right)\\)<\/p>\n<p>\\(\\alpha_{n}\\)\u306f\u30cf\u30a4\u30d1\u30fc\u30d1\u30e9\u30e1\u30fc\u30bf\u3001\\(\\theta \\)\u306f\u30e2\u30c7\u30eb\u306e\u30d1\u30e9\u30e1\u30fc\u30bf\u3067\u3059\u3002\u6700\u521d\u306e\u548c\u306e\\(n=0\\)\u306e\u90e8\u5206\u304c\u30aa\u30ea\u30b8\u30ca\u30eb\u306eTransformer\u3067\u63a1\u7528\u3055\u308c\u3066\u3044\u308bLanguage Model\u306e\u640d\u5931\u95a2\u6570\u3067\u3001\\(n\\ge 1\\)\u306e\u90e8\u5206\u304c\u4eca\u56de\u8ffd\u52a0\u3055\u308c\u305f\u640d\u5931\u95a2\u6570\u306e\u90e8\u5206\u3067\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E3%83%87%E3%82%B3%E3%83%BC%E3%83%80%E3%81%AE%E6%B3%A8%E6%84%8F%E6%A9%9F%E6%A7%8B\"><\/span>\u30c7\u30b3\u30fc\u30c0\u306e\u6ce8\u610f\u6a5f\u69cb<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u56f32\u306e\u53f3\u4e2d\u306e\u30c7\u30b3\u30fc\u30c0\u306e\u5177\u4f53\u7684\u306a\u69cb\u6210\u3092\u7d39\u4ecb\u3057\u307e\u3059\u3002\u69cb\u6210\u3092\u307e\u3068\u3081\u305f\u3082\u306e\u304c\u56f33\u306b\u306a\u308a\u307e\u3059\u3002<\/p>\n<p><img decoding=\"async\" class=\"size-full wp-image-1878 aligncenter\" src=\"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020-p\/09\/p3.png\" alt=\"\" width=\"688\" height=\"474\" \/><\/p>\n<p style=\"text-align: center;\">\u56f33\uff1a\u53c2\u8003\u6587\u732e[1]\u3088\u308a\u629c\u7c8b<\/p>\n<p>\u307e\u305aAttention\u6a5f\u69cb\u3092\u5fa9\u7fd2\u3059\u308b\u3068\u30013\u7a2e\u985e\u306e\u30d9\u30af\u30c8\u30eb\\(Q\\)(query), \\(K\\)(key), \\(V\\)(value)\u3092\u5165\u529b\u3068\u3057\u305f\u3068\u304d\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u6f14\u7b97\u3059\u308b\u3082\u306e\u3067\u3059\u3002<\/p>\n<p style=\"text-align: center;\">\\( Attention(Q,K,V)=Softmax(\\frac{QK^{T}}{\\sqrt{d_k}})V\\)<\/p>\n<p>Transformer\u306e\u30aa\u30ea\u30b8\u30ca\u30eb\u306e\u30c7\u30b3\u30fc\u30c0\u306f2\u7a2e\u985e\u306eAttention\u6a5f\u69cb\u3092\u6301\u3061\u307e\u3059\u304c\u3001ProphetNet\u3067\u306fMasked Attention [2]\u3068\u547c\u3070\u308c\u308b\u65b9\u306b\u5909\u66f4\u3092\u52a0\u3048\u307e\u3059\u3002<\/p>\n<p>\\(k+1\\)\u5c64\u76ee\u3067\u306eTransformer\u306e\u30aa\u30ea\u30b8\u30ca\u30eb\u306eMasked Attention\u306e\u51fa\u529b\u306f\\(k\\)\u5c64\u76ee\u3067\u306e\\(t\\)\u756a\u76ee\u5358\u8a9e\u57cb\u3081\u8fbc\u307f\u3092\\(h^{(k)}_{t}\\)\u3068\u66f8\u304f\u3068\u3001\u56f33(a)\u306e\u3088\u3046\u306b\u4ee5\u4e0b\u3068\u306a\u308a\u307e\u3059\u3002<\/p>\n<p style=\"text-align: center;\">\\( h^{(k+1)}_{t}=Attention(h^{(k)}_t, h^{(k)}_{\\le t}, h^{(k)}_{\\le t} )\\)<\/p>\n<p>\u3053\u308c\u3092\u8ad6\u6587[1]\u3067\u306f<strong>main stream<\/strong>\u3068\u547c\u3093\u3067\u3044\u307e\u3059\u3002<\/p>\n<p>ProphetNet\u3067\u306f\u4eca\u307e\u3067\u306e\u5358\u8a9e\u306e\u60c5\u5831\u3092\u7528\u3044\u3066\\(n\\)\u500b\u5148\u307e\u3067\u306e\u5358\u8a9e\u306e\u4e88\u6e2c\u306e\u305f\u3081\u306e\u30d9\u30af\u30c8\u30eb\u3092\u305d\u308c\u305e\u308c\u306b\u51fa\u529b\u3059\u308b\u3088\u3046\u306b\u3057\u307e\u3059\u3002<\/p>\n<p>\u305d\u306e\u305f\u3081\u3001main stream\u3067\u8a08\u7b97\u3055\u308c\u3066\u3044\u305f\\(h^{(k)}_{l}\\)\u306b\u52a0\u3048\u3066\u30011\u500b\u5148\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3059\u308b\u305f\u3081\u306e\u65b0\u305f\u306a\u30d9\u30af\u30c8\u30eb\\(g^{(k)}_{l}\\)\u3092\u5c0e\u5165\u3057\u307e\u3059\u3002<\/p>\n<p>\\(k+1\\)\u5c64\u76ee\u3067\u306e\\(g\\)\u306e\u51fa\u529b\u306f\u56f33(b)\u306e\u3088\u3046\u306b\u4ee5\u4e0b\u3067\u8a08\u7b97\u3055\u308c\u307e\u3059\u3002<\/p>\n<p style=\"text-align: center;\">\\( g^{(k+1)}_{t}=Attention(g^{(k)}_t, h^{(k)}_{\\le t}\\oplus g^{(k)}, h^{(k)}_{\\le t}\\oplus g^{(k)} )\\)<\/p>\n<p>\u3053\u306e\u8a08\u7b97\u904e\u7a0b\u3092<strong>first predicting stream<\/strong>\u3068\u547c\u3073\u307e\u3059\u3002<\/p>\n<p>2\u500b\u5148\u306e\u5358\u8a9e\u307e\u3067\u4e88\u6e2c\u3059\u308b\u5834\u5408\u306f\u3001\u3055\u3089\u306b<strong>second predicting stream<\/strong>\u3092\u5c0e\u5165\u3057\u307e\u3059\u3002\u65b0\u305f\u306a\u30d9\u30af\u30c8\u30eb\\(s^{(k)}_{l}\\)\u3092\u5c0e\u5165\u3057\u3001\\(k+1\\)\u5c64\u76ee\u3067\u306e\\(s\\)\u306e\u51fa\u529b\u306f\\(g\\)\u540c\u69d8\u3001\u56f33(c)\u306e\u3088\u3046\u306b\u4ee5\u4e0b\u3067\u8a08\u7b97\u3055\u308c\u307e\u3059\u3002<\/p>\n<p style=\"text-align: center;\">\\( s^{(k+1)}_{t}=Attention(s^{(k)}_t, h^{(k)}_{\\le t}\\oplus s^{(k)}, h^{(k)}_{\\le t}\\oplus s^{(k)} )\\)<\/p>\n<p>\u4ee5\u964d\u540c\u69d8\u306b\\(N\\)\u500b\u5148\u307e\u3067\u4e88\u6e2c\u3055\u305b\u308b\u3068\u304d\u306f\\(N\\)-th predicting stream\u307e\u3067\u5c0e\u5165\u3057\u307e\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E3%83%87%E3%82%B3%E3%83%BC%E3%83%80%E3%81%B8%E3%81%AE%E5%85%A5%E5%8A%9B\"><\/span>\u30c7\u30b3\u30fc\u30c0\u3078\u306e\u5165\u529b<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u56f33(d)\u4e0b\u90e8\u306f\u30c7\u30b3\u30fc\u30c0\u3078\u306e\u5165\u529b\u3067\u3059\u3002<\/p>\n<p>main stream\u306b\u95a2\u3057\u3066\u306f\u30aa\u30ea\u30b8\u30ca\u30eb\u3068\u5909\u66f4\u3042\u308a\u307e\u305b\u3093\u304c\u3001\\(n\\)-th prediction stream (\\(1 \\leq n \\leq N\\))\u306b\u95a2\u3057\u3066\u306f\u5b66\u7fd2\u53ef\u80fd\u306a\u30d1\u30e9\u30e1\u30fc\u30bf\\(p_{n}\\)\u3092\u5165\u529b\u3068\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n<p>&nbsp;<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E7%B2%BE%E5%BA%A6\"><\/span>\u7cbe\u5ea6<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u809d\u5fc3\u306e\u4e8b\u524d\u5b66\u7fd2\u3067\u3059\u304c\u3001BERT\u306b\u5023\u3063\u3066\u82f1\u8a9e\u306eWikipedia\u3068BookCorpus\u3092\u7528\u3044\u3066\u3044\u307e\u3059\u3002<\/p>\n<p>\u305d\u306e\u4e8b\u524d\u5b66\u7fd2\u3057\u305f\u30e2\u30c7\u30eb\u3092\u500b\u3005\u306e\u30bf\u30b9\u30af\u306b\u30d5\u30a1\u30a4\u30f3\u30c1\u30e5\u30fc\u30cb\u30f3\u30b0\u3057\u305f\u7cbe\u5ea6\u3092\u4ee5\u4e0b\u3067\u898b\u3066\u3044\u304d\u307e\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E8%A6%81%E7%B4%84\"><\/span>\u8981\u7d04<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u307e\u305a\u306f\u88681\u3001\u88682\u306b\u6587\u7ae0\u8981\u7d04\u306e\u30bf\u30b9\u30af\u3067\u3042\u308b<strong>CNN\/Daily Mail<\/strong> \u3068 <strong>Giwaword<\/strong>\u306e\u7cbe\u5ea6\u3092\u305d\u308c\u305e\u308c\u63b2\u8f09\u3057\u307e\u3059\u3002\u3053\u308c\u3089\u306f\u8a18\u4e8b\u304b\u3089\u305d\u306e\u8981\u7d04\u6587\u3092\u63a8\u5b9a\u3059\u308b\u30bf\u30b9\u30af\u3067\u3059\u3002<\/p>\n<p><img decoding=\"async\" class=\"alignnone size-full wp-image-1900 aligncenter\" src=\"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020-p\/09\/p4.png\" alt=\"\" width=\"566\" height=\"257\" \/><\/p>\n<p style=\"text-align: center;\">\u88681\uff1a\u53c2\u8003\u6587\u732e[1]\u3088\u308a\u629c\u7c8b<\/p>\n<p><img decoding=\"async\" class=\"alignnone size-full wp-image-1904 aligncenter\" src=\"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020-p\/09\/p5.png\" alt=\"\" width=\"372\" height=\"180\" \/><\/p>\n<p style=\"text-align: center;\">\u88682\uff1a\u53c2\u8003\u6587\u732e[1]\u3088\u308a\u629c\u7c8b<\/p>\n<p>\u8a18\u4e8b\u57f7\u7b46\u6642\u306b\u306f\u3055\u3089\u306b\u7cbe\u5ea6\u306e\u9ad8\u3044\u30e2\u30c7\u30eb\u304c\u8003\u6848\u3055\u308c\u3066\u3044\u307e\u3059\u304c\u3001\u8ad6\u6587[1]\u3067\u6bd4\u8f03\u3055\u308c\u3066\u3044\u308b\u30e2\u30c7\u30eb\u306e\u4e2d\u3067\u306f\u6700\u3082\u7cbe\u5ea6\u304c\u9ad8\u3044\u7d50\u679c\u306b\u306a\u3063\u3066\u3044\u307e\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E6%96%87%E7%AB%A0%E7%94%9F%E6%88%90\"><\/span>\u6587\u7ae0\u751f\u6210<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u6b21\u306b\u88683\u306b\u6587\u7ae0\u751f\u6210\u30bf\u30b9\u30af\u306e\u7cbe\u5ea6\u3092\u63b2\u8f09\u3057\u307e\u3059\u3002<\/p>\n<p>\u4f7f\u7528\u3057\u305f\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8SQuAD1.1\u3067\u306f\u3001\u8a18\u4e8b\u3068\u305d\u306e\u5185\u5bb9\u306b\u57fa\u3065\u304f\u8cea\u554f\u6587\u3068\u305d\u306e\u7b54\u3048\u306e\uff13\u3064\u304c\u4e0e\u3048\u3089\u308c\u307e\u3059\u3002<\/p>\n<p>\u88683\u306f\u8a18\u4e8b\u3068\u7b54\u3048\u3092\u4e0e\u3048\u3066\u3001\u305d\u306e\u7b54\u3048\u3092\u5c0e\u304f\u8cea\u554f\u3092\u751f\u6210\u3059\u308b\u30bf\u30b9\u30af\u306e\u7cbe\u5ea6\u3067\u3059\u3002<\/p>\n<p><img decoding=\"async\" class=\"alignnone size-full wp-image-1915 aligncenter\" src=\"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020-p\/09\/p6.png\" alt=\"\" width=\"390\" height=\"294\" \/><\/p>\n<p style=\"text-align: center;\">\u88683\uff1a\u53c2\u8003\u6587\u732e[1]\u3088\u308a\u629c\u7c8b<\/p>\n<p>\u3053\u3061\u3089\u3082\u8a18\u4e8b\u57f7\u7b46\u6642\u306b\u306f\u3055\u3089\u306b\u7cbe\u5ea6\u306e\u9ad8\u3044\u30e2\u30c7\u30eb\u304c\u8003\u6848\u3055\u308c\u3066\u3044\u307e\u3059\u304c\u3001\u8ad6\u6587[1]\u3067\u6bd4\u8f03\u3055\u308c\u3066\u3044\u308b\u30e2\u30c7\u30eb\u306e\u4e2d\u3067\u306f\u6700\u3082\u7cbe\u5ea6\u304c\u9ad8\u3044\u7d50\u679c\u306b\u306a\u3063\u3066\u3044\u307e\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E3%83%87%E3%83%BC%E3%82%BF%E3%82%B5%E3%82%A4%E3%82%BA%E3%81%A8%E7%B2%BE%E5%BA%A6\"><\/span>\u30c7\u30fc\u30bf\u30b5\u30a4\u30ba\u3068\u7cbe\u5ea6<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u4e8b\u524d\u5b66\u7fd2\u3067\u7528\u3044\u308b\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u3092\u5927\u304d\u304f\u3059\u308b\u3068\u7cbe\u5ea6\u304c\u4e0a\u304c\u308b\u3053\u3068\u304c\u77e5\u3089\u308c\u3066\u3044\u307e\u3059\u304c\u3001ProphetNet\u3067\u3082\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u3092\u5927\u304d\u304f\u3057\u3066\u307f\u3066\u7cbe\u5ea6\u3092\u691c\u8a3c\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n<p>\u7d50\u679c\u3092\u88684\u306b\u63b2\u8f09\u3057\u307e\u3059\u3002<\/p>\n<p><img decoding=\"async\" class=\"alignnone size-full wp-image-1917 aligncenter\" src=\"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020-p\/09\/p7.png\" alt=\"\" width=\"797\" height=\"263\" \/><\/p>\n<p style=\"text-align: center;\">\u88684\uff1a\u53c2\u8003\u6587\u732e[1]\u3088\u308a\u629c\u7c8b<\/p>\n<p>\u88682\u3001\u88683\u3068\u6bd4\u3079\u3066\u7cbe\u5ea6\u304c\u5411\u4e0a\u3057\u3066\u3044\u308b\u3053\u3068\u304c\u5206\u304b\u308a\u307e\u3059\u3002\u307e\u305f\u3001\u305d\u306e\u307b\u304b\u306e\u30e2\u30c7\u30eb\u3068\u6bd4\u8f03\u3059\u308b\u3068\u3001\u5c0f\u3055\u306a\u30c7\u30fc\u30bf\u30b5\u30a4\u30b9\u3067\u540c\u7b49\u306e\u7cbe\u5ea6\u3092\u9054\u6210\u3057\u3066\u3044\u308b\u3053\u3068\u304c\u5206\u304b\u308a\u307e\u3059\u3002<\/p>\n<p>\u4ed6\u306e\u30e2\u30c7\u30eb\u306b\u6bd4\u3079\u3066\u52b9\u7387\u7684\u306b\u5b66\u7fd2\u3067\u304d\u3066\u3044\u308b\u3088\u3046\u3067\u3059\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E4%BA%88%E6%B8%AC%E5%8D%98%E8%AA%9E%E6%95%B0%E3%81%AE%E9%81%95%E3%81%84\"><\/span>\u4e88\u6e2c\u5358\u8a9e\u6570\u306e\u9055\u3044<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u6700\u5f8c\u306b\u3069\u306e\u304f\u3089\u3044\u5148\u307e\u3067\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3055\u305b\u308b\u304b\u3067\u3001\u3069\u306e\u304f\u3089\u3044\u7cbe\u5ea6\u306b\u9055\u3044\u304c\u751f\u3058\u308b\u304b\u306e\u7d50\u679c\u3092\u63b2\u8f09\u3057\u307e\u3059\u3002<\/p>\n<p>\u88685\u3067\u306f1\uff5e3\u5358\u8a9e\u5148\u307e\u3067\u3092\u4e88\u6e2c\u3055\u305b\u305f\u3082\u306e\u3067\u306e\u7cbe\u5ea6\u306e\u9055\u3044\u3092\u898b\u3066\u3044\u307e\u3059\u3002\u8a55\u4fa1\u306b\u7528\u3044\u305f\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306f\u6587\u7ae0\u8981\u7d04\u30bf\u30b9\u30af\u3067\u3042\u308b<strong>CNN\/DailyMail<\/strong>\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u3067\u3059\u3002<\/p>\n<p><img decoding=\"async\" class=\"alignnone size-full wp-image-1919 aligncenter\" src=\"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020-p\/09\/p8.png\" alt=\"\" width=\"363\" height=\"145\" \/><\/p>\n<p style=\"text-align: center;\">\u88685\uff1a\u53c2\u8003\u6587\u732e[1]\u3088\u308a\u629c\u7c8b<\/p>\n<p>\u4e88\u6e2c\u5358\u8a9e\u6570\u3092\u5927\u304d\u304f\u3059\u308c\u3070\u7cbe\u5ea6\u304c\u4e0a\u304c\u3063\u3066\u3044\u308b\u3053\u3068\u304c\u5206\u304b\u308a\u307e\u3059\u3002<\/p>\n<p>\u4e88\u6e2c\u3059\u308b\u5358\u8a9e\u6570\u304c\u591a\u304f\u306a\u308c\u3070\u3001\u305d\u308c\u3060\u3051\u6b63\u3057\u304f\u6587\u8108\u3092\u7406\u89e3\u3057\u306a\u3044\u3068\u3044\u3051\u306a\u3044\u306e\u3067\u81ea\u7136\u306a\u7d50\u679c\u3067\u3059\u306d\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E3%81%8A%E3%82%8F%E3%82%8A%E3%81%AB\"><\/span>\u304a\u308f\u308a\u306b<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u4e00\u3064\u5148\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3059\u308b\u3060\u3051\u3067\u306f\u306a\u304f\u3001N\u500b\u5148\u307e\u3067\u4e88\u6e2c\u3055\u305b\u3066\u307f\u308b\u3068\u3044\u3046\u767a\u60f3\u3067\u7cbe\u5ea6\u3092\u5411\u4e0a\u3055\u305b\u305f\u3068\u3044\u3046\u8ad6\u6587\u3067\u3057\u305f\u3002<\/p>\n<p>\u3069\u306e\u3088\u3046\u306a\u624b\u6cd5\u304c\u7cbe\u5ea6\u304c\u51fa\u308b\u304b\u7b49\u3092\u5b9f\u9a13\u3067\u78ba\u304b\u3081\u308b\u3068\u3044\u3046\u306e\u306f\u3053\u306e\u5206\u91ce\u306e\u738b\u9053\u3067\u3059\u304c\u3001\u305d\u308c\u306b\u52a0\u3048\u6570\u5b66\u7684\u306a\u89e3\u6790\u304c\u306a\u3055\u308c\u308b\u3088\u3046\u306b\u306a\u308b\u3068\u3088\u308a\u9762\u767d\u3044\u304b\u306a\u3068\u601d\u3044\u307e\u3057\u305f\u3002<\/p>\n<p>\u305d\u306e\u305f\u3081\u306b\u306f\u3082\u3046\u5c11\u3057\u5206\u91ce\u306e\u6210\u719f\u304c\u5fc5\u8981\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u304c\u3001\u4eca\u5f8c\u306e\u7814\u7a76\u306b\u671f\u5f85\u3057\u305f\u3044\u3067\u3059\u3002<\/p>\n<p>\u672c\u8a18\u4e8b\u3067\u306f\u7701\u7565\u3057\u305f\u90e8\u5206\u3082\u3042\u308b\u306e\u3067\u3001\u8a73\u7d30\u304c\u6c17\u306b\u306a\u308b\u65b9\u306f\u5143\u8ad6\u6587[1]\u3092\u53c2\u7167\u3055\u308c\u308b\u3053\u3068\u3092\u304a\u52e7\u3081\u3057\u307e\u3059\u3002<\/p>\n<div class=\"st-header-flexwrap\" style=\"height: auto; width: 100%; box-sizing: border-box; background-color: #efefef; border-radius: 5px; border: solid #ccc 1px; padding: 20px; margin-bottom: 0px;\">\n<div class=\"st-header-flexbox\">\n<p class=\"st-header-flextitle\" style=\"font-size: 110%; color: #000;\">\u7269\u4f53\u691c\u51fa\u306a\u3069AI\u6a5f\u68b0\u5b66\u7fd2\u3092\u6d3b\u7528\u3057\u305f\u30b7\u30b9\u30c6\u30e0\u306e\u69cb\u7bc9\u306b\u95a2\u3059\u308b\u3054\u76f8\u8ac7\uff08\u7121\u6599\uff09\u3092\u627f\u3063\u3066\u3044\u307e\u3059<\/p>\n<p><span class=\"st-mybtn st-mybtn-mini\"><a style=\"background: #01579b; border-radius: 5px; font-weight: normal; color: #fff; box-shadow: 0 3px 0 #01579b;\" href=\"https:\/\/www.informatix.co.jp\/contact-us\/\" target=\"_blank\" rel=\"noopener\">\u304a\u6c17\u8efd\u306b\u304a\u554f\u3044\u5408\u308f\u305b\u304f\u3060\u3055\u3044<i class=\"fa fa-after fa-angle-right st-css-no\" aria-hidden=\"true\"><\/i><\/a><\/span><\/p>\n<\/div>\n<\/div>\n<p>&nbsp;<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E5%8F%82%E8%80%83%E6%96%87%E7%8C%AE\"><\/span>\u53c2\u8003\u6587\u732e<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>[1]Yan, Y., Qi, W., Gong, Y., Liu, D., Duan, N., Chen, J., Zhang, R., &amp; Zhou, M. (2020). ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training.\u00a0<em>ArXiv, abs\/2001.04063<\/em>.<\/p>\n<p>[2]Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, L., &amp; Polosukhin, I. (2017). Attention is All you Need. <em>ArXiv, abs\/1706.03762<\/em>.<br \/>\n<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u3053\u3093\u306b\u3061\u306f\u3001\u30a4\u30f3\u30d5\u30a9\u30de\u30c6\u30a3\u30af\u30b9\u3067\u6a5f\u68b0\u5b66\u7fd2\u696d\u52d9\u3092\u62c5\u5f53\u3057\u3066\u3044\u308b\u5927\u6a4b\u3067\u3059\u3002 \u4eca\u56de\u306f\u30c7\u30a3\u30fc\u30d7\u30e9\u30fc\u30cb\u30f3\u30b0\u3092\u7528\u3044\u305f\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306e\u8ad6\u6587\u3001ProphetNet [1] \u3092\u7d39\u4ecb\u3057\u305f\u3044\u3068\u601d\u3044\u307e\u3059\u3002 \u65e2\u5b58\u306e\u69cb\u9020\u306b\u5bfe\u3059\u308b\u4fee\u6b63 &#8230; <\/p>\n","protected":false},"author":11,"featured_media":11235,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[589,4],"tags":[732,608],"class_list":["post-1854","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-ai","category-4","tag-transformer","tag-608"],"jetpack_featured_media_url":"https:\/\/club.informatix.co.jp\/wp-content\/uploads\/2020\/11\/p1.png","_links":{"self":[{"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=\/wp\/v2\/posts\/1854","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=\/wp\/v2\/users\/11"}],"replies":[{"embeddable":true,"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1854"}],"version-history":[{"count":15,"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=\/wp\/v2\/posts\/1854\/revisions"}],"predecessor-version":[{"id":18241,"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=\/wp\/v2\/posts\/1854\/revisions\/18241"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=\/wp\/v2\/media\/11235"}],"wp:attachment":[{"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1854"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1854"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/club.informatix.co.jp\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1854"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}