(* Content-type: application/vnd.wolfram.mathematica *) (*** Wolfram Notebook File ***) (* http://www.wolfram.com/nb *) (* CreatedBy='Mathematica 11.1' *) (*CacheID: 234*) (* Internal cache information: NotebookFileLineBreakTest NotebookFileLineBreakTest NotebookDataPosition[ 158, 7] NotebookDataLength[ 399577, 6721] NotebookOptionsPosition[ 396880, 6666] NotebookOutlinePosition[ 397274, 6682] CellTagsIndexPosition[ 397231, 6679] WindowFrame->Normal*) (* Beginning of Notebook Content *) Notebook[{ Cell[CellGroupData[{ Cell[TextData[{ StyleBox["Construction Notebook for:", FontSize->24], "\nBERT Trained on BookCorpus and Wikipedia Data" }], "Title", CellChangeTimes->{ 3.703596290184906*^9, 3.704015777970223*^9, {3.7112829422331343`*^9, 3.71128294621737*^9}, {3.759063852165167*^9, 3.759063852467972*^9}, 3.760082541657879*^9, {3.760084783168932*^9, 3.760084783873008*^9}, { 3.760976072757799*^9, 3.760976073508733*^9}, 3.763380994552088*^9, 3.7970812599393682`*^9},ExpressionUUID->"a33618b8-7416-413f-9875-\ 525e49421225"], Cell[CellGroupData[{ Cell["NetModel Access", "Subsection",ExpressionUUID->"05498e76-0a7f-48f3-8742-2a22e96acb4d"], Cell[CellGroupData[{ Cell["This Notebook", "Subsubsection",ExpressionUUID->"7b1b33a9-3717-469b-8b65-a7035e35aaa9"], Cell[BoxData[ RowBox[{"NetModel", "[", RowBox[{ "\"\\"", ",", " ", "\"\\""}], "]"}]], "Input", CellChangeTimes->{ 3.703596295151729*^9, 3.7040157830563927`*^9, 3.7112829386133738`*^9, { 3.75906385453332*^9, 3.759063856020535*^9}, {3.7600794761205378`*^9, 3.7600794835667553`*^9}, {3.760079576083949*^9, 3.76007957919051*^9}, { 3.760084772747191*^9, 3.760084773594036*^9}, {3.760160008262457*^9, 3.7601600090532417`*^9}, {3.7609760767176313`*^9, 3.76097608673199*^9}, 3.763380994555273*^9, 3.7970812568967648`*^9},ExpressionUUID->"b4e81b34-93b5-4f07-89a8-\ 04ddeb31a697"] }, Open ]], Cell[CellGroupData[{ Cell["Untrained Net", "Subsubsection",ExpressionUUID->"950658ac-daca-42dc-9c33-5fd10a6cd280"], Cell[BoxData[ RowBox[{"NetModel", "[", RowBox[{ "\"\\"", ",", " ", "\"\\""}], "]"}]], "Input", CellChangeTimes->{ 3.703596299712925*^9, 3.704015788088829*^9, 3.711282939761187*^9, { 3.7590638588363953`*^9, 3.759063859138372*^9}, 3.760079589046831*^9, { 3.760084775697905*^9, 3.7600847761538677`*^9}, {3.760160011180855*^9, 3.76016001172514*^9}, {3.760976080172661*^9, 3.760976085283339*^9}, { 3.76338099455842*^9, 3.7633810106849337`*^9}, 3.797081257839881*^9},ExpressionUUID->"ca3c3f5b-06b7-49d4-823d-\ 9a7faa05773d"] }, Open ]], Cell[CellGroupData[{ Cell["Trained Net", "Subsubsection",ExpressionUUID->"11624637-737b-4e92-b609-e11fcbf10ae2"], Cell[BoxData[ RowBox[{"NetModel", "[", "\"\\"", "]"}]], "Input", CellChangeTimes->{ 3.703596303209467*^9, 3.704015790751824*^9, 3.711282940752515*^9, { 3.759063863948758*^9, 3.7590638642527237`*^9}, {3.760079590533895*^9, 3.760079594340671*^9}, {3.760084778625613*^9, 3.760084780993493*^9}, { 3.7601600133247433`*^9, 3.760160015196162*^9}, {3.7609760910207043`*^9, 3.760976093659471*^9}, 3.763380994559218*^9, 3.797081258501999*^9},ExpressionUUID->"12df8018-c460-4308-a6fa-\ ea80f797b729"] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell["Net Construction", "Section",ExpressionUUID->"cd545e68-ff54-4416-8f56-6fd64aedca73"], Cell[CellGroupData[{ Cell["Internal Functions", "Subsection",ExpressionUUID->"55fb2a0d-2d0b-4adf-a72e-6c7fc391c78a"], Cell[BoxData[ RowBox[{ RowBox[{"multiHeadAttention", "[", " ", RowBox[{"numHeads_", ",", "numHiddens_", ",", RowBox[{"causal_", ":", "False"}]}], "]"}], " ", ":=", "\[IndentingNewLine]", RowBox[{"NetGraph", "[", RowBox[{ RowBox[{"<|", "\[IndentingNewLine]", RowBox[{ RowBox[{"\"\\"", "\[Rule]", RowBox[{"NetMapOperator", "[", RowBox[{"{", RowBox[{"numHeads", ",", "numHiddens"}], "}"}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"NetMapOperator", "[", RowBox[{"{", RowBox[{"numHeads", ",", "numHiddens"}], "}"}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"NetMapOperator", "[", RowBox[{"{", RowBox[{"numHeads", ",", "numHiddens"}], "}"}], "]"}]}], ",", "\[IndentingNewLine]", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"ElementwiseLayer", "[", RowBox[{ RowBox[{"#", "/", RowBox[{"Sqrt", "[", "numHiddens", "]"}]}], " ", "&"}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"AttentionLayer", "[", RowBox[{"\"\\"", ",", " ", RowBox[{"\"\\"", " ", "->", " ", "None"}], " ", ",", " ", "\n", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"If", "[", RowBox[{"causal", ",", " ", "\"\\"", ",", " ", "None"}], "]"}]}], ",", RowBox[{"\"\\"", "\[Rule]", "True"}]}], "]"}]}], ",", "\n", RowBox[{"\"\\"", "\[Rule]", RowBox[{"NetMapOperator", "[", RowBox[{"numHeads", "*", "numHiddens"}], "]"}]}]}], "|>"}], ",", "\[IndentingNewLine]", RowBox[{"{", "\[IndentingNewLine]", RowBox[{ RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], " ", "\[Rule]", " ", RowBox[{"\"\\"", " ", "\[Rule]", " ", RowBox[{"NetPort", "[", RowBox[{"\"\\"", ",", "\"\\""}], "]"}]}]}], ",", "\[IndentingNewLine]", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], " ", "\[Rule]", " ", RowBox[{"\"\\"", " ", "\[Rule]", " ", RowBox[{"NetPort", "[", RowBox[{"\"\\"", ",", " ", "\"\\""}], "]"}]}]}], ",", "\[IndentingNewLine]", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], " ", "\[Rule]", " ", RowBox[{"\"\\"", " ", "\[Rule]", " ", RowBox[{"\"\\"", " ", "\[Rule]", " ", RowBox[{"NetPort", "[", RowBox[{"\"\\"", ",", " ", "\"\\""}], "]"}]}]}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", "\"\\""}]}], "\[IndentingNewLine]", "}"}]}], "]"}]}]], "Input", CellChangeTimes->{{3.797702822827237*^9, 3.79770294952463*^9}, { 3.7977031777544394`*^9, 3.7977032491988993`*^9}, {3.797703280325777*^9, 3.797703358490405*^9}, {3.7977034188091383`*^9, 3.79770350025205*^9}, { 3.797705056586336*^9, 3.7977050761358643`*^9}, {3.797705376109312*^9, 3.797705377739538*^9}}, CellLabel->"In[1]:=",ExpressionUUID->"bd9a6749-eda1-4474-b3c5-5e12945db7c9"], Cell[BoxData[ RowBox[{ RowBox[{ RowBox[{"multiHeadAttentionBlock", "[", "args__", "]"}], " ", ":=", " ", RowBox[{"NetGraph", "[", "\n", " \t", RowBox[{ RowBox[{"<|", "\n", " \t\t", RowBox[{ RowBox[{"\"\\"", " ", "->", " ", RowBox[{"multiHeadAttention", "[", "args", "]"}]}], ",", "\n", " \t\t", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"DropoutLayer", "[", "0.1", "]"}]}], ",", "\n", " \t\t", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"ThreadingLayer", "[", "Plus", "]"}]}], ",", "\n", " \t\t", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"NormalizationLayer", "[", RowBox[{ RowBox[{"2", " ", ";;"}], ",", " ", "\"\\"", ",", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"10", "^", RowBox[{"-", "12"}]}]}]}], "]"}]}]}], "\n", " \t", "|>"}], ",", "\n", " \t", RowBox[{"{", RowBox[{ RowBox[{"\"\\"", " ", "->", " ", "\"\\""}], ",", " ", RowBox[{ RowBox[{"{", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], ",", " ", "\"\\""}], "}"}], " ", "->", " ", "\n", " ", RowBox[{"\"\\"", " ", "->", " ", "\"\\""}]}]}], "}"}]}], "\n", " ", "]"}]}], ";"}]], "Input", CellChangeTimes->{{3.760079663274143*^9, 3.7600796686566343`*^9}}, CellLabel->"In[2]:=",ExpressionUUID->"6930d9bf-6317-42e4-a088-5b32a5c15609"], Cell[BoxData[ RowBox[{ RowBox[{ RowBox[{"selfAttentionBlock", "[", "args__", "]"}], " ", ":=", " ", "\n", " ", RowBox[{"With", "[", RowBox[{ RowBox[{"{", RowBox[{"multihead", " ", "=", " ", RowBox[{"multiHeadAttentionBlock", "[", "args", "]"}]}], "}"}], ",", "\n", " \t", RowBox[{"NetGraph", "[", "\n", " \t\t", RowBox[{ RowBox[{"Normal", "[", "multihead", "]"}], ",", "\n", " \t\t", RowBox[{"ReplaceAll", "[", RowBox[{ RowBox[{"EdgeList", "[", "multihead", "]"}], ",", " ", "\n", " ", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], " ", "->", " ", RowBox[{"NetPort", "[", "\"\\"", "]"}]}]}], "]"}]}], "\n", " \t", "]"}]}], "\n", " ", "]"}]}], ";"}]], "Input", CellChangeTimes->{{3.760079663274143*^9, 3.7600796720081253`*^9}}, CellLabel->"In[3]:=",ExpressionUUID->"0c232b85-aae7-4dd6-bb24-e20693260987"], Cell[BoxData[ RowBox[{ RowBox[{ RowBox[{"feedForwardBlock", "[", RowBox[{"inputDim_", ",", " ", "numHiddens_", ",", " ", RowBox[{"normalization_", ":", " ", "True"}]}], "]"}], " ", ":=", " ", "\n", " ", RowBox[{"NetGraph", "[", "\n", " \t", RowBox[{ RowBox[{"<|", "\n", " \t\t", RowBox[{ RowBox[{"\"\\"", " ", "->", " ", RowBox[{"NetMapOperator", "[", "numHiddens", "]"}]}], ",", "\n", " \t", "\n", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"ElementwiseLayer", "[", RowBox[{ RowBox[{"(", RowBox[{"0.5", " ", "#", " ", "*", RowBox[{"(", RowBox[{"1.0", " ", "+", " ", RowBox[{"Erf", "[", RowBox[{"#", "/", RowBox[{"Sqrt", "[", "2.0", "]"}]}], "]"}]}], ")"}]}], ")"}], " ", "&"}], "]"}]}], ",", "\n", " \t", "\n", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"NetMapOperator", "[", "inputDim", "]"}]}], ",", "\n", " \t\t", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"DropoutLayer", "[", "0.1", "]"}]}], ",", "\n", " \t\t", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"ThreadingLayer", "[", "Plus", "]"}]}], ",", " ", RowBox[{"(*", " ", RowBox[{"Residual", " ", "connection"}], " ", "*)"}], "\n", " \t\t", "\n", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"NormalizationLayer", "[", RowBox[{ RowBox[{"2", " ", ";;"}], ",", " ", "\"\\"", ",", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"10", "^", RowBox[{"-", "12"}]}]}]}], "]"}]}]}], " ", RowBox[{"(*", " ", "\n", " ", "LayerNorm", " ", "*)"}], "\n", " \t", "|>"}], ",", "\n", " \t", RowBox[{"{", RowBox[{ RowBox[{"\"\\"", " ", "->", " ", "\n", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"\"\\"", " ", "->", " ", "\"\\""}]}]}], ",", " ", RowBox[{ RowBox[{"{", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], ",", " ", "\n", " ", "\"\\""}], "}"}], " ", "->", " ", RowBox[{"\"\\"", " ", "->", " ", "\"\\""}]}]}], "}"}], ",", "\n", " \t", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"{", RowBox[{"\"\\"", ",", " ", "Automatic"}], "}"}]}]}], "\n", " ", "]"}]}], ";"}]], "Input", CellChangeTimes->{{3.760079663274143*^9, 3.7600796756479692`*^9}}, CellLabel->"In[4]:=",ExpressionUUID->"29f94bc3-a119-4807-9a4a-84a475ef73b8"], Cell[BoxData[ RowBox[{ RowBox[{"encoderBlock", "[", RowBox[{ "attentionHeads_", ",", " ", "attentionHiddens_", ",", " ", "\n", " ", "feedForwardHiddens_"}], "]"}], " ", ":=", " ", RowBox[{"NetChain", "[", RowBox[{"{", "\n", " \t", RowBox[{ RowBox[{"selfAttentionBlock", "[", RowBox[{"attentionHeads", ",", " ", "attentionHiddens"}], "]"}], ",", "\n", " \t", RowBox[{"feedForwardBlock", "[", RowBox[{ RowBox[{"attentionHeads", " ", "*", " ", "attentionHiddens"}], ",", " ", "\n", " ", "feedForwardHiddens"}], "]"}]}], "\n", " ", "}"}], "]"}]}]], "Input", CellChangeTimes->{{3.760079663274143*^9, 3.7600796756479692`*^9}}, CellLabel->"In[5]:=",ExpressionUUID->"7bedcbe1-0b2b-474a-8d94-195c1df43fdc"] }, Open ]], Cell[CellGroupData[{ Cell["Final Net", "Subsection", CellChangeTimes->{ 3.760082607940754*^9},ExpressionUUID->"b9859337-b9ae-415f-a290-\ 86e9a29b57c5"], Cell["\<\ Returns the Final net. Argument name is one of: \t\[OpenCurlyDoubleQuote]BaseUncased\[CloseCurlyDoubleQuote], \t\[OpenCurlyDoubleQuote]LargeUncased\[CloseCurlyDoubleQuote], \t\[OpenCurlyDoubleQuote]BaseCased\[CloseCurlyDoubleQuote], \t\[OpenCurlyDoubleQuote]LargeCased\[CloseCurlyDoubleQuote], \t\[OpenCurlyDoubleQuote]MultilingualUncased\[CloseCurlyDoubleQuote], \t\[OpenCurlyDoubleQuote]MultilingualCased\[CloseCurlyDoubleQuote], \t\[OpenCurlyDoubleQuote]BaseChinese\[CloseCurlyDoubleQuote].\ \>", "Text", CellChangeTimes->{{3.760083845237187*^9, 3.7600838834056187`*^9}, { 3.763912488184597*^9, 3.7639125031105423`*^9}, {3.793521520628243*^9, 3.793521522975457*^9}},ExpressionUUID->"d2676916-2cca-4a30-b4e1-\ 373a4e987547"], Cell[BoxData[ RowBox[{ RowBox[{"createBERT", "[", "modelname_", "]"}], ":=", " ", RowBox[{"Module", "[", RowBox[{ RowBox[{"{", RowBox[{ "numStackEncoder", ",", " ", "embeddingSize", ",", " ", "feedForwardHiddens", ",", "attentionHeads", ",", "numWords", ",", "\[IndentingNewLine]", "attentionHiddens", ",", "numTokens", " ", ",", "numPos", ",", " ", "encoder", ",", "embedding", ",", " ", "transformer"}], "}"}], ",", "\[IndentingNewLine]", "\n", "\t\t ", RowBox[{ RowBox[{"Which", "[", "\[IndentingNewLine]", "\t\t ", RowBox[{ RowBox[{"modelname", " ", "==", " ", "\"\\""}], ",", "\n", "\t\t\t", RowBox[{ RowBox[{"{", RowBox[{ "numStackEncoder", ",", " ", "embeddingSize", ",", " ", "feedForwardHiddens", ",", "attentionHeads", ",", "numWords"}], "}"}], "=", "\n", "\t\t\t", RowBox[{"{", RowBox[{ "12", ",", " ", "768", ",", " ", "3072", ",", " ", "12", ",", " ", "30522"}], "}"}]}], ",", "\n", "\t\t\t", RowBox[{"modelname", " ", "==", " ", "\"\\""}], ",", "\n", "\t\t\t", RowBox[{ RowBox[{"{", RowBox[{ "numStackEncoder", ",", " ", "embeddingSize", ",", " ", "feedForwardHiddens", ",", "attentionHeads", ",", "numWords"}], "}"}], "=", "\n", "\t\t\t", RowBox[{"{", RowBox[{ "12", ",", " ", "768", ",", " ", "3072", ",", " ", "12", ",", " ", "28996"}], "}"}]}], ",", "\n", "\t\t\t", RowBox[{"modelname", " ", "==", " ", "\"\\""}], ",", "\n", "\t\t\t", RowBox[{ RowBox[{"{", RowBox[{ "numStackEncoder", ",", " ", "embeddingSize", ",", " ", "feedForwardHiddens", ",", "attentionHeads", ",", "numWords"}], "}"}], "=", "\n", "\t\t\t", RowBox[{"{", RowBox[{ "24", ",", " ", "1024", ",", " ", "4096", ",", " ", "16", ",", " ", "30522"}], "}"}]}], ",", "\n", "\t\t\t", RowBox[{"modelname", " ", "==", " ", "\"\\""}], ",", "\n", "\t\t\t", RowBox[{ RowBox[{"{", RowBox[{ "numStackEncoder", ",", " ", "embeddingSize", ",", " ", "feedForwardHiddens", ",", "attentionHeads", ",", "numWords"}], "}"}], "=", "\n", "\t\t\t", RowBox[{"{", RowBox[{ "24", ",", " ", "1024", ",", " ", "4096", ",", " ", "16", ",", " ", "28996"}], "}"}]}]}], "\n", "\t\t\t", "]"}], ";", "\[IndentingNewLine]", "\n", "\t", RowBox[{"attentionHiddens", " ", "=", " ", "64"}], ";", "\n", "\t", RowBox[{"numTokens", " ", "=", " ", "2"}], ";", "\n", "\t", RowBox[{"numPos", " ", "=", " ", "512"}], ";", "\[IndentingNewLine]", "\n", "\t", RowBox[{"encoder", " ", "=", " ", RowBox[{"NetGraph", "[", "\n", "\t\t", RowBox[{ RowBox[{"Table", "[", "\n", "\t\t\t", RowBox[{ RowBox[{"encoderBlock", "[", RowBox[{ "attentionHeads", ",", " ", "attentionHiddens", ",", " ", "feedForwardHiddens"}], "]"}], ",", "\n", "\t\t\t", "numStackEncoder"}], "\n", "\t\t", "]"}], ",", "\n", "\t\t", RowBox[{"Table", "[", RowBox[{ RowBox[{"i", " ", "->", " ", RowBox[{"i", " ", "+", " ", "1"}]}], ",", " ", RowBox[{"{", RowBox[{"i", ",", " ", RowBox[{"numStackEncoder", " ", "-", " ", "1"}]}], "}"}]}], "]"}]}], "\n", "\t\t", "]"}]}], ";", "\n", "\n", "\t", RowBox[{"embedding", " ", ":=", " ", RowBox[{"NetGraph", "[", RowBox[{ RowBox[{"<|", "\n", "\t\t", RowBox[{ RowBox[{"\"\\"", " ", "->", " ", RowBox[{"PartLayer", "[", RowBox[{"{", RowBox[{"All", ",", " ", "1"}], "}"}], "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"PartLayer", "[", RowBox[{"{", RowBox[{"All", ",", " ", "2"}], "}"}], "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"ElementwiseLayer", "[", "\n", "\t\t\t\t\t", RowBox[{ RowBox[{ RowBox[{"Min", "[", RowBox[{"2", ",", "#"}], "]"}], "&"}], ",", "\n", "\t\t\t\t\t", RowBox[{"\"\\"", "->", " ", RowBox[{"{", RowBox[{"\"\\"", ",", " ", RowBox[{"Restricted", "[", RowBox[{"\"\\"", ",", "numWords"}], "]"}]}], "}"}]}]}], "\n", "\t\t\t\t\t", "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", "->", RowBox[{"EmbeddingLayer", "[", RowBox[{"embeddingSize", ",", "numWords"}], "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", "->", " ", RowBox[{"NeuralNetworks`SequenceIndicesLayer", "[", "512", "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", "->", RowBox[{"EmbeddingLayer", "[", RowBox[{"embeddingSize", ",", "numPos"}], "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", "->", RowBox[{"EmbeddingLayer", "[", RowBox[{"embeddingSize", ",", "numTokens"}], "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", "->", " ", RowBox[{"ThreadingLayer", "[", RowBox[{ RowBox[{"#1", "+", "#2", "+", "#3"}], "&"}], "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", "->", " ", RowBox[{"NormalizationLayer", "[", RowBox[{ RowBox[{"2", ";;"}], ",", "\"\\"", ",", RowBox[{"\"\\"", "->", RowBox[{"10", "^", RowBox[{"-", "12"}]}]}]}], "]"}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", "->", " ", RowBox[{"DropoutLayer", "[", "0.1", "]"}]}]}], "\n", "\t\t", "|>"}], ",", "\t", "\n", "\t\t", RowBox[{"{", "\n", "\t\t", RowBox[{ RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], " ", "->", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{"{", RowBox[{"\"\\"", ",", " ", "\"\\""}], " ", "}"}]}]}], ",", " ", "\n", "\t\t", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], " ", "->", " ", RowBox[{"\"\\"", " ", "->", " ", RowBox[{ "\"\\"", " ", "->", " ", "\"\\""}]}]}], ",", "\n", "\t\t", RowBox[{"\"\\"", " ", "->", "\"\\""}], ",", "\n", "\t\t", RowBox[{ RowBox[{"{", RowBox[{ "\"\\"", ",", "\"\\"", ",", "\"\\""}], "}"}], "->", RowBox[{"\"\\"", "->", RowBox[{"\"\\"", "->", "\"\\""}]}]}]}], "}"}]}], "\n", "\t", "]"}]}], ";", "\[IndentingNewLine]", "\n", " ", RowBox[{"transformer", " ", "=", " ", RowBox[{"NetChain", "[", RowBox[{ RowBox[{"<|", "\n", "\t\t", RowBox[{ RowBox[{"\"\\"", " ", "->", " ", "embedding"}], ",", "\n", "\t\t", RowBox[{"\"\\"", " ", "->", " ", "encoder"}]}], "\n", "\t\t", "|>"}], ",", " ", RowBox[{"\"\\"", "\[Rule]", " ", RowBox[{"{", RowBox[{"\"\\"", ",", "2", ",", RowBox[{"Restricted", "[", RowBox[{"\"\\"", ",", " ", "numWords"}], "]"}]}], "}"}]}]}], "\n", "\t", "]"}]}]}]}], "\n", "]"}]}]], "Input", CellChangeTimes->{{3.760082769713105*^9, 3.760083012092045*^9}, { 3.760083043019932*^9, 3.7600831474160423`*^9}, {3.760083250355433*^9, 3.760083301834972*^9}, {3.760083350136086*^9, 3.760083432559209*^9}, { 3.7600835945550823`*^9, 3.760083610977531*^9}, {3.760083715024948*^9, 3.760083715695142*^9}, {3.763912505035767*^9, 3.763912525290502*^9}, { 3.797704437462492*^9, 3.797704449788377*^9}}, CellLabel->"In[6]:=",ExpressionUUID->"61578ba9-8bc9-40a4-a284-f99381c11fd1"], Cell[CellGroupData[{ Cell[BoxData[ RowBox[{"createBERT", "[", "\"\\"", "]"}]], "Input", CellChangeTimes->{{3.760083159278599*^9, 3.760083171524016*^9}, { 3.763912535897531*^9, 3.7639125388077097`*^9}, {3.797704495242385*^9, 3.797704496343207*^9}, 3.7977054267210712`*^9}, CellLabel->"In[11]:=",ExpressionUUID->"3ef45ba1-be99-49d1-b315-f7e8450c70ca"], Cell[BoxData[ TagBox[ TemplateBox[{ RowBox[{ StyleBox[ TagBox["NetChain", "SummaryHead"], "NonInterpretableSummary"], StyleBox["[", "NonInterpretableSummary"], DynamicModuleBox[{Typeset`open = False}, PanelBox[ PaneSelectorBox[{False -> GridBox[{{ PaneBox[ ButtonBox[ DynamicBox[ FEPrivate`FrontEndResource[ "FEBitmaps", "SquarePlusIconMedium"]], ButtonFunction :> (Typeset`open = True), Appearance -> None, Evaluator -> Automatic, Method -> "Preemptive"], Alignment -> {Center, Center}, ImageSize -> {Automatic, 24}], StyleBox[ OverlayBox[{ GraphicsBox[{ Thickness[0.014925373134328358`], { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{"Miter", 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{23., 91.5}, {30., 91.5}}}, CurveClosed -> {0}]}, { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{"Miter", 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{37., 91.5}, {44., 91.5}}}, CurveClosed -> {0}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 15.5, 103.5}, {23.5, 103.5}, {23.5, 79.5}, {15.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{24., 104.}, {15., 104.}, { 15., 79.}, {24., 79.}}, {{23., 80.}, {16., 80.}, {16., 103.}, {23., 103.}}}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 29.5, 103.5}, {37.5, 103.5}, {37.5, 79.5}, {29.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{38., 104.}, {29., 104.}, { 29., 79.}, {38., 79.}}, {{37., 80.}, {30., 80.}, {30., 103.}, {37., 103.}}}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 43.5, 103.5}, {51.5, 103.5}, {51.5, 79.5}, {43.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{52., 104.}, {43., 104.}, { 43., 79.}, {52., 79.}}, {{51., 80.}, {44., 80.}, {44., 103.}, {51., 103.}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{28., 91.5}, { 25., 95.}, {25., 88.}}}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{42., 91.5}, { 39., 95.}, {39., 88.}}}]}}, ImageMargins -> 0, ImageSize -> {36, Automatic}, PlotRangePadding -> 0, BaseStyle -> GrayLevel[0.6]], StyleBox["uninitialized", RGBColor[0.66, 0, 0], FontSize -> 8, FontFamily -> "Roboto", Background -> GrayLevel[1, 0.65]]}, Alignment -> {Center, Scaled[0.0001]}], CacheGraphics -> False], GridBox[{{ StyleBox[ TemplateBox[{ TemplateBox[{ StyleBox["\"Input\"", GrayLevel[0], StripOnInput -> False], "\" \"", "\"port\""}, "RowDefault"], "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox[ TemplateBox[{ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{ "\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"2\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], "\" of \"", TemplateBox[{"\"indices\"", "\" \"", StyleBox[ TemplateBox[{ "\"(\"", "\"\[VeryThinSpace]\"", "\"range\"", "\":\"", "\" \"", TemplateBox[{"1", "\"..\"", "30522"}, "RowDefault"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"]}, "RowDefault"], "SummaryItem"]}, { StyleBox[ TemplateBox[{ TemplateBox[{ StyleBox["\"Output\"", GrayLevel[0], StripOnInput -> False], "\" \"", "\"port\""}, "RowDefault"], "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox[ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{ "\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"768\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], "SummaryItem"]}}, BaselinePosition -> Automatic, GridBoxAlignment -> { "Columns" -> {{Left}}, "Rows" -> {{Automatic}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> { "Columns" -> {{2}}, "Rows" -> {{Automatic}}}]}}, GridBoxAlignment -> {"Rows" -> {{Top}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridFrameMargins -> {{0, 0}, {0, 0}}, BaselinePosition -> {1, 3}], True -> GridBox[{{ PaneBox[ ButtonBox[ DynamicBox[ FEPrivate`FrontEndResource[ "FEBitmaps", "SquareMinusIconMedium"]], ButtonFunction :> (Typeset`open = False), Appearance -> None, Evaluator -> Automatic, Method -> "Preemptive"], Alignment -> {Center, Center}, ImageSize -> {Automatic, 24}], StyleBox[ OverlayBox[{ GraphicsBox[{ Thickness[0.014925373134328358`], { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{"Miter", 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{23., 91.5}, {30., 91.5}}}, CurveClosed -> {0}]}, { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{"Miter", 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{37., 91.5}, {44., 91.5}}}, CurveClosed -> {0}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 15.5, 103.5}, {23.5, 103.5}, {23.5, 79.5}, {15.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{24., 104.}, {15., 104.}, { 15., 79.}, {24., 79.}}, {{23., 80.}, {16., 80.}, {16., 103.}, {23., 103.}}}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 29.5, 103.5}, {37.5, 103.5}, {37.5, 79.5}, {29.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{38., 104.}, {29., 104.}, { 29., 79.}, {38., 79.}}, {{37., 80.}, {30., 80.}, {30., 103.}, {37., 103.}}}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 43.5, 103.5}, {51.5, 103.5}, {51.5, 79.5}, {43.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{52., 104.}, {43., 104.}, { 43., 79.}, {52., 79.}}, {{51., 80.}, {44., 80.}, {44., 103.}, {51., 103.}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{28., 91.5}, { 25., 95.}, {25., 88.}}}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{42., 91.5}, { 39., 95.}, {39., 88.}}}]}}, ImageMargins -> 0, ImageSize -> {36, Automatic}, PlotRangePadding -> 0, BaseStyle -> GrayLevel[0.6]], StyleBox["uninitialized", RGBColor[0.66, 0, 0], FontSize -> 8, FontFamily -> "Roboto", Background -> GrayLevel[1, 0.65]]}, Alignment -> {Center, Scaled[0.0001]}], CacheGraphics -> False], GridBox[{{ DynamicModuleBox[{ NeuralNetworks`Private`NetChain`assoc3 = Association[ "Type" -> "Chain", "Nodes" -> Association[ "embedding" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 2}, NeuralNetworks`IndexIntegerT[30522]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association[ "part1" -> Association[ "Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[{ Span[1, -1], 1}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 2}, NeuralNetworks`IndexIntegerT[30522]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[30522]]]], "embeddingwords" -> Association[ "Type" -> "Embedding", "Arrays" -> Association[ "Weights" -> NeuralNetworks`TensorT[{30522, 768}, NeuralNetworks`RealT]], "Parameters" -> Association[ "OutputDimension" -> 768, "ClassCount" -> 30522, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[30522]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "posembed" -> Association[ "Type" -> "SequenceIndices", "Arrays" -> Association[], "Parameters" -> Association[ "Maximum" -> 512, "$Length" -> NeuralNetworks`LengthVar[1956562679]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[30522]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[512]]]], "embeddingpos" -> Association[ "Type" -> "Embedding", "Arrays" -> Association[ "Weights" -> NeuralNetworks`TensorT[{512, 768}, NeuralNetworks`RealT]], "Parameters" -> Association[ "OutputDimension" -> 768, "ClassCount" -> 512, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[512]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "part2" -> Association[ "Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[{ Span[1, -1], 2}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 2}, NeuralNetworks`IndexIntegerT[30522]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[30522]]]], "Clip" -> Association[ "Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Min, 2., NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[30522]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[2]]]], "embeddingtokens" -> Association[ "Type" -> "Embedding", "Arrays" -> Association[ "Weights" -> NeuralNetworks`TensorT[{2, 768}, NeuralNetworks`RealT]], "Parameters" -> Association[ "OutputDimension" -> 768, "ClassCount" -> 2, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679]}, NeuralNetworks`IndexIntegerT[2]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "inputCombine" -> Association[ "Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[# + #2 + #3& ]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "3" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "normalize" -> Association[ "Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association[ "Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association[ "DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath[ "Nodes", "part1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "part2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "embeddingwords", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "part1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "posembed", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "part1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "inputCombine", "Inputs", "1"] -> NeuralNetworks`NetPath[ "Nodes", "embeddingwords", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "embeddingpos", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "posembed", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "inputCombine", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "embeddingpos", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Clip", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "part2", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "embeddingtokens", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "Clip", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "inputCombine", "Inputs", "3"] -> NeuralNetworks`NetPath[ "Nodes", "embeddingtokens", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "normalize", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "inputCombine", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "normalize", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"]}], "encoder" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association[ "1" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "2" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "3" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "4" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "5" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "6" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "7" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "8" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "9" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "10" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "11" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "12" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["attention" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["key" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "query" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "elem" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[2], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> {Times, 0.125, NeuralNetworks`Private`ScalarSymbol[1]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "value" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {12, 64}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "attention" -> Association["Type" -> "Attention", "Arrays" -> Association[], "Parameters" -> Association["ScoringNet" -> Association["Type" -> "Graph", "Inputs" -> Association["Query" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association[ "Type" -> "Dot", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association[ "1" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "1", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath["Nodes", "1", "Inputs", "2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"]}], "Mask" -> None, "ScoreRescaling" -> None, "MultiHead" -> True, "$InputPorts" -> "KeyValueQuery", "$KeyAndValueShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryShape" -> { NeuralNetworks`LengthVar[1956562679], 12}, "$QueryChannels" -> {64}, "$KeyChannels" -> {64}, "$ValueChannels" -> {64}], "Inputs" -> Association["Key" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Value" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT], "Query" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]]], "Merge" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 768, "$InputDimensions" -> {12, 64}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{12, 64}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 12, 64}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "key", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "value", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Key"] -> NeuralNetworks`NetPath[ "Nodes", "key", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "query", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Query"], NeuralNetworks`NetPath[ "Nodes", "elem", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "query", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath[ "Nodes", "elem", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Value"] -> NeuralNetworks`NetPath[ "Nodes", "value", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Merge", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "Merge", "Outputs", "Output"]}], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "attention", "Inputs", "Query"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "attention", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}], "2" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Nodes" -> Association["linear1" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 768}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 768, "$InputDimensions" -> {768}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "gelu" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[ NeuralNetworks`Private`ScalarFunctionObject[{ NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[5], Association[ NeuralNetworks`Private`ScalarSymbol[2] -> { Times, 0.7071067811865475, NeuralNetworks`Private`ScalarSymbol[1]}, NeuralNetworks`Private`ScalarSymbol[3] -> {Erf, NeuralNetworks`Private`ScalarSymbol[2]}, NeuralNetworks`Private`ScalarSymbol[4] -> {Plus, 1., NeuralNetworks`Private`ScalarSymbol[3]}, NeuralNetworks`Private`ScalarSymbol[5] -> {Times, 0.5, NeuralNetworks`Private`ScalarSymbol[1], NeuralNetworks`Private`ScalarSymbol[4]}]]], "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 3072}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]]], "linear2" -> Association["Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{768, 3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {768}, "$OutputSize" -> 768, "$InputSize" -> 3072, "$InputDimensions" -> {3072}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1956562679], "$InputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "dropout" -> Association["Type" -> "Dropout", "Arrays" -> Association[], "Parameters" -> Association["DropoutProbability" -> 0.1, "Method" -> "Dropout", "OutputPorts" -> NeuralNetworks`ValidatedParameter[{"Output"}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "add" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Plus]], "Inputs" -> Association["1" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT], "2" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], "norm" -> Association["Type" -> "Normalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{768}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "AggregationLevels" -> NeuralNetworks`ValidatedParameter[ Span[2, All]], "ScalingLevels" -> NeuralNetworks`ValidatedParameter["Same"], "Epsilon" -> 1.*^-12, "$Dimensions" -> { NeuralNetworks`LengthVar[1956562679], 768}, "$StatsDimensions" -> {768}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "add", "Inputs", "1"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "linear1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "gelu", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear1", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "linear2", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "gelu", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "dropout", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "linear2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "add", "Inputs", "2"] -> NeuralNetworks`NetPath[ "Nodes", "dropout", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "norm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "add", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "norm", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "4", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "3", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "5", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "4", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "6", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "5", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "7", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "6", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "8", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "7", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "9", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "8", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "10", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "9", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "11", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "10", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "12", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "11", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "12", "Outputs", "Output"]}]], "Edges" -> { NeuralNetworks`NetPath[ "Nodes", "embedding", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "encoder", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "embedding", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "encoder", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 2}, NeuralNetworks`IndexIntegerT[30522]]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1956562679], 768}, NeuralNetworks`RealT]]], NeuralNetworks`Private`NetChain`opart, NeuralNetworks`Private`NetChain`part, NeuralNetworks`Private`NetChain`selected = Null}, DynamicBox[ GridBox[{{ TagBox[ TagBox[ GridBox[{{ TagBox[ TagBox[ "\"\"", Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ "\"Input\"", Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"2\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], "\" of \"", TemplateBox[{"\"indices\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"range\"", "\":\"", "\" \"", TemplateBox[{"1", "\"..\"", "30522"}, "RowDefault"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"]}, "RowDefault"], Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"embedding\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "embedding"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ TemplateBox[{"NetGraph", "\" \"", TemplateBox[{"\"(\"", "10", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification]}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> {"SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"], RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "embedding"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"768\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], Annotation[#, {"Nodes", "embedding"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"encoder\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "encoder"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ TemplateBox[{"NetGraph", "\" \"", TemplateBox[{"\"(\"", "12", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification]}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> {"SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"], RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "encoder"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"768\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], Annotation[#, {"Nodes", "encoder"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ "\"\"", Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ "\"Output\"", Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"768\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}}, GridBoxAlignment -> {"Columns" -> {{Left}}}, AutoDelete -> False, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> {"Columns" -> {{1.1}}}], "Grid"], EventHandlerTag[{"MouseClicked" :> If[ ListQ[ NeuralNetworks`Private`NetChain`part = MouseAnnotation[]], If[NeuralNetworks`Private`NetChain`opart === NeuralNetworks`Private`NetChain`part, NeuralNetworks`Private`NetChain`selected = Null; NeuralNetworks`Private`NetChain`opart = Null, NeuralNetworks`Private`NetChain`selected = Part[NeuralNetworks`Private`NetChain`assoc3, Apply[Sequence, NeuralNetworks`Private`NetChain`part]]; NeuralNetworks`Private`NetChain`opart = NeuralNetworks`Private`NetChain`part; Null]; Null], Method -> "Preemptive", PassEventsDown -> Automatic, PassEventsUp -> True}]]}, NeuralNetworks`FormatSelectedParameterChain[ NeuralNetworks`Private`NetChain`selected, NeuralNetworks`Private`NetChain`part, Association[], {}]}, GridBoxSpacings -> {"Columns" -> {{1}}}, GridBoxAlignment -> {"Columns" -> {{Left}}}, GridFrameMargins -> {{0, 0}, {0, 0}}], TrackedSymbols :> { NeuralNetworks`Private`NetChain`selected}], Initialization :> {NetChain}]}}, BaselinePosition -> Automatic, GridBoxAlignment -> { "Columns" -> {{Left}}, "Rows" -> {{Automatic}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> { "Columns" -> {{2}}, "Rows" -> {{Automatic}}}]}}, GridBoxAlignment -> {"Rows" -> {{Top}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridFrameMargins -> {{0, 0}, {0, 0}}, BaselinePosition -> {1, 1}]}, Dynamic[Typeset`open], ImageSize -> Automatic], BaselinePosition -> Baseline, BaseStyle -> { ShowStringCharacters -> False, NumberMarks -> False, PrintPrecision -> 3, ShowSyntaxStyles -> False}]], StyleBox["]", "NonInterpretableSummary"]}]}, "CopyTag", DisplayFunction->(#& ), InterpretationFunction->("NetChain[<>]"& )], False, Editable->False, SelectWithContents->True, Selectable->False]], "Output", CellChangeTimes->{ 3.761015191480523*^9, 3.763912566595449*^9, 3.797703693798815*^9, { 3.797705405907714*^9, 3.797705434896901*^9}}, CellLabel->"Out[11]=",ExpressionUUID->"02c3a0f3-8a6b-4669-8cd0-04371abb898a"] }, Open ]] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell["Training", "Section",ExpressionUUID->"77cc71e7-c40d-420d-8050-3de316a83d0a"], Cell["(Performed separately)", "Text", CellChangeTimes->{{3.703671715995969*^9, 3.703671721081246*^9}},ExpressionUUID->"799c8263-2b8e-4167-841b-\ 78a369bf1d15"] }, Open ]] }, Open ]] }, WindowSize->{898, 773}, WindowMargins->{{Automatic, 129}, {Automatic, 0}}, FrontEndVersion->"12.1 for Mac OS X x86 (64-bit) (March 13, 2020)", StyleDefinitions->"Default.nb", ExpressionUUID->"275f6411-7bad-424a-ab9a-61bb622764e2" ] (* End of Notebook Content *) (* Internal cache information *) (*CellTagsOutline CellTagsIndex->{} *) (*CellTagsIndex CellTagsIndex->{} *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[580, 22, 529, 11, 195, "Title",ExpressionUUID->"a33618b8-7416-413f-9875-525e49421225"], Cell[CellGroupData[{ Cell[1134, 37, 92, 0, 54, "Subsection",ExpressionUUID->"05498e76-0a7f-48f3-8742-2a22e96acb4d"], Cell[CellGroupData[{ Cell[1251, 41, 93, 0, 45, "Subsubsection",ExpressionUUID->"7b1b33a9-3717-469b-8b65-a7035e35aaa9"], Cell[1347, 43, 676, 13, 30, "Input",ExpressionUUID->"b4e81b34-93b5-4f07-89a8-04ddeb31a697"] }, Open ]], Cell[CellGroupData[{ Cell[2060, 61, 93, 0, 45, "Subsubsection",ExpressionUUID->"950658ac-daca-42dc-9c33-5fd10a6cd280"], Cell[2156, 63, 625, 12, 30, "Input",ExpressionUUID->"ca3c3f5b-06b7-49d4-823d-9a7faa05773d"] }, Open ]], Cell[CellGroupData[{ Cell[2818, 80, 91, 0, 45, "Subsubsection",ExpressionUUID->"11624637-737b-4e92-b609-e11fcbf10ae2"], Cell[2912, 82, 562, 10, 30, "Input",ExpressionUUID->"12df8018-c460-4308-a6fa-ea80f797b729"] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell[3523, 98, 90, 0, 67, "Section",ExpressionUUID->"cd545e68-ff54-4416-8f56-6fd64aedca73"], Cell[CellGroupData[{ Cell[3638, 102, 95, 0, 54, "Subsection",ExpressionUUID->"55fb2a0d-2d0b-4adf-a72e-6c7fc391c78a"], Cell[3736, 104, 3364, 74, 325, "Input",ExpressionUUID->"bd9a6749-eda1-4474-b3c5-5e12945db7c9"], Cell[7103, 180, 1579, 35, 220, "Input",ExpressionUUID->"6930d9bf-6317-42e4-a088-5b32a5c15609"], Cell[8685, 217, 971, 22, 178, "Input",ExpressionUUID->"0c232b85-aae7-4dd6-bb24-e20693260987"], Cell[9659, 241, 2783, 62, 409, "Input",ExpressionUUID->"29f94bc3-a119-4807-9a4a-84a475ef73b8"], Cell[12445, 305, 786, 18, 136, "Input",ExpressionUUID->"7bedcbe1-0b2b-474a-8d94-195c1df43fdc"] }, Open ]], Cell[CellGroupData[{ Cell[13268, 328, 133, 3, 54, "Subsection",ExpressionUUID->"b9859337-b9ae-415f-a290-86e9a29b57c5"], Cell[13404, 333, 747, 13, 196, "Text",ExpressionUUID->"d2676916-2cca-4a30-b4e1-373a4e987547"], Cell[14154, 348, 8571, 188, 1270, "Input",ExpressionUUID->"61578ba9-8bc9-40a4-a284-f99381c11fd1"], Cell[CellGroupData[{ Cell[22750, 540, 352, 5, 30, "Input",ExpressionUUID->"3ef45ba1-be99-49d1-b315-f7e8450c70ca"], Cell[23105, 547, 373437, 6103, 64, "Output",ExpressionUUID->"02c3a0f3-8a6b-4669-8cd0-04371abb898a"] }, Open ]] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell[396603, 6657, 82, 0, 67, "Section",ExpressionUUID->"77cc71e7-c40d-420d-8050-3de316a83d0a"], Cell[396688, 6659, 164, 3, 35, "Text",ExpressionUUID->"799c8263-2b8e-4167-841b-78a369bf1d15"] }, Open ]] }, Open ]] } ] *)