(* Content-type: application/vnd.wolfram.mathematica *) (*** Wolfram Notebook File ***) (* http://www.wolfram.com/nb *) (* CreatedBy='Mathematica 11.1' *) (*CacheID: 234*) (* Internal cache information: NotebookFileLineBreakTest NotebookFileLineBreakTest NotebookDataPosition[ 158, 7] NotebookDataLength[ 266730, 4743] NotebookOptionsPosition[ 264214, 4690] NotebookOutlinePosition[ 264609, 4706] CellTagsIndexPosition[ 264566, 4703] WindowFrame->Normal*) (* Beginning of Notebook Content *) Notebook[{ Cell[CellGroupData[{ Cell[TextData[{ StyleBox["Construction Notebook for:", FontSize->24], "\nDeep Speech 2 Trained on Baidu English Data" }], "Title", CellChangeTimes->{{3.700390059931024*^9, 3.700390069370269*^9}, { 3.713527335930427*^9, 3.71352733920855*^9}, {3.725196329204988*^9, 3.7251963296657867`*^9}, 3.725709769145364*^9, 3.725975379788035*^9, 3.725979717913773*^9, {3.7306357844249887`*^9, 3.7306357846969*^9}, { 3.731150927772277*^9, 3.7311509306364307`*^9}, {3.7332151707195063`*^9, 3.733215170865391*^9}},ExpressionUUID->"cf0af03b-7c0d-44fe-a36a-\ 801fb2cc9d1e"], Cell[CellGroupData[{ Cell["NetModel Access", "Subsection", CellChangeTimes->{{3.695474314080426*^9, 3.695474346087426*^9}},ExpressionUUID->"20a938bc-d1c4-4bd6-a5fd-\ 63b43d74e3fb"], Cell[CellGroupData[{ Cell["This Notebook", "Subsubsection", CellChangeTimes->{{3.6954750910664797`*^9, 3.695475092737351*^9}},ExpressionUUID->"aece8991-ede8-4c4f-b8b7-\ d3deda75843b"], Cell[BoxData[ RowBox[{"NetModel", "[", RowBox[{ "\"\\"", ",", "\"\\""}], "]"}]], "Input", CellChangeTimes->{{3.696656935596855*^9, 3.6966569364445457`*^9}, 3.696768427361453*^9, 3.696768595763381*^9, 3.700390080550715*^9, 3.713527332938039*^9, 3.7251895368569007`*^9, 3.7257097745186977`*^9, { 3.725974160027033*^9, 3.7259741712693043`*^9}, 3.7259797223664417`*^9, { 3.730635787025857*^9, 3.730635788177823*^9}, 3.73115092198287*^9, { 3.733215172935543*^9, 3.73321517308002*^9}},ExpressionUUID->"f9e334a7-6b84-4b97-a015-\ 364f975bd6c8"] }, Open ]], Cell[CellGroupData[{ Cell["Untrained Net", "Subsubsection", CellChangeTimes->{{3.6954750978333807`*^9, 3.695475133273511*^9}},ExpressionUUID->"278efdc3-d956-4587-8723-\ 2a3485f9bac0"], Cell[BoxData[ RowBox[{"NetModel", "[", RowBox[{ "\"\\"", ",", "\"\\""}], "]"}]], "Input", CellChangeTimes->{{3.696656940213286*^9, 3.6966569411083603`*^9}, 3.6967685960955133`*^9, 3.700390083629743*^9, {3.713527345914727*^9, 3.713527346642221*^9}, {3.7251895437188444`*^9, 3.725189544052307*^9}, { 3.7257097811399565`*^9, 3.725709791683883*^9}, 3.7259741741281953`*^9, 3.725979726480075*^9, {3.7306357894097147`*^9, 3.730635790706603*^9}, 3.7311509229558277`*^9, 3.733215174959585*^9},ExpressionUUID->"4997fe3c-81ac-42f4-9497-\ 390d3d9b0fce"] }, Open ]], Cell[CellGroupData[{ Cell["Trained Net", "Subsubsection", CellChangeTimes->{{3.6954751072255173`*^9, 3.6954751086815357`*^9}},ExpressionUUID->"71c6cef7-94d1-4660-857a-\ d6696698fd69"], Cell[BoxData[ RowBox[{"NetModel", "[", "\"\\"", "]"}]], "Input", CellChangeTimes->{{3.6966569441649303`*^9, 3.696656946349728*^9}, 3.696768596439824*^9, 3.70039008606037*^9, 3.713527350999201*^9, 3.725189549129289*^9, 3.725709788369228*^9, 3.725974177559237*^9, 3.725979728776238*^9, {3.730635792761504*^9, 3.730635793977479*^9}, 3.731150924044902*^9, {3.733215176775631*^9, 3.7332151769051723`*^9}},ExpressionUUID->"908d5d25-3cee-40c2-a5c9-\ 2b89750c8f92"] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell["Net Construction", "Section", CellChangeTimes->{ 3.695625386014464*^9},ExpressionUUID->"e6f0ec16-daa9-4151-8b11-\ 649c83fbd9a1"], Cell[CellGroupData[{ Cell["Intermediate Functions", "Subsection", CellChangeTimes->{{3.7251952635053062`*^9, 3.7251952804601765`*^9}},ExpressionUUID->"075f3fef-d0b5-4896-9f18-\ 27f1ae7779eb"], Cell[BoxData[ RowBox[{ RowBox[{"gruBNStep", "[", RowBox[{"inSize_", ",", "outSize_"}], "]"}], ":=", RowBox[{"{", "\[IndentingNewLine]", RowBox[{ RowBox[{"NetMapOperator", "[", RowBox[{"LinearLayer", "[", RowBox[{ RowBox[{"3", "*", "outSize"}], ",", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", "inSize", "}"}]}]}], "]"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"BatchNormalizationLayer", "[", RowBox[{ RowBox[{"\"\\"", "\[Rule]", "True"}], ",", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", RowBox[{"\"\\"", ",", RowBox[{"3", "*", "outSize"}]}], "}"}]}]}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"NetGraph", "[", "\[IndentingNewLine]", RowBox[{ RowBox[{"<|", "\[IndentingNewLine]", RowBox[{ RowBox[{"\"\\"", "\[Rule]", RowBox[{"NetArrayLayer", "[", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", RowBox[{"3", ",", "outSize"}], "}"}]}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\<{Wis,Wrs}.s[t-1]\>\"", "\[Rule]", RowBox[{"LinearLayer", "[", RowBox[{"2", "*", "outSize"}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"LinearLayer", "[", "outSize", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"PartLayer", "[", RowBox[{"1", ";;", "outSize"}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"PartLayer", "[", RowBox[{ RowBox[{"outSize", "+", "1"}], ";;", RowBox[{"2", "*", "outSize"}]}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"PartLayer", "[", RowBox[{ RowBox[{ RowBox[{"2", "*", "outSize"}], "+", "1"}], ";;", RowBox[{"3", "*", "outSize"}]}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"PartLayer", "[", RowBox[{"1", ";;", "outSize"}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"PartLayer", "[", RowBox[{ RowBox[{"outSize", "+", "1"}], ";;", RowBox[{"2", "*", "outSize"}]}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"PartLayer", "[", "1", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"PartLayer", "[", "2", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"PartLayer", "[", "3", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\<+i\>\"", "\[Rule]", RowBox[{"TotalLayer", "[", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\<+r\>\"", "\[Rule]", RowBox[{"TotalLayer", "[", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"ElementwiseLayer", "[", "LogisticSigmoid", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"ElementwiseLayer", "[", "LogisticSigmoid", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"ElementwiseLayer", "[", "\"\\"", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"ThreadingLayer", "[", "Times", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\<+m\>\"", "\[Rule]", RowBox[{"TotalLayer", "[", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\<(1-i[t])*s[t-1]\>\"", "\[Rule]", RowBox[{"ThreadingLayer", "[", RowBox[{ RowBox[{ RowBox[{"(", RowBox[{"1", "-", "#1"}], ")"}], "*", "#2"}], "&"}], "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"ThreadingLayer", "[", "Times", "]"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"TotalLayer", "[", "]"}]}]}], "\[IndentingNewLine]", "|>"}], ",", "\[IndentingNewLine]", RowBox[{"{", "\[IndentingNewLine]", RowBox[{ RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], "\[Rule]", "\"\<{Wis,Wrs}.s[t-1]\>\""}], ",", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], "\[Rule]", "\"\\""}], ",", RowBox[{"\"\<{Wis,Wrs}.s[t-1]\>\"", "\[Rule]", "\"\\""}], ",", RowBox[{"\"\\"", "\[Rule]", "\"\\""}], ",", RowBox[{ RowBox[{"{", RowBox[{ "\"\\"", ",", "\"\\"", ",", "\"\\""}], "}"}], "\[Rule]", RowBox[{"\"\<+i\>\"", "\[Rule]", "\"\\""}]}], ",", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], "\[Rule]", "\"\\""}], ",", RowBox[{"\"\<{Wis,Wrs}.s[t-1]\>\"", "\[Rule]", "\"\\""}], ",", RowBox[{"\"\\"", "\[Rule]", "\"\\""}], ",", RowBox[{ RowBox[{"{", RowBox[{ "\"\\"", ",", "\"\\"", ",", "\"\\""}], "}"}], "\[Rule]", RowBox[{"\"\<+r\>\"", "\[Rule]", "\"\\""}]}], ",", RowBox[{ RowBox[{"NetPort", "[", "\"\\"", "]"}], "\[Rule]", "\"\\""}], ",", RowBox[{ RowBox[{"{", RowBox[{"\"\\"", ",", RowBox[{"NetPort", "[", "\"\\"", "]"}]}], "}"}], "\[Rule]", RowBox[{ "\"\\"", "\[Rule]", "\"\\""}]}], ",", RowBox[{"\"\\"", "\[Rule]", "\"\\""}], ",", RowBox[{ RowBox[{"{", RowBox[{ "\"\\"", ",", "\"\\"", ",", "\"\\""}], "}"}], "\[Rule]", RowBox[{"\"\<+m\>\"", "\[Rule]", "\"\\""}]}], ",", RowBox[{ RowBox[{"{", RowBox[{"\"\\"", ",", RowBox[{"NetPort", "[", "\"\\"", "]"}]}], "}"}], "\[Rule]", "\"\<(1-i[t])*s[t-1]\>\""}], ",", RowBox[{ RowBox[{"{", RowBox[{"\"\\"", ",", "\"\\""}], "}"}], "\[Rule]", "\"\\""}], ",", RowBox[{ RowBox[{"{", RowBox[{"\"\\"", ",", "\"\<(1-i[t])*s[t-1]\>\""}], "}"}], "\[Rule]", "\"\\""}]}], "\[IndentingNewLine]", "}"}]}], "\[IndentingNewLine]", "]"}]}], "\[IndentingNewLine]", "}"}]}]], "Input", CellChangeTimes->{{3.73321557991497*^9, 3.733215596896578*^9}, { 3.7332156611331453`*^9, 3.733215840944017*^9}, {3.733216163645007*^9, 3.733216176878696*^9}, {3.7332164580893087`*^9, 3.733216463552476*^9}, 3.8092503999028473`*^9, {3.814026481075951*^9, 3.814026487762102*^9}}, CellLabel->"In[17]:=",ExpressionUUID->"74e06088-1beb-4cc7-81cf-ba7ba9bf7fcd"], Cell[BoxData[ RowBox[{ RowBox[{"gruBNLayer", "[", RowBox[{"inSize_", ",", "outSize_"}], "]"}], ":=", RowBox[{ RowBox[{ RowBox[{"NetChain", "[", RowBox[{"{", RowBox[{"#1", ",", "#2", ",", RowBox[{"NetFoldOperator", "[", "#3", "]"}]}], "}"}], "]"}], "&"}], "@@", RowBox[{"gruBNStep", "[", RowBox[{"inSize", ",", "outSize"}], "]"}]}]}]], "Input", CellChangeTimes->{{3.73321585017839*^9, 3.733215874791708*^9}, { 3.733215932179358*^9, 3.733215934474948*^9}}, CellLabel->"In[18]:=",ExpressionUUID->"eca2777e-8639-4a73-8fdb-8fb65b27c0c8"], Cell[BoxData[ RowBox[{ RowBox[{"biGRUBNLayer", "[", RowBox[{"inSize_", ",", "outSize_"}], "]"}], ":=", RowBox[{"NetBidirectionalOperator", "[", RowBox[{"{", RowBox[{ RowBox[{"gruBNLayer", "[", RowBox[{"inSize", ",", "outSize"}], "]"}], ",", RowBox[{"gruBNLayer", "[", RowBox[{"inSize", ",", "outSize"}], "]"}]}], "}"}], "]"}]}]], "Input", CellChangeTimes->{{3.7332158419501047`*^9, 3.733215845244082*^9}, { 3.7332158780667067`*^9, 3.73321587819409*^9}, {3.733215909079764*^9, 3.733215935594493*^9}}, CellLabel->"In[19]:=",ExpressionUUID->"6f3bb86d-ffc0-4703-ab7c-e0825f587d5f"], Cell[BoxData[ RowBox[{ RowBox[{"enc", " ", "=", " ", RowBox[{"NetEncoder", "[", "\n", " \t", RowBox[{"{", "\n", " \t", RowBox[{"\"\\"", ",", "\n", " \t", RowBox[{"Function", "[", RowBox[{ RowBox[{"{", "input", "}"}], ",", "\n", "\t\t\t", RowBox[{"Module", "[", "\n", "\t\t\t\t", RowBox[{ RowBox[{"{", RowBox[{ RowBox[{"a", " ", "=", " ", "input"}], ",", " ", "power"}], "}"}], ",", "\n", "\t\t\t\t", RowBox[{ RowBox[{"If", "[", RowBox[{ RowBox[{ RowBox[{"Head", "[", "a", "]"}], " ", "===", " ", "File"}], ",", " ", RowBox[{"a", " ", "=", " ", RowBox[{"Audio", "[", "a", "]"}]}]}], "]"}], ";", "\n", " \t\t\t", RowBox[{"If", "[", RowBox[{ RowBox[{"AudioQ", "[", "a", "]"}], ",", "\n", " \t\t\t\t", RowBox[{ RowBox[{"If", "[", RowBox[{ RowBox[{ RowBox[{"AudioChannels", "[", "a", "]"}], " ", "=!=", " ", "1"}], ",", " ", RowBox[{"a", " ", "=", " ", RowBox[{"Mean", "[", RowBox[{"AudioChannelSeparate", "[", "a", "]"}], "]"}]}]}], "]"}], ";", "\n", "\t \t\t\t", RowBox[{"If", "[", RowBox[{ RowBox[{ RowBox[{"AudioSampleRate", "[", "a", "]"}], " ", "=!=", " ", RowBox[{"Quantity", "[", RowBox[{"16000", ",", "\"\\""}], "]"}]}], ",", " ", RowBox[{"a", "=", RowBox[{"AudioResample", "[", RowBox[{"a", ",", "16000", ",", RowBox[{"Resampling", "->", RowBox[{"{", RowBox[{"\"\\"", ",", "2"}], "}"}]}]}], "]"}]}]}], "]"}], ";", "\n", "\t \t\t\t", RowBox[{"power", " ", "=", " ", RowBox[{"AudioMeasurements", "[", RowBox[{"a", ",", "\"\\""}], "]"}]}], ";", "\n", "\t \t\t\t", RowBox[{"If", "[", RowBox[{ RowBox[{"power", "=!=", "0.01"}], ",", RowBox[{"a", "=", RowBox[{"AudioAmplify", "[", RowBox[{"a", ",", RowBox[{"Sqrt", "[", RowBox[{"0.01", "/", "power"}], "]"}]}], "]"}]}]}], "]"}], ";", "\n", "\t \t\t\t", RowBox[{ RowBox[{"Abs", "[", RowBox[{ RowBox[{"SpectrogramArray", "[", RowBox[{ "a", ",", " ", "320", ",", " ", "160", ",", " ", "HannWindow"}], "]"}], "[", RowBox[{"[", RowBox[{"All", ",", " ", RowBox[{"1", ";;", "161"}]}], "]"}], "]"}], "]"}], "^", "2"}]}], "\n", "\t\t \t\t", ",", "\n", "\t\t \t\t", "$Failed"}], "\t", "\n", "\t \t\t", "]"}]}]}], "\n", "\t \t", "]"}]}], "\n", " \t", "]"}], ",", "\n", " \t", RowBox[{"{", RowBox[{"\"\\"", ",", " ", "161"}], "}"}]}], "\n", " \t", "}"}], "\n", "]"}]}], ";"}]], "Input", CellChangeTimes->{3.733215248996581*^9}, CellLabel->"In[20]:=",ExpressionUUID->"399601e9-fdc8-44d8-ae12-419eebdaedde"], Cell[BoxData[ RowBox[{ RowBox[{"dec", "=", RowBox[{"NetDecoder", "[", RowBox[{"{", RowBox[{"\"\\"", ",", RowBox[{"Join", "[", RowBox[{ RowBox[{"{", RowBox[{"\"\<'\>\"", ",", "\"\< \>\""}], "}"}], ",", RowBox[{"Alphabet", "[", "]"}]}], "]"}]}], "}"}], "]"}]}], ";"}]], "Input", CellChangeTimes->{{3.733215269429811*^9, 3.733215272049913*^9}}, CellLabel->"In[21]:=",ExpressionUUID->"0057b0bc-5c66-42bc-b74d-c408ad38ccb4"] }, Open ]], Cell[CellGroupData[{ Cell["Final Net", "Subsection", CellChangeTimes->{{3.695106779945572*^9, 3.695106780985426*^9}},ExpressionUUID->"791564d7-bbb0-42be-870f-\ 225e70c83a2d"], Cell[CellGroupData[{ Cell[BoxData[ RowBox[{"deepSpeech2", "=", RowBox[{"NetChain", "[", "\[IndentingNewLine]", RowBox[{ RowBox[{"{", "\[IndentingNewLine]", RowBox[{ RowBox[{"NetMapOperator", "[", RowBox[{"NetChain", "[", RowBox[{"{", RowBox[{ RowBox[{"NetGraph", "[", RowBox[{ RowBox[{"{", RowBox[{ RowBox[{"NetArrayLayer", "[", "]"}], ",", RowBox[{"ThreadingLayer", "[", "Times", "]"}]}], "}"}], ",", RowBox[{"{", RowBox[{ RowBox[{"{", RowBox[{"1", ",", RowBox[{"NetPort", "[", "\"\\"", "]"}]}], "}"}], "\[Rule]", " ", "2"}], "}"}]}], "]"}], ",", RowBox[{"ElementwiseLayer", "[", RowBox[{ RowBox[{"Log", "[", RowBox[{"#", "+", RowBox[{"10", "^", RowBox[{"-", "14"}]}]}], "]"}], "&"}], "]"}], ",", RowBox[{"NetGraph", "[", RowBox[{ RowBox[{"{", RowBox[{ RowBox[{"NetArrayLayer", "[", "]"}], ",", RowBox[{"ThreadingLayer", "[", "Plus", "]"}]}], "}"}], ",", RowBox[{"{", RowBox[{ RowBox[{"{", RowBox[{"1", ",", RowBox[{"NetPort", "[", "\"\\"", "]"}]}], "}"}], "\[Rule]", " ", "2"}], "}"}]}], "]"}], ",", RowBox[{"NetGraph", "[", RowBox[{ RowBox[{"{", RowBox[{ RowBox[{"NetArrayLayer", "[", "]"}], ",", RowBox[{"ThreadingLayer", "[", "Times", "]"}]}], "}"}], ",", RowBox[{"{", RowBox[{ RowBox[{"{", RowBox[{"1", ",", RowBox[{"NetPort", "[", "\"\\"", "]"}]}], "}"}], "\[Rule]", " ", "2"}], "}"}]}], "]"}], ",", RowBox[{"ReshapeLayer", "[", RowBox[{"{", RowBox[{"161", ",", "1"}], "}"}], "]"}]}], "}"}], "]"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"ConvolutionLayer", "[", RowBox[{"32", ",", RowBox[{"{", RowBox[{"11", ",", "41"}], "}"}], ",", RowBox[{"\"\\"", "\[Rule]", "True"}], ",", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", RowBox[{"5", ",", "20"}], "}"}]}], ",", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", RowBox[{"3", ",", "2"}], "}"}]}]}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"BatchNormalizationLayer", "[", RowBox[{"\"\\"", "\[Rule]", "True"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"ElementwiseLayer", "[", RowBox[{"Function", "[", RowBox[{ RowBox[{"{", "x", "}"}], ",", RowBox[{"Clip", "[", RowBox[{"x", ",", RowBox[{"{", RowBox[{"0", ",", "24"}], "}"}]}], "]"}]}], "]"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"ConvolutionLayer", "[", RowBox[{"32", ",", RowBox[{"{", RowBox[{"11", ",", "21"}], "}"}], ",", RowBox[{"\"\\"", "\[Rule]", "True"}], ",", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", RowBox[{"5", ",", "10"}], "}"}]}], ",", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", RowBox[{"1", ",", "2"}], "}"}]}]}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"BatchNormalizationLayer", "[", RowBox[{"\"\\"", "\[Rule]", "True"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"ElementwiseLayer", "[", RowBox[{"Function", "[", RowBox[{ RowBox[{"{", "x", "}"}], ",", RowBox[{"Clip", "[", RowBox[{"x", ",", RowBox[{"{", RowBox[{"0", ",", "24"}], "}"}]}], "]"}]}], "]"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"FlattenLayer", "[", RowBox[{"-", "1"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"biGRUBNLayer", "[", RowBox[{"1312", ",", "1024"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"biGRUBNLayer", "[", RowBox[{"2048", ",", "1024"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"biGRUBNLayer", "[", RowBox[{"2048", ",", "1024"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"NetMapOperator", "[", RowBox[{"NetChain", "[", RowBox[{"{", RowBox[{ RowBox[{"LinearLayer", "[", "29", "]"}], ",", RowBox[{"SoftmaxLayer", "[", "]"}]}], "}"}], "]"}], "]"}]}], "\[IndentingNewLine]", "}"}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", "enc"}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", "dec"}]}], "\[IndentingNewLine]", "]"}]}]], "Input", CellChangeTimes->{{3.733215378494*^9, 3.733215475971236*^9}, { 3.733216000376679*^9, 3.733216002345433*^9}, {3.7332160695525627`*^9, 3.7332161236635303`*^9}, {3.7332161911433363`*^9, 3.7332161955154657`*^9}, 3.733216271011199*^9, {3.809250367489215*^9, 3.8092503790873613`*^9}, { 3.81722779452046*^9, 3.817227836001773*^9}}, CellLabel->"In[22]:=",ExpressionUUID->"0dac311f-6bd1-4d70-89ac-edd4b16cb5d0"], Cell[BoxData[ TagBox[ TemplateBox[{ RowBox[{ StyleBox[ TagBox["NetChain", "SummaryHead"], "NonInterpretableSummary"], StyleBox["[", "NonInterpretableSummary"], DynamicModuleBox[{Typeset`open = True}, PanelBox[ PaneSelectorBox[{False -> GridBox[{{ PaneBox[ ButtonBox[ DynamicBox[ FEPrivate`FrontEndResource[ "FEBitmaps", "SquarePlusIconMedium"], ImageSizeCache -> {12., {0., 12.}}], Appearance -> None, ButtonFunction :> (Typeset`open = True), Evaluator -> Automatic, Method -> "Preemptive"], Alignment -> {Center, Center}, ImageSize -> {Automatic, 24}], StyleBox[ OverlayBox[{ GraphicsBox[{ Thickness[0.014925373134328358`], { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{ Style[ "Miter", NeuralNetworks`Private`FormattingCommon`foreground], 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{23., 91.5}, {30., 91.5}}}, CurveClosed -> {0}]}, { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{ Style[ "Miter", NeuralNetworks`Private`FormattingCommon`foreground], 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{37., 91.5}, {44., 91.5}}}, CurveClosed -> {0}]}, { FaceForm[{ RGBColor[0.84, 0.84, 0.84], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 15.5, 103.5}, {23.5, 103.5}, {23.5, 79.5}, {15.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{24., 104.}, {15., 104.}, { 15., 79.}, {24., 79.}}, {{23., 80.}, {16., 80.}, {16., 103.}, {23., 103.}}}]}, { FaceForm[{ RGBColor[0.84, 0.84, 0.84], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 29.5, 103.5}, {37.5, 103.5}, {37.5, 79.5}, {29.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{38., 104.}, {29., 104.}, { 29., 79.}, {38., 79.}}, {{37., 80.}, {30., 80.}, {30., 103.}, {37., 103.}}}]}, { FaceForm[{ RGBColor[0.84, 0.84, 0.84], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 43.5, 103.5}, {51.5, 103.5}, {51.5, 79.5}, {43.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{52., 104.}, {43., 104.}, { 43., 79.}, {52., 79.}}, {{51., 80.}, {44., 80.}, {44., 103.}, {51., 103.}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{28., 91.5}, { 25., 95.}, {25., 88.}}}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{42., 91.5}, { 39., 95.}, {39., 88.}}}]}}, ImageMargins -> 0, ImageSize -> {36, Automatic}, PlotRangePadding -> 0], StyleBox[ "uninitialized", LineColor -> RGBColor[0.66, 0, 0], FrontFaceColor -> RGBColor[0.66, 0, 0], BackFaceColor -> RGBColor[0.66, 0, 0], GraphicsColor -> RGBColor[0.66, 0, 0], FontFamily -> "Roboto", FontSize -> 8, FontColor -> RGBColor[0.66, 0, 0], Background -> GrayLevel[1, 0.65]]}, Alignment -> {Center, Scaled[0.0001]}], CacheGraphics -> False], GridBox[{{ StyleBox[ TemplateBox[{ TemplateBox[{"\[ThinSpace]", "\" \"", StyleBox["\"Input\"", GrayLevel[0], StripOnInput -> False], "\"port\""}, "RowWithSeparators"], "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox["\"expression\"", "SummaryItem"]}, { StyleBox[ TemplateBox[{ TemplateBox[{"\[ThinSpace]", "\" \"", StyleBox["\"Output\"", GrayLevel[0], StripOnInput -> False], "\"port\""}, "RowWithSeparators"], "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox["\"ctcbeam search\"", "SummaryItem"]}}, BaselinePosition -> Automatic, GridBoxAlignment -> { "Columns" -> {{Left}}, "Rows" -> {{Automatic}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> { "Columns" -> {{2}}, "Rows" -> {{Automatic}}}]}}, BaselinePosition -> {1, 3}, GridBoxAlignment -> {"Rows" -> {{Top}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> {"Columns" -> { Offset[0.], { Offset[0.5599999999999999]}, Offset[0.]}, "Rows" -> { Offset[0.], { Offset[0.4]}, Offset[0.]}}], True -> GridBox[{{ PaneBox[ ButtonBox[ DynamicBox[ FEPrivate`FrontEndResource[ "FEBitmaps", "SquareMinusIconMedium"], ImageSizeCache -> {12., {0., 12.}}], Appearance -> None, ButtonFunction :> (Typeset`open = False), Evaluator -> Automatic, Method -> "Preemptive"], Alignment -> {Center, Center}, ImageSize -> {Automatic, 24}], StyleBox[ OverlayBox[{ GraphicsBox[{ Thickness[0.014925373134328358`], { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{ Style[ "Miter", NeuralNetworks`Private`FormattingCommon`foreground], 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{23., 91.5}, {30., 91.5}}}, CurveClosed -> {0}]}, { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{ Style[ "Miter", NeuralNetworks`Private`FormattingCommon`foreground], 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{37., 91.5}, {44., 91.5}}}, CurveClosed -> {0}]}, { FaceForm[{ RGBColor[0.84, 0.84, 0.84], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 15.5, 103.5}, {23.5, 103.5}, {23.5, 79.5}, {15.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{24., 104.}, {15., 104.}, { 15., 79.}, {24., 79.}}, {{23., 80.}, {16., 80.}, {16., 103.}, {23., 103.}}}]}, { FaceForm[{ RGBColor[0.84, 0.84, 0.84], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 29.5, 103.5}, {37.5, 103.5}, {37.5, 79.5}, {29.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{38., 104.}, {29., 104.}, { 29., 79.}, {38., 79.}}, {{37., 80.}, {30., 80.}, {30., 103.}, {37., 103.}}}]}, { FaceForm[{ RGBColor[0.84, 0.84, 0.84], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 43.5, 103.5}, {51.5, 103.5}, {51.5, 79.5}, {43.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{52., 104.}, {43., 104.}, { 43., 79.}, {52., 79.}}, {{51., 80.}, {44., 80.}, {44., 103.}, {51., 103.}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{28., 91.5}, { 25., 95.}, {25., 88.}}}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{42., 91.5}, { 39., 95.}, {39., 88.}}}]}}, ImageMargins -> 0, ImageSize -> {36, Automatic}, PlotRangePadding -> 0], StyleBox[ "uninitialized", LineColor -> RGBColor[0.66, 0, 0], FrontFaceColor -> RGBColor[0.66, 0, 0], BackFaceColor -> RGBColor[0.66, 0, 0], GraphicsColor -> RGBColor[0.66, 0, 0], FontFamily -> "Roboto", FontSize -> 8, FontColor -> RGBColor[0.66, 0, 0], Background -> GrayLevel[1, 0.65]]}, Alignment -> {Center, Scaled[0.0001]}], CacheGraphics -> False], GridBox[{{ DynamicModuleBox[{ NeuralNetworks`Private`NetChain`assoc3 = Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association["Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Dimensions" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {161}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT]]], "2" -> Association[ "Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "Input2" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "2", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input1"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}], "2" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Log[1.*^-14 + #]& ], "$Dimensions" -> {161}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]]], "3" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association["Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Dimensions" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {161}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT]]], "2" -> Association[ "Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[Plus], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "Input2" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "2", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input1"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}], "4" -> Association["Type" -> "Graph", "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Nodes" -> Association["1" -> Association["Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Dimensions" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {161}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT]]], "2" -> Association[ "Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "Input2" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "2", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input1"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}], "5" -> Association["Type" -> "Reshape", "Arrays" -> Association[], "Parameters" -> Association["Dimensions" -> NeuralNetworks`ValidatedParameter[{161, 1}]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161, 1}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "4", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "3", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "5", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "4", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "5", "Outputs", "Output"]}, "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161, 1}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1510227809], "$InputShape" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{161, 1}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1510227809], 161}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1510227809], 161, 1}, NeuralNetworks`RealT]]], "2" -> Association[ "Type" -> "Convolution", "Arrays" -> Association[ "Weights" -> NeuralNetworks`TensorT[{32, 1, 11, 41}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputChannels" -> 32, "KernelSize" -> {11, 41}, "Stride" -> {3, 2}, "PaddingSize" -> {{5, 5}, {20, 20}}, "Dilation" -> {1, 1}, "ChannelGroups" -> 1, "Dimensionality" -> 2, "Interleaving" -> True, "$InputChannels" -> 1, "$InputSize" -> { NeuralNetworks`LengthVar[1510227809], 161}, "$OutputSize" -> { NeuralNetworks`LengthVar[206861180], 81}, "$WeightsInputChannels" -> 1], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1510227809], 161, 1}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[206861180], 81, 32}, NeuralNetworks`RealT]]], "3" -> Association[ "Type" -> "BatchNormalization", "Arrays" -> Association[ "Scaling" -> NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT], "MovingMean" -> NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT], "MovingVariance" -> NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Momentum" -> 0.9, "Epsilon" -> 0.001, "Interleaving" -> True, "$Channels" -> 32, "$SpatialDimensions" -> { NeuralNetworks`LengthVar[206861180], 81}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[206861180], 81, 32}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[206861180], 81, 32}, NeuralNetworks`RealT]]], "4" -> Association[ "Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[Clip[#, {0, 24}]& ], "$Dimensions" -> { NeuralNetworks`LengthVar[206861180], 81, 32}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[206861180], 81, 32}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[206861180], 81, 32}, NeuralNetworks`RealT]]], "5" -> Association[ "Type" -> "Convolution", "Arrays" -> Association[ "Weights" -> NeuralNetworks`TensorT[{32, 32, 11, 21}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputChannels" -> 32, "KernelSize" -> {11, 21}, "Stride" -> {1, 2}, "PaddingSize" -> {{5, 5}, {10, 10}}, "Dilation" -> {1, 1}, "ChannelGroups" -> 1, "Dimensionality" -> 2, "Interleaving" -> True, "$InputChannels" -> 32, "$InputSize" -> { NeuralNetworks`LengthVar[206861180], 81}, "$OutputSize" -> { NeuralNetworks`LengthVar[2134392381], 41}, "$WeightsInputChannels" -> 32], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[206861180], 81, 32}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 41, 32}, NeuralNetworks`RealT]]], "6" -> Association[ "Type" -> "BatchNormalization", "Arrays" -> Association[ "Scaling" -> NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT], "MovingMean" -> NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT], "MovingVariance" -> NeuralNetworks`TensorT[{32}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Momentum" -> 0.9, "Epsilon" -> 0.001, "Interleaving" -> True, "$Channels" -> 32, "$SpatialDimensions" -> { NeuralNetworks`LengthVar[2134392381], 41}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 41, 32}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 41, 32}, NeuralNetworks`RealT]]], "7" -> Association[ "Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[Clip[#, {0, 24}]& ], "$Dimensions" -> { NeuralNetworks`LengthVar[2134392381], 41, 32}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 41, 32}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 41, 32}, NeuralNetworks`RealT]]], "8" -> Association[ "Type" -> "Flatten", "Arrays" -> Association[], "Parameters" -> Association["Level" -> -1], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 41, 32}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1312}, NeuralNetworks`RealT]]], "9" -> Association[ "Type" -> "NetBidirectional", "Arrays" -> Association[], "Parameters" -> Association[ "ForwardNet" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 1312}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 1312, "$InputDimensions" -> {1312}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1312}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381], "$InputShape" -> NeuralNetworks`TensorT[{1312}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1312}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "2" -> Association["Type" -> "BatchNormalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingMean" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingVariance" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Momentum" -> 0.9, "Epsilon" -> 0.001, "Interleaving" -> True, "$Channels" -> 3072, "$SpatialDimensions" -> { NeuralNetworks`LengthVar[2134392381]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "3" -> Association["Type" -> "NetFold", "Arrays" -> Association[], "Parameters" -> Association["Net" -> Association["Type" -> "Graph", "Inputs" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Nodes" -> Association[ "biases" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`RealT]], "Parameters" -> Association["Dimensions" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {3, 1024}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]]], "{Wis,Wrs}.s[t-1]" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{2048, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {2048}, "$OutputSize" -> 2048, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Wix.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wmx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[2049, 3072]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "bi" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[1]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "br" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[2]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "bm" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[3]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "Wis.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrs.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+i" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+r" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t] * s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "(1-i[t])*s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[#2 Subtract[1, #]& ], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wms.(r[t] * s[t-1])" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {1024}, "$OutputSize" -> 1024, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+m" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "m[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Ramp], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]*m[t]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "state" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "bi", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "br", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "bm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bi", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "br", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bm", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+i", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+r", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "r[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "m[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+m", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "m[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "state", "Outputs", "Output"]}], "Recurrence" -> NeuralNetworks`ValidatedParameter[{"Output" -> "State"}], "ConstantPorts" -> {}, "OutputPorts" -> {"Output"}, "$InputNames" -> {"Input"}, "$StateNames" -> {"State"}, "$OutputNames" -> {"Output"}, "$ForceStateRank" -> False, "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "States" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "3", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1312}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "InteriorStates" -> Association[{3, "State"} -> NeuralNetworks`NetPath["Nodes", "3", "States", "State"]]], "BackwardNet" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 1312}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 1312, "$InputDimensions" -> {1312}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1312}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381], "$InputShape" -> NeuralNetworks`TensorT[{1312}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1312}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "2" -> Association["Type" -> "BatchNormalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingMean" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingVariance" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Momentum" -> 0.9, "Epsilon" -> 0.001, "Interleaving" -> True, "$Channels" -> 3072, "$SpatialDimensions" -> { NeuralNetworks`LengthVar[2134392381]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "3" -> Association["Type" -> "NetFold", "Arrays" -> Association[], "Parameters" -> Association["Net" -> Association["Type" -> "Graph", "Inputs" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Nodes" -> Association[ "biases" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`RealT]], "Parameters" -> Association["Dimensions" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {3, 1024}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]]], "{Wis,Wrs}.s[t-1]" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{2048, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {2048}, "$OutputSize" -> 2048, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Wix.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wmx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[2049, 3072]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "bi" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[1]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "br" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[2]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "bm" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[3]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "Wis.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrs.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+i" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+r" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t] * s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "(1-i[t])*s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[#2 Subtract[1, #]& ], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wms.(r[t] * s[t-1])" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {1024}, "$OutputSize" -> 1024, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+m" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "m[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Ramp], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]*m[t]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "state" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "bi", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "br", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "bm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bi", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "br", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bm", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+i", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+r", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "r[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "m[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+m", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "m[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "state", "Outputs", "Output"]}], "Recurrence" -> NeuralNetworks`ValidatedParameter[{"Output" -> "State"}], "ConstantPorts" -> {}, "OutputPorts" -> {"Output"}, "$InputNames" -> {"Input"}, "$StateNames" -> {"State"}, "$OutputNames" -> {"Output"}, "$ForceStateRank" -> False, "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "States" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "3", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1312}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "InteriorStates" -> Association[{3, "State"} -> NeuralNetworks`NetPath["Nodes", "3", "States", "State"]]], "Aggregation" -> Catenate, "$InputTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1312}, NeuralNetworks`RealT], "$ForwardTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT], "$BackwardTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT], "$SequentialOutput" -> True], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1312}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]]], "10" -> Association[ "Type" -> "NetBidirectional", "Arrays" -> Association[], "Parameters" -> Association[ "ForwardNet" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 2048}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 2048, "$InputDimensions" -> {2048}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381], "$InputShape" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "2" -> Association["Type" -> "BatchNormalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingMean" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingVariance" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Momentum" -> 0.9, "Epsilon" -> 0.001, "Interleaving" -> True, "$Channels" -> 3072, "$SpatialDimensions" -> { NeuralNetworks`LengthVar[2134392381]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "3" -> Association["Type" -> "NetFold", "Arrays" -> Association[], "Parameters" -> Association["Net" -> Association["Type" -> "Graph", "Inputs" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Nodes" -> Association[ "biases" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`RealT]], "Parameters" -> Association["Dimensions" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {3, 1024}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]]], "{Wis,Wrs}.s[t-1]" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{2048, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {2048}, "$OutputSize" -> 2048, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Wix.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wmx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[2049, 3072]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "bi" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[1]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "br" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[2]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "bm" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[3]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "Wis.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrs.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+i" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+r" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t] * s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "(1-i[t])*s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[#2 Subtract[1, #]& ], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wms.(r[t] * s[t-1])" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {1024}, "$OutputSize" -> 1024, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+m" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "m[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Ramp], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]*m[t]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "state" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "bi", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "br", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "bm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bi", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "br", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bm", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+i", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+r", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "r[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "m[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+m", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "m[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "state", "Outputs", "Output"]}], "Recurrence" -> NeuralNetworks`ValidatedParameter[{"Output" -> "State"}], "ConstantPorts" -> {}, "OutputPorts" -> {"Output"}, "$InputNames" -> {"Input"}, "$StateNames" -> {"State"}, "$OutputNames" -> {"Output"}, "$ForceStateRank" -> False, "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "States" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "3", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "InteriorStates" -> Association[{3, "State"} -> NeuralNetworks`NetPath["Nodes", "3", "States", "State"]]], "BackwardNet" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 2048}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 2048, "$InputDimensions" -> {2048}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381], "$InputShape" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "2" -> Association["Type" -> "BatchNormalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingMean" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingVariance" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Momentum" -> 0.9, "Epsilon" -> 0.001, "Interleaving" -> True, "$Channels" -> 3072, "$SpatialDimensions" -> { NeuralNetworks`LengthVar[2134392381]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "3" -> Association["Type" -> "NetFold", "Arrays" -> Association[], "Parameters" -> Association["Net" -> Association["Type" -> "Graph", "Inputs" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Nodes" -> Association[ "biases" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`RealT]], "Parameters" -> Association["Dimensions" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {3, 1024}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]]], "{Wis,Wrs}.s[t-1]" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{2048, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {2048}, "$OutputSize" -> 2048, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Wix.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wmx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[2049, 3072]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "bi" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[1]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "br" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[2]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "bm" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[3]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "Wis.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrs.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+i" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+r" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t] * s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "(1-i[t])*s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[#2 Subtract[1, #]& ], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association[ "Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wms.(r[t] * s[t-1])" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {1024}, "$OutputSize" -> 1024, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+m" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "m[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Ramp], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]*m[t]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "state" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "bi", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "br", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "bm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bi", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "br", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bm", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+i", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+r", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "r[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "m[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+m", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "m[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "state", "Outputs", "Output"]}], "Recurrence" -> NeuralNetworks`ValidatedParameter[{"Output" -> "State"}], "ConstantPorts" -> {}, "OutputPorts" -> {"Output"}, "$InputNames" -> {"Input"}, "$StateNames" -> {"State"}, "$OutputNames" -> {"Output"}, "$ForceStateRank" -> False, "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "States" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "3", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "InteriorStates" -> Association[{3, "State"} -> NeuralNetworks`NetPath["Nodes", "3", "States", "State"]]], "Aggregation" -> Catenate, "$InputTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT], "$ForwardTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT], "$BackwardTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT], "$SequentialOutput" -> True], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]]], "11" -> Association[ "Type" -> "NetBidirectional", "Arrays" -> Association[], "Parameters" -> Association[ "ForwardNet" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 2048}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 2048, "$InputDimensions" -> {2048}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381], "$InputShape" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "2" -> Association["Type" -> "BatchNormalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingMean" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingVariance" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Momentum" -> 0.9, "Epsilon" -> 0.001, "Interleaving" -> True, "$Channels" -> 3072, "$SpatialDimensions" -> { NeuralNetworks`LengthVar[2134392381]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "3" -> Association["Type" -> "NetFold", "Arrays" -> Association[], "Parameters" -> Association["Net" -> Association["Type" -> "Graph", "Inputs" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Nodes" -> Association[ "biases" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`RealT]], "Parameters" -> Association["Dimensions" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {3, 1024}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]]], "{Wis,Wrs}.s[t-1]" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{2048, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {2048}, "$OutputSize" -> 2048, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Wix.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wmx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[2049, 3072]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "bi" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[1]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "br" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[2]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "bm" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[3]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "Wis.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrs.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+i" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+r" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t] * s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "(1-i[t])*s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[#2 Subtract[1, #]& ], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wms.(r[t] * s[t-1])" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {1024}, "$OutputSize" -> 1024, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+m" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "m[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Ramp], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]*m[t]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "state" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "bi", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "br", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "bm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bi", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "br", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bm", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+i", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+r", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "r[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "m[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+m", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "m[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "state", "Outputs", "Output"]}], "Recurrence" -> NeuralNetworks`ValidatedParameter[{"Output" -> "State"}], "ConstantPorts" -> {}, "OutputPorts" -> {"Output"}, "$InputNames" -> {"Input"}, "$StateNames" -> {"State"}, "$OutputNames" -> {"Output"}, "$ForceStateRank" -> False, "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "States" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "3", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "InteriorStates" -> Association[{3, "State"} -> NeuralNetworks`NetPath["Nodes", "3", "States", "State"]]], "BackwardNet" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{3072, 2048}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {3072}, "$OutputSize" -> 3072, "$InputSize" -> 2048, "$InputDimensions" -> {2048}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381], "$InputShape" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "2" -> Association["Type" -> "BatchNormalization", "Arrays" -> Association["Scaling" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingMean" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT], "MovingVariance" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Momentum" -> 0.9, "Epsilon" -> 0.001, "Interleaving" -> True, "$Channels" -> 3072, "$SpatialDimensions" -> { NeuralNetworks`LengthVar[2134392381]}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]]], "3" -> Association["Type" -> "NetFold", "Arrays" -> Association[], "Parameters" -> Association["Net" -> Association["Type" -> "Graph", "Inputs" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Nodes" -> Association[ "biases" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`RealT]], "Parameters" -> Association["Dimensions" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {3, 1024}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]]], "{Wis,Wrs}.s[t-1]" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{2048, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {2048}, "$OutputSize" -> 2048, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]]], "Wix.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wmx.x[t]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[2049, 3072]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "bi" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[1]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "br" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[2]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "bm" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[3]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{3, 1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]]], "Wis.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1, 1024]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wrs.s[t-1]" -> Association["Type" -> "Part", "Arrays" -> Association[], "Parameters" -> Association[ "Specification" -> NeuralNetworks`ValidatedParameter[ Span[1025, 2048]]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+i" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+r" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[LogisticSigmoid], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "r[t] * s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "(1-i[t])*s[t-1]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[#2 Subtract[1, #]& ], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Wms.(r[t] * s[t-1])" -> Association["Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {1024}, "$OutputSize" -> 1024, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "+m" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input3" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`AtomT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "m[t]" -> Association["Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Ramp], "$Dimensions" -> {1024}], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "i[t]*m[t]" -> Association["Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association["Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "state" -> Association["Type" -> "Total", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "Input2" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> {NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "State"], NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "bi", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "br", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "bm", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "biases", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "{Wis,Wrs}.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wix.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wrx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "Wmx.x[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bi", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "br", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input3"] -> NeuralNetworks`NetPath[ "Nodes", "bm", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+i", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wis.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+r", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wrs.s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+i", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+r", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "r[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "r[t] * s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "(1-i[t])*s[t-1]", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "+m", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "Wms.(r[t] * s[t-1])", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "m[t]", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "+m", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Inputs", "Input2"] -> NeuralNetworks`NetPath[ "Nodes", "m[t]", "Outputs", "Output"], NeuralNetworks`NetPath[ "Nodes", "state", "Inputs", "Input1"] -> NeuralNetworks`NetPath[ "Nodes", "i[t]*m[t]", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "state", "Outputs", "Output"]}], "Recurrence" -> NeuralNetworks`ValidatedParameter[{"Output" -> "State"}], "ConstantPorts" -> {}, "OutputPorts" -> {"Output"}, "$InputNames" -> {"Input"}, "$StateNames" -> {"State"}, "$OutputNames" -> {"Output"}, "$ForceStateRank" -> False, "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 3072}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "States" -> Association["State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "3", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT]], "InteriorStates" -> Association[{3, "State"} -> NeuralNetworks`NetPath["Nodes", "3", "States", "State"]]], "Aggregation" -> Catenate, "$InputTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT], "$ForwardTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT], "$BackwardTensor" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 1024}, NeuralNetworks`RealT], "$SequentialOutput" -> True], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]]], "12" -> Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Linear", "Arrays" -> Association["Weights" -> NeuralNetworks`TensorT[{29, 2048}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{29}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {29}, "$OutputSize" -> 29, "$InputSize" -> 2048, "$InputDimensions" -> {2048}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{29}, NeuralNetworks`RealT]]], "2" -> Association["Type" -> "Softmax", "Arrays" -> Association[], "Parameters" -> Association["Level" -> -1], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{29}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{29}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}, "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{29}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[2134392381], "$InputShape" -> NeuralNetworks`TensorT[{2048}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{29}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 2048}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 29}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "4", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "3", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "5", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "4", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "6", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "5", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "7", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "6", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "8", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "7", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "9", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "8", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "10", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "9", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "11", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "10", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "12", "Inputs", "Input"] -> NeuralNetworks`NetPath[ "Nodes", "11", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "12", "Outputs", "Output"]}, "Inputs" -> Association[ "Input" -> Association["Form" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1510227809], 161}, NeuralNetworks`RealT], "Type" -> "Function", "Function" -> NeuralNetworks`ValidatedParameter[ Tooltip[ Row[{Function, "[", Row[{ Style["x", Italic], ",", "\[Ellipsis]"}], "]"}], Function[{$CellContext`input}, Module[{$CellContext`a = $CellContext`input, \ $CellContext`power}, If[Head[$CellContext`a] === File, $CellContext`a = Audio[$CellContext`a]]; If[ AudioQ[$CellContext`a], If[AudioChannels[$CellContext`a] =!= 1, $CellContext`a = Mean[ AudioChannelSeparate[$CellContext`a]]]; If[AudioSampleRate[$CellContext`a] =!= Quantity[16000, "Hertz"], $CellContext`a = AudioResample[$CellContext`a, 16000, Resampling -> {"Hermite", 2}]]; $CellContext`power = AudioMeasurements[$CellContext`a, "Power"]; If[$CellContext`power =!= 0.01, $CellContext`a = AudioAmplify[$CellContext`a, Sqrt[0.01/$CellContext`power]]]; Abs[ Part[ SpectrogramArray[$CellContext`a, 320, 160, HannWindow], All, Span[1, 161]]]^2, $Failed]]]]], "OutputType" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[9913298], 161}, NeuralNetworks`RealT], "Pattern" -> NeuralNetworks`ValidatedParameter[None], "Batched" -> False, "$Version" -> "12.2.13"]], "Outputs" -> Association[ "Output" -> Association["Form" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[2134392381], 29}, NeuralNetworks`RealT], "Type" -> "CTCBeamSearch", "Labels" -> Tooltip[{"\"'\"", "\" \"", Skeleton[25], "\"z\""}, { "'", " ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"}], "BeamSize" -> 100, "$AlphabetSize" -> 28, "$InputSize" -> 29, "$Version" -> "12.2.13"]]], NeuralNetworks`Private`NetChain`opart = {"Nodes", "1"}, NeuralNetworks`Private`NetChain`part = {"Nodes", "1"}, NeuralNetworks`Private`NetChain`selected = Association[ "Type" -> "NetMap", "Arrays" -> Association[], "Parameters" -> Association[ "Net" -> Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "Graph", "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Dimensions" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {161}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT]]], "2" -> Association[ "Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "Input2" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input1"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}], "2" -> Association[ "Type" -> "Elementwise", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[ Log[1.*^-14 + #]& ], "$Dimensions" -> {161}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]]], "3" -> Association[ "Type" -> "Graph", "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Dimensions" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {161}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT]]], "2" -> Association[ "Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[Plus], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "Input2" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input1"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}], "4" -> Association[ "Type" -> "Graph", "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Nodes" -> Association[ "1" -> Association[ "Type" -> "NetArray", "Arrays" -> Association["Array" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Parameters" -> Association[ "Dimensions" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "$ArrayDimensions" -> {161}], "Inputs" -> Association[], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT]]], "2" -> Association[ "Type" -> "Threading", "Arrays" -> Association[], "Parameters" -> Association[ "Function" -> NeuralNetworks`ValidatedParameter[Times], "Broadcasting" -> NeuralNetworks`ValidatedParameter[ Automatic]], "Inputs" -> Association["Input1" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`AtomT], "Input2" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input2"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input1"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "2", "Outputs", "Output"]}], "5" -> Association[ "Type" -> "Reshape", "Arrays" -> Association[], "Parameters" -> Association[ "Dimensions" -> NeuralNetworks`ValidatedParameter[{161, 1}]], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161, 1}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "4", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "3", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "5", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "4", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "5", "Outputs", "Output"]}, "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{161, 1}, NeuralNetworks`RealT]]], "$SequenceLength" -> NeuralNetworks`LengthVar[1510227809], "$InputShape" -> NeuralNetworks`TensorT[{161}, NeuralNetworks`RealT], "$OutputShape" -> NeuralNetworks`TensorT[{161, 1}, NeuralNetworks`RealT]], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1510227809], 161}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1510227809], 161, 1}, NeuralNetworks`RealT]]]}, DynamicBox[ GridBox[{{ TagBox[ TagBox[ GridBox[{{ TagBox[ TagBox[ "\"\"", Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ "\"Input\"", Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TagBox[ GridBox[{{"\"expression\""}, { TemplateBox[{"\[ThinSpace]", "\" \"", "\"matrix\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"1\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]161\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"]}}, GridBoxAlignment -> {"Columns" -> {{Left}}}, BaselinePosition -> 2, DefaultBaseStyle -> "Column", GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}], "Column"], Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"1\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "1"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ RowBox[{ TagBox["NetMapOperator", HoldForm], "[", TemplateBox[{"NetChain", "\" \"", TemplateBox[{"\"(\"", "5", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 (CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification])}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> {"SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"], "]"}], RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "1"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"array\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"1\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]161\[Times]1\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "1"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"2\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "2"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox["ConvolutionLayer", RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "2"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"array\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"2\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]81\[Times]32\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "2"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"3\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "3"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox["BatchNormalizationLayer", RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "3"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"array\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"2\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]81\[Times]32\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "3"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"4\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "4"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ RowBox[{"Clip", "[", RowBox[{ StyleBox["\"x\"", Italic, StripOnInput -> False], ",", RowBox[{"{", RowBox[{"0", ",", "24"}], "}"}]}], "]"}], GrayLevel[0], StripOnInput -> False], Annotation[#, {"Nodes", "4"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"array\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"2\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]81\[Times]32\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "4"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"5\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "5"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox["ConvolutionLayer", RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "5"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"array\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"3\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]41\[Times]32\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "5"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"6\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "6"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox["BatchNormalizationLayer", RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "6"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"array\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"3\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]41\[Times]32\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "6"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"7\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "7"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ RowBox[{"Clip", "[", RowBox[{ StyleBox["\"x\"", Italic, StripOnInput -> False], ",", RowBox[{"{", RowBox[{"0", ",", "24"}], "}"}]}], "]"}], GrayLevel[0], StripOnInput -> False], Annotation[#, {"Nodes", "7"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"array\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"3\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]41\[Times]32\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "7"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"8\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "8"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox["FlattenLayer", GrayLevel[0], StripOnInput -> False], Annotation[#, {"Nodes", "8"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"matrix\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"3\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]1312\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "8"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"9\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "9"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ RowBox[{ TagBox["NetBidirectionalOperator", HoldForm], "[", RowBox[{"{", RowBox[{ TemplateBox[{"NetChain", "\" \"", TemplateBox[{"\"(\"", "3", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 (CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification])}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> { "SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"], ",", TemplateBox[{"NetChain", "\" \"", TemplateBox[{"\"(\"", "3", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 (CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification])}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> { "SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"]}], "}"}], "]"}], RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "9"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"matrix\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"3\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]2048\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "9"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"10\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "10"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ RowBox[{ TagBox["NetBidirectionalOperator", HoldForm], "[", RowBox[{"{", RowBox[{ TemplateBox[{"NetChain", "\" \"", TemplateBox[{"\"(\"", "3", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 (CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification])}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> { "SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"], ",", TemplateBox[{"NetChain", "\" \"", TemplateBox[{"\"(\"", "3", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 (CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification])}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> { "SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"]}], "}"}], "]"}], RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "10"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"matrix\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"3\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]2048\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "10"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"11\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "11"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ RowBox[{ TagBox["NetBidirectionalOperator", HoldForm], "[", RowBox[{"{", RowBox[{ TemplateBox[{"NetChain", "\" \"", TemplateBox[{"\"(\"", "3", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 (CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification])}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> { "SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"], ",", TemplateBox[{"NetChain", "\" \"", TemplateBox[{"\"(\"", "3", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 (CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification])}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> { "SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"]}], "}"}], "]"}], RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "11"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"matrix\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"3\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]2048\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "11"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"12\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "12"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox[ RowBox[{ TagBox["NetMapOperator", HoldForm], "[", TemplateBox[{"NetChain", "\" \"", TemplateBox[{"\"(\"", "2", "\" nodes)\""}, "Row", DisplayFunction -> (RowBox[{ TemplateSlotSequence[1, "\[InvisibleSpace]"]}]& ), InterpretationFunction -> (RowBox[{"Row", "[", RowBox[{ RowBox[{"{", TemplateSlotSequence[1, ","], "}"}], ",", RowBox[{"BaseStyle", "\[Rule]", InterpretationBox[ ButtonBox[ TooltipBox[ GraphicsBox[{{ GrayLevel[0], RectangleBox[{0, 0}]}, { GrayLevel[0], RectangleBox[{1, -1}]}, { GrayLevel[0.5], RectangleBox[{0, -1}, {2, 1}]}}, DefaultBaseStyle -> "ColorSwatchGraphics", AspectRatio -> 1, Frame -> True, FrameStyle -> GrayLevel[0.33333333333333337`], FrameTicks -> None, PlotRangePadding -> None, ImageSize -> Dynamic[{Automatic, 1.35 (CurrentValue["FontCapHeight"]/ AbsoluteCurrentValue[Magnification])}]], StyleBox[ RowBox[{"GrayLevel", "[", "0.5`", "]"}], NumberMarks -> False]], Appearance -> None, BaseStyle -> {}, BaselinePosition -> Baseline, DefaultBaseStyle -> {}, ButtonFunction :> With[{Typeset`box$ = EvaluationBox[]}, If[ Not[ AbsoluteCurrentValue["Deployed"]], SelectionMove[Typeset`box$, All, Expression]; FrontEnd`Private`$ColorSelectorInitialAlpha = 1; FrontEnd`Private`$ColorSelectorInitialColor = GrayLevel[0.5]; FrontEnd`Private`$ColorSelectorUseMakeBoxes = True; MathLink`CallFrontEnd[ FrontEnd`AttachCell[Typeset`box$, FrontEndResource["GrayLevelColorValueSelector"], { 0, {Left, Bottom}}, {Left, Top}, "ClosingActions" -> {"SelectionDeparture", "ParentChanged", "EvaluatorQuit"}]]]], BaseStyle -> Inherited, Evaluator -> Automatic, Method -> "Preemptive"], GrayLevel[0.5], Editable -> False, Selectable -> False]}]}], "]"}]& ), BaseStyle -> GrayLevel[0.5]]}, "RowDefault"], "]"}], RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "12"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\[ThinSpace]", "\" \"", "\"matrix\"", StyleBox[ TemplateBox[{"\"(\[VeryThinSpace]size: \"", StyleBox[ SubscriptBox["\"n\"", StyleBox["\"3\"", 7, StripOnInput -> False]], Italic, StripOnInput -> False], "\"\[Times]29\[VeryThinSpace])\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowWithSeparators"], Annotation[#, {"Nodes", "12"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ "\"\"", Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ "\"Output\"", Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ "\"ctcbeam search\"", Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}}, GridBoxAlignment -> {"Columns" -> {{Left}}}, AutoDelete -> False, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> {"Columns" -> {{1.1}}}], "Grid"], EventHandlerTag[{"MouseClicked" :> If[ ListQ[ NeuralNetworks`Private`NetChain`part = MouseAnnotation[]], If[NeuralNetworks`Private`NetChain`opart === NeuralNetworks`Private`NetChain`part, NeuralNetworks`Private`NetChain`selected = Null; NeuralNetworks`Private`NetChain`opart = Null, NeuralNetworks`Private`NetChain`selected = Part[NeuralNetworks`Private`NetChain`assoc3, Apply[Sequence, NeuralNetworks`Private`NetChain`part]]; NeuralNetworks`Private`NetChain`opart = NeuralNetworks`Private`NetChain`part; Null]; Null], Method -> "Preemptive", PassEventsDown -> Automatic, PassEventsUp -> True}]]}, NeuralNetworks`FormatSelectedParameterChain[ NeuralNetworks`Private`NetChain`selected, NeuralNetworks`Private`NetChain`part, Association[], {}]}, GridBoxSpacings -> {"Columns" -> {{1}}}, GridBoxAlignment -> {"Columns" -> {{Left}}}, GridFrameMargins -> {{0, 0}, {0, 0}}], ImageSizeCache -> { 472., {328.172119140625, 333.827880859375}}, TrackedSymbols :> { NeuralNetworks`Private`NetChain`selected}], Initialization :> {NetChain}]}}, BaselinePosition -> Automatic, GridBoxAlignment -> { "Columns" -> {{Left}}, "Rows" -> {{Automatic}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> { "Columns" -> {{2}}, "Rows" -> {{Automatic}}}]}}, BaselinePosition -> {1, 1}, GridBoxAlignment -> {"Rows" -> {{Top}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> {"Columns" -> { Offset[0.], { Offset[0.5599999999999999]}, Offset[0.]}, "Rows" -> { Offset[0.], { Offset[0.4]}, Offset[0.]}}]}, Dynamic[Typeset`open], ImageSize -> Automatic], BaseStyle -> { ShowStringCharacters -> False, NumberMarks -> False, PrintPrecision -> 3, ShowSyntaxStyles -> False}, BaselinePosition -> Baseline]], StyleBox["]", "NonInterpretableSummary"]}]}, "CopyTag", DisplayFunction->(#& ), InterpretationFunction->("NetChain[<>]"& )], False, Editable->False, SelectWithContents->True, Selectable->False]], "Output", CellChangeTimes->{{3.733216262907097*^9, 3.733216272012073*^9}, 3.733216477145299*^9, {3.8092503881501226`*^9, 3.809250417879632*^9}, 3.814026495747974*^9, {3.817227836826013*^9, 3.817227854932225*^9}}, CellLabel->"Out[22]=",ExpressionUUID->"ff9e0821-82b3-4ab7-8262-75665a2772ec"] }, Open ]] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell["Training", "Section", CellChangeTimes->{{3.695474798960601*^9, 3.69547479969661*^9}},ExpressionUUID->"5bc6142b-c2c8-49ee-aaf4-\ 033c6d7612ae"], Cell["(Performed separately)", "Text", CellChangeTimes->{{3.695474856418449*^9, 3.695474863168799*^9}, { 3.7003902300515137`*^9, 3.7003902345796556`*^9}, {3.7135276003953753`*^9, 3.713527600730994*^9}, 3.725189606493602*^9},ExpressionUUID->"da4acdae-b62b-490e-b846-\ 47ec715675fd"] }, Open ]] }, Open ]] }, WindowSize->{856, 995}, WindowMargins->{{0, Automatic}, {Automatic, 0}}, FrontEndVersion->"12.2 for Mac OS X x86 (64-bit) (November 19, 2020)", StyleDefinitions->"Default.nb", ExpressionUUID->"6a434178-0a46-4aaa-a4a8-5d5fe4938981" ] (* End of Notebook Content *) (* Internal cache information *) (*CellTagsOutline CellTagsIndex->{} *) (*CellTagsIndex CellTagsIndex->{} *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[580, 22, 578, 11, 195, "Title",ExpressionUUID->"cf0af03b-7c0d-44fe-a36a-801fb2cc9d1e"], Cell[CellGroupData[{ Cell[1183, 37, 163, 3, 54, "Subsection",ExpressionUUID->"20a938bc-d1c4-4bd6-a5fd-63b43d74e3fb"], Cell[CellGroupData[{ Cell[1371, 44, 166, 3, 45, "Subsubsection",ExpressionUUID->"aece8991-ede8-4c4f-b8b7-d3deda75843b"], Cell[1540, 49, 637, 12, 30, "Input",ExpressionUUID->"f9e334a7-6b84-4b97-a015-364f975bd6c8"] }, Open ]], Cell[CellGroupData[{ Cell[2214, 66, 166, 3, 45, "Subsubsection",ExpressionUUID->"278efdc3-d956-4587-8723-2a3485f9bac0"], Cell[2383, 71, 651, 12, 30, "Input",ExpressionUUID->"4997fe3c-81ac-42f4-9497-390d3d9b0fce"] }, Open ]], Cell[CellGroupData[{ Cell[3071, 88, 166, 3, 45, "Subsubsection",ExpressionUUID->"71c6cef7-94d1-4660-857a-d6696698fd69"], Cell[3240, 93, 528, 9, 30, "Input",ExpressionUUID->"908d5d25-3cee-40c2-a5c9-2b89750c8f92"] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell[3817, 108, 137, 3, 67, "Section",ExpressionUUID->"e6f0ec16-daa9-4151-8b11-649c83fbd9a1"], Cell[CellGroupData[{ Cell[3979, 115, 174, 3, 54, "Subsection",ExpressionUUID->"075f3fef-d0b5-4896-9f18-27f1ae7779eb"], Cell[4156, 120, 7392, 165, 850, "Input",ExpressionUUID->"74e06088-1beb-4cc7-81cf-ba7ba9bf7fcd"], Cell[11551, 287, 583, 14, 52, "Input",ExpressionUUID->"eca2777e-8639-4a73-8fdb-8fb65b27c0c8"], Cell[12137, 303, 628, 14, 52, "Input",ExpressionUUID->"6f3bb86d-ffc0-4703-ab7c-e0825f587d5f"], Cell[12768, 319, 3593, 82, 472, "Input",ExpressionUUID->"399601e9-fdc8-44d8-ae12-419eebdaedde"], Cell[16364, 403, 500, 13, 30, "Input",ExpressionUUID->"0057b0bc-5c66-42bc-b74d-c408ad38ccb4"] }, Open ]], Cell[CellGroupData[{ Cell[16901, 421, 157, 3, 54, "Subsection",ExpressionUUID->"791564d7-bbb0-42be-870f-225e70c83a2d"], Cell[CellGroupData[{ Cell[17083, 428, 5364, 127, 535, "Input",ExpressionUUID->"0dac311f-6bd1-4d70-89ac-edd4b16cb5d0"], Cell[22450, 557, 241227, 4112, 701, "Output",ExpressionUUID->"ff9e0821-82b3-4ab7-8262-75665a2772ec"] }, Open ]] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell[263738, 4676, 152, 3, 67, "Section",ExpressionUUID->"5bc6142b-c2c8-49ee-aaf4-033c6d7612ae"], Cell[263893, 4681, 293, 5, 35, "Text",ExpressionUUID->"da4acdae-b62b-490e-b846-47ec715675fd"] }, Open ]] }, Open ]] } ] *)