(* Content-type: application/vnd.wolfram.mathematica *) (*** Wolfram Notebook File ***) (* http://www.wolfram.com/nb *) (* CreatedBy='Mathematica 11.1' *) (*CacheID: 234*) (* Internal cache information: NotebookFileLineBreakTest NotebookFileLineBreakTest NotebookDataPosition[ 158, 7] NotebookDataLength[ 46239, 1027] NotebookOptionsPosition[ 43940, 975] NotebookOutlinePosition[ 44280, 990] CellTagsIndexPosition[ 44237, 987] WindowFrame->Normal*) (* Beginning of Notebook Content *) Notebook[{ Cell[CellGroupData[{ Cell[TextData[{ StyleBox["Construction Notebook for:", FontSize->24], "\nSketch-RNN Trained on QuickDraw Data" }], "Title", CellChangeTimes->{{3.700390059931024*^9, 3.700390069370269*^9}, { 3.713527335930427*^9, 3.71352733920855*^9}, {3.725196329204988*^9, 3.7251963296657867`*^9}, 3.725709769145364*^9, 3.725975379788035*^9, 3.725979717913773*^9, {3.7414487891539593`*^9, 3.741448824686091*^9}, { 3.761543486499277*^9, 3.761543487395276*^9}, {3.7622500684987946`*^9, 3.76225006873069*^9}, {3.762250223999448*^9, 3.762250227917072*^9}, { 3.762250258561981*^9, 3.762250261842757*^9}, {3.766235582149639*^9, 3.766235599045607*^9}, 3.767007119701935*^9, {3.769245203453505*^9, 3.769245216173695*^9}, {3.769247521976131*^9, 3.7692475237708273`*^9}, { 3.77779716805066*^9, 3.777797192977129*^9}},ExpressionUUID->"cf0af03b-7c0d-44fe-a36a-\ 801fb2cc9d1e"], Cell[CellGroupData[{ Cell["NetModel Access", "Subsection", CellChangeTimes->{{3.695474314080426*^9, 3.695474346087426*^9}},ExpressionUUID->"20a938bc-d1c4-4bd6-a5fd-\ 63b43d74e3fb"], Cell[CellGroupData[{ Cell["This Notebook", "Subsubsection", CellChangeTimes->{{3.6954750910664797`*^9, 3.695475092737351*^9}},ExpressionUUID->"aece8991-ede8-4c4f-b8b7-\ d3deda75843b"], Cell[BoxData[ RowBox[{"NetModel", "[", RowBox[{ "\"\\"", ",", "\"\\""}], "]"}]], "Input", CellChangeTimes->{{3.696656935596855*^9, 3.6966569364445457`*^9}, 3.696768427361453*^9, 3.696768595763381*^9, 3.700390080550715*^9, 3.713527332938039*^9, 3.7251895368569007`*^9, 3.7257097745186977`*^9, { 3.725974160027033*^9, 3.7259741712693043`*^9}, 3.7259797223664417`*^9, { 3.7414488366470466`*^9, 3.7414488401761193`*^9}, 3.761543503231561*^9, 3.762250232214696*^9, 3.762257437121704*^9, 3.766235575869665*^9, 3.767007114244733*^9, 3.7692452236216927`*^9, 3.7692475279954233`*^9, 3.777797277831686*^9},ExpressionUUID->"f9e334a7-6b84-4b97-a015-\ 364f975bd6c8"] }, Open ]], Cell[CellGroupData[{ Cell["Untrained Net", "Subsubsection", CellChangeTimes->{{3.6954750978333807`*^9, 3.695475133273511*^9}},ExpressionUUID->"278efdc3-d956-4587-8723-\ 2a3485f9bac0"], Cell[BoxData[ RowBox[{"NetModel", "[", RowBox[{ "\"\\"", ",", "\"\\""}], "]"}]], "Input", CellChangeTimes->{{3.696656940213286*^9, 3.6966569411083603`*^9}, 3.6967685960955133`*^9, 3.700390083629743*^9, {3.713527345914727*^9, 3.713527346642221*^9}, {3.7251895437188444`*^9, 3.725189544052307*^9}, { 3.7257097811399565`*^9, 3.725709791683883*^9}, 3.7259741741281953`*^9, 3.725979726480075*^9, 3.7414488429385967`*^9, 3.7615435115559664`*^9, 3.762250233443425*^9, 3.7622574401486087`*^9, 3.766235575870553*^9, 3.767007113239605*^9, 3.769245225632111*^9, 3.7692475313003817`*^9, 3.777797280484914*^9},ExpressionUUID->"4997fe3c-81ac-42f4-9497-\ 390d3d9b0fce"] }, Open ]], Cell[CellGroupData[{ Cell["Trained Net", "Subsubsection", CellChangeTimes->{{3.6954751072255173`*^9, 3.6954751086815357`*^9}},ExpressionUUID->"71c6cef7-94d1-4660-857a-\ d6696698fd69"], Cell[BoxData[ RowBox[{"NetModel", "[", "\"\\"", "]"}]], "Input", CellChangeTimes->{{3.6966569441649303`*^9, 3.696656946349728*^9}, 3.696768596439824*^9, 3.70039008606037*^9, 3.713527350999201*^9, 3.725189549129289*^9, 3.725709788369228*^9, 3.725974177559237*^9, 3.725979728776238*^9, 3.7414488460285764`*^9, 3.7615435148854666`*^9, 3.762250234778886*^9, 3.762257444218728*^9, 3.766235575871327*^9, 3.7670071132404833`*^9, 3.7692452283212633`*^9, 3.769247533245225*^9, 3.777797281621447*^9},ExpressionUUID->"908d5d25-3cee-40c2-a5c9-\ 2b89750c8f92"] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell["Net Construction", "Section", CellChangeTimes->{ 3.695625386014464*^9},ExpressionUUID->"e6f0ec16-daa9-4151-8b11-\ 649c83fbd9a1"], Cell[CellGroupData[{ Cell["Decoder definition", "Subsection", CellChangeTimes->{{3.777797389522964*^9, 3.777797391458592*^9}},ExpressionUUID->"021566a9-cbd1-49a8-ac13-\ 02dba4447d9e"], Cell[BoxData[ RowBox[{ RowBox[{"decodeFunc", "=", RowBox[{"Function", "[", RowBox[{ RowBox[{"{", RowBox[{"input", ",", "prop"}], "}"}], ",", "\[IndentingNewLine]", RowBox[{"Block", "[", RowBox[{ RowBox[{"{", RowBox[{ "softmax", ",", "sample", ",", "top", ",", "n", ",", "logits", ",", "pi", ",", "mu1", ",", "mu2", ",", "s1", ",", "s2", ",", "corr", ",", "dists", ",", "temp1", ",", "temp2"}], "}"}], ",", "\[IndentingNewLine]", RowBox[{ RowBox[{"softmax", "=", RowBox[{"Function", "[", RowBox[{"Normalize", "[", RowBox[{ RowBox[{"Chop", "[", RowBox[{"Exp", "[", RowBox[{"#", "+", RowBox[{"-", RowBox[{"Max", "[", "#", "]"}]}]}], "]"}], "]"}], ",", "Total"}], "]"}], "]"}]}], ";", "\[IndentingNewLine]", RowBox[{"sample", "=", RowBox[{"Function", "[", RowBox[{"RandomChoice", "[", RowBox[{"#", "\[Rule]", RowBox[{"Range", "[", RowBox[{"Length", "[", "#", "]"}], "]"}]}], "]"}], "]"}]}], ";", "\[IndentingNewLine]", RowBox[{"top", "=", RowBox[{"Function", "[", RowBox[{"Part", "[", RowBox[{ RowBox[{"Position", "[", RowBox[{"#", ",", RowBox[{"Max", "@", "#"}]}], "]"}], ",", "1", ",", "1"}], "]"}], "]"}]}], ";", "\[IndentingNewLine]", RowBox[{"n", "=", RowBox[{ RowBox[{"(", RowBox[{ RowBox[{"Length", "[", "input", "]"}], "-", "3"}], ")"}], "/", "6"}]}], ";", "\[IndentingNewLine]", RowBox[{ RowBox[{"{", RowBox[{ "logits", ",", "pi", ",", "mu1", ",", "mu2", ",", "s1", ",", "s2", ",", "corr"}], "}"}], "=", RowBox[{"TakeList", "[", RowBox[{ RowBox[{"Normal", "@", "input"}], ",", RowBox[{"{", RowBox[{ "3", ",", "n", ",", "n", ",", "n", ",", "n", ",", "n", ",", "n"}], "}"}]}], "]"}]}], ";", "\[IndentingNewLine]", RowBox[{"dists", "=", RowBox[{"MapThread", "[", RowBox[{ RowBox[{ RowBox[{"BinormalDistribution", "[", RowBox[{ RowBox[{"{", RowBox[{"#", ",", "#2"}], "}"}], ",", RowBox[{"Exp", "@", RowBox[{"{", RowBox[{"#3", ",", "#4"}], "}"}]}], ",", RowBox[{"Tanh", "@", "#5"}]}], "]"}], "&"}], ",", RowBox[{"{", RowBox[{"mu1", ",", "mu2", ",", "s1", ",", "s2", ",", "corr"}], "}"}]}], "]"}]}], ";", "\[IndentingNewLine]", RowBox[{"Switch", "[", RowBox[{ "prop", ",", "\[IndentingNewLine]", "\"\\"", ",", "\[IndentingNewLine]", RowBox[{"{", RowBox[{ RowBox[{"MixtureDistribution", "[", RowBox[{ RowBox[{"softmax", "@", "pi"}], ",", "dists"}], "]"}], ",", RowBox[{"softmax", "@", "logits"}]}], "}"}], ",", "\[IndentingNewLine]", RowBox[{"Alternatives", "[", RowBox[{"\"\\"", ",", RowBox[{"{", RowBox[{"\"\\"", ",", RowBox[{"\"\\"", "\[Rule]", RowBox[{"0", "|", "0."}]}]}], "}"}]}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"{", RowBox[{ RowBox[{"First", "[", RowBox[{"Part", "[", RowBox[{"dists", ",", RowBox[{"top", "@", "pi"}]}], "]"}], "]"}], ",", RowBox[{"top", "@", "logits"}]}], "}"}], ",", "\[IndentingNewLine]", RowBox[{"{", RowBox[{"\"\\"", ",", RowBox[{"\"\\"", "\[Rule]", "_"}]}], "}"}], ",", "\[IndentingNewLine]", RowBox[{ RowBox[{"temp1", "=", RowBox[{"prop", "[", RowBox[{"[", RowBox[{"2", ",", "2"}], "]"}], "]"}]}], ";", "\[IndentingNewLine]", RowBox[{"temp2", "=", RowBox[{ RowBox[{"(", RowBox[{"1", "+", "temp1"}], ")"}], "/", "2."}]}], ";", "\[IndentingNewLine]", RowBox[{"{", "\[IndentingNewLine]", RowBox[{ RowBox[{"RandomVariate", "[", RowBox[{"MapAt", "[", RowBox[{ RowBox[{ RowBox[{"#", "*", RowBox[{"Sqrt", "[", "temp1", "]"}]}], "&"}], ",", RowBox[{"Part", "[", RowBox[{"dists", ",", RowBox[{"sample", "[", RowBox[{"softmax", "[", RowBox[{"pi", "/", "temp2"}], "]"}], "]"}]}], "]"}], ",", "2"}], "]"}], "]"}], ",", "\[IndentingNewLine]", RowBox[{"sample", "[", RowBox[{"softmax", "[", RowBox[{"logits", "/", "temp2"}], "]"}], "]"}]}], "\[IndentingNewLine]", "}"}]}]}], "\[IndentingNewLine]", "]"}]}]}], "\[IndentingNewLine]", "]"}]}], "\[IndentingNewLine]", "]"}]}], ";"}]], "Input", CellChangeTimes->{{3.777797409244815*^9, 3.7777974696754704`*^9}, { 3.777797656002253*^9, 3.777797672606804*^9}, {3.77779770883818*^9, 3.777797711033559*^9}}, CellLabel-> "In[204]:=",ExpressionUUID->"a35b6757-b029-4068-8c43-d854bd289c71"], Cell[CellGroupData[{ Cell[BoxData[ RowBox[{"decoder", "=", RowBox[{"NetDecoder", "@", RowBox[{"{", RowBox[{"\"\\"", ",", "decodeFunc", ",", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", RowBox[{"\"\\"", ",", "\"\\"", ",", RowBox[{"{", RowBox[{"\"\\"", ",", RowBox[{"\"\\"", "\[Rule]", "_Real"}]}], "}"}]}], "}"}]}]}], "}"}]}]}]], "Input", CellChangeTimes->{{3.7777973595110493`*^9, 3.777797359511662*^9}, { 3.7777973979890413`*^9, 3.7777973986756907`*^9}}, CellLabel-> "In[205]:=",ExpressionUUID->"f0686720-b26a-4c5f-bd3c-42c42d331737"], Cell[BoxData[ TagBox[ TemplateBox[{RowBox[{ StyleBox[ TagBox["NetDecoder", "SummaryHead"], "NonInterpretableSummary"], StyleBox["[", "NonInterpretableSummary"], DynamicModuleBox[{Typeset`open = False}, PanelBox[ PaneSelectorBox[{False -> GridBox[{{ PaneBox[ ButtonBox[ DynamicBox[ FEPrivate`FrontEndResource[ "FEBitmaps", "SquarePlusIconMedium"]], ButtonFunction :> (Typeset`open = True), Appearance -> None, Evaluator -> Automatic, Method -> "Preemptive"], Alignment -> {Center, Center}, ImageSize -> {Automatic, 24}], GridBox[{{ StyleBox[ TemplateBox[{"\"Type\"", "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox["\"Function\"", "SummaryItem"]}, { StyleBox[ TemplateBox[{"\"Input\"", "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox["\"array\"", "SummaryItem"]}}, BaselinePosition -> Automatic, GridBoxAlignment -> { "Columns" -> {{Left}}, "Rows" -> {{Automatic}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> { "Columns" -> {{2}}, "Rows" -> {{Automatic}}}]}}, GridBoxAlignment -> {"Rows" -> {{Top}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridFrameMargins -> {{0, 0}, {0, 0}}, BaselinePosition -> {1, 3}], True -> GridBox[{{ PaneBox[ ButtonBox[ DynamicBox[ FEPrivate`FrontEndResource[ "FEBitmaps", "SquareMinusIconMedium"]], ButtonFunction :> (Typeset`open = False), Appearance -> None, Evaluator -> Automatic, Method -> "Preemptive"], Alignment -> {Center, Center}, ImageSize -> {Automatic, 24}], GridBox[{{ StyleBox[ TemplateBox[{"\"Type\"", "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox["\"Function\"", "SummaryItem"]}, { StyleBox[ TemplateBox[{"\"Function\"", "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox[ StyleBox[ RowBox[{"Function", "[", RowBox[{ RowBox[{"{", RowBox[{"input", ",", " ", "prop"}], "}"}], ",", " ", "\"\[Ellipsis]\""}], "]"}], FontFamily -> "Source Code Pro", StripOnInput -> False], "SummaryItem"]}, { StyleBox[ TemplateBox[{"\"Properties\"", "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox[ RowBox[{"{", RowBox[{ "\"\\\"Distribution\\\"\"", ",", "\"\\\"Prediction\\\"\"", ",", RowBox[{"{", RowBox[{"\"Sample\"", ",", RowBox[{"\"Temperature\"", "\[Rule]", "_Real"}]}], "}"}]}], "}"}], "SummaryItem"]}, { StyleBox[ TemplateBox[{"\"Input\"", "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox["\"array\"", "SummaryItem"]}}, BaselinePosition -> {2, 1}, GridBoxAlignment -> { "Columns" -> {{Left}}, "Rows" -> {{Automatic}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> { "Columns" -> {{2}}, "Rows" -> {{Automatic}}}]}}, GridBoxAlignment -> {"Rows" -> {{Top}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridFrameMargins -> {{0, 0}, {0, 0}}, BaselinePosition -> {1, 3}]}, Dynamic[Typeset`open], ImageSize -> Automatic], BaselinePosition -> Baseline, BaseStyle -> { ShowStringCharacters -> False, NumberMarks -> False, PrintPrecision -> 3, ShowSyntaxStyles -> False}]], StyleBox["]", "NonInterpretableSummary"]}]}, "CopyTag", DisplayFunction->(#& ), InterpretationFunction->("NetDecoder[<>]"& )], False, Editable->False, SelectWithContents->True, Selectable->False]], "Output", CellChangeTimes->{3.7777977303584137`*^9}, CellLabel-> "Out[205]=",ExpressionUUID->"0a6d43b3-a65a-42f6-9e39-7b9266b4af52"] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell["Final Net", "Subsection", CellChangeTimes->{{3.7449565337928762`*^9, 3.744956534945279*^9}},ExpressionUUID->"b4b988fc-c50c-4908-9341-\ f93d9871644d"], Cell[CellGroupData[{ Cell[BoxData[ RowBox[{"sketchRNN", "=", RowBox[{"NetChain", "[", "\[IndentingNewLine]", RowBox[{ RowBox[{"{", RowBox[{ RowBox[{"LongShortTermMemoryLayer", "[", "1024", "]"}], ",", RowBox[{"SequenceLastLayer", "[", "]"}], ",", RowBox[{"LinearLayer", "[", "123", "]"}]}], "}"}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", RowBox[{"{", RowBox[{"\"\\"", ",", "5"}], "}"}]}], ",", "\[IndentingNewLine]", RowBox[{"\"\\"", "\[Rule]", "decoder"}]}], "\[IndentingNewLine]", "]"}]}]], "Input", CellChangeTimes->{{3.744956316402637*^9, 3.744956326935472*^9}, { 3.744956536486313*^9, 3.74495660476466*^9}, {3.7615444082918167`*^9, 3.761544781089865*^9}, 3.761545087687909*^9, {3.7615462993538413`*^9, 3.761546404227684*^9}, {3.761546743980044*^9, 3.761546744842285*^9}, { 3.7615640278228407`*^9, 3.7615640298554044`*^9}, {3.7622508859074097`*^9, 3.7622509185352106`*^9}, {3.762250977159768*^9, 3.7622509784224095`*^9}, { 3.76225140414182*^9, 3.762251409906753*^9}, {3.762251445538254*^9, 3.762251467799245*^9}, {3.769245239619438*^9, 3.7692452981828823`*^9}, { 3.777797284674946*^9, 3.7777973239503717`*^9}, {3.777797734302114*^9, 3.777797764606984*^9}}, CellLabel-> "In[209]:=",ExpressionUUID->"43c8819d-41c0-45fd-8e4a-6f34e70549c2"], Cell[BoxData[ TagBox[ TemplateBox[{RowBox[{ StyleBox[ TagBox["NetChain", "SummaryHead"], "NonInterpretableSummary"], StyleBox["[", "NonInterpretableSummary"], DynamicModuleBox[{Typeset`open = False}, PanelBox[ PaneSelectorBox[{False -> GridBox[{{ PaneBox[ ButtonBox[ DynamicBox[ FEPrivate`FrontEndResource[ "FEBitmaps", "SquarePlusIconMedium"]], ButtonFunction :> (Typeset`open = True), Appearance -> None, Evaluator -> Automatic, Method -> "Preemptive"], Alignment -> {Center, Center}, ImageSize -> {Automatic, 24}], OverlayBox[{ GraphicsBox[{ Thickness[0.014925373134328358`], { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{"Miter", 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{23., 91.5}, {30., 91.5}}}, CurveClosed -> {0}]}, { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{"Miter", 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{37., 91.5}, {44., 91.5}}}, CurveClosed -> {0}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 15.5, 103.5}, {23.5, 103.5}, {23.5, 79.5}, {15.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{24., 104.}, {15., 104.}, { 15., 79.}, {24., 79.}}, {{23., 80.}, {16., 80.}, {16., 103.}, {23., 103.}}}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 29.5, 103.5}, {37.5, 103.5}, {37.5, 79.5}, {29.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{38., 104.}, {29., 104.}, { 29., 79.}, {38., 79.}}, {{37., 80.}, {30., 80.}, {30., 103.}, {37., 103.}}}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 43.5, 103.5}, {51.5, 103.5}, {51.5, 79.5}, {43.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{52., 104.}, {43., 104.}, { 43., 79.}, {52., 79.}}, {{51., 80.}, {44., 80.}, {44., 103.}, {51., 103.}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{28., 91.5}, { 25., 95.}, {25., 88.}}}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{42., 91.5}, { 39., 95.}, {39., 88.}}}]}}, ImageMargins -> 0, ImageSize -> {36, Automatic}, PlotRangePadding -> 0, BaseStyle -> GrayLevel[0.6]], StyleBox["uninitialized", RGBColor[0.66, 0, 0], FontSize -> 8, FontFamily -> "Roboto", Background -> GrayLevel[1, 0.65]]}, Alignment -> {Center, Scaled[0.0001]}], GridBox[{{ StyleBox[ TemplateBox[{ TemplateBox[{ StyleBox["\"Input\"", GrayLevel[0], StripOnInput -> False], "\" \"", "\"port\""}, "RowDefault"], "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox[ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{ "\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"5\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], "SummaryItem"]}, { StyleBox[ TemplateBox[{ TemplateBox[{ StyleBox["\"Output\"", GrayLevel[0], StripOnInput -> False], "\" \"", "\"port\""}, "RowDefault"], "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox["\"expression\"", "SummaryItem"]}, { StyleBox[ TemplateBox[{"\"Number of layers\"", "\":\""}, "RowDefault"], "SummaryItemAnnotation"], StyleBox["3", "SummaryItem"]}}, BaselinePosition -> {2, 1}, GridBoxAlignment -> { "Columns" -> {{Left}}, "Rows" -> {{Automatic}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> { "Columns" -> {{2}}, "Rows" -> {{Automatic}}}]}}, GridBoxAlignment -> {"Rows" -> {{Top}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridFrameMargins -> {{0, 0}, {0, 0}}, BaselinePosition -> {1, 3}], True -> GridBox[{{ PaneBox[ ButtonBox[ DynamicBox[ FEPrivate`FrontEndResource[ "FEBitmaps", "SquareMinusIconMedium"]], ButtonFunction :> (Typeset`open = False), Appearance -> None, Evaluator -> Automatic, Method -> "Preemptive"], Alignment -> {Center, Center}, ImageSize -> {Automatic, 24}], OverlayBox[{ GraphicsBox[{ Thickness[0.014925373134328358`], { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{"Miter", 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{23., 91.5}, {30., 91.5}}}, CurveClosed -> {0}]}, { GrayLevel[0.6], Thickness[0.018686567164179105`], Opacity[1.], JoinForm[{"Miter", 10.}], JoinedCurveBox[{{{0, 2, 0}}}, {{{37., 91.5}, {44., 91.5}}}, CurveClosed -> {0}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 15.5, 103.5}, {23.5, 103.5}, {23.5, 79.5}, {15.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{24., 104.}, {15., 104.}, { 15., 79.}, {24., 79.}}, {{23., 80.}, {16., 80.}, {16., 103.}, {23., 103.}}}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 29.5, 103.5}, {37.5, 103.5}, {37.5, 79.5}, {29.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{38., 104.}, {29., 104.}, { 29., 79.}, {38., 79.}}, {{37., 80.}, {30., 80.}, {30., 103.}, {37., 103.}}}]}, { FaceForm[{ GrayLevel[0.9], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{ 43.5, 103.5}, {51.5, 103.5}, {51.5, 79.5}, {43.5, 79.5}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}, {{0, 2, 0}, {0, 1, 0}, {0, 1, 0}}}, {{{52., 104.}, {43., 104.}, { 43., 79.}, {52., 79.}}, {{51., 80.}, {44., 80.}, {44., 103.}, {51., 103.}}}]}, { FaceForm[{ GrayLevel[0.6], Opacity[1.]}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{28., 91.5}, { 25., 95.}, {25., 88.}}}], FilledCurveBox[{{{0, 2, 0}, {0, 1, 0}}}, {{{42., 91.5}, { 39., 95.}, {39., 88.}}}]}}, ImageMargins -> 0, ImageSize -> {36, Automatic}, PlotRangePadding -> 0, BaseStyle -> GrayLevel[0.6]], StyleBox["uninitialized", RGBColor[0.66, 0, 0], FontSize -> 8, FontFamily -> "Roboto", Background -> GrayLevel[1, 0.65]]}, Alignment -> {Center, Scaled[0.0001]}], GridBox[{{ DynamicModuleBox[{ NeuralNetworks`Private`NetChain`assoc3 = Association[ "Type" -> "Chain", "Nodes" -> Association[ "1" -> Association[ "Type" -> "LongShortTermMemory", "Arrays" -> Association[ "InputGateInputWeights" -> NeuralNetworks`TensorT[{1024, 5}, NeuralNetworks`RealT], "InputGateStateWeights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "InputGateBiases" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "OutputGateInputWeights" -> NeuralNetworks`TensorT[{1024, 5}, NeuralNetworks`RealT], "OutputGateStateWeights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "OutputGateBiases" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "ForgetGateInputWeights" -> NeuralNetworks`TensorT[{1024, 5}, NeuralNetworks`RealT], "ForgetGateStateWeights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "ForgetGateBiases" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "MemoryGateInputWeights" -> NeuralNetworks`TensorT[{1024, 5}, NeuralNetworks`RealT], "MemoryGateStateWeights" -> NeuralNetworks`TensorT[{1024, 1024}, NeuralNetworks`RealT], "MemoryGateBiases" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Parameters" -> Association[ "OutputSize" -> 1024, "Dropout" -> NeuralNetworks`ValidatedParameter[None], "$InputSize" -> 5, "$SequenceLength" -> NeuralNetworks`LengthVar[1356412463], "$CellStateConnectedQ" -> False], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1356412463], 5}, NeuralNetworks`RealT]], "Outputs" -> Association["Output" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1356412463], 1024}, NeuralNetworks`RealT]], "States" -> Association[ "State" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT], "CellState" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "2" -> Association[ "Type" -> "SequenceLast", "Arrays" -> Association[], "Parameters" -> Association[], "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1356412463], 1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]]], "3" -> Association[ "Type" -> "Linear", "Arrays" -> Association[ "Weights" -> NeuralNetworks`TensorT[{123, 1024}, NeuralNetworks`RealT], "Biases" -> NeuralNetworks`Nullable[ NeuralNetworks`TensorT[{123}, NeuralNetworks`RealT]]], "Parameters" -> Association[ "OutputDimensions" -> {123}, "$OutputSize" -> 123, "$InputSize" -> 1024, "$InputDimensions" -> {1024}], "Inputs" -> Association[ "Input" -> NeuralNetworks`TensorT[{1024}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> NeuralNetworks`TensorT[{123}, NeuralNetworks`RealT]]]], "Edges" -> { NeuralNetworks`NetPath["Nodes", "1", "Inputs", "Input"] -> NeuralNetworks`NetPath["Inputs", "Input"], NeuralNetworks`NetPath["Nodes", "2", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "1", "Outputs", "Output"], NeuralNetworks`NetPath["Nodes", "3", "Inputs", "Input"] -> NeuralNetworks`NetPath["Nodes", "2", "Outputs", "Output"], NeuralNetworks`NetPath["Outputs", "Output"] -> NeuralNetworks`NetPath[ "Nodes", "3", "Outputs", "Output"]}, "Inputs" -> Association["Input" -> NeuralNetworks`TensorT[{ NeuralNetworks`LengthVar[1356412463], 5}, NeuralNetworks`RealT]], "Outputs" -> Association[ "Output" -> Association[ "Form" -> NeuralNetworks`TensorT[{123}, NeuralNetworks`RealT], "Type" -> "Function", "Function" -> NeuralNetworks`ValidatedParameter[ Function[{$CellContext`input, $CellContext`prop}, "\[Ellipsis]"]], "Properties" -> { "\"Distribution\"", "\"Prediction\"", { "Sample", "Temperature" -> Blank[Real]}}, "$Version" -> "12.0.12"]], "InteriorStates" -> Association[{1, "State"} -> NeuralNetworks`NetPath["Nodes", "1", "States", "State"], { 1, "CellState"} -> NeuralNetworks`NetPath[ "Nodes", "1", "States", "CellState"]]], NeuralNetworks`Private`NetChain`opart, NeuralNetworks`Private`NetChain`part, NeuralNetworks`Private`NetChain`selected = Null}, DynamicBox[ GridBox[{{ NeuralNetworks`Private`NetChain`MouseClickBoxes[ TagBox[ GridBox[{{ TagBox[ TagBox[ "\"\"", Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ "\"Input\"", Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"5\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], Annotation[#, {"Inputs", "Input"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"1\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "1"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox["LongShortTermMemoryLayer", RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "1"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\"matrix\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", StyleBox["\"n\"", Italic, StripOnInput -> False], "\"1024\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], Annotation[#, {"Nodes", "1"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"2\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "2"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox["SequenceLastLayer", GrayLevel[0], StripOnInput -> False], Annotation[#, {"Nodes", "2"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\"vector\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", "\"1024\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], Annotation[#, {"Nodes", "2"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ StyleBox["\"3\"", GrayLevel[0.5], StripOnInput -> False], Annotation[#, {"Nodes", "3"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ StyleBox["LinearLayer", RGBColor[0.66, 0, 0], StripOnInput -> False], Annotation[#, {"Nodes", "3"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ TemplateBox[{"\"vector\"", "\" \"", StyleBox[ TemplateBox[{"\"(\"", "\"\[VeryThinSpace]\"", "\"size\"", "\":\"", "\" \"", TemplateBox[{"\[Times]", "\"\[Times]\"", "\"123\""}, "RowWithSeparators"], "\"\[VeryThinSpace]\"", "\")\""}, "RowDefault"], GrayLevel[0.5], StripOnInput -> False]}, "RowDefault"], Annotation[#, {"Nodes", "3"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}, { TagBox[ TagBox[ "\"\"", Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ "\"Output\"", Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]], TagBox[ TagBox[ "\"expression\"", Annotation[#, {"Outputs", "Output"}, "Mouse"]& ], MouseAppearanceTag["LinkHand"]]}}, GridBoxAlignment -> {"Columns" -> {{Left}}}, AutoDelete -> False, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> {"Columns" -> {{1.1}}}], "Grid"], If[ ListQ[ NeuralNetworks`Private`NetChain`part = MouseAnnotation[]], If[NeuralNetworks`Private`NetChain`opart === NeuralNetworks`Private`NetChain`part, NeuralNetworks`Private`NetChain`selected = Null; NeuralNetworks`Private`NetChain`opart = Null, NeuralNetworks`Private`NetChain`selected = Part[NeuralNetworks`Private`NetChain`assoc3, Apply[Sequence, NeuralNetworks`Private`NetChain`part]]; NeuralNetworks`Private`NetChain`opart = NeuralNetworks`Private`NetChain`part; Null]; Null]]}, NeuralNetworks`Private`NetChain`fmtSelected[ NeuralNetworks`Private`NetChain`selected, NeuralNetworks`Private`NetChain`part, NeuralNetworks`Private`GetSharedArrays[ NeuralNetworks`Private`NetChain`assoc3]]}, GridBoxSpacings -> {"Columns" -> {{1}}}, GridBoxAlignment -> {"Columns" -> {{Left}}}, GridFrameMargins -> {{0, 0}, {0, 0}}], TrackedSymbols :> { NeuralNetworks`Private`NetChain`selected}], Initialization :> {NetChain}]}}, BaselinePosition -> Automatic, GridBoxAlignment -> { "Columns" -> {{Left}}, "Rows" -> {{Automatic}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridBoxSpacings -> { "Columns" -> {{2}}, "Rows" -> {{Automatic}}}]}}, GridBoxAlignment -> {"Rows" -> {{Top}}}, GridBoxItemSize -> { "Columns" -> {{Automatic}}, "Rows" -> {{Automatic}}}, GridFrameMargins -> {{0, 0}, {0, 0}}, BaselinePosition -> {1, 1}]}, Dynamic[Typeset`open], ImageSize -> Automatic], BaselinePosition -> Baseline, BaseStyle -> { ShowStringCharacters -> False, NumberMarks -> False, PrintPrecision -> 3, ShowSyntaxStyles -> False}]], StyleBox["]", "NonInterpretableSummary"]}]}, "CopyTag", DisplayFunction->(#& ), InterpretationFunction->("NetChain[<>]"& )], False, Editable->False, SelectWithContents->True, Selectable->False]], "Output", CellChangeTimes->{ 3.744956605282753*^9, 3.7615448822724056`*^9, 3.761545102455099*^9, 3.761546444357603*^9, {3.76154673920253*^9, 3.761546755871623*^9}, 3.7615640656307526`*^9, 3.7622509423950047`*^9, 3.766235632577178*^9, 3.7692452928721457`*^9, {3.77779774129296*^9, 3.7777977648956413`*^9}}, CellLabel-> "Out[209]=",ExpressionUUID->"3551ac2f-538e-40cf-968b-2c7bbdeac792"] }, Open ]] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell["Training", "Section", CellChangeTimes->{{3.695474798960601*^9, 3.69547479969661*^9}},ExpressionUUID->"5bc6142b-c2c8-49ee-aaf4-\ 033c6d7612ae"], Cell["(Performed separately)", "Text", CellChangeTimes->{{3.695474856418449*^9, 3.695474863168799*^9}, { 3.7003902300515137`*^9, 3.7003902345796556`*^9}, {3.7135276003953753`*^9, 3.713527600730994*^9}, 3.725189606493602*^9},ExpressionUUID->"da4acdae-b62b-490e-b846-\ 47ec715675fd"] }, Open ]] }, Open ]] }, WindowSize->{695.25, 762.}, WindowMargins->{{0, Automatic}, {0, Automatic}}, FrontEndVersion->"12.1 for Linux x86 (64-bit) (September 2, 2019)", StyleDefinitions->"Default.nb" ] (* End of Notebook Content *) (* Internal cache information *) (*CellTagsOutline CellTagsIndex->{} *) (*CellTagsIndex CellTagsIndex->{} *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[580, 22, 889, 16, 136, "Title",ExpressionUUID->"cf0af03b-7c0d-44fe-a36a-801fb2cc9d1e"], Cell[CellGroupData[{ Cell[1494, 42, 163, 3, 55, "Subsection",ExpressionUUID->"20a938bc-d1c4-4bd6-a5fd-63b43d74e3fb"], Cell[CellGroupData[{ Cell[1682, 49, 166, 3, 47, "Subsubsection",ExpressionUUID->"aece8991-ede8-4c4f-b8b7-d3deda75843b"], Cell[1851, 54, 752, 13, 34, "Input",ExpressionUUID->"f9e334a7-6b84-4b97-a015-364f975bd6c8"] }, Open ]], Cell[CellGroupData[{ Cell[2640, 72, 166, 3, 47, "Subsubsection",ExpressionUUID->"278efdc3-d956-4587-8723-2a3485f9bac0"], Cell[2809, 77, 760, 13, 34, "Input",ExpressionUUID->"4997fe3c-81ac-42f4-9497-390d3d9b0fce"] }, Open ]], Cell[CellGroupData[{ Cell[3606, 95, 166, 3, 47, "Subsubsection",ExpressionUUID->"71c6cef7-94d1-4660-857a-d6696698fd69"], Cell[3775, 100, 615, 10, 34, "Input",ExpressionUUID->"908d5d25-3cee-40c2-a5c9-2b89750c8f92"] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell[4439, 116, 137, 3, 70, "Section",ExpressionUUID->"e6f0ec16-daa9-4151-8b11-649c83fbd9a1"], Cell[CellGroupData[{ Cell[4601, 123, 166, 3, 55, "Subsection",ExpressionUUID->"021566a9-cbd1-49a8-ac13-02dba4447d9e"], Cell[4770, 128, 5645, 139, 583, "Input",ExpressionUUID->"a35b6757-b029-4068-8c43-d854bd289c71"], Cell[CellGroupData[{ Cell[10440, 271, 663, 15, 34, "Input",ExpressionUUID->"f0686720-b26a-4c5f-bd3c-42c42d331737"], Cell[11106, 288, 4900, 107, 66, "Output",ExpressionUUID->"0a6d43b3-a65a-42f6-9e39-7b9266b4af52"] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell[16055, 401, 159, 3, 55, "Subsection",ExpressionUUID->"b4b988fc-c50c-4908-9341-f93d9871644d"], Cell[CellGroupData[{ Cell[16239, 408, 1373, 27, 133, "Input",ExpressionUUID->"43c8819d-41c0-45fd-8e4a-6f34e70549c2"], Cell[17615, 437, 25788, 517, 81, "Output",ExpressionUUID->"3551ac2f-538e-40cf-968b-2c7bbdeac792"] }, Open ]] }, Open ]] }, Open ]], Cell[CellGroupData[{ Cell[43464, 961, 152, 3, 70, "Section",ExpressionUUID->"5bc6142b-c2c8-49ee-aaf4-033c6d7612ae"], Cell[43619, 966, 293, 5, 36, "Text",ExpressionUUID->"da4acdae-b62b-490e-b846-47ec715675fd"] }, Open ]] }, Open ]] } ] *)