In[]:=
$Version
Out[]=
14.0.0 for Linux x86 (64-bit) (December 13, 2023)
Ordinary NN Plots
Ordinary NN Plots
In[]:=
Clear[LayerGraph]LayerGraph[l_LinearLayer,n_:0]:=Block[{weights,biases,nIn,nOut,edges},weights=Normal@NetExtract[l,"Weights"];biases=Normal@NetExtract[l,"Biases"];{nOut,nIn}=Dimensions[weights];edges=Flatten@Outer[Annotation[DirectedEdge[#,#2],EdgeWeight->weights[[#2-n-nIn,#1-n]]]&,n+Range[nIn],n+nIn+Range[nOut],1]]LayerGraph[l_ElementwiseLayer,n_:0]:=Block[{weights,nIn,f,nOut,dx=1,dy=1,r=.2,layerPts,edges},nIn=First@Values[Information[l,"InputPorts"]];f=NetExtract[l,"Function"];edges=MapThread[Annotation[DirectedEdge[#,#2],EdgeWeight->0]&,{n+Range[nIn],n+nIn+Range[nIn]}];edges]
In[]:=
Clear[AddArrow]AddArrow[g_,in_,out_,h_:.6]:=Module[{newedges,newvertices,coordin,coordout,coordold},newedges=Join[DirectedEdge[-#,#]&/@in,DirectedEdge[#,-#]&/@out];newvertices=-Join[in,out];coordin=AnnotationValue[{g,#},VertexCoordinates]+{0,h}&/@in;coordout=AnnotationValue[{g,#},VertexCoordinates]-{0,h}&/@out;coordold=AnnotationValue[{g,VertexList[g]},VertexCoordinates];Graph[EdgeAdd[g,newedges],VertexStyle->Thread[newvertices->Directive[Transparent,EdgeForm[Transparent]]],EdgeStyle->Thread[newedges->GrayLevel[.3,.5]],VertexCoordinates->Thread[Join[VertexList[g],newvertices]->Join[coordold,coordin,coordout]],VertexSize->1/h0.25]]
In[]:=
colorFun:=FunctionBlend-1,,-,,0,,,,1,,#
1
2
1
2
In[]:=
colorFunLighter:=Function[Lighter[colorFun[#],.2]]
In[]:=
colorFunDarker:=Function[Darker[colorFun[#],.15]]
In[]:=
colorFunOrange:=FunctionBlendColorConvert[#,RGBColor]&/@,,,,,,,#
In[]:=
colorFunOrange2:=FunctionBlendColorConvert[#,RGBColor]&/@,,,,#
In[]:=
ChatTechColors[key_]:=Association"Normal"->colorFun,"Light"->colorFunLighter,"Dark"->colorFunDarker,"Orange"->colorFunOrange,"OrangeShort"->colorFunOrange2,"Data"->Association"Orange"->,"Teal"->[key]
In[]:=
Clear[NetGraphPlot]Options[NetGraphPlot]=Join[Options[Graph],{"LayerTypes"->{LinearLayer},RandomSeeding->1234,"AddArrows"->False}];NetGraphPlot[net_,input:{__?NumericQ}:None,opts:OptionsPattern[]]:=Module{initnet,layerpattern,values,colors,linear,n,edges,g,edgestyle,cfun=ChatTechColors["Dark"]},initnet=NetInitialize[net,RandomSeeding->OptionValue[RandomSeeding]];layerpattern=Alternatives@@Blank/@OptionValue["LayerTypes"];linear=Cases[Information[initnet,"LayersList"],layerpattern];n=Replace[First@Values[Information[#,"InputPorts"]],"Real"|{}->1]&/@linear;edges=Flatten@MapThread[LayerGraph,{linear,Prepend[Most@Accumulate[n],0]}];g=Graph[edges];Ifinput=!=None,values=Flatten@Prepend[Values@initnet[input,NetPort[#,"Output"]&/@Keys[Select[Information[initnet,"Layers"],MatchQ[layerpattern]]]],input];colors={EdgeForm[GrayLevel[.3,.5]],cfun[#]}&/@values;g=Graph[g,VertexLabels->Thread[Range[Length[values]]->(NumberForm[#,2]&/@values)],VertexStyle->Thread[Range[Length[values]]->colors]],g=Graphg,VertexStyle->DirectiveEdgeForm[GrayLevel[.3,.5]],;edgestyle=Thread[EdgeList[g]->(Directive[AbsoluteThickness[Clip[Abs[#],{1.5,3}]],cfun[#]]&/@AnnotationValue[g,EdgeWeight])];g=Graph[g,FilterRules[{opts},Options[Graph]],EdgeStyle->edgestyle,GraphLayout->"LayeredDigraphEmbedding",VertexSize->.25];Which[TrueQ[OptionValue["AddArrows"]],g=AddArrow[g,Select[VertexList[g],VertexInComponent[g,#,{1}]=={}&],Select[VertexList[g],VertexOutComponent[g,#,{1}]=={}&]],NumericQ@OptionValue["AddArrows"],g=AddArrow[g,Select[VertexList[g],VertexInComponent[g,#,{1}]=={}&],Select[VertexList[g],VertexOutComponent[g,#,{1}]=={}&],OptionValue["AddArrows"]]];g
In[]:=
piecewise=Piecewise[{{0,#<-2},{1,#<-1},{0,#<0},{1,#<2}}]&;
In[]:=
SeedRandom[23049829048];trainingx=RandomReal[{-3,3},5000];
In[]:=
trainingy=piecewise/@trainingx;
In[]:=
TrainPiecewise[net_,opts___]:=Module[{res,checkpoints={}},res=NetTrain[net,trainingx->trainingy,All,opts,TrainingProgressFunction{Function[AppendTo[checkpoints,#Net];],"Interval"Quantity[10,"Rounds"]},MaxTrainingRounds->15000,BatchSize->5000];{res,Prepend[checkpoints,NetInitialize[net]]}]
In[]:=
RRnet[lws_List,opts___]:=NetChain[Join[Riffle[LinearLayer/@lws,Ramp],{Ramp,LinearLayer[]}],"Input"->"Real","Output"->"Real"]
In[]:=
RRnetSELU[lws_List,opts___]:=NetChain[Join[Riffle[LinearLayer/@lws,ElementwiseLayer["SELU"]],{ElementwiseLayer["SELU"],LinearLayer[]}],"Input"->"Real","Output"->"Real"]
In[]:=
RRnetELU[lws_List,opts___]:=NetChain[Join[Riffle[LinearLayer/@lws,ElementwiseLayer["ELU"]],{ElementwiseLayer["ELU"],LinearLayer[]}],"Input"->"Real","Output"->"Real"]
In[]:=
RRnetSwish[lws_List,opts___]:=NetChain[Join[Riffle[LinearLayer/@lws,ElementwiseLayer["Swish"]],{ElementwiseLayer["Swish"],LinearLayer[]}],"Input"->"Real","Output"->"Real"]
Architecture Variations Plots
Architecture Variations Plots
variations=TrainPiecewise[RRnet[#],TargetDevice->"GPU"]&/@{{},{2,2},{3,3},{4,4}};
In[]:=
variations=TrainPiecewise[RRnet[#],TargetDevice->"GPU"]&/@{{3,3},{5,5},{7,7},{5,5,5},{7,7,7}};
In[]:=
GraphicsGrid[Transpose[With[{experiment=Last[#]},{NetGraphPlot[experiment,ImageSize->{Automatic,140}],Plot[{experiment[x],piecewise[x]},{x,-3,3},Frame->True,FrameTicks->None,PlotStyle->{Automatic,Directive[Gray,Dotted]},Exclusions->None,PlotRange->{{-3,3},{-0.4,1.4}},ImageSize->160]}]&/@Last/@variations],Spacings->0]
SELU
SELU
More training points:
More training points:
Local NN Plots
Local NN Plots
Function Definitions
Function Definitions
Range of n_steps
Range of n_steps
[[[ Need to fix scaling of weights etc. ]]]
[[[ Need to fix scaling of weights etc. ]]]
Lifetime vs. Position
Lifetime vs. Position
[[ Initialized RuleArray_Functions_Cache.nb ; RuleArray_Functions.nb ]]