In[]:=
generator=Function[Table[{a,b}=RandomReal[{-1,1},2];{a,b}(a*b),#BatchSize]];net=NetChain[{32,Tanh,1},"Input"2,"Output""Scalar"];data=generator[<|"BatchSize"->80|>];schedule1[b_,bmax_]:=1;schedule2[b_,bmax_]:=1/(2-b/bmax);schedule3=With[{n1=0.15#2,n2=0.75#2,sc=0.5},((1-sc)Which[#1<n1,Cos[Pi(#1-n1)/n1],#1<n2,Cos[Pi(#1-n1)/(n2-n1)],True,-1]+1+sc)/2]&;lab={"auto","const","example","cyclicLR"};schedules={Automatic,schedule1,schedule2,schedule3};plot=ListLinePlot[#1,PlotLegends->lab,PlotLabel->#2,PlotRange->{0,0.012},ImageSize->200,GridLines->{None,{0.01}}]&;train=(leng=1;{met,round,batch}=#;If[Length[batch]===2,{batch,leng}=batch];(NetTrain[net,If[leng===1,data,{generator,"RoundLength"leng}],All,LearningRate0.01,Method{met,"LearningRateSchedule"#},MaxTrainingRounds->round,BatchSize->batch,TrainingProgressReporting->File["log.json"]];"LearningRate"/.Import["log.json"])&/@schedules)&;
tl1=train[{"SGD",100,80}];tl2=train[{"ADAM",100,80}];tl3=train[{"ADAM",100,20}];tl4=train[{"ADAM",100,{20,80}}];tl5=train[{"ADAM",500,{20,80}}];tl6=train[{"ADAM",500,{20,240}}];
In[]:=
Grid@Partition[{plot[tl1,"SGD"],plot[tl2,"ADAM - 100/80"],plot[tl3,"ADAM - 100/20"],plot[tl4,"ADAM - 100/gen 80-20"],plot[tl5,"ADAM - 500/gen 80-20"],plot[tl6,"ADAM - 500/gen 240-20"]},2]
Out[]=
