1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Model Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    3. Composite Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    4. Results
  3. Results

Target Description: The type Re lu activation layer.

Report Description: The type Basic.

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 829706264506413056

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:445 executed in 0.02 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.08645210515158785 ], [ 0.46371328451778027 ], [ 0.7118133195882037 ], [ 0.858771275693855 ], [ 0.31522834462325333 ], [ -0.8553338797352668 ], [ -0.5696554003085794 ], [ -0.0712819884245266 ], ... ],
    	[ [ 0.21245711354565633 ], [ -0.5744314817709455 ], [ 0.6720724363511505 ], [ 0.05406824165281021 ], [ -0.6420307535442074 ], [ -0.9390114848851809 ], [ -0.9042633573801411 ], [ 0.07880291138213136 ], ... ],
    	[ [ 0.9480204684646694 ], [ 0.5116638121447667 ], [ -0.8747270409471488 ], [ -0.11953416438937481 ], [ -0.8444672287035084 ], [ -0.22501287759124855 ], [ -0.4625267811556568 ], [ 0.259796816973211 ], ... ],
    	[ [ -0.8466699589000137 ], [ 0.6411738483841547 ], [ 0.9993051751975219 ], [ 0.2567318274988477 ], [ -0.6424147769577733 ], [ 0.2394416967153623 ], [ 0.5822618420970047 ], [ 0.024639635619943645 ], ... ],
    	[ [ -0.87397947007601 ], [ -0.3314246190080985 ], [ -0.3705101860006308 ], [ -0.9559903545715192 ], [ -0.22086532314400076 ], [ -0.8651751863226864 ], [ 0.642328885289158 ], [ -0.7564653692190122 ], ... ],
    	[ [ -0.612010153231126 ], [ 0.8158182361934978 ], [ 0.5337603145231153 ], [ 0.24323223681671968 ], [ -0.8588239754963292 ], [ -0.9906203504305526 ], [ -0.3645437467595958 ], [ -0.34387994216055007 ], ... ],
    	[ [ 0.9457729786637556 ], [ -0.12291229967067352 ], [ 0.19189509807413718 ], [ 0.8447982118356383 ], [ 0.8317430452235834 ], [ -0.897170562800587 ], [ 0.347201568693686 ], [ -0.8018251692123695 ], ... ],
    	[ [ 0.58914675233618 ], [ -0.89491664170846 ], [ -0.9377631215895865 ], [ 0.12177052080501571 ], [ 0.9329674102559711 ], [ 0.6485429018716153 ], [ -0.4672454596992286 ], [ -0.49998177990885484 ], ... ],
    	...
    ]
    [
    	[ [ -0.5710755420703773 ], [ -0.01635445763261134 ], [ 0.42660136123210735 ], [ -0.10917549148147399 ], [ 0.1503880974544698 ], [ -0.8447504443945066 ], [ 0.29098779988829326 ], [ -0.2202448363003464 ], ... ],
    	[ [ -0.618495386399716 ], [ -0.4557809123471188 ], [ -0.583836663042377 ], [ 0.8200199058821496 ], [ 0.44761247401285376 ], [ 0.09348043017346375 ], [ -0.2701823976156339 ], [ 0.9177087886916713 ], ... ],
    	[ [ 0.3077728035081988 ], [ 0.9471660320000617 ], [ -0.3358871749693102 ], [ -0.8610653932440655 ], [ 0.40281813453519644 ], [ 0.18936003749488695 ], [ 0.5209234480556086 ], [ 0.5291346601453306 ], ... ],
    	[ [ -0.8027454069379667 ], [ -0.33490488164511123 ], [ -0.9798220415046697 ], [ -0.2043110305999447 ], [ -0.9856899540731956 ], [ -0.07021325070054574 ], [ -0.4581032123851234 ], [ 0.0955745514713291 ], ... ],
    	[ [ -0.9081677707822513 ], [ -0.7130601912598933 ], [ -0.12630960629555776 ], [ 0.5059821307244314 ], [ -0.12774045047478944 ], [ -0.15227779969941624 ], [ -0.23910775603096068 ], [ 0.4350022863891079 ], ... ],
    	[ [ 0.03741528064494992 ], [ 0.6772647534182625 ], [ -0.042123664765186364 ], [ -0.3515222763160428 ], [ -0.0444153225152899 ], [ -0.9660098656667819 ], [ -0.7357731541116119 ], [ -0.5737962585790783 ], ... ],
    	[ [ 0.3527284560546058 ], [ 0.16802238941211756 ], [ 0.3756775470899052 ], [ 0.38238955506512484 ], [ 0.9324841164273538 ], [ -0.23137066258045547 ], [ -0.05485413813548967 ], [ -0.5563665096921995 ], ... ],
    	[ [ 0.8975655403221185 ], [ 0.8320824472182611 ], [ -0.6777305252833268 ], [ 0.9658287456742278 ], [ -0.5470387967755681 ], [ -0.7597136508151869 ], [ 0.39690554756249985 ], [ 0.1243795670243042 ], ... ],
    	...
    ]
    [
    	[ [ 0.2271746924508764 ], [ 0.6018534345465394 ], [ -0.82527264832845 ], [ -0.8307858463180755 ], [ 0.037444588589903516 ], [ 0.5614895307627936 ], [ -0.5360954937063342 ], [ 0.17047485265823004 ], ... ],
    	[ [ 0.32061890704269924 ], [ 0.18753426503559134 ], [ 0.06151861009359383 ], [ -0.498875726073976 ], [ -0.41881758316485374 ], [ -0.4619628987548523 ], [ -0.2543547308815144 ], [ 0.3065385447876099 ], ... ],
    	[ [ 0.05024392285762502 ], [ -0.9366066137038191 ], [ -0.7453915694260682 ], [ -0.3687083931309556 ], [ -0.3258382794453214 ], [ -0.23932479256101136 ], [ -0.6540359618934408 ], [ 0.7521817734988852 ], ... ],
    	[ [ 0.5264329625351756 ], [ -0.871982430472535 ], [ 0.914521881072555 ], [ 0.6995367051179304 ], [ -0.38175760738216624 ], [ -0.5800821350171443 ], [ 0.5428838595572292 ], [ 0.5966977544159424 ], ... ],
    	[ [ 0.7800314348451538 ], [ 0.8781629553697572 ], [ -0.09574007306150989 ], [ 0.775820766227574 ], [ -0.5259927758601195 ], [ -0.5451367399864682 ], [ 0.24837918023966465 ], [ -0.6175748555406371 ], ... ],
    	[ [ -0.8585696983187043 ], [ 0.43473926208769953 ], [ -0.8441728175263545 ], [ 0.9281683672108951 ], [ 0.908981087676747 ], [ -0.43729416012659605 ], [ -0.9579781512576724 ], [ 0.34516425141845275 ], ... ],
    	[ [ -0.7626646648665836 ], [ -0.3974643292036453 ], [ 0.7281877900088602 ], [ -0.6773058082127514 ], [ 0.9868276652162944 ], [ -0.5915216976072151 ], [ 0.6467660242811812 ], [ 0.2567993796274659 ], ... ],
    	[ [ -0.12447784958158281 ], [ 0.3287617682611932 ], [ 0.7164794283679279 ], [ 0.7946080918454841 ], [ 0.2941874260195694 ], [ 0.7978359863889006 ], [ 0.7739633567998148 ], [ 0.9345540118016662 ], ... ],
    	...
    ]
    [
    	[ [ -0.03887835500324366 ], [ -0.6814827410942659 ], [ -0.5336323975102621 ], [ -0.04456590994606491 ], [ 0.09539903705803221 ], [ 0.5492338914095902 ], [ 0.37755885534982947 ], [ -0.5975607969241523 ], ... ],
    	[ [ -0.5568402383788262 ], [ -0.9844768522227669 ], [ 0.15763816449284218 ], [ -0.341660771509388 ], [ -0.6732804979080038 ], [ 0.8585946706213781 ], [ 0.9295840182072443 ], [ 0.5878686541363398 ], ... ],
    	[ [ 0.8964346183944294 ], [ -0.3480436195357772 ], [ -0.0688512131289647 ], [ -0.32239948948017094 ], [ 0.9147069742839493 ], [ -0.2541491808916647 ], [ 0.2910544012505625 ], [ -0.7374421122231862 ], ... ],
    	[ [ -0.08721130679470313 ], [ -0.27331473505845794 ], [ -0.8664285312132451 ], [ 0.4925616015347434 ], [ -0.7187947097421603 ], [ 0.45163308898828114 ], [ -0.5155800617643371 ], [ 0.17904539693732113 ], ... ],
    	[ [ 0.358935905841534 ], [ 0.0073079722785884815 ], [ 0.7534299822023596 ], [ 0.31946690686340107 ], [ 0.5849695075218289 ], [ -0.40988843108054285 ], [ 0.9565083515092399 ], [ 0.33920337143659585 ], ... ],
    	[ [ -0.5533842914394329 ], [ 0.9111114535099225 ], [ 0.7044027706240619 ], [ -0.7544471258737686 ], [ 0.7580498290596829 ], [ -0.22840600295287294 ], [ -0.5212609563617656 ], [ 0.1542098042040272 ], ... ],
    	[ [ 0.10476714805403864 ], [ -0.2544955089731422 ], [ 0.76162909527586 ], [ -0.15937668550708461 ], [ -0.5736855477082738 ], [ -0.1412102886411799 ], [ 0.696693384482828 ], [ -0.9252488271175665 ], ... ],
    	[ [ 0.8650633960621825 ], [ 0.7969138273672638 ], [ -0.1787445869420916 ], [ 0.05111697267308135 ], [ -0.43249826413355086 ], [ -0.8326422365709534 ], [ 0.16113355410443542 ], [ -0.950185701046053 ], ... ],
    	...
    ]
    [
    	[ [ -0.3031242426058993 ], [ -0.7481765773406999 ], [ -0.13760189964334013 ], [ -0.026624657010618114 ], [ -0.6255239037285085 ], [ -0.16036605972027673 ], [ -0.6868225484332358 ], [ -0.26106405122441667 ], ... ],
    	[ [ -0.6367545319341559 ], [ -0.8252037096896754 ], [ -0.6312486919089972 ], [ 0.14089034659429522 ], [ 0.9626053449495251 ], [ 0.10316374571225895 ], [ 0.6805915053173912 ], [ -0.896702324557064 ], ... ],
    	[ [ -0.4113658834925298 ], [ 0.29707047070330295 ], [ 0.48198565103802804 ], [ -0.1688878336003916 ], [ 0.1283001925295968 ], [ 0.9603402374050569 ], [ 0.19546276356567205 ], [ -0.21246874894190504 ], ... ],
    	[ [ -0.13818225400927386 ], [ -0.5500372344120117 ], [ 0.17307736140901242 ], [ -0.8570653694166326 ], [ -0.4416279238826635 ], [ -0.7372306686927836 ], [ -0.17922307339857335 ], [ 0.5672376147954878 ], ... ],
    	[ [ 0.967530786880419 ], [ 0.7404550355652426 ], [ -0.9111971144371291 ], [ -0.44528492520888086 ], [ 0.7343278255082881 ], [ 0.8310357715620751 ], [ 0.012322744663007423 ], [ -0.33969025525998564 ], ... ],
    	[ [ -0.4719273355882536 ], [ -0.8601943770684144 ], [ 0.44339196643571444 ], [ 0.4528586535530936 ], [ 0.9510938847002839 ], [ 0.810765246525617 ], [ 0.8236545763570498 ], [ 0.2852973293086887 ], ... ],
    	[ [ -0.5332084536724091 ], [ 0.28981419149298215 ], [ 0.8806937161895676 ], [ -0.7607509909837702 ], [ 0.8456346382972904 ], [ 0.7815559902306262 ], [ 0.7831346374940338 ], [ 0.17983052011525213 ], ... ],
    	[ [ -0.21827141768161118 ], [ -0.4193745866068481 ], [ 0.6960549952704249 ], [ 0.7273048529294488 ], [ -0.8232041771742831 ], [ 0.3634959739247059 ], [ -0.8159846737292891 ], [ -0.784660501534503 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:638 executed in 2.43 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2734647232854
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 2734744677728
Constructing line search parameters: GD
th(0)=0.012695985971682134;dx=-3.703891504480856E-8
New Minimum: 0.012695985971682134 > 0.01269590617396819
WOLFE (weak): th(2.154434690031884)=0.01269590617396819; dx=-3.7038720557997534E-8 evalInputDelta=7.97977139435635E-8
New Minimum: 0.01269590617396819 > 0.012695826376673233
WOLFE (weak): th(4.308869380063768)=0.012695826376673233; dx=-3.703852607118651E-8 evalInputDelta=1.5959500890110057E-7
New Minimum: 0.012695826376673233 > 0.012695507191683572
WOLFE (weak): th(12.926608140191302)=0.012695507191683572; dx=-3.7037748123942403E-8 evalInputDelta=4.787799985622038E-7
New Minimum: 0.012695507191683572 > 0.012694070942193852
WOLFE (weak): th(51.70643256076521)=0.012694070942193852; dx=-3.703424736134393E-8 evalInputDelta=1.915029488281511E-6
New Minimum: 0.012694070942193852 > 0.012686413237733434
WOLFE (weak): th(258.53216280382605)=0.012686413237733434; dx=-3.701557662748545E-8 evalInputDelta=9.572733948699785E-6
New Minimum: 0.012686413237733434 > 0.012638640073962612
WOLFE (weak): th(1551.1929768229563)=0.012638640073962612; dx=-3.689888454086992E-8 evalInputDelta=5.7345897719522296E-5
New Minimum: 0.012638640073962612 > 0.012299126188664704
WOLFE (weak): th(10858.350837760694)=0.012299126188664704; dx=-3.605870151723808E-8 evalInputDelta=3.968597830174298E-4
New Minimum: 0.012299126188664704 > 0.00981912577413471
END: th(86866.80670208555)=0.00981912577413471; dx=-2.9197206824244733E-8 evalInputDelta=0.0028768601975474248
Fitness changed from 0.012695985971682134 to 0.00981912577413471
Iteration 1 complete. Error: 0.00981912577413471 Total: 0.9149; Orientation: 0.0104; Line Search: 0.7323
th(0)=0.00981912577413471;dx=-2.301570889175405E-8
New Minimum: 0.00981912577413471 > 0.006494114441218358
END: th(187148.86177126726)=0.006494114441218358; dx=-1.2517623253008661E-8 evalInputDelta=0.003325011332916351
Fitness changed from 0.00981912577413471 to 0.006494114441218358
Iteration 2 complete. Error: 0.006494114441218358 Total: 0.0447; Orientation: 0.0051; Line Search: 0.0304
th(0)=0.006494114441218358;dx=-6.807997643748508E-9
New Minimum: 0.006494114441218358 > 0.005097872199788118
END: th(403200.00000000006)=0.005097872199788118; dx=-1.1780712525071616E-10 evalInputDelta=0.0013962422414302402
Fitness changed from 0.006494114441218358 to 0.005097872199788118
Iteration 3 complete. Error: 0.005097872199788118 Total: 0.2052; Orientation: 0.0034; Line Search: 0.1937
Low gradient: 1.427781860237443E-6
th(0)=0.005097872199788118;dx=-2.0385610404230935E-12
New Minimum: 0.005097872199788118 > 0.005097626994221243
WOLF (strong): th(868668.0670208557)=0.005097626994221243; dx=7.504349947796447E-13 evalInputDelta=2.452055668754338E-7
New Minimum: 0.005097626994221243 > 0.005097454468114826
WOLF (strong): th(434334.03351042786)=0.005097454468114826; dx=3.9863699854217757E-14 evalInputDelta=4.1773167329221816E-7
END: th(144778.01117014262)=0.005097629132164982; dx=-1.319236475549105E-12 evalInputDelta=2.43067623135855E-7
Fitness changed from 0.005097872199788118 to 0.005097454468114826
Iteration 4 complete. Error: 0.005097454468114826 Total: 0.1389; Orientation: 0.0044; Line Search: 0.1250
Low gradient: 4.832280299739577E-8
th(0)=0.005097454468114826;dx=-2.335093289525122E-15
New Minimum: 0.005097454468114826 > 0.005097454016614841
END: th(311914.76961877884)=0.005097454016614841; dx=-5.599282313767193E-16 evalInputDelta=4.5149998496862853E-10
Fitness changed from 0.005097454468114826 to 0.005097454016614841
Iteration 5 complete. Error: 0.005097454016614841 Total: 0.0344; Orientation: 0.0038; Line Search: 0.0229
Low gradient: 1.1587246530522972E-8
th(0)=0.005097454016614841;dx=-1.3426428215911667E-16
New Minimum: 0.005097454016614841 > 0.005097454000276174
WOLF (strong): th(672000.0000000002)=0.005097454000276174; dx=8.563728911973967E-17 evalInputDelta=1.6338667398196893E-11
New Minimum: 0.005097454000276174 > 0.005097453989973774
END: th(336000.0000000001)=0.005097453989973774; dx=-2.4313496519687354E-17 evalInputDelta=2.6641066956656356E-11
Fitness changed from 0.005097454016614841 to 0.005097453989973774
Iteration 6 complete. Error: 0.005097453989973774 Total: 0.1086; Orientation: 0.0033; Line Search: 0.0990
Low gradient: 2.098298025820117E-9
th(0)=0.005097453989973774;dx=-4.402854605160601E-18
New Minimum: 0.005097453989973774 > 0.005097453989598153
WOLF (strong): th(723890.0558507132)=0.005097453989598153; dx=3.3650792713821846E-18 evalInputDelta=3.75621408976734E-13
New Minimum: 0.005097453989598153 > 0.005097453989083057
END: th(361945.0279253566)=0.005097453989083057; dx=-5.188876668892631E-19 evalInputDelta=8.907171875072173E-13
Fitness changed from 0.005097453989973774 to 0.005097453989083057
Iteration 7 complete. Error: 0.005097453989083057 Total: 0.1327; Orientation: 0.0174; Line Search: 0.1077
Low gradient: 2.472897846274513E-10
th(0)=0.005097453989083057;dx=-6.115223758109126E-20
New Minimum: 0.005097453989083057 > 0.005097453989080682
WOLF (strong): th(779786.9240469472)=0.005097453989080682; dx=5.506936678531995E-20 evalInputDelta=2.374836438612249E-15
New Minimum: 0.005097453989080682 > 0.005097453989070531
END: th(389893.4620234736)=0.005097453989070531; dx=-3.0414353978597295E-21 evalInputDelta=1.2525570858290536E-14
Fitness changed from 0.005097453989083057 to 0.005097453989070531
Iteration 8 complete. Error: 0.005097453989070531 Total: 0.3059; Orientation: 0.0034; Line Search: 0.2760
Zero gradient: 1.2299074150764996E-11
th(0)=0.005097453989070531;dx=-1.5126722496601572E-22
WOLF (strong): th(840000.0000000003)=0.005097453989070531; dx=1.58419572575524E-22 evalInputDelta=0.0
New Minimum: 0.005097453989070531 > 0.005097453989070528
WOLF (strong): th(420000.0000000002)=0.005097453989070528; dx=3.57617380451544E-24 evalInputDelta=3.469446951953614E-18
END: th(140000.00000000006)=0.0050974539890705295; dx=-9.965275871015604E-23 evalInputDelta=1.734723475976807E-18
Fitness changed from 0.005097453989070531 to 0.005097453989070528
Iteration 9 complete. Error: 0.005097453989070528 Total: 0.4772; Orientation: 0.0036; Line Search: 0.4642
Zero gradient: 2.9076772451957344E-13
th(0)=0.005097453989070528;dx=-8.454586962229055E-26
END: th(301620.8566044639)=0.005097453989070528; dx=-2.2394267302949764E-26 evalInputDelta=0.0
Fitness changed from 0.005097453989070528 to 0.005097453989070528
Static Iteration Total: 0.0523; Orientation: 0.0034; Line Search: 0.0416
Iteration 10 failed. Error: 0.005097453989070528
Previous Error: 0.0 -> 0.005097453989070528
Optimization terminated 10
Final threshold in iteration 10: 0.005097453989070528 (> 0.0) after 2.416s (< 30.000s)

Returns

    0.005097453989070528

This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:797 executed in 0.01 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.08645210736072692 ], [ -0.710851331074656 ], [ 0.7118133183668041 ], [ -0.888708624872665 ], [ 0.3152283458967086 ], [ -0.30810884642026215 ], [ -0.7233629671695123 ], [ -4.1653778867581457E-4 ], ... ],
    	[ [ -0.3959343999923348 ], [ -0.2958558215451197 ], [ 0.6720724369188132 ], [ 0.05406824366578505 ], [ -0.15006219442230018 ], [ -9.711349135853417E-5 ], [ -2.562332278814375E-4 ], [ 0.07880291207851724 ], ... ],
    	[ [ 0.9480204674095962 ], [ 0.5116638114410909 ], [ -1.484614425550138E-4 ], [ -0.39144977573267936 ], [ -3.294763254180766E-4 ], [ -0.37830121691534124 ], [ -2.5127835143891157E-4 ], [ -0.3451331706051841 ], ... ],
    	[ [ -3.588607844282653E-4 ], [ 0.6411738470881434 ], [ -0.19396667629153108 ], [ -0.7260461997613561 ], [ -4.130691754290726E-4 ], [ -0.30702367029412403 ], [ -0.4169083958269233 ], [ -0.190216204029529 ], ... ],
    	[ [ -3.834110807805109E-4 ], [ -0.06934885921235057 ], [ -1.8727426254149318E-4 ], [ -0.18705588169274656 ], [ -2.1651346320112654E-4 ], [ -0.4135782201259133 ], [ -0.11265745828239293 ], [ -0.1344171343203029 ], ... ],
    	[ [ -1.458529724768648E-4 ], [ 0.8158182351604897 ], [ 0.5337603140525852 ], [ 0.2432322364534065 ], [ -3.837285784689926E-5 ], [ -0.9755946459884148 ], [ -1.5545444422875986E-4 ], [ -2.3388335244554047E-4 ], ... ],
    	[ [ -0.52560013812054 ], [ -9.316238554598548E-5 ], [ 0.19189509986946407 ], [ -0.4038943319057626 ], [ 0.8317430446485077 ], [ -0.9510686432968569 ], [ 0.34720156781284334 ], [ -0.927494229969759 ], ... ],
    	[ [ 0.5891467509836 ], [ -1.258643400638181E-4 ], [ -2.108846722881216E-4 ], [ -0.24305425978260198 ], [ 0.9329674086410903 ], [ 0.6485429014355696 ], [ -1.0322407101733683E-4 ], [ -0.5158390101476158 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:808 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.0213400224067787 ], [ 0.0 ], [ 0.17570551635033474 ], [ 0.0 ], [ 0.0778116366396807 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.1658957924875418 ], [ 0.013346320483648205 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.019451878743200018 ], ... ],
    	[ [ 0.23401139236770008 ], [ 0.12630018555048975 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.15826871873012524 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.2013783116418542 ], [ 0.13175453334184203 ], [ 0.06003996356397142 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.04736779537228122 ], [ 0.0 ], [ 0.20530922555098544 ], [ 0.0 ], [ 0.08570397487106983 ], [ 0.0 ], ... ],
    	[ [ 0.14542623946009478 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.23029566326383788 ], [ 0.16008771180839326 ], [ 0.0 ], [ 0.0 ], ... ],
    	...
    ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:603 executed in 2.43 seconds (0.026 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2737089468144
Reset training subject: 2737097908613
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.012695985971682134}, derivative=-3.703891504480856E-8}
New Minimum: 0.012695985971682134 > 0.012695985971682127
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.012695985971682127}, derivative=-3.703891504480855E-8}, evalInputDelta = -6.938893903907228E-18
New Minimum: 0.012695985971682127 > 0.012695985971682103
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.012695985971682103}, derivative=-3.7038915044808494E-8}, evalInputDelta = -3.122502256758253E-17
New Minimum: 0.012695985971682103 > 0.012695985971681941
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.012695985971681941}, derivative=-3.703891504480812E-8}, evalInputDelta = -1.9255430583342559E-16
New Minimum: 0.012695985971681941 > 0.01269598597168085
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.01269598597168085}, derivative=-3.703891504480546E-8}, evalInputDelta = -1.2836953722228372E-15
New Minimum: 0.01269598597168085 > 0.012695985971673226
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.012695985971673226}, derivative=-3.703891504478688E-8}, evalInputDelta = -8.907805049140904E-15
New Minimum: 0.012695985971673226 > 0.012695985971619871
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.012695985971619871}, derivative=-3.7038915044656835E-8}, evalInputDelta = -6.226269499975956E-14
New Minimum: 0.012695985971619871 > 0.012695985971246377
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.012695985971246377}, derivative=-3.7038915043746507E-8}, evalInputDelta = -4.35757332994946E-13
New Minimum: 0.012695985971246377 > 0.012695985968631807
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.012695985968631807}, derivative=-3.7038915037374205E-8}, evalInputDelta = -3.0503273518167617E-12
New Minimum: 0.012695985968631807 > 0.012695985950329933
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.012695985950329933}, derivative=-3.70389149927681E-8}, evalInputDelta = -2.135220125709658E-11
New Minimum: 0.012695985950329933 > 0.012695985822216736
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.012695985822216736}, derivative=-3.7038914680525325E-8}, evalInputDelta = -1.4946539839133521E-10
New Minimum: 0.012695985822216736 > 0.012695984925424494
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.012695984925424494}, derivative=-3.703891249482593E-8}, evalInputDelta = -1.0462576395531276E-9
New Minimum: 0.012695984925424494 > 0.012695978647880154
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.012695978647880154}, derivative=-3.703889719493015E-8}, evalInputDelta = -7.323801979805533E-9
New Minimum: 0.012695978647880154 > 0.01269593470514252
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.01269593470514252}, derivative=-3.7038790095659683E-8}, evalInputDelta = -5.1266539614208684E-8
New Minimum: 0.01269593470514252 > 0.012695627109536764
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.012695627109536764}, derivative=-3.703804040076645E-8}, evalInputDelta = -3.5886214536963446E-7
New Minimum: 0.012695627109536764 > 0.012693474114625783
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.012693474114625783}, derivative=-3.703279253651379E-8}, evalInputDelta = -2.511857056351416E-6
New Minimum: 0.012693474114625783 > 0.012678411692383115
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.012678411692383115}, derivative=-3.699605748674517E-8}, evalInputDelta = -1.7574279299019044E-5
New Minimum: 0.012678411692383115 > 0.012573393301264396
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.012573393301264396}, derivative=-3.6738912138364884E-8}, evalInputDelta = -1.2259267041773818E-4
New Minimum: 0.012573393301264396 > 0.011858774227855171
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.011858774227855171}, derivative=-3.493889469970282E-8}, evalInputDelta = -8.372117438269627E-4
New Minimum: 0.011858774227855171 > 0.007861414270651085
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.007861414270651085}, derivative=-2.2338772629068407E-8}, evalInputDelta = -0.0048345717010310485
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.00960317811384135}, derivative=1.1425704812052044E-8}, evalInputDelta = -0.003092807857840784
0.00960317811384135 <= 0.012695985971682134
New Minimum: 0.007861414270651085 > 0.007047264230111846
F(871156.5501239488) = LineSearchPoint{point=PointSample{avg=0.007047264230111846}, derivative=7.596320354572978E-9}, evalInputDelta = -0.005648721741570288
Right bracket at 871156.5501239488
New Minimum: 0.007047264230111846 > 0.00607103913702543
F(722897.3514322843) = LineSearchPoint{point=PointSample{avg=0.00607103913702543}, derivative=5.563779361577957E-9}, evalInputDelta = -0.006624946834656704
Right bracket at 722897.3514322843
New Minimum: 0.00607103913702543 > 0.005610228458229039
F(628489.2061147109) = LineSearchPoint{point=PointSample{avg=0.005610228458229039}, derivative=4.197360706112715E-9}, evalInputDelta = -0.007085757513453095
Right bracket at 628489.2061147109
New Minimum: 0.005610228458229039 > 0.005371804561555259
F(564516.5061091114) = LineSearchPoint{point=PointSample{avg=0.005371804561555259}, derivative=3.229312563166222E-9}, evalInputDelta = -0.007324181410126875
Right bracket at 564516.5061091114
New Minimum: 0.005371804561555259 > 0.005242670586188206
F(519245.0761609051) = LineSearchPoint{point=PointSample{avg=0.005242670586188206}, derivative=2.4546307865351367E-9}, evalInputDelta = -0.007453315385493928
Right bracket at 519245.0761609051
New Minimum: 0.005242670586188206 > 0.005173185515645957
F(486972.5890273289) = LineSearchPoint{point=PointSample{avg=0.005173185515645957}, derivative=1.8457587268906866E-9}, evalInputDelta = -0.007522800456036177
Right bracket at 486972.5890273289
New Minimum: 0.005173185515645957 > 0.005136015187762401
F(463857.2117136088) = LineSearchPoint{point=PointSample{avg=0.005136015187762401}, derivative=1.3638736186409923E-9}, evalInputDelta = -0.007559970783919733
Right bracket at 463857.2117136088
New Minimum: 0.005136015187762401 > 0.0051165596372680705
F(447383.340000363) = LineSearchPoint{point=PointSample{avg=0.0051165596372680705}, derivative=9.911689909408076E-10}, evalInputDelta = -0.0075794263344140635
Right bracket at 447383.340000363
New Minimum: 0.0051165596372680705 > 0.005106648579691744
F(435723.294935644) = LineSearchPoint{point=PointSample{avg=0.005106648579691744}, derivative=7.031897298512973E-10}, evalInputDelta = -0.00758933739199039
Right bracket at 435723.294935644
New Minimum: 0.005106648579691744 > 0.005101791210277136
F(427605.1428642357) = LineSearchPoint{point=PointSample{avg=0.005101791210277136}, derivative=4.916573599011519E-10}, evalInputDelta = -0.007594194761404998
Right bracket at 427605.1428642357
New Minimum: 0.005101791210277136 > 0.0050994615615554215
F(422003.437317256) = LineSearchPoint{point=PointSample{avg=0.0050994615615554215}, derivative=3.3882450617093923E-10}, evalInputDelta = -0.0075965244101267124
Right bracket at 422003.437317256
Converged to right
Fitness changed from 0.012695985971682134 to 0.0050994615615554215
Iteration 1 complete. Error: 0.0050994615615554215 Total: 2.0255; Orientation: 0.0043; Line Search: 1.8275
Low gradient: 3.1088147791984515E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0050994615615554215}, derivative=-9.664729331362716E-12}
New Minimum: 0.0050994615615554215 > 0.005097480455918504
F(422003.437317256) = LineSearchPoint{point=PointSample{avg=0.005097480455918504}, derivative=2.756795666998735E-13}, evalInputDelta = -1.981105636917474E-6
0.005097480455918504 <= 0.0050994615615554215
New Minimum: 0.005097480455918504 > 0.005097478842708532
F(410299.9223071086) = LineSearchPoint{point=PointSample{avg=0.005097478842708532}, derivative=-6.5280812280244685E-28}, evalInputDelta = -1.982718846889331E-6
Left bracket at 410299.9223071086
Converged to left
Fitness changed from 0.0050994615615554215 to 0.005097478842708532
Iteration 2 complete. Error: 0.005097478842708532 Total: 0.1930; Orientation: 0.0029; Line Search: 0.1837
Zero gradient: 1.1304214998978426E-20
F(0.0) = LineSearchPoint{point=PointSample{avg=0.005097478842708532}, derivative=-1.2778527674312883E-40}
F(410299.9223071086) = LineSearchPoint{point=PointSample{avg=0.005097478842708532}, derivative=0.0}, evalInputDelta = 0.0
0.005097478842708532 <= 0.005097478842708532
Converged to right
Fitness changed from 0.005097478842708532 to 0.005097478842708532
Static Iteration Total: 0.2137; Orientation: 0.0348; Line Search: 0.1674
Iteration 3 failed. Error: 0.005097478842708532
Previous Error: 0.0 -> 0.005097478842708532
Optimization terminated 3
Final threshold in iteration 3: 0.005097478842708532 (> 0.0) after 2.433s (< 30.000s)

Returns

    0.005097478842708532

This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:797 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.08645210515158785 ], [ -0.710851331074656 ], [ 0.7118133195882036 ], [ -0.888708624872665 ], [ 0.31522834462325333 ], [ -0.30810884642026215 ], [ -0.7233629671695123 ], [ -0.027340718640375017 ], ... ],
    	[ [ -0.3959343999923348 ], [ -0.2958558215451197 ], [ 0.6720724363511505 ], [ 0.054068241652810205 ], [ -0.15006219442230018 ], [ -0.006374337972693822 ], [ -0.016818643542744316 ], [ 0.07880291138213136 ], ... ],
    	[ [ 0.9480204684646694 ], [ 0.5116638121447667 ], [ -0.009744716182281288 ], [ -0.39144977573267936 ], [ -0.021626175960066774 ], [ -0.37830121691534124 ], [ -0.01649341522878134 ], [ -0.3451331706051841 ], ... ],
    	[ [ -0.023554913875421746 ], [ 0.6411738483841547 ], [ -0.19396667629153108 ], [ -0.7260461997613561 ], [ -0.027113045710259742 ], [ -0.30702367029412403 ], [ -0.4169083958269233 ], [ -0.190216204029529 ], ... ],
    	[ [ -0.02516634689147168 ], [ -0.06934885921235057 ], [ -0.012292313110433328 ], [ -0.18705588169274656 ], [ -0.014211516554245351 ], [ -0.4135782201259133 ], [ -0.11265745828239293 ], [ -0.1344171343203029 ], ... ],
    	[ [ -0.009573501352732994 ], [ 0.8158182361934978 ], [ 0.5337603145231152 ], [ 0.24323223681671968 ], [ -0.0025187186813336315 ], [ -0.9755946459884148 ], [ -0.01020372301530198 ], [ -0.015351641814322803 ], ... ],
    	[ [ -0.52560013812054 ], [ -0.00611499518249306 ], [ 0.19189509807413718 ], [ -0.4038943319057626 ], [ 0.8317430452235836 ], [ -0.9510686432968569 ], [ 0.347201568693686 ], [ -0.927494229969759 ], ... ],
    	[ [ 0.58914675233618 ], [ -0.008261486957716557 ], [ -0.013842053824040024 ], [ -0.24305425978260198 ], [ 0.9329674102559711 ], [ 0.6485429018716153 ], [ -0.006775424365628335 ], [ -0.5158390101476158 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:808 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.021340021861470104 ], [ 0.0 ], [ 0.17570551665182763 ], [ 0.0 ], [ 0.07781163632533829 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.16589579234741875 ], [ 0.013346319986761205 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.019451878571302643 ], ... ],
    	[ [ 0.23401139262813658 ], [ 0.1263001857241866 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.15826871905003545 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.20137831189684413 ], [ 0.13175453345798868 ], [ 0.06003996365365242 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.047367794929118905 ], [ 0.0 ], [ 0.20530922569293839 ], [ 0.0 ], [ 0.08570397508849892 ], [ 0.0 ], ... ],
    	[ [ 0.1454262397939685 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.2302956636624585 ], [ 0.16008771191602772 ], [ 0.0 ], [ 0.0 ], ... ],
    	...
    ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:674 executed in 30.30 seconds (0.077 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2739538944605
Reset training subject: 2739604717263
Adding measurement 709bd454 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.012695985971682134 < 0.012695985971682134. Total: 1
th(0)=0.012695985971682134;dx=-3.703891504480856E-8
Adding measurement 418bdb9e to history. Total: 1
New Minimum: 0.012695985971682134 > 0.01269590617396819
WOLFE (weak): th(2.154434690031884)=0.01269590617396819; dx=-3.7038720557997534E-8 evalInputDelta=7.97977139435635E-8
Adding measurement b79cf28 to history. Total: 2
New Minimum: 0.01269590617396819 > 0.012695826376673233
WOLFE (weak): th(4.308869380063768)=0.012695826376673233; dx=-3.703852607118651E-8 evalInputDelta=1.5959500890110057E-7
Adding measurement b8a5169 to history. Total: 3
New Minimum: 0.012695826376673233 > 0.012695507191683572
WOLFE (weak): th(12.926608140191302)=0.012695507191683572; dx=-3.7037748123942403E-8 evalInputDelta=4.787799985622038E-7
Adding measurement 527d0d92 to history. Total: 4
New Minimum: 0.012695507191683572 > 0.012694070942193852
WOLFE (weak): th(51.70643256076521)=0.012694070942193852; dx=-3.703424736134393E-8 evalInputDelta=1.915029488281511E-6
Adding measurement 351de280 to history. Total: 5
New Minimum: 0.012694070942193852 > 0.012686413237733434
WOLFE (weak): th(258.53216280382605)=0.012686413237733434; dx=-3.701557662748545E-8 evalInputDelta=9.572733948699785E-6
Adding measurement 6a768599 to history. Total: 6
New Minimum: 0.012686413237733434 > 0.012638640073962612
WOLFE (weak): th(1551.1929768229563)=0.012638640073962612; dx=-3.689888454086992E-8 evalInputDelta=5.7345897719522296E-5
Adding measurement 48ebbed8 to history. Total: 7
New Minimum: 0.012638640073962612 > 0.012299126188664704
WOLFE (weak): th(10858.350837760694)=0.012299126188664704; dx=-3.605870151723808E-8 evalInputDelta=3.968597830174298E-4
Adding measurement 24d9e6d3 to history. Total: 8
New Minimum: 0.012299126188664704 > 0.00981912577413471
END: th(86866.80670208555)=0.00981912577413471; dx=-2.9197206824244733E-8 evalInputDelta=0.0028768601975474248
Fitness changed from 0.012695985971682134 to 0.00981912577413471
Iteration 1 complete. Error: 0.00981912577413471 Total: 0.6935; Orientation: 0.0269; Line Search: 0.5649
Non-optimal measurement 0.00981912577413471 < 0.00981912577413471. Total: 9
Rejected: LBFGS Orientation magnitude: 6.225e+01, gradient 1.517e-04, dot -1.000; [fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00981912577413471, 0.012299126188664704, 0.012638640073962612, 0.012686413237733434, 0.012694070942193852, 0.012695507191683572, 0.012695826376673233, 0.01269590617396819, 0.012695985971682134
Rejected: LBFGS Orientation magnitude: 6.225e+01, gradient 1.517e-04, dot -1.000; [fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00981912577413471, 0.012299126188664704, 0.012638640073962612, 0.012686413237733434, 0.012694070942193852, 0.012695507191683572, 0.012695826376673233, 0.01269590617396819
Rejected: LBFGS Orientation magnitude: 6.225e+01, gradient 1.517e-04, dot -1.000; [729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00981912577413471, 0.012299126188664704, 0.012638640073962612, 0.012686413237733434, 0.012694070942193852, 0.012695507191683572, 0.012695826376673233
Rejected: LBFGS Orientation magnitude: 6.225e+01, gradient 1.517e-04, dot -1.000; [729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00981912577413471, 0.012299126188664704, 0.012638640073962612, 0.012686413237733434, 0.012694070942193852, 0.012695507191683572
Rejected: LBFGS Orientation magnitude: 6.225e+01, gradient 1.517e-04, dot -1.000; [729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00981912577413471, 0.012299126188664704, 0.012638640073962612, 0.012686413237733434, 0.012694070942193852
Rejected: LBFGS Orientation magnitude: 6.225e+01, gradient 1.517e-04, dot -1.000; [fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00981912577413471, 0.012299126188664704, 0.012638640073962612, 0.012686413237733434
LBFGS Accumulation History: 3 points
Removed measurement 24d9e6d3 to history. Total: 8
Removed measurement 48ebbed8 to history. Total: 7
Removed measurement 6a768599 to history. Total: 6
Removed measurement 351de280 to history. Total: 5
Removed measurement 527d0d92 to history. Total: 4
Removed measurement b8a5169 to history. Total: 3
Adding measurement 682dd5b5 to history. Total: 3
th(0)=0.00981912577413471;dx=-2.301570889175405E-8
Adding measurement 78fb7af1 to history. Total: 4
New Minimum: 0.00981912577413471 > 0.006494114441218358
END: th(187148.86177126726)=0.006494114441218358; dx=-1.2517623253008661E-8 evalInputDelta=0.003325011332916351
Fitness changed from 0.00981912577413471 to 0.006494114441218358
Iteration 2 complete. Error: 0.006494114441218358 Total: 8.8016; Orientation: 8.7574; Line Search: 0.0269
Non-optimal measurement 0.006494114441218358 < 0.006494114441218358. Total: 5
Rejected: LBFGS Orientation magnitude: 3.385e+01, gradient 8.251e-05, dot -1.000; [6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.006494114441218358, 0.00981912577413471, 0.012695826376673233, 0.01269590617396819, 0.012695985971682134
Rejected: LBFGS Orientation magnitude: 3.385e+01, gradient 8.251e-05, dot -1.000; [729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.006494114441218358, 0.00981912577413471, 0.012695826376673233, 0.01269590617396819
LBFGS Accumulation History: 3 points
Removed measurement 78fb7af1 to history. Total: 4
Removed measurement 682dd5b5 to history. Total: 3
Adding measurement 6fa29289 to history. Total: 3
th(0)=0.006494114441218358;dx=-6.807997643748508E-9
Adding measurement 51cef2f0 to history. Total: 4
New Minimum: 0.006494114441218358 > 0.005097872199788118
END: th(403200.00000000006)=0.005097872199788118; dx=-1.1780712525071616E-10 evalInputDelta=0.0013962422414302402
Fitness changed from 0.006494114441218358 to 0.005097872199788118
Iteration 3 complete. Error: 0.0050978721997881

...skipping 7974 bytes...

8f4c5b300618 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989973774, 0.005097454000276174, 0.005097454016614841, 0.012695826376673233
LBFGS Accumulation History: 3 points
Removed measurement 738a1676 to history. Total: 5
Removed measurement 756ed82d to history. Total: 4
Removed measurement 5bbe6915 to history. Total: 3
Adding measurement 5ee2813b to history. Total: 3
th(0)=0.005097453989973774;dx=-4.402854605160601E-18
Adding measurement db771f0 to history. Total: 4
New Minimum: 0.005097453989973774 > 0.005097453989598153
WOLF (strong): th(723890.0558507132)=0.005097453989598153; dx=3.3650792713821846E-18 evalInputDelta=3.75621408976734E-13
Adding measurement 66213e83 to history. Total: 5
New Minimum: 0.005097453989598153 > 0.005097453989083057
END: th(361945.0279253566)=0.005097453989083057; dx=-5.188876668892631E-19 evalInputDelta=8.907171875072173E-13
Fitness changed from 0.005097453989973774 to 0.005097453989083057
Iteration 7 complete. Error: 0.005097453989083057 Total: 2.9311; Orientation: 2.7409; Line Search: 0.1840
Non-optimal measurement 0.005097453989083057 < 0.005097453989083057. Total: 6
Rejected: LBFGS Orientation magnitude: 1.015e-04, gradient 2.473e-10, dot -1.000; [fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989083057, 0.005097453989598153, 0.005097453989973774, 0.012695826376673233, 0.01269590617396819, 0.012695985971682134
Rejected: LBFGS Orientation magnitude: 1.015e-04, gradient 2.473e-10, dot -1.000; [59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989083057, 0.005097453989598153, 0.005097453989973774, 0.012695826376673233, 0.01269590617396819
Rejected: LBFGS Orientation magnitude: 1.015e-04, gradient 2.473e-10, dot -1.000; [729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989083057, 0.005097453989598153, 0.005097453989973774, 0.012695826376673233
LBFGS Accumulation History: 3 points
Removed measurement 66213e83 to history. Total: 5
Removed measurement db771f0 to history. Total: 4
Removed measurement 5ee2813b to history. Total: 3
Adding measurement 2e4fe35c to history. Total: 3
th(0)=0.005097453989083057;dx=-6.115223758109126E-20
Adding measurement 7bb57f5a to history. Total: 4
New Minimum: 0.005097453989083057 > 0.005097453989080682
WOLF (strong): th(779786.9240469472)=0.005097453989080682; dx=5.506936678531995E-20 evalInputDelta=2.374836438612249E-15
Adding measurement 6143c8fd to history. Total: 5
New Minimum: 0.005097453989080682 > 0.005097453989070531
END: th(389893.4620234736)=0.005097453989070531; dx=-3.0414353978597295E-21 evalInputDelta=1.2525570858290536E-14
Fitness changed from 0.005097453989083057 to 0.005097453989070531
Iteration 8 complete. Error: 0.005097453989070531 Total: 3.4461; Orientation: 3.0616; Line Search: 0.2792
Non-optimal measurement 0.005097453989070531 < 0.005097453989070531. Total: 6
Rejected: LBFGS Orientation magnitude: 5.046e-06, gradient 1.230e-11, dot -1.000; [59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989070531, 0.005097453989080682, 0.005097453989083057, 0.012695826376673233, 0.01269590617396819, 0.012695985971682134
Rejected: LBFGS Orientation magnitude: 5.046e-06, gradient 1.230e-11, dot -1.000; [59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989070531, 0.005097453989080682, 0.005097453989083057, 0.012695826376673233, 0.01269590617396819
Rejected: LBFGS Orientation magnitude: 5.046e-06, gradient 1.230e-11, dot -1.000; [fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989070531, 0.005097453989080682, 0.005097453989083057, 0.012695826376673233
LBFGS Accumulation History: 3 points
Removed measurement 6143c8fd to history. Total: 5
Removed measurement 7bb57f5a to history. Total: 4
Removed measurement 2e4fe35c to history. Total: 3
Adding measurement 7a66cf5a to history. Total: 3
th(0)=0.005097453989070531;dx=-1.5126722496601572E-22
Non-optimal measurement 0.005097453989070531 < 0.005097453989070531. Total: 4
WOLF (strong): th(840000.0000000003)=0.005097453989070531; dx=1.58419572575524E-22 evalInputDelta=0.0
Adding measurement 2b1be5ee to history. Total: 4
New Minimum: 0.005097453989070531 > 0.005097453989070528
WOLF (strong): th(420000.0000000002)=0.005097453989070528; dx=3.57617380451544E-24 evalInputDelta=3.469446951953614E-18
Non-optimal measurement 0.0050974539890705295 < 0.005097453989070528. Total: 5
END: th(140000.00000000006)=0.0050974539890705295; dx=-9.965275871015604E-23 evalInputDelta=1.734723475976807E-18
Fitness changed from 0.005097453989070531 to 0.005097453989070528
Iteration 9 complete. Error: 0.005097453989070528 Total: 3.2893; Orientation: 3.1225; Line Search: 0.1606
Non-optimal measurement 0.005097453989070528 < 0.005097453989070528. Total: 5
Rejected: LBFGS Orientation magnitude: 1.193e-07, gradient 2.908e-13, dot -1.000; [6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989070528, 0.005097453989070531, 0.012695826376673233, 0.01269590617396819, 0.012695985971682134
Rejected: LBFGS Orientation magnitude: 1.193e-07, gradient 2.908e-13, dot -1.000; [9642d24e-a048-4861-a77e-d14bf10316ce = 1.000/1.000e+00, 6f774963-fafc-4253-9432-150d344ccce5 = 1.000/1.000e+00, 59bc051f-617c-4510-8943-0387c6be112a = 1.000/1.000e+00, fadbc993-9c86-44e2-9871-8f4c5b300618 = 1.000/1.000e+00, 729e848d-4e1e-4c32-8834-f2e84f59d9a4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.005097453989070528, 0.005097453989070531, 0.012695826376673233, 0.01269590617396819
LBFGS Accumulation History: 3 points
Removed measurement 2b1be5ee to history. Total: 4
Removed measurement 7a66cf5a to history. Total: 3
Adding measurement 445cf448 to history. Total: 3
th(0)=0.005097453989070528;dx=-8.454586962229055E-26
Non-optimal measurement 0.005097453989070528 < 0.005097453989070528. Total: 4
END: th(301620.8566044639)=0.005097453989070528; dx=-2.2394267302949764E-26 evalInputDelta=0.0
Fitness changed from 0.005097453989070528 to 0.005097453989070528
Static Iteration Total: 2.0463; Orientation: 2.0182; Line Search: 0.0217
Iteration 10 failed. Error: 0.005097453989070528
Previous Error: 0.0 -> 0.005097453989070528
Optimization terminated 10
Final threshold in iteration 10: 0.005097453989070528 (> 0.0) after 30.302s (< 30.000s)

Returns

    0.005097453989070528

This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:797 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.08645210736072692 ], [ -0.710851331074656 ], [ 0.7118133183668041 ], [ -0.888708624872665 ], [ 0.3152283458967086 ], [ -0.30810884642026215 ], [ -0.7233629671695123 ], [ -4.1653778867581457E-4 ], ... ],
    	[ [ -0.3959343999923348 ], [ -0.2958558215451197 ], [ 0.6720724369188132 ], [ 0.05406824366578505 ], [ -0.15006219442230018 ], [ -9.711349135853417E-5 ], [ -2.562332278814375E-4 ], [ 0.07880291207851724 ], ... ],
    	[ [ 0.9480204674095962 ], [ 0.5116638114410909 ], [ -1.484614425550138E-4 ], [ -0.39144977573267936 ], [ -3.294763254180766E-4 ], [ -0.37830121691534124 ], [ -2.5127835143891157E-4 ], [ -0.3451331706051841 ], ... ],
    	[ [ -3.588607844282653E-4 ], [ 0.6411738470881434 ], [ -0.19396667629153108 ], [ -0.7260461997613561 ], [ -4.130691754290726E-4 ], [ -0.30702367029412403 ], [ -0.4169083958269233 ], [ -0.190216204029529 ], ... ],
    	[ [ -3.834110807805109E-4 ], [ -0.06934885921235057 ], [ -1.8727426254149318E-4 ], [ -0.18705588169274656 ], [ -2.1651346320112654E-4 ], [ -0.4135782201259133 ], [ -0.11265745828239293 ], [ -0.1344171343203029 ], ... ],
    	[ [ -1.458529724768648E-4 ], [ 0.8158182351604897 ], [ 0.5337603140525852 ], [ 0.2432322364534065 ], [ -3.837285784689926E-5 ], [ -0.9755946459884148 ], [ -1.5545444422875986E-4 ], [ -2.3388335244554047E-4 ], ... ],
    	[ [ -0.52560013812054 ], [ -9.316238554598548E-5 ], [ 0.19189509986946407 ], [ -0.4038943319057626 ], [ 0.8317430446485077 ], [ -0.9510686432968569 ], [ 0.34720156781284334 ], [ -0.927494229969759 ], ... ],
    	[ [ 0.5891467509836 ], [ -1.258643400638181E-4 ], [ -2.108846722881216E-4 ], [ -0.24305425978260198 ], [ 0.9329674086410903 ], [ 0.6485429014355696 ], [ -1.0322407101733683E-4 ], [ -0.5158390101476158 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:808 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.0213400224067787 ], [ 0.0 ], [ 0.17570551635033474 ], [ 0.0 ], [ 0.0778116366396807 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.1658957924875418 ], [ 0.013346320483648205 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.019451878743200018 ], ... ],
    	[ [ 0.23401139236770008 ], [ 0.12630018555048975 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.15826871873012524 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.2013783116418542 ], [ 0.13175453334184203 ], [ 0.06003996356397142 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.04736779537228122 ], [ 0.0 ], [ 0.20530922555098544 ], [ 0.0 ], [ 0.08570397487106983 ], [ 0.0 ], ... ],
    	[ [ 0.14542623946009478 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.23029566326383788 ], [ 0.16008771180839326 ], [ 0.0 ], [ 0.0 ], ... ],
    	...
    ]

TrainingTester.java:576 executed in 0.10 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -2.2926466855857144], [9.0, -2.0079271770165503]; valueStats=DoubleSummaryStatistics{count=20, sum=0.114189, min=0.005097, average=0.005709, max=0.009819}
Plotting 9 points for GD
Plotting 2 points for CjGD
Plotting 9 points for LBFGS

Returns

Result

TrainingTester.java:579 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -2.2926466855857144], [27.561, -2.0079271770165503]; valueStats=DoubleSummaryStatistics{count=20, sum=0.114189, min=0.005097, average=0.005709, max=0.009819}
Plotting 9 points for GD
Plotting 2 points for CjGD
Plotting 9 points for LBFGS

Returns

Result

Model Learning

In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:

TrainingTester.java:492 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> temp_18_0042 = network_target.state();
    assert temp_18_0042 != null;
    String temp_18_0041 = temp_18_0042.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
    temp_18_0042.freeRef();
    return temp_18_0041;

Returns

    [-0.2602369953763708]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:638 executed in 0.00 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2770168198761

Nothing to optimize
java.lang.AssertionError: Nothing to optimize
at com.simiacryptus.mindseye.opt.IterativeTrainer.measure(IterativeTrainer.java:240)
at com.simiacryptus.mindseye.opt.IterativeTrainer.run(IterativeTrainer.java:286)
at com.simiacryptus.mindseye.test.unit.TrainingTester.lambda$trainGD$892d3ca8$1(TrainingTester.java:647)
at com.simiacryptus.notebook.MarkdownNotebookOutput.lambda$eval$dc58be99$1(MarkdownNotebookOutput.java:657)
at com.simiacryptus.util.test.SysOutInterceptor.withOutput(SysOutInterceptor.java:102)
at com.simiacryptus.notebook.MarkdownNotebookOutput.eval(MarkdownNotebookOutput.java:649)
at com.simiacryptus.notebook.NotebookOutput.eval(NotebookOutput.java:125)
at com.simiacryptus.mindseye.test.unit.TrainingTester.trainGD(TrainingTester.java:638)
at com.simiacryptus.mindseye.test.unit.TrainingTester.runOpt(TrainingTester.java:833)
at com.simiacryptus.mindseye.test.unit.TrainingTester.train(TrainingTester.java:781)
at com.simiacryptus.mindseye.test.unit.TrainingTester.trainAll(TrainingTester.java:558)
at com.simiacryptus.mindseye.test.unit.TrainingTester.testModelLearning(TrainingTester.java:516)
at com.simiacryptus.mindseye.test.unit.TrainingTester.test(TrainingTester.java:333)
at com.simiacryptus.mindseye.test.unit.TrainingTester.test(TrainingTester.java:61)
at com.simiacryptus.mindseye.test.unit.LayerTests.run(LayerTests.java:541)
at com.simiacryptus.mindseye.test.unit.LayerTests.run(LayerTests.java:598)
at com.simiacryptus.mindseye.test.LayerTestBase.trainingTest(LayerTestBase.java:119)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:686)
at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
at org.junit.jupiter.engine.extension.TimeoutInvocation.proceed(TimeoutInvocation.java:46)
at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84)
at org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
at org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:205)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:201)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:137)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:135)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at java.util.ArrayList.forEach(Unknown Source)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at java.util.ArrayList.forEach(Unknown Source)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:248)
at org.junit.platform.launcher.core.DefaultLauncher.lambda$execute$5(DefaultLauncher.java:211)
at org.junit.platform.launcher.core.DefaultLauncher.withInterceptedStreams(DefaultLauncher.java:226)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:199)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:141)
at org.junit.platform.runner.JUnitPlatform.run(JUnitPlatform.java:139)
at com.simiacryptus.util.test.MacroTestRunner.runTest(MacroTestRunner.java:94)
at com.simiacryptus.util.test.MacroTestRunner.lambda$null$8bb7732f$1(MacroTestRunner.java:177)
at com.simiacryptus.aws.TendrilControl.lambda$null$0(TendrilControl.java:86)
at java.lang.Thread.run(Unknown Source)


This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

To produce the following output:

TrainingTester.java:808 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.08531091656270175 ], [ 0.3881622753500146 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.10124896921344513 ], ... ],
    	[ [ 0.029171835725210163 ], [ 0.0 ], [ 0.36971365799049705 ], [ 0.0 ], [ 0.5064746838443757 ], [ 0.0 ], [ 0.5182643114781256 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.2892825774512323 ], [ 0.14246034906289956 ], [ 0.0 ], [ 0.0 ], [ 0.03038253509615923 ], [ 0.34873302685884355 ], [ 0.0 ], ... ],
    	[ [ 0.0415546683716771 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0036362608536647236 ], [ 0.36364067466322714 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.2348253623762637 ], [ 0.0 ], [ 0.11186062291934198 ], [ 0.31698086923939495 ], [ 0.23949726944515493 ], [ 0.23744729950617882 ], [ 0.3460606374694131 ], [ 0.003996283722330777 ], ... ],
    	[ [ 0.0 ], [ 0.11381992566537869 ], [ 0.0 ], [ 0.34212195659002015 ], [ 0.16110922830239294 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.36715790159217915 ], [ 0.0 ], [ 0.036470184282202354 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.08454929227347129 ], [ 0.0 ], [ 0.06408796026069269 ], [ 0.0 ], [ 0.0 ], [ 0.5609576526445887 ], [ 0.0 ], ... ],
    	...
    ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:603 executed in 0.00 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2770179759510

Nothing to optimize
java.lang.AssertionError: Nothing to optimize
at com.simiacryptus.mindseye.opt.IterativeTrainer.measure(IterativeTrainer.java:240)
at com.simiacryptus.mindseye.opt.IterativeTrainer.run(IterativeTrainer.java:286)
at com.simiacryptus.mindseye.test.unit.TrainingTester.lambda$trainCjGD$892d3ca8$1(TrainingTester.java:612)
at com.simiacryptus.notebook.MarkdownNotebookOutput.lambda$eval$dc58be99$1(MarkdownNotebookOutput.java:657)
at com.simiacryptus.util.test.SysOutInterceptor.withOutput(SysOutInterceptor.java:102)
at com.simiacryptus.notebook.MarkdownNotebookOutput.eval(MarkdownNotebookOutput.java:649)
at com.simiacryptus.notebook.NotebookOutput.eval(NotebookOutput.java:125)
at com.simiacryptus.mindseye.test.unit.TrainingTester.trainCjGD(TrainingTester.java:603)
at com.simiacryptus.mindseye.test.unit.TrainingTester.runOpt(TrainingTester.java:833)
at com.simiacryptus.mindseye.test.unit.TrainingTester.train(TrainingTester.java:781)
at com.simiacryptus.mindseye.test.unit.TrainingTester.trainAll(TrainingTester.java:560)
at com.simiacryptus.mindseye.test.unit.TrainingTester.testModelLearning(TrainingTester.java:516)
at com.simiacryptus.mindseye.test.unit.TrainingTester.test(TrainingTester.java:333)
at com.simiacryptus.mindseye.test.unit.TrainingTester.test(TrainingTester.java:61)
at com.simiacryptus.mindseye.test.unit.LayerTests.run(LayerTests.java:541)
at com.simiacryptus.mindseye.test.unit.LayerTests.run(LayerTests.java:598)
at com.simiacryptus.mindseye.test.LayerTestBase.trainingTest(LayerTestBase.java:119)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:686)
at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
at org.junit.jupiter.engine.extension.TimeoutInvocation.proceed(TimeoutInvocation.java:46)
at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84)
at org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
at org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:205)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:201)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:137)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:135)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at java.util.ArrayList.forEach(Unknown Source)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at java.util.ArrayList.forEach(Unknown Source)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:248)
at org.junit.platform.launcher.core.DefaultLauncher.lambda$execute$5(DefaultLauncher.java:211)
at org.junit.platform.launcher.core.DefaultLauncher.withInterceptedStreams(DefaultLauncher.java:226)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:199)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:141)
at org.junit.platform.runner.JUnitPlatform.run(JUnitPlatform.java:139)
at com.simiacryptus.util.test.MacroTestRunner.runTest(MacroTestRunner.java:94)
at com.simiacryptus.util.test.MacroTestRunner.lambda$null$8bb7732f$1(MacroTestRunner.java:177)
at com.simiacryptus.aws.TendrilControl.lambda$null$0(TendrilControl.java:86)
at java.lang.Thread.run(Unknown Source)


This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

To produce the following output:

TrainingTester.java:808 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.08531091656270175 ], [ 0.3881622753500146 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.10124896921344513 ], ... ],
    	[ [ 0.029171835725210163 ], [ 0.0 ], [ 0.36971365799049705 ], [ 0.0 ], [ 0.5064746838443757 ], [ 0.0 ], [ 0.5182643114781256 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.2892825774512323 ], [ 0.14246034906289956 ], [ 0.0 ], [ 0.0 ], [ 0.03038253509615923 ], [ 0.34873302685884355 ], [ 0.0 ], ... ],
    	[ [ 0.0415546683716771 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0036362608536647236 ], [ 0.36364067466322714 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.2348253623762637 ], [ 0.0 ], [ 0.11186062291934198 ], [ 0.31698086923939495 ], [ 0.23949726944515493 ], [ 0.23744729950617882 ], [ 0.3460606374694131 ], [ 0.003996283722330777 ], ... ],
    	[ [ 0.0 ], [ 0.11381992566537869 ], [ 0.0 ], [ 0.34212195659002015 ], [ 0.16110922830239294 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.36715790159217915 ], [ 0.0 ], [ 0.036470184282202354 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.08454929227347129 ], [ 0.0 ], [ 0.06408796026069269 ], [ 0.0 ], [ 0.0 ], [ 0.5609576526445887 ], [ 0.0 ], ... ],
    	...
    ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:674 executed in 0.00 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2770188840724

Nothing to optimize
java.lang.AssertionError: Nothing to optimize
at com.simiacryptus.mindseye.opt.IterativeTrainer.measure(IterativeTrainer.java:240)
at com.simiacryptus.mindseye.opt.IterativeTrainer.run(IterativeTrainer.java:286)
at com.simiacryptus.mindseye.test.unit.TrainingTester.lambda$trainLBFGS$892d3ca8$1(TrainingTester.java:684)
at com.simiacryptus.notebook.MarkdownNotebookOutput.lambda$eval$dc58be99$1(MarkdownNotebookOutput.java:657)
at com.simiacryptus.util.test.SysOutInterceptor.withOutput(SysOutInterceptor.java:102)
at com.simiacryptus.notebook.MarkdownNotebookOutput.eval(MarkdownNotebookOutput.java:649)
at com.simiacryptus.notebook.NotebookOutput.eval(NotebookOutput.java:125)
at com.simiacryptus.mindseye.test.unit.TrainingTester.trainLBFGS(TrainingTester.java:674)
at com.simiacryptus.mindseye.test.unit.TrainingTester.runOpt(TrainingTester.java:833)
at com.simiacryptus.mindseye.test.unit.TrainingTester.train(TrainingTester.java:781)
at com.simiacryptus.mindseye.test.unit.TrainingTester.trainAll(TrainingTester.java:562)
at com.simiacryptus.mindseye.test.unit.TrainingTester.testModelLearning(TrainingTester.java:516)
at com.simiacryptus.mindseye.test.unit.TrainingTester.test(TrainingTester.java:333)
at com.simiacryptus.mindseye.test.unit.TrainingTester.test(TrainingTester.java:61)
at com.simiacryptus.mindseye.test.unit.LayerTests.run(LayerTests.java:541)
at com.simiacryptus.mindseye.test.unit.LayerTests.run(LayerTests.java:598)
at com.simiacryptus.mindseye.test.LayerTestBase.trainingTest(LayerTestBase.java:119)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:686)
at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
at org.junit.jupiter.engine.extension.TimeoutInvocation.proceed(TimeoutInvocation.java:46)
at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84)
at org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
at org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:205)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:201)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:137)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:135)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at java.util.ArrayList.forEach(Unknown Source)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at java.util.ArrayList.forEach(Unknown Source)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:248)
at org.junit.platform.launcher.core.DefaultLauncher.lambda$execute$5(DefaultLauncher.java:211)
at org.junit.platform.launcher.core.DefaultLauncher.withInterceptedStreams(DefaultLauncher.java:226)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:199)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:141)
at org.junit.platform.runner.JUnitPlatform.run(JUnitPlatform.java:139)
at com.simiacryptus.util.test.MacroTestRunner.runTest(MacroTestRunner.java:94)
at com.simiacryptus.util.test.MacroTestRunner.lambda$null$8bb7732f$1(MacroTestRunner.java:177)
at com.simiacryptus.aws.TendrilControl.lambda$null$0(TendrilControl.java:86)
at java.lang.Thread.run(Unknown Source)


This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

To produce the following output:

TrainingTester.java:808 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.08531091656270175 ], [ 0.3881622753500146 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.10124896921344513 ], ... ],
    	[ [ 0.029171835725210163 ], [ 0.0 ], [ 0.36971365799049705 ], [ 0.0 ], [ 0.5064746838443757 ], [ 0.0 ], [ 0.5182643114781256 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.2892825774512323 ], [ 0.14246034906289956 ], [ 0.0 ], [ 0.0 ], [ 0.03038253509615923 ], [ 0.34873302685884355 ], [ 0.0 ], ... ],
    	[ [ 0.0415546683716771 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0036362608536647236 ], [ 0.36364067466322714 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.2348253623762637 ], [ 0.0 ], [ 0.11186062291934198 ], [ 0.31698086923939495 ], [ 0.23949726944515493 ], [ 0.23744729950617882 ], [ 0.3460606374694131 ], [ 0.003996283722330777 ], ... ],
    	[ [ 0.0 ], [ 0.11381992566537869 ], [ 0.0 ], [ 0.34212195659002015 ], [ 0.16110922830239294 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.36715790159217915 ], [ 0.0 ], [ 0.036470184282202354 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.08454929227347129 ], [ 0.0 ], [ 0.06408796026069269 ], [ 0.0 ], [ 0.0 ], [ 0.5609576526445887 ], [ 0.0 ], ... ],
    	...
    ]

TrainingTester.java:576 executed in 0.00 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
No Data

TrainingTester.java:579 executed in 0.00 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
No Data

Composite Learning

In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:

TrainingTester.java:383 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> temp_18_0037 = temp_18_0035.state();
    assert temp_18_0037 != null;
    String temp_18_0036 = temp_18_0037.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
    temp_18_0037.freeRef();
    return temp_18_0036;

Returns

    [-0.9197221878085049]

We simultaneously regress this target input:

TrainingTester.java:391 executed in 0.01 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(x -> {
      RefStream<Tensor> temp_18_0006 = RefArrays.stream(RefUtil.addRef(x));
      if (null != x)
        RefUtil.freeRef(x);
      return temp_18_0006;
    }).map(x -> {
      String temp_18_0007 = x.prettyPrint();
      x.freeRef();
      return temp_18_0007;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.5122149471910227 ], [ -0.11219219362080679 ], [ -0.5723143546349623 ], [ 0.16423430579767673 ], [ -0.05510076450446921 ], [ -0.9822227508609254 ], [ -0.989746341426442 ], [ -0.7161144299777358 ], ... ],
    	[ [ -0.8050907547295163 ], [ -0.698960753827047 ], [ 0.9892271508315151 ], [ 0.168433914051076 ], [ 0.30329779504984167 ], [ 0.9961985908976221 ], [ 0.9312388687451656 ], [ 0.512479623885763 ], ... ],
    	[ [ 0.8055339992780817 ], [ 0.029123344528420247 ], [ 0.4908729185767149 ], [ 0.733473129758289 ], [ -0.4488792841685201 ], [ -0.9349936804794572 ], [ 0.30892873420055134 ], [ 0.0016096694230993691 ], ... ],
    	[ [ 0.22738653791229124 ], [ 0.23422773357237525 ], [ 0.06744121468368447 ], [ 0.6933801750951274 ], [ 0.30711572351709626 ], [ -0.6172782971351143 ], [ -0.7903471380991933 ], [ 0.9868106513194623 ], ... ],
    	[ [ -0.9579016907891553 ], [ -0.2545568644407141 ], [ 0.8012448229018594 ], [ 0.8256905717352381 ], [ 0.2770105011557651 ], [ 0.3391593448591017 ], [ 0.24473242566145936 ], [ 0.8357602466593128 ], ... ],
    	[ [ 0.6632666885117922 ], [ -0.8295829287752499 ], [ -0.8151812004071484 ], [ -0.5380804228289655 ], [ -0.7968288803557027 ], [ -0.05624891233015772 ], [ -0.0016909799720519203 ], [ 0.8537126974864913 ], ... ],
    	[ [ 0.010973421968767383 ], [ 0.8176340382467235 ], [ 0.0659918387698013 ], [ 0.9013830393389668 ], [ 0.3152957661310458 ], [ -0.19966462216665515 ], [ -0.8623088091729325 ], [ 0.9409451587563185 ], ... ],
    	[ [ -0.38179816046097925 ], [ 0.03888426212809826 ], [ -0.9482053407875568 ], [ 0.7326825012626133 ], [ -0.007932552150691663 ], [ 0.9325557088842751 ], [ -0.6770738983623747 ], [ 0.7234708117653523 ], ... ],
    	...
    ]
    [
    	[ [ -0.9825799829812383 ], [ -0.7520366036092345 ], [ 0.7246832362943514 ], [ -0.842528633030805 ], [ -0.7746826048070237 ], [ -0.20999409289113502 ], [ 0.17496309144082867 ], [ 0.9309444717411546 ], ... ],
    	[ [ -0.4273651348965153 ], [ 0.43272631553232777 ], [ -0.42755061069939915 ], [ 0.33291957781733217 ], [ -0.271879561323791 ], [ -0.17691280425449674 ], [ -0.26500562866562305 ], [ -0.933954158157708 ], ... ],
    	[ [ -0.5483521497160035 ], [ -0.0751533165387579 ], [ -0.7172044156799506 ], [ -0.8803427177080347 ], [ -0.9059518398759439 ], [ -0.3365001608374427 ], [ 0.8576114183459562 ], [ 0.8055293611979517 ], ... ],
    	[ [ 0.811493774792917 ], [ -0.8882572305065255 ], [ 0.4936761587071292 ], [ -0.16850018499040464 ], [ 0.23325625172047104 ], [ -0.7439966036804322 ], [ 0.2313543527019566 ], [ 0.8745352830918118 ], ... ],
    	[ [ -0.870991619088819 ], [ 0.1925441677312607 ], [ 0.5576825737413542 ], [ -0.6224445244944179 ], [ -0.21157096586369328 ], [ 0.23617291296000031 ], [ 0.9150588992969213 ], [ 0.781596802355971 ], ... ],
    	[ [ -0.03582029772030437 ], [ -0.12179305383260663 ], [ -0.5164420512102652 ], [ -0.022729798412316482 ], [ 0.5114593109244667 ], [ 0.10885055729153414 ], [ -0.0636172671978168 ], [ 0.052423533014850365 ], ... ],
    	[ [ 0.02701691941341733 ], [ 0.41464296527379685 ], [ -0.8555804575124841 ], [ 0.05726597779384224 ], [ 0.023661582289090433 ], [ -0.2900228828393452 ], [ 0.6074348697913312 ], [ 0.2366001160156519 ], ... ],
    	[ [ -0.92975110792234 ], [ 0.3592783697068038 ], [ -0.8487321105110222 ], [ 0.4854162357359473 ], [ -0.4819291359325739 ], [ 0.6711589665360742 ], [ 0.6857184865319912 ], [ -0.5334235525380895 ], ... ],
    	...
    ]
    [
    	[ [ 0.6981738524845351 ], [ -0.4675599731427438 ], [ 0.9864797457120986 ], [ 0.11098696951302234 ], [ -0.5325888742911351 ], [ -0.7919746175766198 ], [ -0.17607220730494522 ], [ 0.3967290230529896 ], ... ],
    	[ [ -0.9139674588208972 ], [ -0.9525571772748236 ], [ 0.7133543304901544 ], [ 0.6468827817873384 ], [ 0.10174554948606684 ], [ -0.7204268055250502 ], [ -0.37263948576827555 ], [ -0.5394803925830394 ], ... ],
    	[ [ 0.6511519984918632 ], [ 0.26669207500360015 ], [ -0.4050224025772553 ], [ -0.7801880014781926 ], [ -0.4499377100305957 ], [ 0.4289145529859344 ], [ -0.28336522917833995 ], [ 0.6015339051428843 ], ... ],
    	[ [ 0.18493087168279576 ], [ -0.6087093888468802 ], [ -0.18231590032836098 ], [ -0.5347091548378311 ], [ 0.09441790737431033 ], [ 0.7657052153011403 ], [ -0.22778982645701196 ], [ -0.7629101069159245 ], ... ],
    	[ [ 0.3287600564333073 ], [ 0.2601203280646691 ], [ -0.3260281805505387 ], [ -0.7821992309282944 ], [ -0.07798306211359751 ], [ 0.38815494835515274 ], [ -0.3312484791649839 ], [ -0.03433262429907913 ], ... ],
    	[ [ 0.4522846887699963 ], [ -0.6822308729937185 ], [ 0.09063055425619315 ], [ -0.0932306266914551 ], [ 0.7951022550325886 ], [ -0.8898395515522071 ], [ -0.7209406819218664 ], [ -0.2950589932474341 ], ... ],
    	[ [ 0.2542412715972917 ], [ -0.8473816861153372 ], [ 0.4595675007322406 ], [ 0.9453226189859598 ], [ -0.48561942028202165 ], [ 0.9552695271540974 ], [ -0.5106775881439574 ], [ -0.6795785990856975 ], ... ],
    	[ [ -0.14719197155340225 ], [ 0.30858880183412785 ], [ -0.07884233332441881 ], [ -0.41300310325129463 ], [ -0.1462667583768349 ], [ 0.9270667293732933 ], [ -0.9726207213416853 ], [ 0.3336333614673028 ], ... ],
    	...
    ]
    [
    	[ [ -0.2229459969403489 ], [ -0.8395741611906409 ], [ 0.3543288400695641 ], [ -0.27624827851198797 ], [ -0.7436827905218906 ], [ 0.3964170607556443 ], [ -0.9436875106432263 ], [ -0.9459824856963504 ], ... ],
    	[ [ -0.9981783206472477 ], [ 0.02366494388620133 ], [ 0.34515689482998746 ], [ -0.8907481130736457 ], [ -0.19393557594802724 ], [ -0.8826844623900794 ], [ 0.37440595366353135 ], [ 0.3921392448739278 ], ... ],
    	[ [ -0.6092513832231861 ], [ -0.8764442738267828 ], [ 0.12479281264454212 ], [ 0.6078547914580421 ], [ 0.8390036751962728 ], [ -0.5377617154887573 ], [ -0.8612660815009208 ], [ 0.29461310750054026 ], ... ],
    	[ [ -0.2236920612131177 ], [ 0.2984133013606758 ], [ -0.09194219844801155 ], [ -0.7978504087139864 ], [ -0.867031785520485 ], [ -0.3384699413712111 ], [ -0.26139990793487944 ], [ -0.010368501747352266 ], ... ],
    	[ [ -0.057949724679143166 ], [ -0.17771694885426936 ], [ -0.4954211915399298 ], [ -0.6377639750408957 ], [ 0.6419943408517974 ], [ 0.5579140215415599 ], [ -0.4374734440575456 ], [ 0.5860883052654913 ], ... ],
    	[ [ 0.22708236366275036 ], [ -0.5118999151172536 ], [ -0.0991448703031439 ], [ -0.6250912445101655 ], [ -0.5570787203954584 ], [ -0.6754250032049165 ], [ 0.417417556549035 ], [ -0.21671046920145587 ], ... ],
    	[ [ 0.7918703894374588 ], [ -0.9631198979134865 ], [ -0.8671680831952095 ], [ -0.48443118734944535 ], [ -0.07427356559465625 ], [ -0.3609917730578964 ], [ -0.5619072868902202 ], [ 0.8172014362069846 ], ... ],
    	[ [ 0.015000019324445502 ], [ -0.5679550729668763 ], [ 0.9995313767200478 ], [ -0.3032758684245713 ], [ -0.8845728922487384 ], [ 0.01779254789678819 ], [ 0.34463991626913004 ], [ 0.2634578575358344 ], ... ],
    	...
    ]
    [
    	[ [ -0.7783602908601288 ], [ 0.3829632143339674 ], [ 0.7939685100083773 ], [ 0.8463232380151433 ], [ 0.9434599255936102 ], [ 0.6014726795425405 ], [ 0.14226276231851065 ], [ -0.3571875616427487 ], ... ],
    	[ [ 0.9110876724594703 ], [ -0.8683058451149559 ], [ -0.3481398804846607 ], [ 0.6983355269685831 ], [ -0.6213057894966139 ], [ 0.445923997611988 ], [ -0.5936212821132549 ], [ -0.790319942616726 ], ... ],
    	[ [ 0.09277688515850935 ], [ 0.2784894912068243 ], [ -0.07980037094659087 ], [ -0.09424828184364031 ], [ 0.8903855639619305 ], [ 0.8030407543578588 ], [ -0.5854546155834761 ], [ 0.8005128552842742 ], ... ],
    	[ [ -0.013167233675957624 ], [ 0.28352387462899675 ], [ -0.05889882570207661 ], [ 0.6987147510403853 ], [ -0.63113502595101 ], [ -0.9859397418211495 ], [ 0.04413892109490658 ], [ 0.7120991435016255 ], ... ],
    	[ [ 0.24217734107132682 ], [ 0.03478132691301905 ], [ -0.5458835718691586 ], [ -0.9420761673148761 ], [ -0.8093362774933117 ], [ -0.3479821195160122 ], [ -0.06549088485679233 ], [ 0.4851318341533015 ], ... ],
    	[ [ 0.37084107452477943 ], [ 0.9525832471914424 ], [ 0.44968543029229435 ], [ -0.20418285110206646 ], [ 0.9332555806824452 ], [ 0.24739134147344788 ], [ 0.6969531441575842 ], [ 0.9351780412088013 ], ... ],
    	[ [ -0.18526580097903866 ], [ -0.8363106492850514 ], [ 0.5779201411575532 ], [ -0.3991381114398731 ], [ 0.9616298929750047 ], [ -0.5921282880065697 ], [ 0.6291286806449943 ], [ -0.9492027130943113 ], ... ],
    	[ [ 0.09558742618603455 ], [ -0.3674857420371582 ], [ 0.033781924434168165 ], [ -0.753272890073976 ], [ -0.10420001541809243 ], [ -0.29136330325703796 ], [ -0.8502781502460428 ], [ -0.7976239200720376 ], ... ],
    	...
    ]

Which produces the following output:

TrainingTester.java:412 executed in 0.01 seconds (0.000 gc):

    return RefStream.of(RefUtil.addRef(output_target)).map(x -> {
      String temp_18_0008 = x.prettyPrint();
      x.freeRef();
      return temp_18_0008;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.4710954518587452 ], [ 0.1031856497719638 ], [ 0.52637021035908 ], [ 0.0 ], [ 0.050677395679971635 ], [ 0.9033720573370984 ], [ 0.9102916705121907 ], [ 0.6586263302603635 ], ... ],
    	[ [ 0.7404598303242311 ], [ 0.6428497137020935 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.41284423729738684 ], [ 0.8599344333976926 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.5677245459278156 ], [ 0.7268997989807806 ], [ 0.0 ], ... ],
    	[ [ 0.8810034387580679 ], [ 0.2341215962850866 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.7629858262217599 ], [ 0.7497402370988258 ], [ 0.49488450370118153 ], [ 0.7328612011497483 ], [ 0.051733372710141445 ], [ 0.0015552317994359567 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.18363598312707458 ], [ 0.7930845445390761 ], [ 0.0 ], ... ],
    	[ [ 0.35114823944043444 ], [ 0.0 ], [ 0.8720854905208407 ], [ 0.0 ], [ 0.007295744218939197 ], [ 0.0 ], [ 0.6227198871098766 ], [ 0.0 ], ... ],
    	...
    ]
    [
    	[ [ 0.903700611644348 ], [ 0.6916647503835626 ], [ 0.0 ], [ 0.774892277662401 ], [ 0.7124927801503071 ], [ 0.19313622654069712 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.3930571968600999 ], [ 0.0 ], [ 0.39322778307131373 ], [ 0.0 ], [ 0.2500536649611336 ], [ 0.16271063138028352 ], [ 0.24373155657791507 ], [ 0.8589783616536576 ], ... ],
    	[ [ 0.5043316388262996 ], [ 0.06912017270809151 ], [ 0.6596288142950846 ], [ 0.8096707303517188 ], [ 0.8332240082198434 ], [ 0.3094866641233266 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.8169498833781851 ], [ 0.0 ], [ 0.15497335878551277 ], [ 0.0 ], [ 0.6842701840590643 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.8010703174712405 ], [ 0.0 ], [ 0.0 ], [ 0.5724760398574306 ], [ 0.1945865116009145 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.03294472258727034 ], [ 0.11201577393080399 ], [ 0.47498321321541703 ], [ 0.020905099924221997 ], [ 0.0 ], [ 0.0 ], [ 0.0585102121695743 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.7868963302295835 ], [ 0.0 ], [ 0.0 ], [ 0.26674048031953224 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.8551127230957158 ], [ 0.0 ], [ 0.780597753542527 ], [ 0.0 ], [ 0.44324091926856923 ], [ 0.0 ], [ 0.0 ], [ 0.49060147676891663 ], ... ],
    	...
    ]
    [
    	[ [ 0.0 ], [ 0.4300252814305301 ], [ 0.0 ], [ 0.0 ], [ 0.4898338046655116 ], [ 0.7283966279663727 ], [ 0.16193751571477683 ], [ 0.0 ], ... ],
    	[ [ 0.8405961508125352 ], [ 0.8760879710958946 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.6625925177333915 ], [ 0.3427248031146346 ], [ 0.4961720869462641 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.3725080902098103 ], [ 0.7175562156214683 ], [ 0.41381769504688815 ], [ 0.0 ], [ 0.2606172885287612 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.5598435308498306 ], [ 0.16767997872227747 ], [ 0.49178387372868665 ], [ 0.0 ], [ 0.0 ], [ 0.2095033575495627 ], [ 0.7016653526339345 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.2998553515031677 ], [ 0.7194059879715008 ], [ 0.07172275249912444 ], [ 0.0 ], [ 0.30465657596585893 ], [ 0.031576476333556494 ], ... ],
    	[ [ 0.0 ], [ 0.6274628711002891 ], [ 0.0 ], [ 0.08574627595142308 ], [ 0.0 ], [ 0.8184051791521347 ], [ 0.6630651412573344 ], [ 0.271372302802105 ], ... ],
    	[ [ 0.0 ], [ 0.7793557382628576 ], [ 0.0 ], [ 0.0 ], [ 0.4466349556640788 ], [ 0.0 ], [ 0.46968150863253105 ], [ 0.6250235159389366 ], ... ],
    	[ [ 0.13537572210494234 ], [ 0.0 ], [ 0.07251304329706186 ], [ 0.3798481176939825 ], [ 0.13452478301800055 ], [ 0.0 ], [ 0.894540857740261 ], [ 0.0 ], ... ],
    	...
    ]
    [
    	[ [ 0.20504838006912593 ], [ 0.7721749843577466 ], [ 0.0 ], [ 0.25407167109137874 ], [ 0.6839815631343272 ], [ 0.0 ], [ 0.8679303418963499 ], [ 0.870041081373175 ], ... ],
    	[ [ 0.918046748888706 ], [ 0.0 ], [ 0.0 ], [ 0.8192408033423909 ], [ 0.17836685220482207 ], [ 0.8118244848939777 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.5603420151033865 ], [ 0.806085245016205 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.4945913814889746 ], [ 0.7921255247632849 ], [ 0.0 ], ... ],
    	[ [ 0.2057345519343226 ], [ 0.0 ], [ 0.08456127990852891 ], [ 0.7338007234463374 ], [ 0.7974283706784149 ], [ 0.31129831498534666 ], [ 0.24041529521880908 ], [ 0.00953614111137113 ], ... ],
    	[ [ 0.05329764756480206 ], [ 0.1634502210109008 ], [ 0.4556498621698006 ], [ 0.5865656784300612 ], [ 0.0 ], [ 0.0 ], [ 0.40235403307672746 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.4708057098706285 ], [ 0.09118573702519797 ], [ 0.5749102869808305 ], [ 0.5123576595036733 ], [ 0.6212033616481922 ], [ 0.0 ], [ 0.1993134268549706 ], ... ],
    	[ [ 0.0 ], [ 0.8858027396308957 ], [ 0.7975537266740057 ], [ 0.4455421114717036 ], [ 0.06831104624505575 ], [ 0.3320121432976798 ], [ 0.5167985992442146 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.5223608822860345 ], [ 0.0 ], [ 0.278929545216971 ], [ 0.8135613157351065 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	...
    ]
    [
    	[ [ 0.7158752296131419 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.328513325652054 ], ... ],
    	[ [ 0.0 ], [ 0.79860015155604 ], [ 0.32019197254274356 ], [ 0.0 ], [ 0.5714287200139161 ], [ 0.0 ], [ 0.5459666643148925 ], [ 0.7268747866921473 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.07339417175492881 ], [ 0.08668223597442545 ], [ 0.0 ], [ 0.0 ], [ 0.5384555999070219 ], [ 0.0 ], ... ],
    	[ [ 0.012110196963837567 ], [ 0.0 ], [ 0.05417055683406569 ], [ 0.0 ], [ 0.5804688868702405 ], [ 0.9067906563951001 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.5020612330082238 ], [ 0.866448353685089 ], [ 0.7443645318089399 ], [ 0.3200468762795074 ], [ 0.060233419902003926 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.18779149852857077 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.1703930678025365 ], [ 0.7691734600479987 ], [ 0.0 ], [ 0.36709617709123493 ], [ 0.0 ], [ 0.5445935245087068 ], [ 0.0 ], [ 0.8730027959608685 ], ... ],
    	[ [ 0.0 ], [ 0.33798479065484704 ], [ 0.0 ], [ 0.6928017904756726 ], [ 0.0958350661500079 ], [ 0.2679732947186758 ], [ 0.7820196805900591 ], [ 0.7335924168170505 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:638 executed in 2.90 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2770227382017
Reset training subject: 2770234860122
Constructing line search parameters: GD
th(0)=0.12206570803184764;dx=-2.7300513067236125E-7
New Minimum: 0.12206570803184764 > 0.12206511986177429
WOLFE (weak): th(2.154434690031884)=0.12206511986177429; dx=-2.7300359842233973E-7 evalInputDelta=5.881700733539885E-7
New Minimum: 0.12206511986177429 > 0.12206453169500191
WOLFE (weak): th(4.308869380063768)=0.12206453169500191; dx=-2.730020661723182E-7 evalInputDelta=1.176336845737369E-6
New Minimum: 0.12206453169500191 > 0.12206217906092373
WOLFE (weak): th(12.926608140191302)=0.12206217906092373; dx=-2.729959371722322E-7 evalInputDelta=3.528970923913355E-6
New Minimum: 0.12206217906092373 > 0.12205159286119618
WOLFE (weak): th(51.70643256076521)=0.12205159286119618; dx=-2.72968356671845E-7 evalInputDelta=1.4115170651460285E-5
New Minimum: 0.12205159286119618 > 0.12199515119311326
WOLFE (weak): th(258.53216280382605)=0.12199515119311326; dx=-2.7282126066977986E-7 evalInputDelta=7.05568387343869E-5
New Minimum: 0.12199515119311326 > 0.12164308004408202
WOLFE (weak): th(1551.1929768229563)=0.12164308004408202; dx=-2.7190191065687303E-7 evalInputDelta=4.2262798776562804E-4
New Minimum: 0.12164308004408202 > 0.11914324956742646
WOLFE (weak): th(10858.350837760694)=0.11914324956742646; dx=-2.6528259056394366E-7 evalInputDelta=0.002922458464421185
New Minimum: 0.11914324956742646 > 0.10103395371244235
END: th(86866.80670208555)=0.10103395371244235; dx=-2.112248098050203E-7 evalInputDelta=0.021031754319405294
Fitness changed from 0.12206570803184764 to 0.10103395371244235
Iteration 1 complete. Error: 0.10103395371244235 Total: 0.4290; Orientation: 0.0028; Line Search: 0.4035
th(0)=0.10103395371244235;dx=-1.634252080438423E-7
New Minimum: 0.10103395371244235 > 0.07790482062918866
END: th(187148.86177126726)=0.07790482062918866; dx=-8.374843612741587E-8 evalInputDelta=0.023129133083253692
Fitness changed from 0.10103395371244235 to 0.07790482062918866
Iteration 2 complete. Error: 0.07790482062918866 Total: 0.1878; Orientation: 0.0037; Line Search: 0.1776
th(0)=0.07790482062918866;dx=-4.2917495028712176E-8
New Minimum: 0.07790482062918866 > 0.06968637956139398
WOLF (strong): th(403200.00000000006)=0.06968637956139398; dx=1.9383683598258734E-9 evalInputDelta=0.00821844106779468
END: th(201600.00000000003)=0.07152466719240506; dx=-2.0377677955252916E-8 evalInputDelta=0.006380153436783595
Fitness changed from 0.07790482062918866 to 0.06968637956139398
Iteration 3 complete. Error: 0.06968637956139398 Total: 0.1766; Orientation: 0.0036; Line Search: 0.1660
Low gradient: 9.881953147106661E-6
th(0)=0.06968637956139398;dx=-9.765299800161122E-11
New Minimum: 0.06968637956139398 > 0.0696679609854943
WOLF (strong): th(434334.03351042786)=0.0696679609854943; dx=1.2840045395395675E-11 evalInputDelta=1.8418575899681322E-5
END: th(217167.01675521393)=0.06967117141229244; dx=-4.240647630310777E-11 evalInputDelta=1.5208149101539337E-5
Fitness changed from 0.06968637956139398 to 0.0696679609854943
Iteration 4 complete. Error: 0.0696679609854943 Total: 0.1026; Orientation: 0.0040; Line Search: 0.0922
Low gradient: 1.2993428732411179E-6
th(0)=0.0696679609854943;dx=-1.6882919022424838E-12
New Minimum: 0.0696679609854943 > 0.06966765247113359
WOLF (strong): th(467872.15442816826)=0.06966765247113359; dx=3.6949420116019475E-13 evalInputDelta=3.085143607112206E-7
END: th(233936.07721408413)=0.0696676863807117; dx=-6.593988505411453E-13 evalInputDelta=2.7460478259355714E-7
Fitness changed from 0.0696679609854943 to 0.06966765247113359
Iteration 5 complete. Error: 0.06966765247113359 Total: 0.2819; Orientation: 0.0033; Line Search: 0.2700
Low gradient: 2.8437005256242943E-7
th(0)=0.06966765247113359;dx=-8.086632679435887E-14
New Minimum: 0.06966765247113359 > 0.06966763847069571
WOLF (strong): th(504000.0000000002)=0.06966763847069571; dx=2.5309034333312708E-14 evalInputDelta=1.4000437878469718E-8
END: th(252000.0000000001)=0.06966763878186685; dx=-2.7778646230522783E-14 evalInputDelta=1.3689266734062677E-8
Fitness changed from 0.06966765247113359 to 0.06966763847069571
Iteration 6 complete. Error: 0.06966763847069571 Total: 0.1751; Orientation: 0.0027; Line Search: 0.1668
Low gradient: 8.900035044216363E-8
th(0)=0.06966763847069571;dx=-7.921062378827935E-15
New Minimum: 0.06966763847069571 > 0.069667637211424
WOLF (strong): th(542917.5418880349)=0.069667637211424; dx=3.2821559949668998E-15 evalInputDelta=1.2592717146819865E-9
New Minimum: 0.069667637211424 > 0.06966763708075689
END: th(271458.77094401745)=0.06966763708075689; dx=-2.319453191930578E-15 evalInputDelta=1.3899388168203686E-9
Fitness changed from 0.06966763847069571 to 0.06966763708075689
Iteration 7 complete. Error: 0.06966763708075689 Total: 0.3082; Orientation: 0.0028; Line Search: 0.1930
Low gradient: 2.6061169202225353E-8
th(0)=0.06966763708075689;dx=-6.791845401870193E-16
New Minimum: 0.06966763708075689 > 0.06966763698613464
WOLF (strong): th(584840.1930352104)=0.06966763698613464; dx=3.5560134798374524E-16 evalInputDelta=9.462225447620654E-11
New Minimum: 0.06966763698613464 > 0.06966763695779771
END: th(292420.0965176052)=0.06966763695779771; dx=-1.617915961016471E-16 evalInputDelta=1.229591845008926E-10
Fitness changed from 0.06966763708075689 to 0.06966763695779771
Iteration 8 complete. Error: 0.06966763695779771 Total: 0.3073; Orientation: 0.0030; Line Search: 0.2976
Low gradient: 6.208148024603278E-9
th(0)=0.06966763695779771;dx=-3.854110189538559E-17
New Minimum: 0.06966763695779771 > 0.06966763695344187
WOLF (strong): th(630000.0000000002)=0.06966763695344187; dx=2.4713215046803733E-17 evalInputDelta=4.355835137026531E-12
New Minimum: 0.06966763695344187 > 0.06966763695063836
END: th(315000.0000000001)=0.06966763695063836; dx=-6.913943424294191E-18 evalInputDelta=7.159342563234361E-12
Fitness changed from 0.06966763695779771 to 0.06966763695063836
Iteration 9 complete. Error: 0.06966763695063836 Total: 0.3058; Orientation: 0.0029; Line Search: 0.1880
Low gradient: 1.1136885584708678E-9
th(0)=0.06966763695063836;dx=-1.2403022052689196E-18
New Minimum: 0.06966763695063836 > 0.06966763695054062
WOLF (strong): th(678646.9273600436)=0.06966763695054062; dx=9.524870619056675E-19 evalInputDelta=9.774125953043722E-14
New Minimum: 0.06966763695054062 > 0.06966763695040248
END: th(339323.4636800218)=0.06966763695040248; dx=-1.4390757168150145E-19 evalInputDelta=2.358807593694223E-13
Fitness changed from 0.06966763695063836 to 0.06966763695040248
Iteration 10 complete. Error: 0.06966763695040248 Total: 0.2356; Orientation: 0.0032; Line Search: 0.1298
Low gradient: 1.2921706933856934E-10
th(0)=0.06966763695040248;dx=-1.6697051008448635E-20
New Minimum: 0.06966763695040248 > 0.06966763695040193
WOLF (strong): th(731050.241294013)=0.06966763695040193; dx=1.5101877955840146E-20 evalInputDelta=5.551115123125783E-16
New Minimum: 0.06966763695040193 > 0.06966763695040051
END: th(365525.1206470065)=0.06966763695040051; dx=-7.97586526343283E-22 evalInputDelta=1.970645868709653E-15
Fitness changed from 0.06966763695040248 to 0.06966763695040051
Iteration 11 complete. Error: 0.06966763695040051 Total: 0.0590; Orientation: 0.0028; Line Search: 0.0502
Zero gradient: 6.1724548500127276E-12
th(0)=0.06966763695040051;dx=-3.809919887544564E-23
WOLF (strong): th(787500.0000000003)=0.06966763695040051; dx=4.006212379401493E-23 evalInputDelta=0.0
WOLF (strong): th(393750.0000000002)=0.06966763695040051; dx=9.81462459944723E-25 evalInputDelta=0.0
END: th(131250.00000000006)=0.06966763695040051; dx=-2.5072311762982175E-23 evalInputDelta=0.0
Fitness changed from 0.06966763695040051 to 0.06966763695040051
Static Iteration Total: 0.3278; Orientation: 0.0036; Line Search: 0.3180
Iteration 12 failed. Error: 0.06966763695040051
Previous Error: 0.0 -> 0.06966763695040051
Optimization terminated 12
Final threshold in iteration 12: 0.06966763695040051 (> 0.0) after 2.897s (< 30.000s)

Returns

    0.06966763695040051

This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:797 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 1.8459764478135163 ], [ 0.40433055322285144 ], [ -0.7732142854922412 ], [ -0.6594736281960136 ], [ -0.4527534899846086 ], [ 3.539842156736257 ], [ 3.5669564981224906 ], [ -0.8309168644426477 ], ... ],
    	[ [ 2.9014744319341554 ], [ -0.09445568451065034 ], [ -0.3782167702493058 ], [ -0.008379216395091982 ], [ -0.1327128255849992 ], [ -0.004122958128716311 ], [ -0.7772035073668606 ], [ -0.9363149247886509 ], ... ],
    	[ [ -0.06514583169910049 ], [ -0.01614173720574902 ], [ -0.46858670628239363 ], [ -0.016378390015454825 ], [ -0.12290256442888103 ], [ 3.3696328459767946 ], [ -0.5616579873403837 ], [ -0.25376445767822986 ], ... ],
    	[ [ -0.9661225019625506 ], [ -0.002813726536229154 ], [ -0.39232257023225614 ], [ -0.015342723427375526 ], [ -0.008385109060576879 ], [ -0.43972243316796034 ], [ 2.8483397626931533 ], [ -0.5169848123486458 ], ... ],
    	[ [ 3.452191252836367 ], [ -0.6432085795253306 ], [ -0.7723507200283684 ], [ -0.01147550231083791 ], [ -0.2667536123480314 ], [ -0.010619480698426581 ], [ -0.013854490304692213 ], [ -0.7703905634315851 ], ... ],
    	[ [ -0.01670105619927148 ], [ 2.9897420190403787 ], [ 2.937839490571362 ], [ 1.9391932928579059 ], [ 2.8716993771997337 ], [ -0.5869750225097201 ], [ -0.4175657429514301 ], [ -0.014092103692636415 ], ... ],
    	[ [ -0.2066998368929054 ], [ -2.5059327409261006E-4 ], [ -0.0035537814306240006 ], [ -0.708770532447424 ], [ -0.7242269393360852 ], [ -0.8412188651245902 ], [ 3.1076831284841298 ], [ -0.006642036382156102 ], ... ],
    	[ [ -0.4924430857225839 ], [ -0.002855889423960409 ], [ 3.417246475920538 ], [ -0.9722809911328547 ], [ 0.028588212975918063 ], [ -0.013217721935045001 ], [ -0.4302370389801906 ], [ -0.013313974026900988 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:808 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.4710954470372734 ], [ 0.10318565166254477 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.9033720474842792 ], [ 0.9102916605657997 ], [ 0.0 ], ... ],
    	[ [ 0.7404598234166222 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.8599344232192943 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.7268997908447461 ], [ 0.0 ], ... ],
    	[ [ 0.8810034296154823 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.7629858195938544 ], [ 0.7497402308538477 ], [ 0.49488449989306976 ], [ 0.7328611930346983 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.7930845349611528 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.8720854798105001 ], [ 0.0 ], [ 0.007295746913693797 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	...
    ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:603 executed in 3.05 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2773135465622
Reset training subject: 2773269846141
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.12206570803184764}, derivative=-2.7300513067236125E-7}
New Minimum: 0.12206570803184764 > 0.12206570803184762
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.12206570803184762}, derivative=-2.730051306723612E-7}, evalInputDelta = -2.7755575615628914E-17
New Minimum: 0.12206570803184762 > 0.12206570803184746
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.12206570803184746}, derivative=-2.730051306723608E-7}, evalInputDelta = -1.8041124150158794E-16
New Minimum: 0.12206570803184746 > 0.12206570803184638
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.12206570803184638}, derivative=-2.7300513067235776E-7}, evalInputDelta = -1.2628786905111156E-15
New Minimum: 0.12206570803184638 > 0.12206570803183843
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.12206570803183843}, derivative=-2.7300513067233685E-7}, evalInputDelta = -9.2148511043888E-15
New Minimum: 0.12206570803183843 > 0.12206570803178214
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.12206570803178214}, derivative=-2.7300513067219047E-7}, evalInputDelta = -6.550315845288424E-14
New Minimum: 0.12206570803178214 > 0.12206570803138919
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.12206570803138919}, derivative=-2.7300513067116593E-7}, evalInputDelta = -4.584527202311506E-13
New Minimum: 0.12206570803138919 > 0.12206570802863599
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.12206570802863599}, derivative=-2.73005130663994E-7}, evalInputDelta = -3.211653165635653E-12
New Minimum: 0.12206570802863599 > 0.1220657080093647
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.1220657080093647}, derivative=-2.730051306137903E-7}, evalInputDelta = -2.2482946060442544E-11
New Minimum: 0.1220657080093647 > 0.12206570787446594
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.12206570787446594}, derivative=-2.730051302623643E-7}, evalInputDelta = -1.5738170489054681E-10
New Minimum: 0.12206570787446594 > 0.12206570693017352
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.12206570693017352}, derivative=-2.7300512780238265E-7}, evalInputDelta = -1.1016741269243013E-9
New Minimum: 0.12206570693017352 > 0.12206570032012902
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.12206570032012902}, derivative=-2.730051105825109E-7}, evalInputDelta = -7.711718624792141E-9
New Minimum: 0.12206570032012902 > 0.12206565404982725
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.12206565404982725}, derivative=-2.730049900434084E-7}, evalInputDelta = -5.398202039541555E-8
New Minimum: 0.12206565404982725 > 0.122065330158287
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.122065330158287}, derivative=-2.730041462696913E-7}, evalInputDelta = -3.778735606502215E-7
New Minimum: 0.122065330158287 > 0.12206306294553546
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.12206306294553546}, derivative=-2.7299823985367145E-7}, evalInputDelta = -2.645086312189071E-6
New Minimum: 0.12206306294553546 > 0.12204719382971423
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.12204719382971423}, derivative=-2.7295689494153264E-7}, evalInputDelta = -1.8514202133418367E-5
New Minimum: 0.12204719382971423 > 0.12193617731754261
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.12193617731754261}, derivative=-2.726674805565608E-7}, evalInputDelta = -1.295307143050345E-4
New Minimum: 0.12193617731754261 > 0.12116235936256933
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.12116235936256933}, derivative=-2.7064157986175835E-7}, evalInputDelta = -9.033486692783116E-4
New Minimum: 0.12116235936256933 > 0.11590721755886935
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.11590721755886935}, derivative=-2.564602749981408E-7}, evalInputDelta = -0.006158490472978298
New Minimum: 0.11590721755886935 > 0.08703883510760693
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.08703883510760693}, derivative=-1.571911409528182E-7}, evalInputDelta = -0.03502687292424071
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.25065160399854564}, derivative=4.785032816245554E-7}, evalInputDelta = 0.128585895966698
F(87683.80911825495) = LineSearchPoint{point=PointSample{avg=0.10086161989545842}, derivative=-2.106437515926073E-7}, evalInputDelta = -0.02120408813638923
New Minimum: 0.08703883510760693 > 0.08644166995222505
F(613786.6638277847) = LineSearchPoint{point=PointSample{avg=0.08644166995222505}, derivative=1.4575546987476873E-7}, evalInputDelta = -0.0356240380796226
0.08644166995222505 <= 0.12206570803184764
New Minimum: 0.08644166995222505 > 0.06975219560411362
F(400149.6514818328) = LineSearchPoint{point=PointSample{avg=0.06975219560411362}, derivative=1.0379085993857324E-8}, evalInputDelta = -0.05231351242773402
Right bracket at 400149.6514818328
New Minimum: 0.06975219560411362 > 0.06966848452437956
F(385493.974140304) = LineSearchPoint{point=PointSample{avg=0.06966848452437956}, derivative=1.0412800386079435E-9}, evalInputDelta = -0.05239722350746809
Right bracket at 385493.974140304
Converged to right
Fitness changed from 0.12206570803184764 to 0.06966848452437956
Iteration 1 complete. Error: 0.06966848452437956 Total: 2.1218; Orientation: 0.0026; Line Search: 1.8684
Low gradient: 2.1043866506326034E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=0.06966848452437956}, derivative=-4.428443175360707E-12}
New Minimum: 0.06966848452437956 > 0.06966763458543027
F(385493.974140304) = LineSearchPoint{point=PointSample{avg=0.06966763458543027}, derivative=1.8833654953779628E-14}, evalInputDelta = -8.499389492905296E-7
0.06966763458543027 <= 0.06966848452437956
Converged to right
Fitness changed from 0.06966848452437956 to 0.06966763458543027
Iteration 2 complete. Error: 0.06966763458543027 Total: 0.0403; Orientation: 0.0028; Line Search: 0.0317
Low gradient: 8.949712234735005E-9
F(0.0) = LineSearchPoint{point=PointSample{avg=0.06966763458543027}, derivative=-8.009734908456542E-17}
New Minimum: 0.06966763458543027 > 0.0696676345700572
F(385493.974140304) = LineSearchPoint{point=PointSample{avg=0.0696676345700572}, derivative=3.4064473126490836E-19}, evalInputDelta = -1.5373063932955233E-11
0.0696676345700572 <= 0.06966763458543027
Converged to right
Fitness changed from 0.06966763458543027 to 0.0696676345700572
Iteration 3 complete. Error: 0.0696676345700572 Total: 0.4439; Orientation: 0.0029; Line Search: 0.3295
Zero gradient: 3.806208762141218E-11
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0696676345700572}, derivative=-1.4487225141000587E-21}
New Minimum: 0.0696676345700572 > 0.06966763457005719
F(385493.974140304) = LineSearchPoint{point=PointSample{avg=0.06966763457005719}, derivative=6.161248720194545E-24}, evalInputDelta = -1.3877787807814457E-17
0.06966763457005719 <= 0.0696676345700572
Converged to right
Fitness changed from 0.0696676345700572 to 0.06966763457005719
Iteration 4 complete. Error: 0.06966763457005719 Total: 0.2004; Orientation: 0.0027; Line Search: 0.1062
Zero gradient: 1.6187364133775835E-13
F(0.0) = LineSearchPoint{point=PointSample{avg=0.06966763457005719}, derivative=-2.6203075759945227E-26}
F(385493.974140304) = LineSearchPoint{point=PointSample{avg=0.06966763457005719}, derivative=1.114387674945607E-28}, evalInputDelta = 0.0
0.06966763457005719 <= 0.06966763457005719
Converged to right
Fitness changed from 0.06966763457005719 to 0.06966763457005719
Static Iteration Total: 0.2430; Orientation: 0.0028; Line Search: 0.2342
Iteration 5 failed. Error: 0.06966763457005719
Previous Error: 0.0 -> 0.06966763457005719
Optimization terminated 5
Final threshold in iteration 5: 0.06966763457005719 (> 0.0) after 3.050s (< 30.000s)

Returns

    0.06966763457005719

This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:797 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 1.8459764662109643 ], [ 0.40433054600889956 ], [ -0.7732142854922412 ], [ -0.6594736281960136 ], [ -0.4527534899846086 ], [ 3.539842194331982 ], [ 3.5669565360752618 ], [ -0.8309168644426477 ], ... ],
    	[ [ 2.9014744582917453 ], [ -0.09445568451065034 ], [ -0.3782167702493058 ], [ -0.0017840453611431273 ], [ -0.1327128255849992 ], [ -8.778320044380283E-4 ], [ -0.7772035073668606 ], [ -0.9363149247886509 ], ... ],
    	[ [ -0.06514583169910049 ], [ -0.003436788122523682 ], [ -0.46858670628239363 ], [ -0.0034871746178055663 ], [ -0.12290256442888103 ], [ 3.369632884814843 ], [ -0.5616579873403837 ], [ -0.25376445767822986 ], ... ],
    	[ [ -0.9661225019625506 ], [ -5.99079381387646E-4 ], [ -0.39232257023225614 ], [ -0.0032666675817017987 ], [ -0.00178529998711624 ], [ -0.43972243316796034 ], [ 2.8483397937380874 ], [ -0.5169848123486458 ], ... ],
    	[ [ 3.452191287722031 ], [ -0.6432085795253306 ], [ -0.7723507200283684 ], [ -0.002443285350218316 ], [ -0.2667536123480314 ], [ -0.002261027091850054 ], [ -0.0029498031789185752 ], [ -0.7703905634315851 ], ... ],
    	[ [ -0.0035558744915518137 ], [ 2.9897420443306957 ], [ 2.93783951440053 ], [ 1.9391933073886432 ], [ 2.871699408164596 ], [ -0.5869750225097201 ], [ -0.4175657429514301 ], [ -0.003000394193939382 ], ... ],
    	[ [ -0.2066998368929054 ], [ -5.33546034734763E-5 ], [ -7.566468004734928E-4 ], [ -0.708770532447424 ], [ -0.7242269393360852 ], [ -0.8412188651245902 ], [ 3.107683165030926 ], [ -0.0014141768916566266 ], ... ],
    	[ [ -0.4924430857225839 ], [ -6.080564146473677E-4 ], [ 3.417246516788336 ], [ -0.9722809911328547 ], [ 0.0285882026934545 ], [ -0.0028142268192177244 ], [ -0.4302370389801906 ], [ -0.002834720155334036 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:808 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.47109545173232464 ], [ 0.10318564982153544 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.9033720570787543 ], [ 0.9102916702513932 ], [ 0.0 ], ... ],
    	[ [ 0.7404598301431115 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.8599344331308117 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.7268997987674513 ], [ 0.0 ], ... ],
    	[ [ 0.8810034385183464 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.7629858260479742 ], [ 0.7497402369350804 ], [ 0.4948845036013317 ], [ 0.732861200936969 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.79308454428794 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.8720854902400121 ], [ 0.0 ], [ 0.00729574428959652 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	...
    ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:674 executed in 30.03 seconds (0.109 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2776197471677
Reset training subject: 2776205682790
Adding measurement 2fb886 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.12206570803184764 < 0.12206570803184764. Total: 1
th(0)=0.12206570803184764;dx=-2.7300513067236125E-7
Adding measurement 6845b3ad to history. Total: 1
New Minimum: 0.12206570803184764 > 0.12206511986177429
WOLFE (weak): th(2.154434690031884)=0.12206511986177429; dx=-2.7300359842233973E-7 evalInputDelta=5.881700733539885E-7
Adding measurement 2ae8162e to history. Total: 2
New Minimum: 0.12206511986177429 > 0.12206453169500191
WOLFE (weak): th(4.308869380063768)=0.12206453169500191; dx=-2.730020661723182E-7 evalInputDelta=1.176336845737369E-6
Adding measurement 286e5959 to history. Total: 3
New Minimum: 0.12206453169500191 > 0.12206217906092373
WOLFE (weak): th(12.926608140191302)=0.12206217906092373; dx=-2.729959371722322E-7 evalInputDelta=3.528970923913355E-6
Adding measurement 786be52a to history. Total: 4
New Minimum: 0.12206217906092373 > 0.12205159286119618
WOLFE (weak): th(51.70643256076521)=0.12205159286119618; dx=-2.72968356671845E-7 evalInputDelta=1.4115170651460285E-5
Adding measurement 447bcccc to history. Total: 5
New Minimum: 0.12205159286119618 > 0.12199515119311326
WOLFE (weak): th(258.53216280382605)=0.12199515119311326; dx=-2.7282126066977986E-7 evalInputDelta=7.05568387343869E-5
Adding measurement 70fd2eb9 to history. Total: 6
New Minimum: 0.12199515119311326 > 0.12164308004408202
WOLFE (weak): th(1551.1929768229563)=0.12164308004408202; dx=-2.7190191065687303E-7 evalInputDelta=4.2262798776562804E-4
Adding measurement 4bed3188 to history. Total: 7
New Minimum: 0.12164308004408202 > 0.11914324956742646
WOLFE (weak): th(10858.350837760694)=0.11914324956742646; dx=-2.6528259056394366E-7 evalInputDelta=0.002922458464421185
Adding measurement 4ad22ca5 to history. Total: 8
New Minimum: 0.11914324956742646 > 0.10103395371244235
END: th(86866.80670208555)=0.10103395371244235; dx=-2.112248098050203E-7 evalInputDelta=0.021031754319405294
Fitness changed from 0.12206570803184764 to 0.10103395371244235
Iteration 1 complete. Error: 0.10103395371244235 Total: 0.6004; Orientation: 0.0048; Line Search: 0.4647
Non-optimal measurement 0.10103395371244235 < 0.10103395371244235. Total: 9
Rejected: LBFGS Orientation magnitude: 1.552e+02, gradient 4.043e-04, dot -1.000; [7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10103395371244235, 0.11914324956742646, 0.12164308004408202, 0.12199515119311326, 0.12205159286119618, 0.12206217906092373, 0.12206453169500191, 0.12206511986177429, 0.12206570803184764
Rejected: LBFGS Orientation magnitude: 1.552e+02, gradient 4.043e-04, dot -1.000; [a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10103395371244235, 0.11914324956742646, 0.12164308004408202, 0.12199515119311326, 0.12205159286119618, 0.12206217906092373, 0.12206453169500191, 0.12206511986177429
Rejected: LBFGS Orientation magnitude: 1.552e+02, gradient 4.043e-04, dot -1.000; [a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10103395371244235, 0.11914324956742646, 0.12164308004408202, 0.12199515119311326, 0.12205159286119618, 0.12206217906092373, 0.12206453169500191
Rejected: LBFGS Orientation magnitude: 1.552e+02, gradient 4.043e-04, dot -1.000; [7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10103395371244235, 0.11914324956742646, 0.12164308004408202, 0.12199515119311326, 0.12205159286119618, 0.12206217906092373
Rejected: LBFGS Orientation magnitude: 1.552e+02, gradient 4.043e-04, dot -1.000; [a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10103395371244235, 0.11914324956742646, 0.12164308004408202, 0.12199515119311326, 0.12205159286119618
Rejected: LBFGS Orientation magnitude: 1.552e+02, gradient 4.043e-04, dot -1.000; [7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10103395371244235, 0.11914324956742646, 0.12164308004408202, 0.12199515119311326
LBFGS Accumulation History: 3 points
Removed measurement 4ad22ca5 to history. Total: 8
Removed measurement 4bed3188 to history. Total: 7
Removed measurement 70fd2eb9 to history. Total: 6
Removed measurement 447bcccc to history. Total: 5
Removed measurement 786be52a to history. Total: 4
Removed measurement 286e5959 to history. Total: 3
Adding measurement 3f27ecb1 to history. Total: 3
th(0)=0.10103395371244235;dx=-1.634252080438423E-7
Adding measurement 729f8239 to history. Total: 4
New Minimum: 0.10103395371244235 > 0.07790482062918866
END: th(187148.86177126726)=0.07790482062918866; dx=-8.374843612741587E-8 evalInputDelta=0.023129133083253692
Fitness changed from 0.10103395371244235 to 0.07790482062918866
Iteration 2 complete. Error: 0.07790482062918866 Total: 8.2187; Orientation: 8.1516; Line Search: 0.0610
Non-optimal measurement 0.07790482062918866 < 0.07790482062918866. Total: 5
Rejected: LBFGS Orientation magnitude: 7.952e+01, gradient 2.072e-04, dot -1.000; [7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.07790482062918866, 0.10103395371244235, 0.12206453169500191, 0.12206511986177429, 0.12206570803184764
Rejected: LBFGS Orientation magnitude: 7.952e+01, gradient 2.072e-04, dot -1.000; [a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.07790482062918866, 0.10103395371244235, 0.12206453169500191, 0.12206511986177429
LBFGS Accumulation History: 3 points
Removed measurement 729f8239 to history. Total: 4
Removed measurement 3f27ecb1 to history. Total: 3
Adding measurement 4dc274a0 to history. Total: 3
th(0)=0.07790482062918866;dx=-4.2917495028712176E-8
Adding measurement 7a7f8024 to history. Total: 4
New Minimum: 0.07790482062918866 > 0.06968637956139398
WOLF (strong): th(403200.00000000006)=0.06968637956139398; dx=1.9383683598258734E-9 evalInputDelta=0.00821844106779468
Non-optimal measurement 0.07152466719240506 < 0.06968637956139398. Total: 5
END: th(201600.00000000003)=0.07152466719240506; dx=-2.0377677955252916E-8 evalInputDelta=0.0063801

...skipping 9822 bytes...

> 0.06966763698613464
WOLF (strong): th(584840.1930352104)=0.06966763698613464; dx=3.5560134798374524E-16 evalInputDelta=9.462225447620654E-11
Adding measurement 14f62c5d to history. Total: 5
New Minimum: 0.06966763698613464 > 0.06966763695779771
END: th(292420.0965176052)=0.06966763695779771; dx=-1.617915961016471E-16 evalInputDelta=1.229591845008926E-10
Fitness changed from 0.06966763708075689 to 0.06966763695779771
Iteration 8 complete. Error: 0.06966763695779771 Total: 3.0181; Orientation: 2.8737; Line Search: 0.1387
Non-optimal measurement 0.06966763695779771 < 0.06966763695779771. Total: 6
Rejected: LBFGS Orientation magnitude: 2.383e-03, gradient 6.208e-09, dot -1.000; [a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695779771, 0.06966763698613464, 0.06966763708075689, 0.12206453169500191, 0.12206511986177429, 0.12206570803184764
Rejected: LBFGS Orientation magnitude: 2.383e-03, gradient 6.208e-09, dot -1.000; [8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695779771, 0.06966763698613464, 0.06966763708075689, 0.12206453169500191, 0.12206511986177429
Rejected: LBFGS Orientation magnitude: 2.379e-03, gradient 6.208e-09, dot -1.000; [a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695779771, 0.06966763698613464, 0.06966763708075689, 0.12206453169500191
LBFGS Accumulation History: 3 points
Removed measurement 14f62c5d to history. Total: 5
Removed measurement 17db9f00 to history. Total: 4
Removed measurement 49908051 to history. Total: 3
Adding measurement 28a62513 to history. Total: 3
th(0)=0.06966763695779771;dx=-3.854110189538559E-17
Adding measurement 75f6c46f to history. Total: 4
New Minimum: 0.06966763695779771 > 0.06966763695344187
WOLF (strong): th(630000.0000000002)=0.06966763695344187; dx=2.4713215046803733E-17 evalInputDelta=4.355835137026531E-12
Adding measurement 6d0270a5 to history. Total: 5
New Minimum: 0.06966763695344187 > 0.06966763695063836
END: th(315000.0000000001)=0.06966763695063836; dx=-6.913943424294191E-18 evalInputDelta=7.159342563234361E-12
Fitness changed from 0.06966763695779771 to 0.06966763695063836
Iteration 9 complete. Error: 0.06966763695063836 Total: 2.9735; Orientation: 2.7062; Line Search: 0.2616
Non-optimal measurement 0.06966763695063836 < 0.06966763695063836. Total: 6
Rejected: LBFGS Orientation magnitude: 4.275e-04, gradient 1.114e-09, dot -1.000; [7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695063836, 0.06966763695344187, 0.06966763695779771, 0.12206453169500191, 0.12206511986177429, 0.12206570803184764
Rejected: LBFGS Orientation magnitude: 4.275e-04, gradient 1.114e-09, dot -1.000; [7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695063836, 0.06966763695344187, 0.06966763695779771, 0.12206453169500191, 0.12206511986177429
Rejected: LBFGS Orientation magnitude: 4.268e-04, gradient 1.114e-09, dot -1.000; [7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695063836, 0.06966763695344187, 0.06966763695779771, 0.12206453169500191
LBFGS Accumulation History: 3 points
Removed measurement 6d0270a5 to history. Total: 5
Removed measurement 75f6c46f to history. Total: 4
Removed measurement 28a62513 to history. Total: 3
Adding measurement 576eeec to history. Total: 3
th(0)=0.06966763695063836;dx=-1.2403022052689196E-18
Adding measurement 1671d826 to history. Total: 4
New Minimum: 0.06966763695063836 > 0.06966763695054062
WOLF (strong): th(678646.9273600436)=0.06966763695054062; dx=9.524870619056675E-19 evalInputDelta=9.774125953043722E-14
Adding measurement 71ee165 to history. Total: 5
New Minimum: 0.06966763695054062 > 0.06966763695040248
END: th(339323.4636800218)=0.06966763695040248; dx=-1.4390757168150145E-19 evalInputDelta=2.358807593694223E-13
Fitness changed from 0.06966763695063836 to 0.06966763695040248
Iteration 10 complete. Error: 0.06966763695040248 Total: 3.2257; Orientation: 3.1170; Line Search: 0.1033
Non-optimal measurement 0.06966763695040248 < 0.06966763695040248. Total: 6
Rejected: LBFGS Orientation magnitude: 4.960e-05, gradient 1.292e-10, dot -1.000; [a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695040248, 0.06966763695054062, 0.06966763695063836, 0.12206453169500191, 0.12206511986177429, 0.12206570803184764
Rejected: LBFGS Orientation magnitude: 4.960e-05, gradient 1.292e-10, dot -1.000; [a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695040248, 0.06966763695054062, 0.06966763695063836, 0.12206453169500191, 0.12206511986177429
Rejected: LBFGS Orientation magnitude: 4.952e-05, gradient 1.292e-10, dot -1.000; [a677ec47-4d35-4e2e-baeb-30652229e9a9 = 1.000/1.000e+00, 8c25d7db-8abe-4d0d-bd13-442a88354d1a = 1.000/1.000e+00, a8eaf0c1-80e6-45de-bb00-9555a0be7cd6 = 1.000/1.000e+00, a42d0fd0-432c-44aa-a0db-06f9a2a7469f = 1.000/1.000e+00, 7d11e8a8-2282-41e0-8729-d10d05a4b5db = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06966763695040248, 0.06966763695054062, 0.06966763695063836, 0.12206453169500191
LBFGS Accumulation History: 3 points
Removed measurement 71ee165 to history. Total: 5
Removed measurement 1671d826 to history. Total: 4
Removed measurement 576eeec to history. Total: 3
Adding measurement 9a79443 to history. Total: 3
th(0)=0.06966763695040248;dx=-1.6697051008448635E-20
Adding measurement 31f7c5d3 to history. Total: 4
New Minimum: 0.06966763695040248 > 0.06966763695040193
WOLF (strong): th(731050.241294013)=0.06966763695040193; dx=1.5101877955840146E-20 evalInputDelta=5.551115123125783E-16
Adding measurement 3002381 to history. Total: 5
New Minimum: 0.06966763695040193 > 0.06966763695040051
END: th(365525.1206470065)=0.06966763695040051; dx=-7.97586526343283E-22 evalInputDelta=1.970645868709653E-15
Fitness changed from 0.06966763695040248 to 0.06966763695040051
Iteration 11 complete. Error: 0.06966763695040051 Total: 2.7443; Orientation: 2.4405; Line Search: 0.2974
Final threshold in iteration 11: 0.06966763695040051 (> 0.0) after 30.034s (< 30.000s)

Returns

    0.06966763695040051

This training apply resulted in the following configuration:

TrainingTester.java:785 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:797 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 1.8459764478135163 ], [ 0.40433055322285144 ], [ -0.7732142854922412 ], [ -0.6594736281960136 ], [ -0.4527534899846086 ], [ 3.539842156736257 ], [ 3.5669564981224906 ], [ -0.8309168644426477 ], ... ],
    	[ [ 2.9014744319341554 ], [ -0.09445568451065034 ], [ -0.3782167702493058 ], [ -0.008379216395091982 ], [ -0.1327128255849992 ], [ -0.004122958128716311 ], [ -0.7772035073668606 ], [ -0.9363149247886509 ], ... ],
    	[ [ -0.06514583169910049 ], [ -0.01614173720574902 ], [ -0.46858670628239363 ], [ -0.016378390015454825 ], [ -0.12290256442888103 ], [ 3.3696328459767946 ], [ -0.5616579873403837 ], [ -0.25376445767822986 ], ... ],
    	[ [ -0.9661225019625506 ], [ -0.002813726536229154 ], [ -0.39232257023225614 ], [ -0.015342723427375526 ], [ -0.008385109060576879 ], [ -0.43972243316796034 ], [ 2.8483397626931533 ], [ -0.5169848123486458 ], ... ],
    	[ [ 3.452191252836367 ], [ -0.6432085795253306 ], [ -0.7723507200283684 ], [ -0.01147550231083791 ], [ -0.2667536123480314 ], [ -0.010619480698426581 ], [ -0.013854490304692213 ], [ -0.7703905634315851 ], ... ],
    	[ [ -0.01670105619927148 ], [ 2.9897420190403787 ], [ 2.937839490571362 ], [ 1.9391932928579059 ], [ 2.8716993771997337 ], [ -0.5869750225097201 ], [ -0.4175657429514301 ], [ -0.014092103692636415 ], ... ],
    	[ [ -0.2066998368929054 ], [ -2.5059327409261006E-4 ], [ -0.0035537814306240006 ], [ -0.708770532447424 ], [ -0.7242269393360852 ], [ -0.8412188651245902 ], [ 3.1076831284841298 ], [ -0.006642036382156102 ], ... ],
    	[ [ -0.4924430857225839 ], [ -0.002855889423960409 ], [ 3.417246475920538 ], [ -0.9722809911328547 ], [ 0.028588212975918063 ], [ -0.013217721935045001 ], [ -0.4302370389801906 ], [ -0.013313974026900988 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:808 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.4710954470372734 ], [ 0.10318565166254477 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.9033720474842792 ], [ 0.9102916605657997 ], [ 0.0 ], ... ],
    	[ [ 0.7404598234166222 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.8599344232192943 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.7268997908447461 ], [ 0.0 ], ... ],
    	[ [ 0.8810034296154823 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.7629858195938544 ], [ 0.7497402308538477 ], [ 0.49488449989306976 ], [ 0.7328611930346983 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.7930845349611528 ], [ 0.0 ], ... ],
    	[ [ 0.0 ], [ 0.0 ], [ 0.8720854798105001 ], [ 0.0 ], [ 0.007295746913693797 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
    	...
    ]

TrainingTester.java:576 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -1.1569689348452379], [11.0, -0.9955326516440571]; valueStats=DoubleSummaryStatistics{count=26, sum=1.890605, min=0.069668, average=0.072716, max=0.101034}
Plotting 11 points for GD
Plotting 4 points for CjGD
Plotting 11 points for LBFGS

Returns

Result

TrainingTester.java:579 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -1.1569689348452379], [29.433, -0.9955326516440571]; valueStats=DoubleSummaryStatistics{count=26, sum=1.890605, min=0.069668, average=0.072716, max=0.101034}
Plotting 11 points for GD
Plotting 4 points for CjGD
Plotting 11 points for LBFGS

Returns

Result

Results

TrainingTester.java:350 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:353 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.005097453989070528 }, "CjGD": { "type": "NonConverged", "value": 0.005097478842708532 }, "GD": { "type": "NonConverged", "value": 0.005097453989070528 } }, "model":{ "LBFGS": { "type": "NonConverged", "value": NaN }, "CjGD": { "type": "NonConverged", "value": NaN }, "GD": { "type": "NonConverged", "value": NaN } }, "complete":{ "LBFGS": { "type": "NonConverged", "value": 0.06966763695040051 }, "CjGD": { "type": "NonConverged", "value": 0.06966763457005719 }, "GD": { "type": "NonConverged", "value": 0.06966763695040051 } }}

LayerTests.java:605 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.005097453989070528 }, "CjGD": { "type": "NonConverged", "value": 0.005097478842708532 }, "GD": { "type": "NonConverged", "value": 0.005097453989070528 } }, "model":{ "LBFGS": { "type": "NonConverged", "value": NaN }, "CjGD": { "type": "NonConverged", "value": NaN }, "GD": { "type": "NonConverged", "value": NaN } }, "complete":{ "LBFGS": { "type": "NonConverged", "value": 0.06966763695040051 }, "CjGD": { "type": "NonConverged", "value": 0.06966763457005719 }, "GD": { "type": "NonConverged", "value": 0.06966763695040051 } }}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "72.205",
      "gc_time": "0.427"
    },
    "created_on": 1587004753944,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "Basic",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.ReLuActivationLayerTest.Basic",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/c9a1867488dc7e77a975f095285b5882c0486db6/src/test/java/com/simiacryptus/mindseye/layers/java/ReLuActivationLayerTest.java",
      "javaDoc": "The type Basic."
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 0.005097453989070528
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 0.005097478842708532
        },
        "GD": {
          "type": "NonConverged",
          "value": 0.005097453989070528
        }
      },
      "model": {
        "LBFGS": {
          "type": "NonConverged",
          "value": "NaN"
        },
        "CjGD": {
          "type": "NonConverged",
          "value": "NaN"
        },
        "GD": {
          "type": "NonConverged",
          "value": "NaN"
        }
      },
      "complete": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 0.06966763695040051
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 0.06966763457005719
        },
        "GD": {
          "type": "NonConverged",
          "value": 0.06966763695040051
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/ReLuActivationLayer/Basic/trainingTest/202004163913",
    "id": "980764b9-4d09-43d6-9f61-7fe7186d6d86",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "ReLuActivationLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.ReLuActivationLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/c9a1867488dc7e77a975f095285b5882c0486db6/src/main/java/com/simiacryptus/mindseye/layers/java/ReLuActivationLayer.java",
      "javaDoc": "The type Re lu activation layer."
    }
  }