2b:[["$","p",null,{"children":"recap:"}],"\n",["$","ul",null,{"children":["\n",["$","li",null,{"children":"each note gets turned into coordinates (a little embedding vector)"}],"\n",["$","li",null,{"children":"everyone in the room mixes those numbers in their own weird way"}],"\n",["$","li",null,{"children":"they pass their result to a final guesser"}],"\n",["$","li",null,{"children":"we see how wrong the guess was"}],"\n",["$","li",null,{"children":"everyone adjusts themselves a tiny bit"}],"\n"]}],"\n",["$","p",null,{"children":"but if we don't invite the right initial people into the room, they'll be completely useless."}],"\n",["$","ol",null,{"children":["\n",["$","li",null,{"children":"bad initialization"}],"\n"]}],"\n",["$","p",null,{"children":"if we don't manage who we're inviting into the room, we get tone-deaf people, literally deaf people, bats, ants..."}],"\n",["$","p",null,{"children":["each guesser does: ",["$","code",null,{"dangerouslySetInnerHTML":{"__html":"$2c"}}],". if their random strengths are too large or small, the \"mix\" number they compute is huge or tiny. then tanh tries to squash it:"]}],"\n",["$","p",null,{"children":"squish = tanh(mix)\ntanh(very large) = 1\ntanh(very negative) = -1"}],"\n",["$","p",null,{"children":"and every guesser outputs -1 or 1, which tells you fuck-all. worse still, these guessers are too fucked to even learn, so they stay that way. they're literally called 'dead neurons'."}],"\n",["$","ol",null,{"start":"2","children":["\n",["$","li",null,{"children":"how big should the starting weights be? ('kaiming init')"}],"\n"]}],"\n",["$","p",null,{"children":["we want ",["$","code",null,{"dangerouslySetInnerHTML":{"__html":"variance_out ≈ variance_in"}}]," so the signal stays alive. for a guesser with N inputs, the general rule is"]}],"\n",["$","p",null,{"children":"$L2d"}],"\n","$L2e","\n","$L2f","\n","$L30","\n","$L31","\n","$L32","\n","$L33","\n","$L34","\n","$L35","\n","$L36","\n","$L37","\n","$L38","\n","$L39","\n","$L3a","\n","$L3b","\n","$L3c"]