@article {67789,
	title = {Organizing memories for generalization in complementary learning systems.},
	journal = {Nature Neuroscience},
	volume = {26},
	year = {2023},
	month = {2023 Aug 01},
	pages = {1438-1448},
	abstract = {<p>Our ability to remember the past is essential for guiding our future behavior. Psychological and neurobiological features of declarative memories are known to transform over time in a process known as systems consolidation. While many theories have sought to explain the time-varying role of hippocampal and neocortical brain areas, the computational principles that govern these transformations remain unclear. Here we propose a theory of systems consolidation in which hippocampal-cortical interactions serve to optimize generalizations that guide future adaptive behavior. We use mathematical analysis of neural network models to characterize fundamental performance tradeoffs in systems consolidation, revealing that memory components should be organized according to their predictability. The theory shows that multiple interacting memory systems can outperform just one, normatively unifying diverse experimental observations and making novel experimental predictions. Our results suggest that the psychological taxonomy and neurobiological organization of declarative memories reflect a system optimized for behaving well in an uncertain future.</p>
},
	doi = {10.1038/s41593-023-01382-9},
	url = {https://doi.org/10.1038/s41593-023-01382-9},
	author = {Weinan Sun and Madhu Advani and Nelson Spruston and Andrew Saxe and James E. Fitzgerald}
}
@conference {68572,
	title = {Exact learning dynamics of deep linear networks with prior knowledge},
	booktitle = {Neural Information Processing Systems},
	year = {2022},
	abstract = {<p>Learning in deep neural networks is known to depend critically on the knowledge embedded in the initial network weights. However, few theoretical results have precisely linked prior knowledge to learning dynamics. Here we derive exact solutions to the dynamics of learning with rich prior knowledge in deep linear networks by generalising Fukumizu{\textquoteright}s matrix Riccati solution \citep{fukumizu1998effect}. We obtain explicit expressions for the evolving network function, hidden representational similarity, and neural tangent kernel over training for a broad class of initialisations and tasks. The expressions reveal a class of task-independent initialisations that radically alter learning dynamics from slow non-linear dynamics to fast exponential trajectories while converging to a global optimum with identical representational similarity, dissociating learning trajectories from the structure of initial internal representations. We characterise how network weights dynamically align with task structure, rigorously justifying why previous solutions successfully described learning from small initial weights without incorporating their fine-scale structure. Finally, we discuss the implications of these findings for continual learning, reversal learning and learning of structured knowledge. Taken together, our results provide a mathematical toolkit for understanding the impact of prior knowledge on deep learning.</p>
},
	author = {Lukas Braun and Cl{\'e}mentine Domin{\'e} and James Fitzgerald and Andrew Saxe}
}
