@article {70293, title = {Machine learning-guided spatial omics for tissue-scale discovery of cell-type-specific architectures}, journal = {bioRxiv}, year = {2026}, month = {2026 Feb 13}, abstract = {

Multiplexed protein imaging enables spatially resolved analysis of molecular organization in tissues, but existing spatial proteomics platforms remain constrained in scalability, throughput, and integration with RNA measurements and interpretable computational analysis. Here, we present an integrated spatial omics framework that combines highly multiplexed protein and RNA imaging with explainable machine learning to map cell-type-specific molecular and structural architectures at tissue scale. Using this platform, we simultaneously profiled up to 46 proteins and 79 RNA species across \~370,000 cells in intact mouse brain tissue at diffraction-limited subcellular resolution (\~260 nm). We developed a scalable, open-source computational pipeline for large-scale image processing and analysis, and show that nuclear protein and chromatin features alone are sufficient to accurately classify brain cell types and their spatial organization. Incorporation of explainable deep learning further enabled identification of human-interpretable, cell-type-specific subnuclear structural features directly from imaging data, with independent quantitative validation. Together, this integrated experimental and computational framework enables tissue-scale spatial proteomics-based cell-type classification and structural feature discovery, providing a broadly applicable platform for mechanistic studies, high-content screening, and translational applications.

}, doi = {10.64898/2026.02.12.705598}, url = {https://www.biorxiv.org/content/early/2026/02/13/2026.02.12.705598}, author = {Lian, Yumin and Adjavon, Diane-Yayra and Kawase, Takashi and Kim, Jun and Fleishman, Greg and Preibisch, Stephan and Funke, Jan and Liu, Zhe J.} } @article {70311, title = {A multi-resolution imaging and analysis pipeline for comparative circuit reconstruction in insects}, journal = {bioRxiv}, year = {2026}, month = {2026 Feb 28}, abstract = {

Connectomics has become essential for the study of brain function, yet for most research groups it remains prohibitively costly in imaging time, data storage, and analysis. Here, we present an imaging, processing, and analysis pipeline for multi-resolution image acquisition and circuit reconstruction. Applied to the central complex of six insect species, we were able to obtain global projectomes at cellular resolution (40-50 nm) with embedded local connectomes describing key computational compartments at synaptic resolution (8-12 nm). We provide standardized protocols for volume EM sample preparation, image acquisition and image alignment, combined with existing methods for {\textmu}CT block trimming, automatic segmentation, synapse detection, collaborative skeleton tracing with CATMAID, and segmentation proofreading via CAVE. We validated our workflow by reconstructing head direction cells across all six insect species, which revealed deep conservation at the level of cell types, cell numbers and projection patterns, while also revealing circuit level specializations. Overall, our pipeline democratizes comparative connectomics by making this method accessible for small research groups with modest resources.

}, doi = {10.64898/2026.02.25.708065}, url = {https://www.biorxiv.org/content/early/2026/02/28/2026.02.25.708065}, author = {Gillet, Valentin and Sayre, Marcel E. and Badalamente, Griffin S. and Schieber, Nicole L. and Tedore, Kevin and Funke, Jan and Heinze, Stanley} } @article {69848, title = {A Bayesian model to count the number of two-state emitters in a diffraction limited spot.}, journal = {Nano Lett}, year = {2025}, month = {2025 Apr 04}, abstract = {

We address the problem of inferring the number of independently blinking fluorescent light emitters, when only their combined intensity contributions can be observed. This problem occurs regularly in light microscopy of objects smaller than the diffraction limit, where one wishes to count the number of fluorescently labeled subunits. Our proposed solution directly models the photophysics of the system, as well as the blinking kinetics of the fluorescent emitters as a fully differentiable hidden Markov model, estimating a posterior distribution of the total number of emitters. We show that our model is more accurate and increases the range of countable subunits by a factor of 2 compared to current state-of-the-art methods. Furthermore, we demonstrate that our model can be used to investigate the effect of blinking kinetics on counting ability and therefore can inform optimal experimental conditions.

}, issn = {1530-6992}, doi = {10.1021/acs.nanolett.4c06304}, url = {https://doi.org/10.1021/acs.nanolett.4c06304}, author = {Hillsley, Alexander and Stein, Johannes and Tillberg, Paul W and Stern, David L and Funke, Jan} } @article {70130, title = {Combinatorial protein barcodes enable self-correcting neuron tracing with nanoscale molecular context}, journal = {bioRxiv}, year = {2025}, month = {2025 Sep 28}, abstract = {

Mapping nanoscale neuronal morphology with molecular annotations is critical for understanding healthy and dysfunctional brain circuits. Current methods are constrained by image segmentation errors and by sample defects (e.g., signal gaps, section loss). Genetic strategies promise to overcome these challenges by using easily distinguishable cell identity labels. However, multicolor approaches are spectrally limited in diversity, whereas nucleic acid barcoding lacks a cell-filling morphology signal for segmentation. Here, we introduce PRISM (Protein-barcode Reconstruction via Iterative Staining with Molecular annotations), a platform that integrates combinatorial delivery of antigenically distinct, cell-filling proteins with tissue expansion, multi-cycle imaging, barcode-augmented reconstruction, and molecular annotation. Protein barcodes increase label diversity by \>750-fold over multicolor labeling and enable morphology reconstruction with intrinsic error correction. We acquired a \~10 million μm3 volume of mouse hippocampal area CA2/3, multiplexed across 23 barcode antigen and synaptic marker channels. By combining barcodes with shape information we achieve an 8x increase in automatic tracing accuracy of genetically labelled neurons. We demonstrate PRISM supports automatic proofreading across micron-scale spatial gaps and reconnects neurites across discontinuities spanning hundreds of microns. Using PRISM{\textquoteright}s molecular annotation capability, we map the distribution of synapses onto traced neural morphology, characterizing challenging synaptic structures such as thorny excrescences (TEs), and discovering a size correlation among spatially proximal TEs on the same dendrite. PRISM thus supports self-correcting neuron reconstruction with molecular context.

}, doi = {10.1101/2025.09.26.678648}, url = {https://www.biorxiv.org/content/early/2025/09/28/2025.09.26.678648}, author = {Park, Sung Yun and Sheridan, Arlo and An, Bobae and Jarvis, Erin and Lyudchik, Julia and Patton, William and Axup, Jun Y. and Chan, Stephanie W. and Damstra, Hugo G.J. and Leible, Daniel and Leung, Kylie S. and Magno, Clarence A. and Meeran, Aashir and Michalska, Julia M. and Rieger, Franz and Wang, Claire and Wu, Michelle and Church, George M. and Funke, Jan and Huffman, Todd and Leeper, Kathleen G.C. and Truckenbrodt, Sven and Winnubst, Johan and Kornfeld, Joergen M.R. and Boyden, Edward S. and Rodriques, Samuel G. and Payne, Andrew C.} } @article {70047, title = {Distributed control circuits across a brain-and-cord connectome}, journal = {bioRxiv}, year = {2025}, month = {2025 Aug 02}, abstract = {

Just as genomes revolutionized molecular genetics, connectomes (maps of neurons and synapses) are transforming neuroscience. To date, the only species with complete connectomes are worms\ and sea squirts\ (103-104 synapses). By contrast, the fruit fly is more complex (108 synaptic connections), with a brain that supports learning and spatial memory and an intricate ventral nerve cord analogous to the vertebrate spinal cord. Here we report the first adult fly connectome that unites the brain and ventral nerve cord, and we leverage this resource to investigate principles of neural control. We show that effector cells (motor neurons, endocrine cells and efferent neurons targeting the viscera) are primarily influenced by local sensory cells in the same body part, forming local feedback loops. These local loops are linked by long-range circuits involving ascending and descending neurons organized into behavior-centric modules. Single ascending and descending neurons are often positioned to influence the voluntary movements of multiple body parts, together with endocrine cells or visceral organs that support those movements. Brain regions involved in learning and navigation supervise these circuits. These results reveal an architecture that is distributed, parallelized and embodied (tightly connected to effectors), reminiscent of distributed control architectures in engineered systems.

}, doi = {10.1101/2025.07.31.667571}, url = {https://www.biorxiv.org/content/early/2025/08/02/2025.07.31.667571}, author = {Bates, Alexander Shakeel and Phelps, Jasper S. and Kim, Minsu and Yang, Helen H. and Matsliah, Arie and Ajabi, Zaki and Perlman, Eric and Delgado, Kevin M. and Osman, Mohammed Abdal Monium and Salmon, Christopher K. and Gager, Jay and Silverman, Benjamin and Renauld, Sophia and Collie, Matthew F. and Fan, Jingxuan and Pacheco, Diego A. and Zhao, Yunzhi and Patel, Janki and Zhang, Wenyi and Serratosa Capdevilla, Laia and Roberts, Ruair{\'\i} J.V. and Munnelly, Eva J. and Griggs, Nina and Langley, Helen and Moya-Llamas, Borja and Maloney, Ryan T. and Yu, Szi-Chieh and Sterling, Amy R. and Sorek, Marissa and Kruk, Krzysztof and Serafetinidis, Nikitas and Dhawan, Serene and St{\"u}rner, Tomke and Klemm, Finja and Brooks, Paul and Lesser, Ellen and Jones, Jessica M. and Pierce-Lundgren, Sara E. and Lee, Su-Yee and Luo, Yichen and Cook, Andrew P. and McKim, Theresa H. and Kophs, Emily C. and Falt, Tjalda and Negr{\'o}n Morales, Alexa M. and Burke, Austin and Hebditch, James and Willie, Kyle P. and Willie, Ryan and Popovych, Sergiy and Kemnitz, Nico and Ih, Dodam and Lee, Kisuk and Lu, Ran and Halageri, Akhilesh and Bae, J. Alexander and Jourdan, Ben and Schwartzman, Gregory and Demarest, Damian D. and Behnke, Emily and Bland, Doug and Kristiansen, Anne and Skelton, Jaime and Stocks, Tom and Garner, Dustin and Salman, Farzaan and Daly, Kevin C. and Hernandez, Anthony and Kumar, Sandeep and The BANC-FlyWire Consortium and Dorkenwald, Sven and Collman, Forrest and Suver, Marie P. and Fenk, Lisa M. and Pankratz, Michael J. and Jefferis, Gregory S.X.E. and Eichler, Katharina and Seeds, Andrew M. and Hampel, Stefanie and Agrawal, Sweta and Zandawala, Meet and Macrina, Thomas and Adjavon, Diane-Yayra and Funke, Jan and Tuthill, John C. and Azevedo, Anthony and Seung, H. Sebastian and de Bivort, Benjamin L. and Murthy, Mala and Drugowitsch, Jan and Wilson, Rachel I. and Lee, Wei-Chung Allen} } @conference {70301, title = {An investigation of unsupervised cell tracking and interactive fine-tuning}, booktitle = {2025 IEEE/CVF International Conference on Computer Vision Workshops (ICCVW)}, year = {2025}, month = {2026 Feb 23}, abstract = {

Most existing deep learning-based cell tracking methods rely on supervised learning, requiring large-scale annotated datasets that are often unavailable in real-world scenarios. Moreover, many approaches lack tools and methods for correcting mispredicted links or incorporating corrections through fine-tuning. These limitations contribute to the limited adoption of deep learning-based tracking methods in the life sciences, where manual tracking remains the predominant approach. To reduce the annotation burden and enable model training without extensive labeled data, we introduce a loss function for unsupervised training. Our method leverages the predictable dynamics inherent in many biological processes, providing an initialization that does not require an annotated dataset. We further investigate how minimal user-provided annotations can refine tracking accuracy. To this end, we propose an active learning framework that selectively identifies uncertain decisions within the tracking graph, allowing for efficient annotation of the most informative data points. We evaluate our approach on two microscopy datasets, demonstrating the effectiveness of both our unsupervised training strategy and active learning scheme in improving tracking performance. Our implementation and reproducible experiments are available at github.com/funkelab/attrackt and github.com/funkelab/attrackt_experiments, respectively.

}, keywords = {Active learning, Annotations, Cell Tracking, Computer vision, Conferences, Data models, Fine-Tuning, Life sciences, Manuals, Microscopy, Supervised learning, Training, Unsupervised Deep Learning}, doi = {10.1109/ICCVW69036.2025.00610}, url = {https://doi.org/10.1109/ICCVW69036.2025.00610}, author = {Lalit, Manan and Funke, Jan} } @article {70155, title = {Sexual dimorphism in the complete connectome of the Drosophila male central nervous system}, journal = {bioRxiv}, year = {2025}, month = {2025 Oct 09}, abstract = {

Sex differences in behaviour exist across the animal kingdom, typically under strong genetic regulation. In Drosophila, previous work has shown that fruitless and doublesex transcription factors identify neurons driving sexually dimorphic behaviour. However, the organisation of dimorphic neurons into functional circuits remains unclear.We now present the connectome of the entire Drosophila male central nervous system. This contains 166,691 neurons spanning the brain and ventral nerve cord, fully proofread and comprehensively annotated including fruitless and doublesex expression and 11,691 cell types. By comparison with a previous female brain connectome, we provide the first comprehensive description of the differences between male and female brains to synaptic resolution. Of 7,319 cross-matched cell types in the central brain, 114 are dimorphic with an additional 262 male- and 69 female-specific (totalling 4.8\% of neurons in males and 2.4\% in females).This resource enables analysis of full sensory-to-motor circuits underlying complex behaviours as well as the impact of dimorphic elements. Sex-specific and dimorphic neurons are concentrated in higher brain centres while the sensory and motor periphery are largely isomorphic. Within higher centres, male-specific connections are organised into hotspots defined by male-specific neurons or the presence of male-specific arbours on neurons that are otherwise similar between sexes. Numerous circuit switches reroute sensory information to form conserved, antagonistic circuits controlling opposing behaviours.

}, doi = {10.1101/2025.10.09.680999}, url = {https://www.biorxiv.org/content/early/2025/10/09/2025.10.09.680999}, author = {Berg, Stuart and Beckett, Isabella R and Costa, Marta and Schlegel, Philipp and Januszewski, Micha{\l} and Marin, Elizabeth C and Nern, Aljoscha and Preibisch, Stephan and Qiu, Wei and Takemura, Shin-ya and Fragniere, Alexandra MC and Champion, Andrew S and Adjavon, Diane-Yayra and Cook, Michael and Gkantia, Marina and Hayworth, Kenneth J and Huang, Gary B and Katz, William T and K{\"a}mpf, Florian and Lu, Zhiyuan and Ordish, Christopher and Paterson, Tyler and St{\"u}rner, Tomke and Trautman, Eric T and Whittle, Catherine R and Burnett, Laura E and Hoeller, Judith and Li, Feng and Loesche, Frank and Morris, Billy J and Pietzsch, Tobias and Pleijzier, Markus W and Silva, Valeria and Yin, Yijie and Ali, Iris and Badalamente, Griffin and Bates, Alexander Shakeel and Bogovic, John and Brooks, Paul and Cachero, Sebastian and Canino, Brandon S and Chaisrisawatsuk, Bhumpanya and Clements, Jody and Crowe, Arthur and de Haan Vicente, In{\^e}s and Dempsey, Georgia and Don{\`a}, Erika and dos Santos, M{\'a}rcia and Dreher, Marisa and Dunne, Christopher R and Eichler, Katharina and Finley-May, Samantha and Flynn, Miriam A and Hameed, Imran and Hopkins, Gary Patrick and Hubbard, Philip M and Kiassat, Ladann and Kovalyak, Julie and Lauchie, Shirley A and Leonard, Meghan and Lohff, Alanna and Longden, Kit D and Maldonado, Charli A and Mitletton, Myrto and Moitra, Ilina and Moon, Sung Soo and Mooney, Caroline and Munnelly, Eva J and Okeoma, Nneoma and Olbris, Donald J and Pai, Anika and Patel, Birava and Phillips, Emily M and Plaza, Stephen M and Richards, Alana and Rivas Salinas, Jennifer and Roberts, Ruair{\'\i} Jv and Rogers, Edward M and Scott, Ashley L and Scuderi, Louis A and Seenivasan, Pavithraa and Serratosa Capdevila, Laia and Smith, Claire and Svirskas, Rob and Takemura, Satoko and Tastekin, Ibrahim and Thomson, Alexander and Umayam, Lowell and Walsh, John J and Whittome, Holly and Xu, C Shan and Yakal, Emily A and Yang, Tansy and Zhao, Arthur and George, Reed and Jain, Viren and Jayaraman, Vivek and Korff, Wyatt and Meissner, Geoffrey W and Romani, Sandro and Funke, Jan and Knecht, Christopher and Saalfeld, Stephan and Scheffer, Louis K and Waddell, Scott and Card, Gwyneth M and Ribeiro, Carlos and Reiser, Michael B and Hess, Harald F and Rubin, Gerald M and Jefferis, Gregory Sxe} } @article {70212, title = {Uncertainty quantification for connectomics.}, journal = {Nat Methods}, volume = {22}, year = {2025}, month = {2025 Dec 08}, pages = {2484-2486}, abstract = {

No abstract available.

}, issn = {1548-7105}, doi = {10.1038/s41592-025-02945-3}, url = {https://doi.org/10.1038/s41592-025-02945-3}, author = {Funke, Jan} } @article {69568, title = {DaCapo: a modular deep learning framework for scalable 3D image segmentation}, journal = {arXiv}, year = {2024}, month = {2024 Aug 05}, abstract = {

DaCapo is a specialized deep learning library tailored to expedite the training and application of existing machine learning approaches on large, near-isotropic image data. In this correspondence, we introduce DaCapo{\textquoteright}s unique features optimized for this specific domain, highlighting its modular structure, efficient experiment management tools, and scalable deployment capabilities. We discuss its potential to improve access to large-scale, isotropic image segmentation and invite the community to explore and contribute to this open-source initiative.

}, doi = {10.48550/arXiv.2408.02834}, url = {https://doi.org/10.48550/arXiv.2408.02834}, author = {Patton, William and Jeff L Rhoades and Zouinkhi, Marwan and Ackerman, David G and Malin-Mayor, Caroline and Adjavon, Diane and Heinrich, Larissa and Bennett, Davis and Zubov, Yurii and Team, CellMap Project and Weigel, Aubrey and Funke, Jan} } @article {69633, title = {A Drosophila computational brain model reveals sensorimotor processing.}, journal = {Nature}, volume = {634}, year = {2024}, month = {2024 Oct 02}, pages = {210-219}, abstract = {

The recent assembly of the adult Drosophila melanogaster central brain connectome, containing more than 125,000 neurons and 50 million synaptic connections, provides a template for examining sensory processing throughout the brain. Here we create a leaky integrate-and-fire computational model of the entire Drosophila brain, on the basis of neural connectivity and neurotransmitter identity, to study circuit properties of feeding and grooming behaviours. We show that activation of sugar-sensing or water-sensing gustatory neurons in the computational model accurately predicts neurons that respond to tastes and are required for feeding initiation. In addition, using the model to activate neurons in the feeding region of the Drosophila brain predicts those that elicit motor neuron firing-a testable hypothesis that we validate by optogenetic activation and behavioural studies. Activating different classes of gustatory neurons in the model makes accurate predictions of how several taste modalities interact, providing circuit-level insight into aversive and appetitive taste processing. Additionally, we applied this model to mechanosensory circuits and found that computational activation of mechanosensory neurons predicts activation of a small set of neurons comprising the antennal grooming circuit, and accurately describes the circuit response upon activation of different mechanosensory subtypes. Our results demonstrate that modelling brain circuits using only synapse-level connectivity and predicted neurotransmitter identity generates experimentally testable hypotheses and can describe complete sensorimotor transformations.

}, keywords = {Animals, Brain, Computer Simulation, Connectome, Drosophila melanogaster, Feeding Behavior, Female, Grooming, Male, Models, Neurological, Motor Neurons, Optogenetics, Synapses, Taste}, issn = {1476-4687}, doi = {10.1038/s41586-024-07763-9}, url = {https://doi.org/10.1038/s41586-024-07763-9}, author = {Shiu, Philip K and Sterne, Gabriella R and Spiller, Nico and Franconville, Romain and Sandoval, Andrea and Zhou, Joie and Simha, Neha and Kang, Chan Hyuk and Yu, Seongbong and Kim, Jinseop S and Dorkenwald, Sven and Matsliah, Arie and Schlegel, Philipp and Yu, Szi-Chieh and McKellar, Claire E and Sterling, Amy and Costa, Marta and Eichler, Katharina and Bates, Alexander Shakeel and Eckstein, Nils and Funke, Jan and Jefferis, Gregory S X E and Murthy, Mala and Bidaye, Salil S and Hampel, Stefanie and Seeds, Andrew M and Scott, Kristin} } @article {68889, title = {Neuronal wiring diagram of an adult brain.}, journal = {Nature}, volume = {634}, year = {2024}, month = {2024 Oct 02}, pages = {124-138}, abstract = {

Connections between neurons can be mapped by acquiring and analysing electron microscopic brain images. In recent years, this approach has been applied to chunks of brains to reconstruct local connectivity maps that are highly informative, but nevertheless inadequate for understanding brain function more globally. Here we present a neuronal wiring diagram of a whole brain containing 5 {\texttimes} 107 chemical synapses between 139,255 neurons reconstructed from an adult female Drosophila melanogaster. The resource also incorporates annotations of cell classes and types, nerves, hemilineages and predictions of neurotransmitter identities. Data products are available for download, programmatic access and interactive browsing and have been made interoperable with other fly data resources. We derive a projectome-a map of projections between regions-from the connectome and report on tracing of synaptic pathways and the analysis of information flow from inputs (sensory and ascending neurons) to outputs (motor, endocrine and descending neurons) across both hemispheres and between the central brain and the optic lobes. Tracing from a subset of photoreceptors to descending motor pathways illustrates how structure can uncover putative circuit mechanisms underlying sensorimotor behaviours. The technologies and open ecosystem reported here set the stage for future large-scale connectome projects in other species.

}, doi = {10.1038/s41586-024-07558-y}, url = {https://doi.org/10.1038/s41586-024-07558-y}, author = {Dorkenwald, Sven and Matsliah, Arie and Sterling, Amy R and Schlegel, Philipp and Yu, Szi-Chieh and McKellar, Claire E and Lin, Albert and Costa, Marta and Eichler, Katharina and Yin, Yijie and Silversmith, Will and Schneider-Mizell, Casey and Jordan, Chris S and Brittain, Derrick and Halageri, Akhilesh and Kuehner, Kai and Ogedengbe, Oluwaseun and Morey, Ryan and Gager, Jay and Kruk, Krzysztof and Perlman, Eric and Yang, Runzhe and Deutsch, David and Bland, Doug and Sorek, Marissa and Lu, Ran and Macrina, Thomas and Lee, Kisuk and Bae, J Alexander and Mu, Shang and Nehoran, Barak and Mitchell, Eric and Popovych, Sergiy and Wu, Jingpeng and Jia, Zhen and Castro, Manuel and Kemnitz, Nico and Ih, Dodam and Bates, Alexander Shakeel and Eckstein, Nils and Funke, Jan and Collman, Forrest and Bock, Davi D and Jefferis, Gregory S X E and Seung, H Sebastian and Murthy, Mala} } @article {69439, title = {Neurotransmitter classification from electron microscopy images at synaptic sites in Drosophila melanogaster}, journal = {Cell}, volume = {187}, year = {2024}, month = {2024 May 09}, pages = {2574-2594.e23}, abstract = {

High-resolution electron microscopy of nervous systems has enabled the reconstruction of synaptic connectomes. However, we do not know the synaptic sign for each connection (i.e., whether a connection is excitatory or inhibitory), which is implied by the released transmitter. We demonstrate that artificial neural networks can predict transmitter types for presynapses from electron micrographs: a network trained to predict six transmitters (acetylcholine, glutamate, GABA, serotonin, dopamine, octopamine) achieves an accuracy of 87\% for individual synapses, 94\% for neurons, and 91\% for known cell types across a D.\ melanogaster whole brain. We visualize the ultrastructural features used for prediction, discovering subtle but significant differences between transmitter phenotypes. We also analyze transmitter distributions across the brain and find that neurons that develop together largely express only one fast-acting transmitter (acetylcholine, glutamate, or GABA). We hope that our publicly available predictions act as an accelerant for neuroscientific hypothesis generation for the fly.

}, keywords = {Animals, Brain, Connectome, Drosophila melanogaster, gamma-Aminobutyric Acid, Microscopy, Electron, Neural Networks, Computer, Neurons, Neurotransmitter Agents, Synapses}, issn = {1097-4172}, doi = {10.1016/j.cell.2024.03.016}, author = {Eckstein, Nils and Bates, Alexander Shakeel and Champion, Andrew and Du, Michelle and Yin, Yijie and Schlegel, Philipp and Lu, Alicia Kun-Yang and Rymer, Thomson and Finley-May, Samantha and Paterson, Tyler and Parekh, Ruchi and Dorkenwald, Sven and Matsliah, Arie and Yu, Szi-Chieh and McKellar, Claire and Sterling, Amy and Eichler, Katharina and Costa, Marta and Seung, Sebastian and Murthy, Mala and Hartenstein, Volker and Jefferis, Gregory S X E and Funke, Jan} } @article {69689, title = {Quantitative Attributions with Counterfactuals}, journal = {bioRxiv}, year = {2024}, month = {2024 Dec 02}, abstract = {

We address the problem of explaining the decision process of deep neural network classifiers on images, which is of particular importance in biomedical datasets where class-relevant differences are not always obvious to a human observer. Our proposed solution, termed quantitative attribution with counterfactuals (QuAC), generates visual explanations that highlight class-relevant differences by attributing the classifier decision to changes of visual features in small parts of an image. To that end, we train a separate network to generate counterfactual images (i.e., to translate images between different classes). We then find the most important differences using novel discriminative attribution methods. Crucially, QuAC allows scoring of the attribution and thus provides a measure to quantify and compare the fidelity of a visual explanation. We demonstrate the suitability and limitations of QuAC on two datasets: (1) a synthetic dataset with known class differences, representing different levels of protein aggregation in cells and (2) an electron microscopy dataset of D. melanogaster synapses with different neurotransmitters, where QuAC reveals so far unknown visual differences. We further discuss how QuAC can be used to interrogate mispredictions to shed light on unexpected inter-class similarities and intra-class differences.

}, doi = {10.1101/2024.11.26.625505}, url = {https://doi.org/10.1101/2024.11.26.625505}, author = {Adjavon, Diane-Yayra and Eckstein, Nils and Bates, Alexander S. and Jefferis, Gregory S.X.E. and Funke, Jan} } @article {69694, title = {Spatial single-cell Organellomics reveals nutrient dependent hepatocyte heterogeneity and predicts pathophysiological status in vivo}, journal = {bioRxiv}, year = {2024}, month = {2024 Dec 08}, abstract = {

Cellular heterogeneity within complex tissues and organs is essential to coordinate biological processes across biological scales. The effect of local cues and tissue microenvironments on cell heterogeneity has been mainly studied at the transcriptional level. However, it is within the subcellular scale - the organelles - that lays the machinery to conduct most metabolic reactions and maintain cells alive, ensuring proper tissue function. How changes in subcellular organization under different microenvironments define the functional diversity of cells within organs remains largely unexplored. Here we determine how organelles adapt to different microenvironments using the mouse liver as model system, in combination with computational approaches and machine-learning. To understand organelle adaptation in response to changing nutritional conditions, we analyzed 3D fluorescent microscopy volumes of liver samples labeled to simultaneously visualize mitochondria, peroxisomes, and lipid droplets from mice subjected to different diets: a control diet, a high-fat diet, and a control diet plus fasting. A Cellpose based pipeline was implemented for cell and organelle segmentation, which allowed us to measure 100 different organelle metrics and helped us define subcellular architectures in liver samples at the single cell level. Our results showed that hepatocytes display distinct subcellular architectures within different regions of the liver-close to the central vein, in the middle region, and near the portal vein- and across the various diet groups, thus reflecting their adaptation to specific nutritional inputs. Principal component analysis and clustering of hepatocytes based on organelle signatures revealed 12 different hepatocyte categories within the different experimental groups, highlighting a reduction in hepatocyte heterogeneity under nutritional perturbations. Finally, using single cell organelle signatures exclusively, we generated machine learning models that were able to predict with high accuracy different hepatocyte categories, diet groups, and the stages of MASLD. Our results demonstrate how organelle signatures can be used as hallmarks to define hepatocyte heterogeneity and their adaptation to different nutritional conditions. In the future, our strategy, which combines subcellular resolution imaging of liver volumes and machine learning, could help establish protocols to better define and predict liver disease progression.

}, doi = {10.1101/2024.12.06.627285}, url = {https://doi.org/10.1101/2024.12.06.627285}, author = {Hillsley, Alexander and Adhikari, Raghabendra and Johnson, Alana Dowdell and Espinosa-Medina, Isabel and Funke, Jan and Feliciano, Daniel} } @article {68587, title = {Automated reconstruction of whole-embryo cell lineages by learning from sparse annotations.}, journal = {Nature Biotechnology}, volume = {41}, year = {2023}, month = {2023 Jan 01}, pages = {44-49}, abstract = {

We present a method to automatically identify and track nuclei in time-lapse microscopy recordings of entire developing embryos. The method combines deep learning and global optimization. On a mouse dataset, it reconstructs 75.8\% of cell lineages spanning 1 h, as compared to 31.8\% for the competing method. Our approach improves understanding of where and when cell fate decisions are made in developing embryos, tissues, and organs.

}, keywords = {Animals, Blastocyst, Cell Lineage, Embryo, Mammalian, Mice, Microscopy}, issn = {1546-1696}, doi = {10.1038/s41587-022-01427-7}, author = {Malin-Mayor, Caroline and Hirsch, Peter and Guignard, Leo and McDole, Katie and Wan, Yinan and Lemon, William C and Kainmueller, Dagmar and Keller, Philipp J and Preibisch, Stephan and Funke, Jan} } @article {68564, title = {Local shape descriptors for neuron segmentation.}, journal = {Nature Methods}, volume = {20}, year = {2023}, month = {2023 Feb 01}, pages = {295-303}, abstract = {

We present an auxiliary learning task for the problem of neuron segmentation in electron microscopy volumes. The auxiliary task consists of the prediction of local shape descriptors (LSDs), which we combine with conventional voxel-wise direct neighbor affinities for neuron boundary detection. The shape descriptors capture local statistics about the neuron to be segmented, such as diameter, elongation, and direction. On a study comparing several existing methods across various specimen, imaging techniques, and resolutions, auxiliary learning of LSDs consistently increases segmentation accuracy of affinity-based methods over a range of metrics. Furthermore, the addition of LSDs promotes affinity-based segmentation methods to be on par with the current state of the art for neuron segmentation (flood-filling networks), while being two orders of magnitudes more efficient-a critical requirement for the processing of future petabyte-sized datasets.

}, issn = {1548-7105}, doi = {10.1038/s41592-022-01711-z}, author = {Sheridan, Arlo and Nguyen, Tri M and Deb, Diptodip and Lee, Wei-Chung Allen and Saalfeld, Stephan and Turaga, Srinivas C and Manor, Uri and Funke, Jan} } @article {68517, title = {Structured cerebellar connectivity supports resilient pattern separation.}, journal = {Nature}, volume = {613}, year = {2023}, month = {2023 Jan 01}, pages = {543-549}, abstract = {

The cerebellum is thought to help\ detect and correct errors between intended and executed commands and is critical for social behaviours, cognition and emotion. Computations for motor control must be performed quickly to correct errors in real time and should be sensitive to small differences between patterns for fine error correction while being resilient to noise. Influential theories of cerebellar information processing have largely assumed random network connectivity, which increases the encoding capacity of the network{\textquoteright}s first layer. However, maximizing encoding capacity reduces the resilience to noise. To understand how neuronal circuits address this fundamental trade-off, we mapped the feedforward connectivity in the mouse cerebellar cortex using automated large-scale transmission electron microscopy and convolutional neural network-based image segmentation. We found that both the input and output layers of the circuit exhibit redundant and selective connectivity motifs, which contrast with prevailing models. Numerical simulations suggest that these redundant, non-random connectivity motifs increase the resilience to noise at a negligible cost to the overall encoding capacity. This work reveals how neuronal network structure can support a trade-off between encoding capacity and redundancy, unveiling principles of biological network architecture with implications for the design of artificial neural networks.

}, issn = {1476-4687}, doi = {10.1038/s41586-022-05471-w}, author = {Nguyen, Tri M and Thomas, Logan A and Jeff L Rhoades and Ricchi, Ilaria and Yuan, Xintong Cindy and Sheridan, Arlo and Hildebrand, David G C and Funke, Jan and Regehr, Wade G and Lee, Wei-Chung Allen} } @article {68958, title = {Towards Generalizable Organelle Segmentation in Volume Electron Microscopy.}, journal = {Microscopy and Microanalysis}, volume = {29}, year = {2023}, month = {2023 Jul 22}, pages = {975}, keywords = {Organelles, Volume Electron Microscopy}, issn = {1435-8115}, doi = {10.1093/micmic/ozad067.487}, author = {Heinrich, Larissa and Patton, Will and Bennett, Davis and Ackerman, David and Park, Grace and Bogovic, John A and Eckstein, Nils and Petruncio, Alyson and Clements, Jody and Pang, Song and Shan Xu, C and Funke, Jan and Korff, Wyatt and Hess, Harald and Lippincott-Schwartz, Jennifer and Saalfeld, Stephan and Weigel, Aubrey} } @conference {69206, title = {Unsupervised Learning of Object-Centric Embeddings for Cell Instance Segmentation in Microscopy Images}, booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)}, year = {2023}, month = {2023 Oct 01}, publisher = {IEEE}, organization = {IEEE}, address = {Paris, France}, abstract = {

Segmentation of objects in microscopy images is required for many biomedical applications. We introduce object-centric embeddings (OCEs), which embed image patches such that the spatial offsets between patches cropped from the same object are preserved. Those learnt embeddings can be used to delineate individual objects and thus obtain instance segmentations. Here, we show theoretically that, under assumptions commonly found in microscopy images, OCEs can be learnt through a self-supervised task that predicts the spatial offset between image patches. Together, this forms an unsupervised cell instance segmentation method which we evaluate on nine diverse large-scale microscopy datasets. Segmentations obtained with our method lead to substantially improved results, compared to state-of-the-art baselines on six out of nine datasets, and perform on par on the remaining three datasets. If ground-truth annotations are available, our method serves as an excellent starting point for supervised training, reducing the required amount of ground-truth needed by one order of magnitude, thus substantially increasing the practical applicability of our method. Source code is available at github.com/funkelab/cellulus.

}, doi = {10.1109/ICCV51070.2023.01944}, url = {https://ieeexplore.ieee.org/document/10377649/}, author = {Wolf, Steffen and Lalit, Manan and McDole, Katie and Funke, Jan} } @article {68313, title = {Automated reconstruction of whole-embryo cell lineages by learning from sparse annotations.}, journal = {Nature Biotechnology}, year = {2022}, month = {2022 Sep 05}, abstract = {

We present a method to automatically identify and track nuclei in time-lapse microscopy recordings of entire developing embryos. The method combines deep learning and global optimization. On a mouse dataset, it reconstructs 75.8\% of cell lineages spanning 1 h, as compared to 31.8\% for the competing method. Our approach improves understanding of where and when cell fate decisions are made in developing embryos, tissues, and organs.

}, issn = {1546-1696}, doi = {10.1038/s41587-022-01427-7}, author = {Malin-Mayor, Caroline and Hirsch, Peter and Guignard, Leo and McDole, Katie and Wan, Yinan and Lemon, William C and Kainmueller, Dagmar and Keller, Philipp J and Preibisch, Stephan and Funke, Jan} } @article {68273, title = {Neural network organization for courtship-song feature detection in Drosophila.}, journal = {Current Biology}, volume = {32}, year = {2022}, month = {2022 Aug 08}, pages = {3317-3333.e7}, abstract = {

Animals communicate using sounds in a wide range of contexts, and auditory systems must encode behaviorally relevant acoustic features to drive appropriate reactions. How feature detection emerges along auditory pathways has been difficult to solve due to challenges in mapping the underlying circuits and characterizing responses to behaviorally relevant features. Here, we study auditory activity in the Drosophila melanogaster brain and investigate feature selectivity for the two main modes of fly courtship song, sinusoids and pulse trains. We identify 24 new cell types of the intermediate layers of the auditory pathway, and using a new connectomic resource, FlyWire, we map all synaptic connections between these cell types, in addition to connections to known early and higher-order auditory neurons-this represents the first circuit-level map of the auditory pathway. We additionally determine the sign (excitatory or inhibitory) of most synapses in this auditory connectome. We find that auditory neurons display a continuum of preferences for courtship song modes and that neurons with different song-mode preferences and response timescales are highly interconnected in a network that lacks hierarchical structure. Nonetheless, we find that the response properties of individual cell types within the connectome are predictable from their inputs. Our study thus provides new insights into the organization of auditory coding within the Drosophila brain.

}, keywords = {Animals, Auditory Perception, Courtship, Drosophila, Drosophila melanogaster, Neural Networks, Computer, Sexual Behavior, Animal, Vocalization, Animal}, issn = {1879-0445}, doi = {10.1016/j.cub.2022.06.019}, author = {Baker, Christa A and McKellar, Claire and Pang, Rich and Nern, Aljoscha and Dorkenwald, Sven and Pacheco, Diego A and Eckstein, Nils and Funke, Jan and Dickson, Barry J and Murthy, Mala} } @article {67988, title = {Transverse endoplasmic reticulum expansion in hereditary spastic paraplegia corticospinal axons.}, journal = {Human Molecular Genetics}, year = {2022}, month = {2022 Mar 26}, abstract = {

Hereditary spastic paraplegias (HSPs) comprise a large group of inherited neurologic disorders affecting the longest corticospinal axons (SPG1-86 plus others), with shared manifestations of lower extremity spasticity and gait impairment. Common autosomal dominant HSPs are caused by mutations in genes encoding the microtubule-severing ATPase spastin (SPAST; SPG4), the membrane-bound GTPase atlastin-1 (ATL1; SPG3A), and the reticulon-like, microtubule-binding protein REEP1 (REEP1; SPG31). These proteins bind one another and function in shaping the tubular endoplasmic reticulum (ER) network. Typically, mouse models of HSPs have mild, later-onset phenotypes, possibly reflecting far shorter lengths of their corticospinal axons relative to humans. Here, we have generated a robust, double mutant mouse model of HSP in which atlastin-1 is genetically modified with a K80A knock-in (KI) missense change that abolishes its GTPase activity, while its binding partner Reep1 is knocked out. Atl1KI/KI/Reep1-/- mice exhibit early-onset and rapidly progressive declines in several motor function tests. Also, ER in mutant corticospinal axons dramatically expands transversely and periodically in a mutation dosage-dependent manner to create a ladder-like appearance, based on reconstructions of focused ion beam-scanning electron microscopy datasets using machine learning-based auto-segmentation. In lockstep with changes in ER morphology, axonal mitochondria are fragmented and proportions of hypophosphorylated neurofilament H and M subunits are dramatically increased in Atl1KI/KI/Reep1-/- spinal cord. Co-occurrence of these findings links ER morphology changes to alterations in mitochondrial morphology and cytoskeletal organization. Atl1KI/KI/Reep1-/- mice represent an early-onset rodent HSP model with robust behavioral and cellular readouts for testing novel therapies.

}, issn = {1460-2083}, doi = {10.1093/hmg/ddac072}, author = {Zhu, Peng-Peng and Hung, Hui-Fang and Batchenkova, Natalia and Nixon-Abell, Jonathon and Henderson, James and Zheng, Pengli and Renvois{\'e}, Benoit and Pang, Song and Xu, C Shan and Saalfeld, Stephan and Funke, Jan and Xie, Yuxiang and Svara, Fabian and Hess, Harald F and Blackstone, Craig} } @article {68298, title = {Transverse endoplasmic reticulum expansion in hereditary spastic paraplegia corticospinal axons.}, journal = {Human Molecular Genetics}, volume = {31}, year = {2022}, month = {2022 Aug 23}, pages = {2779-2795}, abstract = {

Hereditary spastic paraplegias (HSPs) comprise a large group of inherited neurologic disorders affecting the longest corticospinal axons (SPG1-86 plus others), with shared manifestations of lower extremity spasticity and gait impairment. Common autosomal dominant HSPs are caused by mutations in genes encoding the microtubule-severing ATPase spastin (SPAST; SPG4), the membrane-bound GTPase atlastin-1 (ATL1; SPG3A) and the reticulon-like, microtubule-binding protein REEP1 (REEP1; SPG31). These proteins bind one another and function in shaping the tubular endoplasmic reticulum (ER) network. Typically, mouse models of HSPs have mild, later onset phenotypes, possibly reflecting far shorter lengths of their corticospinal axons relative to humans. Here, we have generated a robust, double mutant mouse model of HSP in which atlastin-1 is genetically modified with a K80A knock-in (KI) missense change that abolishes its GTPase activity, whereas its binding partner Reep1 is knocked out. Atl1KI/KI/Reep1-/- mice exhibit early onset and rapidly progressive declines in several motor function tests. Also, ER in mutant corticospinal axons dramatically expands transversely and periodically in a mutation dosage-dependent manner to create a ladder-like appearance, on the basis of reconstructions of focused ion beam-scanning electron microscopy datasets using machine learning-based auto-segmentation. In lockstep with changes in ER morphology, axonal mitochondria are fragmented and proportions of hypophosphorylated neurofilament H and M subunits are dramatically increased in Atl1KI/KI/Reep1-/- spinal cord. Co-occurrence of these findings links ER morphology changes to alterations in mitochondrial morphology and cytoskeletal organization. Atl1KI/KI/Reep1-/- mice represent an early onset rodent HSP model with robust behavioral and cellular readouts for testing novel therapies.

}, keywords = {Animals, Axons, Disease Models, Animal, Endoplasmic Reticulum, GTP Phosphohydrolases, Humans, Membrane Proteins, Membrane Transport Proteins, Mice, Mice, Knockout, Mutation, Spastic Paraplegia, Hereditary, Spastin}, issn = {1460-2083}, doi = {10.1093/hmg/ddac072}, author = {Zhu, Peng-Peng and Hung, Hui-Fang and Batchenkova, Natalia and Nixon-Abell, Jonathon and Henderson, James and Zheng, Pengli and Renvois{\'e}, Benoit and Pang, Song and Xu, C Shan and Saalfeld, Stephan and Funke, Jan and Xie, Yuxiang and Svara, Fabian and Hess, Harald F and Blackstone, Craig} } @article {65832, title = {Automatic Detection of Synaptic Partners in a Whole-Brain Drosophila EM Dataset}, journal = {Nature Methods}, volume = {18}, year = {2021}, month = {2021 Jul 1}, pages = {771-4}, abstract = {

The study of neural circuits requires the reconstruction of neurons and the identification of synaptic connections between them. To scale the reconstruction to the size of whole-brain datasets, semi-automatic methods are needed to solve those tasks. Here, we present an automatic method for synaptic partner identification in insect brains, which uses convolutional neural networks to identify post-synaptic sites and their pre-synaptic partners. The networks can be trained from human generated point annotations alone and requires only simple post-processing to obtain final predictions. We used our method to extract 244 million putative synaptic partners in the fifty-teravoxel full adult fly brain (FAFB) electron microscopy (EM) dataset and evaluated its accuracy on 146,643 synapses from 702 neurons with a total cable length of 312 mm in four different brain regions. The predicted synaptic connections can be used together with a neuron segmentation to infer a connectivity graph with high accuracy: 96\% of edges between connected neurons are correctly classified as weakly connected (less than five synapses) and strongly connected (at least five synapses). Our synaptic partner predictions for the FAFB dataset are publicly available, together with a query library allowing automatic retrieval of up- and downstream neurons.

}, doi = {10.1038/s41592-021-01183-7}, author = {Buhmann, Julia and Sheridan, Arlo and Gerhard, Stephan and Krause, Renate and Nguyen, Tri and Heinrich, Larissa and Schlegel, Philipp and Lee, Wei-Chung Allen and Wilson, Rachel and Saalfeld, Stephan and Jefferis, Gregory and Bock, Davi and Turaga, Srinivas and Cook, Matthew and Funke, Jan} } @article {67228, title = {Local Shape Descriptors for Neuron Segmentation}, journal = {bioRxiv}, year = {2021}, month = {2021/01}, abstract = {

We present a simple, yet effective, auxiliary learning task for the problem of neuron segmentation in electron microscopy volumes. The auxiliary task consists of the prediction of Local Shape Descriptors (LSDs), which we combine with conventional voxel-wise direct neighbor affinities for neuron boundary detection. The shape descriptors are designed to capture local statistics about the neuron to be segmented, such as diameter, elongation, and direction. On a large study comparing several existing methods across various specimen, imaging techniques, and resolutions, we find that auxiliary learning of LSDs consistently increases segmentation accuracy of affinity-based methods over a range of metrics. Furthermore, the addition of LSDs promotes affinitybased segmentation methods to be on par with the current state of the art for neuron segmentation (Flood-Filling Networks, FFN), while being two orders of magnitudes more efficient{\textemdash}a critical requirement for the processing of future petabyte-sized datasets. Implementations of the new auxiliary learning task, network architectures, training, prediction, and evaluation code, as well as the datasets used in this study are publicly available as a benchmark for future method contributions.Competing Interest StatementThe authors have declared no competing interest.

}, doi = {10.1101/2021.01.18.427039}, url = {https://www.biorxiv.org/content/early/2021/01/18/2021.01.18.427039}, author = {Sheridan, Arlo and Nguyen, Tri and Deb, Diptodip and Lee, Wei-Chung Allen and Saalfeld, Stephan and Turaga, Srini and Manor, Uri and Funke, Jan} } @article {65833, title = {Reconstruction of motor control circuits in adult Drosophila using automated transmission electron microscopy}, journal = {Cell}, volume = {184}, year = {2021}, month = {2021 Feb 04}, abstract = {

Many animals use coordinated limb movements to interact with and navigate through the environment. To investigate circuit mechanisms underlying locomotor behavior, we used serial-section electron microscopy (EM) to map synaptic connectivity within a neuronal network that controls limb movements. We present a synapse-resolution EM dataset containing the ventral nerve cord (VNC) of an adult female Drosophila melanogaster. To generate this dataset, we developed GridTape, a technology that combines automated serial-section collection with automated high-throughput transmission EM. Using this dataset, we reconstructed 507 motor neurons, including all those that control the legs and wings. We show that a specific class of leg sensory neurons directly synapse onto the largest-caliber motor neuron axons on both sides of the body, representing a unique feedback pathway for fast limb control. We provide open access to the dataset and reconstructions registered to a standard atlas to permit matching of cells between EM and light microscopy data. We also provide GridTape instrumentation designs and software to make large-scale EM data acquisition more accessible and affordable to the scientific community.

}, doi = {10.1101/2020.01.10.902478}, author = {Maniates-Selvin, Jasper T. and Hildebrand, David Grant Colburn and Graham, Brett J. and Kuan, Aaron T. and Thomas, Logan A. and Nguyen, Tri and Buhmann, Julia and Azevedo, Anthony W. and Shanny, Brendan L. and Funke, Jan and Tuthill, John C. and Lee, Wei-Chung Allen} } @article {67315, title = {Which image-based phenotypes are most promising for using AI to understand cellular functions and why?}, journal = {Cell Systems}, volume = {12}, year = {2021}, month = {2021 May 19}, pages = {384-387}, issn = {2405-4720}, doi = {10.1016/j.cels.2021.04.012}, author = {Lundberg, Emma and Funke, Jan and Uhlmann, Virginie and Gerlich, Daniel and Walter, Thomas and Carpenter, Ann and Coehlo, Luis Pedro} } @article {67750, title = {Whole-cell organelle segmentation in volume electron microscopy.}, journal = {Nature}, volume = {599}, year = {2021}, month = {2021 Nov 01}, pages = {141-46}, abstract = {

Cells contain hundreds of organelles and macromolecular assemblies. Obtaining a complete understanding of their intricate organization requires the nanometre-level, three-dimensional reconstruction of whole cells, which is only feasible with robust and scalable automatic methods. Here, to support the development of such methods, we annotated up to 35 different cellular organelle classes-ranging from endoplasmic reticulum to microtubules to ribosomes-in diverse sample volumes from multiple cell types imaged at a near-isotropic resolution of 4 nm per voxel with focused ion beam scanning electron microscopy (FIB-SEM). We trained deep learning architectures to segment these structures in 4 nm and 8 nm per voxel FIB-SEM volumes, validated their performance and showed that automatic reconstructions can be used to directly quantify previously inaccessible metrics including spatial interactions between cellular components. We also show that such reconstructions can be used to automatically register light and electron microscopy images for correlative studies. We have created an open data and open-source web repository, {\textquoteright}OpenOrganelle{\textquoteright}, to share the data, computer code and trained models, which will enable scientists everywhere to query and further improve automatic reconstruction of these datasets.

}, issn = {1476-4687}, doi = {10.1038/s41586-021-03977-3}, author = {Heinrich, Larissa and Bennett, Davis and Ackerman, David and Park, Woohyun and Bogovic, John and Eckstein, Nils and Petruncio, Alyson and Clements, Jody and Pang, Song and Xu, C Shan and Funke, Jan and Korff, Wyatt and Hess, Harald F and Lippincott-Schwartz, Jennifer and Saalfeld, Stephan and Weigel, Aubrey V} } @article {65725, title = {Dense neuronal reconstruction through X-ray holographic nano-tomography.}, journal = {Nature Neuroscience}, volume = {23}, year = {2020}, month = {2020 Dec -1}, pages = {1637-43}, abstract = {

Imaging neuronal networks provides a foundation for understanding the nervous system, but resolving dense nanometer-scale structures over large volumes remains challenging for light microscopy (LM) and electron microscopy (EM). Here we show that X-ray holographic nano-tomography (XNH) can image millimeter-scale volumes with sub-100-nm resolution, enabling reconstruction of dense wiring in Drosophila melanogaster and mouse nervous tissue. We performed correlative XNH and EM to reconstruct hundreds of cortical pyramidal cells and show that more superficial cells receive stronger synaptic inhibition on their apical dendrites. By combining multiple XNH scans, we imaged an adult Drosophila leg with sufficient resolution to comprehensively catalog mechanosensory neurons and trace individual motor axons from muscles to the central nervous system. To accelerate neuronal reconstructions, we trained a convolutional neural network to automatically segment neurons from XNH volumes. Thus, XNH bridges a key gap between LM and EM, providing a new avenue for neural circuit discovery.

}, issn = {1546-1726}, doi = {10.1038/s41593-020-0704-9}, author = {Kuan, Aaron T and Phelps, Jasper S and Thomas, Logan A and Nguyen, Tri M and Han, Julie and Chen, Chiao-Lin and Azevedo, Anthony W and Tuthill, John C and Funke, Jan and Cloetens, Peter and Pacureanu, Alexandra and Lee, Wei-Chung Allen} } @conference {65835, title = {Inpainting Networks Learn to Separate Cells in Microscopy Images}, booktitle = {British Machine Vision Conference}, year = {2020}, month = {2020 Sep}, abstract = {

Deep neural networks trained to inpaint partially occluded images show a deep understanding of image composition and have even been shown to remove objects from images convincingly. In this work, we investigate how this implicit knowledge of image composition can be be used to separate cells in densely populated microscopy images. We propose a measure for the independence of two image regions given a fully self-supervised inpainting network and separate objects by maximizing this independence. We evaluate our method on two cell segmentation datasets and show that cells can be separated completely unsupervised. Furthermore, combined with simple foreground detection, our method yields instance segmentation of similar quality to fully supervised methods.

}, url = {https://www.bmvc2020-conference.com/assets/papers/0528.pdf}, author = {Wolf, Steffen and Hamprecht, Fred A and Funke, Jan} } @article {48806, title = {Large scale image segmentation with structured loss based deep learning for connectome reconstruction.}, journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}, volume = {41}, year = {2019}, month = {2019 Jul 1}, pages = {1669-80}, abstract = {

We present a method combining affinity prediction with region agglomeration, which improves significantly upon the state of the art of neuron segmentation from electron microscopy (EM) in accuracy and scalability. Our method consists of a 3D U-net, trained to predict affinities between voxels, followed by iterative region agglomeration. We train using a structured loss based on MALIS, encouraging topologically correct segmentations obtained from affinity thresholding. Our extension consists of two parts: First, we present a quasi-linear method to compute the loss gradient, improving over the original quadratic algorithm. Second, we compute the gradient in two separate passes to avoid spurious gradient contributions in early training stages. Our predictions are accurate enough that simple learning-free percentile-based agglomeration outperforms more involved methods used earlier on inferior predictions. We present results on three diverse EM datasets, achieving relative improvements over previous results of 27\%, 15\%, and 250\%. Our findings suggest that a single method can be applied to both nearly isotropic block-face EM data and anisotropic serial sectioned EM data. The runtime of our method scales linearly with the size of the volume and achieves a throughput of ~2.6 seconds per megavoxel, qualifying our method for the processing of very large datasets.

}, doi = {10.1109/TPAMI.2018.2835450}, author = {Funke, Jan and Tschopp, Fabian D and Grisaitis, William and Sheridan, Arlo and Singh, Chandan and Saalfeld, Stephan and Turaga, Srinivas C} } @article {49531, title = {Analyzing image segmentation for connectomics.}, journal = {Frontiers in Neural Circuits}, volume = {12}, year = {2018}, month = {2018}, pages = {102}, abstract = {

Automatic image segmentation is critical to scale up electron microscope (EM) connectome reconstruction. To this end, segmentation competitions, such as CREMI and SNEMI, exist to help researchers evaluate segmentation algorithms with the goal of improving them. Because generating ground truth is time-consuming, these competitions often fail to capture the challenges in segmenting larger datasets required in connectomics. More generally, the common metrics for EM image segmentation do not emphasize impact on downstream analysis and are often not very useful for isolating problem areas in the segmentation. For example, they do not capture connectivity information and often over-rate the quality of a segmentation as we demonstrate later. To address these issues, we introduce a novel strategy to enable evaluation of segmentation at large scales both in a supervised setting, where ground truth is available, or an unsupervised setting. To achieve this, we first introduce new metrics more closely aligned with the use of segmentation in downstream analysis and reconstruction. In particular, these include synapse connectivity and completeness metrics that provide both meaningful and intuitive interpretations of segmentation quality as it relates to the preservation of neuron connectivity. Also, we propose measures of segmentation correctness and completeness with respect to the percentage of "orphan" fragments and the concentrations of self-loops formed by segmentation failures, which are helpful in analysis and can be computed without ground truth. The introduction of new metrics intended to be used for practical applications involving large datasets necessitates a scalable software ecosystem, which is a critical contribution of this paper. To this end, we introduce a scalable, flexible software framework that enables integration of several different metrics and provides mechanisms to evaluate and debug differences between segmentations. We also introduce visualization software to help users to consume the various metrics collected. We evaluate our framework on two relatively large public groundtruth datasets providing novel insights on example segmentations.

}, issn = {1662-5110}, doi = {10.3389/fncir.2018.00102}, author = {Plaza, Stephen M and Funke, Jan} } @article {49220, title = {Synaptic cleft segmentation in non-isotropic volume electron microscopy of the complete Drosophila brain.}, journal = {Medical Image Computing and Computer Assisted Intervention {\textendash} MICCAI 2018}, year = {2018}, month = {2018 Sep 26}, pages = {317-25}, abstract = {

Neural circuit reconstruction at single synapse resolution is increasingly recognized as crucially important to decipher the function of biological nervous systems. Volume electron microscopy in serial transmission or scanning mode has been demonstrated to provide the necessary resolution to segment or trace all neurites and to annotate all synaptic connections.\ 
Automatic annotation of synaptic connections has been done successfully in near isotropic electron microscopy of vertebrate model organisms. Results on non-isotropic data in insect models, however, are not yet on par with human annotation.\ 
We designed a new 3D-U-Net architecture to optimally represent isotropic fields of view in non-isotropic data. We used regression on a signed distance transform of manually annotated synaptic clefts of the CREMI challenge dataset to train this model and observed significant improvement over the state of the art.\ 
We developed open source software for optimized parallel prediction on very large volumetric datasets and applied our model to predict synaptic clefts in a 50 tera-voxels dataset of the complete Drosophila brain. Our model generalizes well to areas far away from where training data was available.

}, doi = {10.1007/978-3-030-00934-2_36}, author = {Heinrich, Larissa and Funke, Jan and Pape, Constantin and Nunez-Iglesias, Juan and Saalfeld, Stephan} } @article {49292, title = {Synaptic partner prediction from point annotations in insect brains.}, journal = {MICCAI 2018: Medical Image Computing and Computer Assisted Intervention}, year = {2018}, month = {2018 Sep 26}, abstract = {

High-throughput electron microscopy allows recording of lar- ge stacks of neural tissue with sufficient resolution to extract the wiring diagram of the underlying neural network. Current efforts to automate this process focus mainly on the segmentation of neurons. However, in order to recover a wiring diagram, synaptic partners need to be identi- fied as well. This is especially challenging in insect brains like Drosophila melanogaster, where one presynaptic site is associated with multiple post- synaptic elements. Here we propose a 3D U-Net architecture to directly identify pairs of voxels that are pre- and postsynaptic to each other. To that end, we formulate the problem of synaptic partner identification as a classification problem on long-range edges between voxels to encode both the presence of a synaptic pair and its direction. This formulation allows us to directly learn from synaptic point annotations instead of more ex- pensive voxel-based synaptic cleft or vesicle annotations. We evaluate our method on the MICCAI 2016 CREMI challenge and improve over the current state of the art, producing 3\% fewer errors than the next best method.

}, doi = {10.1007/978-3-030-00934-2_35}, author = {Buhmann, Julia and Krause, Renate and Lentini, Rodrigo Ceballos and Eckstein, Nils and Cook, Matthew and Turaga, Srinivas C and Funke, Jan} } @article {48647, title = {The candidate multi-cut for cell segmentation.}, journal = {2018 IEEE 15th International Symposium on Biomedical Imaging (ISBI 2018)}, year = {2017}, month = {2017 Jul 04}, abstract = {

Two successful approaches for the segmentation of biomedical images are (1) the selection of segment candidates from a merge-tree, and (2) the clustering of small superpixels by solving a Multi-Cut problem. In this paper, we introduce a model that unifies both approaches. Our model, the Candidate Multi-Cut (CMC), allows joint selection and clustering of segment candidates from a merge-tree. This way, we overcome the respective limitations of the individual methods: (1) the space of possible segmentations is not constrained to candidates of a merge-tree, and (2) the decision for clustering can be made on candidates larger than superpixels, using features over larger contexts. We solve the optimization problem of selecting and clustering of candidates using an integer linear program. On datasets of 2D light microscopy of cell populations and 3D electron microscopy of neurons, we show that our method generalizes well and generates more accurate segmentations than merge-tree or Multi-Cut methods alone.

}, doi = {10.1109/ISBI.2018.8363658}, author = {Funke, Jan and Zhang, Chong and Pietzsch, Tobias and Gonzalez Ballester, Miguel A and Saalfeld, Stephan} } @article {48193, title = {TED: A Tolerant Edit Distance for segmentation evaluation.}, journal = {Methods}, volume = {115}, year = {2017}, month = {2017 Jan 17}, pages = {119-27}, abstract = {

In this paper, we present a novel error measure to compare a computer-generated segmentation of images or volumes against ground truth. This measure, which we call Tolerant Edit Distance (TED), is motivated by two observations that we usually encounter in biomedical image processing: (1) Some errors, like small boundary shifts, are tolerable in practice. Which errors are tolerable is application dependent and should be explicitly expressible in the measure. (2) Non-tolerable errors have to be corrected manually. The effort needed to do so should be reflected by the error measure. Our measure is the minimal weighted sum of split and merge operations to apply to one segmentation such that it resembles another segmentation within specified tolerance bounds. This is in contrast to other commonly used measures like Rand index or variation of information, which integrate small, but tolerable, differences. Additionally, the TED provides intuitive numbers and allows the localization and classification of errors in images or volumes. We demonstrate the applicability of the TED on 3D segmentations of neurons in electron microscopy images where topological correctness is arguable more important than exact boundary locations. Furthermore, we show that the TED is not just limited to evaluation tasks. We use it as the loss function in a max-margin learning framework to find parameters of an automatic neuron segmentation algorithm. We show that training to minimize the TED, i.e., to minimize crucial errors, leads to higher segmentation accuracy compared to other learning methods.

}, doi = {10.1016/j.ymeth.2016.12.013}, author = {Funke, Jan and Klein, Jonas and Moreno-Noguer, Francesc and Cardona, Albert and Cook, Matthew} } @conference {47736, title = {Efficient convolutional neural networks for pixelwise classification on heterogeneous hardware systems.}, booktitle = { IEEE 13th International Symposium on Biomedical Imaging: From Nano to Macro}, year = {2016}, month = {2016 Jun 15}, address = {Clarion Congress Hotel in Prague Prague; Czech Republic}, abstract = {

With recent advances in high-throughput Electron Microscopy (EM) imaging it is now possible to image an entire nervous system of organisms like Drosophila melanogaster. One of the bottlenecks to reconstruct a connectome from these large volumes ({\oe} 100 TiB) is the pixel-wise prediction of membranes. The time it would typically take to process such a volume using a convolutional neural network (CNN) with a sliding window approach is in the order of years on a current GPU. With sliding windows, however, a lot of redundant computations are carried out. In this paper, we present an extension to the Caffe library to increase throughput by predicting many pixels at once. On a sliding window network successfully used for membrane classification, we show that our method achieves a speedup of up to 57{\texttimes}, maintaining identical prediction results.

}, isbn = {978-147992350-2}, doi = {10.1109/ISBI.2016.7493487}, author = {Tschopp, F and Martel, J N P and Turaga, Srinivas C and Cook, M and Funke, Jan} } @article {47560, title = {Structured learning of assignment models for neuron reconstruction to minimize topological errors.}, journal = {IEEE 13th International Symposium on Biomedical Imaging (ISBI)}, year = {2016}, month = {2016 Ap 13}, pages = {607-11}, abstract = {

Structured learning provides a powerful framework for empirical risk minimization on the predictions of structured models. It allows end-to-end learning of model parameters to minimize an application specific loss function. This framework is particularly well suited for discrete optimization models that are used for neuron reconstruction from anisotropic electron microscopy (EM) volumes. However, current methods are still learning unary potentials by training a classifier that is agnostic about the model it is used in. We believe the reason for that lies in the difficulties of (1) finding a representative training sample, and (2) designing an application specific loss function that captures the quality of a proposed solution. In this paper, we show how to find a representative training sample from human generated ground truth, and propose a loss function that is suitable to minimize topological errors in the reconstruction. We compare different training methods on two challenging EM-datasets. Our structured learning approach shows consistently higher reconstruction accuracy than other current learning methods.

}, doi = {10.1109/ ISBI.2016.7493341}, author = {Funke, Jan and Klein, Jonas and Moreno-Noguer, Francesc and Cardona, Albert and Cook, Matthew} } @inbook {47184, title = {Who is talking to whom: Synaptic partner detection in anisotropic volumes of insect brain.}, booktitle = {Medical Image Computing and Computer-Assisted Intervention -- MICCAI 2015}, number = {9349}, year = {2015}, pages = {661-8}, abstract = {

Automated reconstruction of neural connectivity graphs from electron microscopy image stacks is an essential step towards large-scale neural circuit mapping. While significant progress has recently been made in automated segmentation of neurons and detection of synapses, the problem of synaptic partner assignment for polyadic (one-to-many) synapses, prevalent in the\ Drosophila\ brain, remains unsolved. In this contribution, we propose a method which automatically assigns pre- and postsynaptic roles to neurites adjacent to a synaptic site. The method constructs a probabilistic graphical model over potential synaptic partner pairs which includes factors to account for a high rate of one-to-many connections, as well as the possibility of the same neuron to be pre-synaptic in one synapse and post-synaptic in another. The algorithm has been validated on a publicly available stack of ssTEM images of\ Drosophila\ neural tissue and has been shown to reconstruct most of the synaptic relations correctly.

}, doi = {10.1007/978-3-319-24553-9_81}, author = {Kreshuk, Anna and Funke, Jan and Cardona, Albert and Hamprecht, Fred A} } @conference {5685, title = {Efficient automatic 3D-reconstruction of branching neurons from EM data.}, booktitle = {Computer Vision and Pattern Recognition (CVPR), 2012 IEEE Conference on}, year = {2012}, month = {2012}, abstract = {

We present an approach for the automatic reconstruction of neurons from 3D stacks of electron microscopy sections. The core of our system is a set of possible assignments, each of which proposes with some cost a link between neuron regions in consecutive sections. These can model the continuation, branching, and end of neurons. The costs are trainable on positive assignment samples. An optimal and consistent set of assignments is found for the whole volume at once by solving an integer linear program. This set of assignments determines both the segmentation into neuron regions and the correspondence between such regions in neighboring slices. For each picked assignment, a confidence value helps to prioritize decisions to be reviewed by a human expert. We evaluate the performance of our method on an annotated volume of neural tissue and compare to the current state of the art [26]. Our method is superior in accuracy and can be trained using a small number of samples. The observed inference times are linear with about 2 milliseconds per neuron and section.

}, author = {Funke, Jan and Andres, Bjoern and Hamprecht, Fred and Cardona, Albert and Cook, Matthew} }