@article {69301, title = {Motor neurons generate pose-targeted movements via proprioceptive sculpting.}, journal = {Nature}, year = {2024}, month = {2024 Mar 20}, abstract = {

Motor neurons are the final common pathway through which the brain controls movement of the body, forming the basic elements from which all movement is composed. Yet how a single motor neuron contributes to control during natural movement remains unclear. Here we anatomically and functionally characterize the individual roles of the motor neurons that control head movement in the fly, Drosophila melanogaster. Counterintuitively, we find that activity in a single motor neuron rotates the head in different directions, depending on the starting posture of the head, such that the head converges towards a pose determined by the identity of the stimulated motor neuron. A feedback model predicts that this convergent behaviour results from motor neuron drive interacting with proprioceptive feedback. We identify and genetically suppress a single class of proprioceptive neuron that changes the motor neuron-induced convergence as predicted by the feedback model. These data suggest a framework for how the brain controls movements: instead of directly generating movement in a given direction by activating a fixed set of motor neurons, the brain controls movements by adding bias to a continuing proprioceptive-motor loop.

}, issn = {1476-4687}, doi = {10.1038/s41586-024-07222-5}, author = {Gorko, Benjamin and Siwanowicz, Igor and Close, Kari and Christoforou, Christina and Hibbard, Karen L and Kabra, Mayank and Lee, Allen and Park, Jin-Yong and Li, Si Ying and Chen, Alex B and Namiki, Shigehiro and Chen, Chenghao and Tuthill, John C and Bock, Davi D and Rouault, Herv{\'e} and Branson, Kristin and Ihrke, Gudrun and Huston, Stephen J} } @article {65259, title = {Cortical pattern generation during dexterous movement is input-driven.}, journal = {Nature}, volume = {577}, year = {2020}, month = {2020 Jan 16}, pages = {386-91}, abstract = {

The motor cortex controls skilled arm movement by sending temporal patterns of activity to lower motor centres. Local cortical dynamics are thought to shape these patterns throughout movement execution. External inputs have been implicated in setting the initial state of the motor cortex, but they may also have a pattern-generating role. Here we dissect the contribution of local dynamics and inputs to cortical pattern generation during a prehension task in mice. Perturbing cortex to an aberrant state prevented movement initiation, but after the perturbation was released, cortex either bypassed the normal initial state and immediately generated the pattern that controls reaching or failed to generate this pattern. The difference in these two outcomes was probably a result of external inputs. We directly investigated the role of inputs by inactivating the thalamus; this perturbed cortical activity and disrupted limb kinematics at any stage of the movement. Activation of thalamocortical axon terminals at different frequencies disrupted cortical activity and arm movement in a graded manner. Simultaneous recordings revealed that both thalamic activity and the current state of cortex predicted changes in cortical activity. Thus, the pattern generator for dexterous arm movement is distributed across multiple, strongly interacting brain regions.

}, issn = {1476-4687}, doi = {10.1038/s41586-019-1869-9}, author = {Sauerbrei, Britton A and Guo, Jian-Zhong and Cohen, Jeremy D and Mischiati, Matteo and Guo, Wendy and Kabra, Mayank and Verma, Nakul and Mensh, Brett and Branson, Kristin and Hantman, Adam W} } @conference {65612, title = {Detecting the Starting Frame of Actions in Video}, booktitle = {2020 IEEE Winter Conference on Applications of Computer Vision (WACV)}, year = {2020}, month = {2020 May 14}, publisher = {IEEE}, organization = {IEEE}, address = {Snowmass Village, CO, USA}, abstract = {

In this work, we address the problem of precisely localizing key frames of an action, for example, the precise time that a pitcher releases a baseball, or the precise time that a crowd begins to applaud. Key frame localization is a largely overlooked and important action-recognition problem, for example in the field of neuroscience, in which we would like to understand the neural activity that produces the start of a bout of an action. To address this problem, we introduce a novel structured loss function that properly weights the types of errors that matter in such applications: it more heavily penalizes extra and missed action start detections over small misalignments. Our structured loss is based on the best matching between predicted and labeled action starts. We train recurrent neural networks (RNNs) to minimize differentiable approximations of this loss. To evaluate these methods, we introduce the Mouse Reach Dataset, a large, annotated video dataset of mice performing a sequence of actions. The dataset was collected and labeled by experts for the purpose of neuroscience research. On this dataset, we demonstrate that our method outperforms related approaches and baseline methods using an unstructured loss.

}, doi = {10.1109/WACV45572.202010.1109/WACV45572.2020.9093405}, url = {https://ieeexplore.ieee.org/xpl/mostRecentIssue.jsp?punumber=9087828}, author = {Kwak, Iljung S. and Guo, Jian-Zhong and Hantman, Adam and Branson, Kristin and Kriegman, David} } @article {65090, title = {An automatic behavior recognition system classifies animal behaviors using movements and their temporal context.}, journal = {Journal of Neuroscience Methods}, volume = {326}, year = {2019}, month = {2019 Aug 12}, pages = {108352}, abstract = {

Animals can perform complex and purposeful behaviors by executing simpler movements in flexible sequences. It is particularly challenging to analyze behavior sequences when they are highly variable, as is the case in language production, certain types of birdsong and, as in our experiments, flies grooming. High sequence variability necessitates rigorous quantification of large amounts of data to identify organizational principles and temporal structure of such behavior. To cope with large amounts of data, and minimize human effort and subjective bias, researchers often use automatic behavior recognition software. Our standard grooming assay involves coating flies in dust and videotaping them as they groom to remove it. The flies move freely and so perform the same movements in various orientations. As the dust is removed, their appearance changes. These conditions make it difficult to rely on precise body alignment and anatomical landmarks such as eyes or legs and thus present challenges to existing behavior classification software. Human observers use speed, location, and shape of the movements as the diagnostic features of particular grooming actions. We applied this intuition to design a new automatic behavior recognition system (ABRS) based on spatiotemporal features in the video data, heavily weighted for temporal dynamics and invariant to the animal{\textquoteright}s position and orientation in the scene. We use these spatiotemporal features in two steps of supervised classification that reflect two time-scales at which the behavior is structured. As a proof of principle, we show results from quantification and analysis of a large data set of stimulus-induced fly grooming behaviors that would have been difficult to assess in a smaller dataset of human-annotated ethograms. While we developed and validated this approach to analyze fly grooming behavior, we propose that the strategy of combining alignment-invariant features and multi-timescale analysis may be generally useful for movement-based classification of behavior from video data.

}, issn = {01650270}, doi = {10.1016/j.jneumeth.2019.108352}, author = {Ravbar, Primoz and Branson, Kristin and Simpson, Jie H.} } @article {65179, title = {Computational neuroethology: A call to action.}, journal = {Neuron}, volume = {104}, year = {2019}, month = {2019 Oct 09}, pages = {11-24}, abstract = {

The brain is worthy of study because it is in charge of behavior. A flurry of recent technical advances in measuring and quantifying naturalistic behaviors provide an important opportunity for advancing brain science. However, the problem of understanding unrestrained behavior in the context of neural recordings and manipulations remains unsolved, and developing approaches to addressing this challenge is critical. Here we discuss considerations in computational neuroethology-the science of quantifying naturalistic behaviors for understanding the brain-and propose strategies to evaluate progress. We point to open questions that require resolution and call upon the broader systems neuroscience community to further develop and leverage measures of naturalistic, unrestrained behavior, which will enable us to more effectively probe the richness and complexity of the brain.

}, issn = {1097-4199}, doi = {10.1016/j.neuron.2019.09.038}, author = {Datta, Sandeep Robert and Anderson, David J and Branson, Kristin and Perona, Pietro and Leifer, Andrew} } @article {65050, title = {State-dependent decoupling of sensory and motor circuits underlies behavioral flexibility in Drosophila.}, journal = {Nature Neuroscience}, volume = {22}, year = {2019}, month = {2019 Jul 01}, pages = {1132-1139}, abstract = {

An approaching predator and self-motion toward an object can generate similar looming patterns on the retina, but these situations demand different rapid responses. How central circuits flexibly process visual cues to activate appropriate, fast motor pathways remains unclear. Here we identify two descending neuron (DN) types that control landing and contribute to visuomotor flexibility in Drosophila. For each, silencing impairs visually evoked landing, activation drives landing, and spike rate determines leg extension amplitude. Critically, visual responses of both DNs are severely attenuated during non-flight periods, effectively decoupling visual stimuli from the landing motor pathway when landing is inappropriate. The flight-dependence mechanism differs between DN types. Octopamine exposure mimics flight effects in one, whereas the other probably receives neuronal feedback from flight motor circuits. Thus, this sensorimotor flexibility arises from distinct mechanisms for gating action-specific descending pathways, such that sensory and motor networks are coupled or decoupled according to the behavioral state.

}, issn = {1546-1726}, doi = {10.1038/s41593-019-0413-4}, author = {Ache, Jan M and Namiki, Shigehiro and Lee, Allen and Branson, Kristin and Card, Gwyneth M} } @article {49127, title = {A deep (learning) dive into a cell.}, journal = {Nature Methods}, volume = {15}, year = {2018}, month = {2018 Apr 03}, pages = {253-4}, issn = {1548-7105}, doi = {10.1038/nmeth.4658}, author = {Branson, Kristin} } @article {49466, title = {In toto imaging and reconstruction of post-implantation mouse development at the single-cell level.}, journal = {Cell}, volume = {175}, year = {2018}, month = {2018 Oct 10}, pages = {859-876}, abstract = {

The mouse embryo has long been central to the study of mammalian development; however, elucidating the cell behaviors governing gastrulation and the formation of tissues and organs remains a fundamental challenge. A major obstacle is the lack of live imaging and image analysis technologies capable of systematically following cellular dynamics across the developing embryo. We developed a light-sheet microscope that adapts itself to the dramatic changes in size, shape, and optical properties of the post-implantation\ mouse embryo and captures its development from gastrulation to early organogenesis at\ the cellular level. We furthermore developed a computational framework for reconstructing long-term cell tracks, cell divisions, dynamic fate maps, and maps of tissue morphogenesis across the entire embryo. By jointly analyzing cellular dynamics in multiple embryos registered in space and time, we built a dynamic atlas of post-implantation mouse development that, together with our microscopy and computational methods, is provided as a resource.

}, issn = {1097-4172}, doi = {10.1016/j.cell.2018.09.031}, author = {McDole, Katie and Guignard, Leo and Amat, Fernando and Berger, Andrew and Malandain, Gr{\'e}goire and Royer, Lo{\"\i}c A and Turaga, Srinivas C and Branson, Kristin and Keller, Philipp J} } @conference {49417, title = {Multiple animals tracking in video using part affinity fields}, booktitle = {Visual observation and analysis of Vertebrate And Insect Behavior 2018}, year = {2018}, month = {2018 Aug 20}, abstract = {

In this work, we address the problem of pose detection and tracking of multiple individuals for the study of behaviour in insects and animals. Using a Deep Neural Network architecture, precise detection and association of the body parts can be performed. The models are learned based on user-annotated training videos, which gives flexibility to the approach. This is illustrated on two different animals: honeybees and mice, where very good performance in part recognition and association are observed despite the presence of multiple interacting individuals.

}, url = {http://homepages.inf.ed.ac.uk/rbf/VAIB18PAPERS/vaib18rodriguez.pdf}, author = {Rodriguez, Ivan F and Megret, Remi and Egnor, Roian and Branson, Kristin and Agosto, Jose L and Giray, Tugrul and Acuna, Edgar} } @article {48160, title = {Machine vision methods for analyzing social interactions.}, journal = {The Journal of Experimental Biology}, volume = {220}, year = {2017}, month = {2017 Jan 01}, pages = {25-34}, abstract = {

Recent developments in machine vision methods for automatic, quantitative analysis of social behavior have immensely improved both the scale and level of resolution with which we can dissect interactions between members of the same species. In this paper, we review these methods, with a particular focus on how biologists can apply them to their own work. We discuss several components of machine vision-based analyses: methods to record high-quality video for automated analyses, video-based tracking algorithms for estimating the positions of interacting animals, and machine learning methods for recognizing patterns of interactions. These methods are extremely general in their applicability, and we review a subset of successful applications of them to biological questions in several model systems with very different types of social behaviors.

}, issn = {1477-9145}, doi = {10.1242/jeb.142281}, author = {Robie, Alice A and Seagraves, Kelly M and Egnor, S E Roian and Branson, Kristin} } @article {48279, title = {Moonwalker descending neurons mediate visually evoked retreat in Drosophila.}, journal = {Current Biology : CB}, volume = {27}, year = {2017}, month = {2017 Mar 6}, pages = {766-71}, abstract = {

Insects, like most animals, tend to steer away from imminent threats [1-7]. Drosophila melanogaster, for example, generally initiate an escape take-off in response to a looming visual stimulus, mimicking a potential predator [8]. The escape response to a visual threat is, however, flexible [9-12] and can alternatively consist of walking backward away from the perceived threat [11], which may be a more effective response to ambush predators such as nymphal praying mantids [7]. Flexibility in escape behavior may also add an element of unpredictability that makes it difficult for predators to anticipate or learn the prey{\textquoteright}s likely response [3-6]. Whereas the fly{\textquoteright}s escape jump has been well studied [8, 9, 13-18], the neuronal underpinnings of evasive walking remain largely unexplored. We previously reported the identification of a cluster of descending neurons-the moonwalker descending neurons (MDNs)-the activity of which is necessary and sufficient to trigger backward walking [19], as well as a population of visual projection neurons-the lobula columnar 16 (LC16) cells-that respond to looming visual stimuli and elicit backward walking and turning [11]. Given the similarity of their activation phenotypes, we hypothesized that LC16 neurons induce backward walking via MDNs and that turning while walking backward might reflect asymmetric activation of the left and right MDNs. Here, we present data from functional imaging, behavioral epistasis, and unilateral activation experiments that support these hypotheses. We conclude that LC16 and MDNs are critical components of the neural circuit that transduces threatening visual stimuli into directional locomotor output.

}, issn = {1879-0445}, doi = {10.1016/j.cub.2017.02.008}, author = {Sen, Rajyashree and Wu, Ming and Branson, Kristin and Robie, Alice and Rubin, Gerald M and Dickson, Barry J} } @article {47433, title = {Computational Analysis of Behavior.}, journal = {Annual Review of Neuroscience}, volume = {39}, year = {2016}, month = {2016 Apr 18}, pages = {217-36}, abstract = {

In this review, we discuss the emerging field of computational behavioral analysis-the use of modern methods from computer science and engineering to quantitatively measure animal behavior. We discuss aspects of experiment design important to both obtaining biologically relevant behavioral data and enabling the use of machine vision and learning techniques for automation. These two goals are often in conflict. Restraining or restricting the environment of the animal can simplify automatic behavior quantification, but it can also degrade the quality or alter important aspects of behavior. To enable biologists to design experiments to obtain better behavioral measurements, and computer scientists to pinpoint fruitful directions for algorithm improvement, we review known effects of artificial manipulation of the animal on behavior. We also review machine vision and learning techniques for tracking, feature extraction, automated behavior classification, and automated behavior discovery, the assumptions they make, and the types of data they work best with. Expected final online publication date for the Annual Review of Neuroscience Volume 39 is July 08, 2016. Please see http://www.annualreviews.org/catalog/pubdates.aspx for revised estimates.

}, issn = {1545-4126}, doi = {10.1146/annurev-neuro-070815-013845}, author = {Egnor, S E Roian and Branson, Kristin} } @article {48132, title = {An empirical analysis of deep network loss surfaces.}, journal = {arXiv}, year = {2016}, month = {2016 Dec 13}, pages = {arXiv:1612.04010}, abstract = {

The training of deep neural networks is a high-dimension optimization problem with respect to the loss function of a model. Unfortunately, these functions are of high dimension and non-convex and hence difficult to characterize. In this paper, we empirically investigate the geometry of the loss functions for state-of-the-art networks with multiple stochastic optimization methods. We do this through several experiments that are visualized on polygons to understand how and when these stochastic optimization methods find minima.

}, url = {https://arxiv.org/pdf/1612.04010.pdf}, author = {Im, Daniel Jiwoong and Tao, Michael and Branson, Kristin} } @article {48005, title = {Learning recurrent representations for hierarchical behavior modeling.}, journal = {arXiv}, volume = {arXiv:1611.00094}, year = {2016}, month = {2016 Nov 1}, abstract = {

We propose a framework for detecting action patterns from motion sequences and modeling the sensory-motor relationship of animals, using a generative recurrent neural network. The network has a discriminative part (classifying actions) and a generative part (predicting motion), whose recurrent cells are laterally connected, allowing higher levels of the network to represent high level phenomena. We test our framework on two types of data, fruit fly behavior and online handwriting. Our results show that 1) taking advantage of unlabeled sequences, by predicting future motion, significantly improves action detection performance when training labels are scarce, 2) the network learns to represent high level phenomena such as writer identity and fly gender, without supervision, and 3) simulated motion trajectories, generated by treating motion prediction as input to the network, look realistic and may be used to qualitatively evaluate whether the model has learnt generative control rules.

}, author = {Eyjolfsdottir, Eyrun and Branson, Kristin and Yue, Yisong and Perona, Pietro} } @article {47277, title = {Cortex commands the performance of skilled movement.}, journal = {eLife}, volume = {4}, year = {2015}, month = {2015 Dec 3}, abstract = {

Mammalian cerebral cortex is accepted as being critical for voluntary motor control, but what functions depend on cortex is still unclear. Here we used rapid, reversible optogenetic inhibition to test the role of cortex during a head-fixed task in which mice reach, grab, and eat a food pellet. Sudden cortical inhibition blocked initiation or froze execution of this skilled prehension behavior, but left untrained forelimb movements unaffected. Unexpectedly, kinematically normal prehension occurred immediately after cortical inhibition even during rest periods lacking cue and pellet. This {\textquoteright}rebound{\textquoteright} prehension was only evoked in trained and food-deprived animals, suggesting that a motivation-gated motor engram sufficient to evoke prehension is activated at inhibition{\textquoteright}s end. These results demonstrate the necessity and sufficiency of cortical activity for enacting a learned skill.

}, issn = {2050-084X}, doi = {10.7554/eLife.10774}, author = {Guo, Jian-Zhong and Graves, Austin R and Guo, Wendy W and Zheng, Jihong and Lee, Allen and Rodr{\'\i}guez-Gonz{\'a}lez, Juan and Li, Nuo and Macklin, John J and Phillips, James W and Mensh, Brett D and Branson, Kristin and Hantman, Adam W} } @article {47127, title = {Imaging the neural basis of locomotion.}, journal = {Cell}, volume = {163}, year = {2015}, month = {2015 Oct 22}, pages = {541-2}, abstract = {

To investigate the fundamental question of how nervous systems encode, organize, and sequence behaviors, Kato et\ al. imaged neural activity with cellular resolution across the brain of the worm Caenorhabditis elegans. Locomotion behavior seems to be continuously represented by cyclical patterns of distributed neural activity that are present even in immobilized animals.

}, issn = {1097-4172}, doi = {10.1016/j.cell.2015.10.014}, author = {Branson, Kristin and Freeman, Jeremy} } @article {45098, title = {Automated image-based tracking and its application in ecology.}, journal = {Trends in Ecology and Evolution}, volume = {29}, year = {2014}, month = {2014 Jul}, pages = {417-428}, abstract = {

The behavior of individuals determines the strength and outcome of ecological interactions, which drive population, community, and ecosystem organization. Bio-logging, such as telemetry and animal-borne imaging, provides essential individual viewpoints, tracks, and life histories, but requires capture of individuals and is often impractical to scale. Recent developments in automated image-based tracking offers opportunities to remotely quantify and understand individual behavior at scales and resolutions not previously possible, providing an essential supplement to other tracking methodologies in ecology. Automated image-based tracking should continue to advance the field of ecology by enabling better understanding of the linkages between individual and higher-level ecological processes, via high-throughput quantitative analysis of complex ecological patterns and processes across scales, including analysis of environmental drivers.

}, issn = {1872-8383}, doi = {10.1016/j.tree.2014.05.004}, author = {Dell, Anthony I and Bender, John A and Branson, Kristin and Couzin, Iain D and de Polavieja, Gonzalo G and Noldus, Lucas P J J and P{\'e}rez-Escudero, Alfonso and Perona, Pietro and Straw, Andrew D and Wikelski, Martin and Brose, Ulrich} } @article {45458, title = {Behavioral variability through stochastic choice and its gating by anterior cingulate cortex.}, journal = {Cell}, volume = {159}, year = {2014}, month = {2014 Sep 25}, pages = {21-32}, abstract = {

Behavioral choices that ignore prior experience promote exploration and unpredictability but are seemingly at odds with the brain{\textquoteright}s tendency to use experience to optimize behavioral choice. Indeed, when faced with virtual competitors, primates resort to strategic counterprediction rather than to stochastic choice. Here, we show that rats also use history- and model-based strategies when faced with similar competitors but can switch to a "stochastic" mode when challenged with a competitor that they cannot defeat by counterprediction. In this mode, outcomes associated with an animal{\textquoteright}s actions are ignored, and normal engagement of anterior cingulate cortex (ACC) is suppressed. Using circuit perturbations in\ transgenic rats, we demonstrate that switching between strategic and stochastic behavioral modes is controlled by locus coeruleus input into ACC. Our\ findings suggest that, under conditions of uncertainty about environmental rules, changes in noradrenergic input alter ACC output and prevent erroneous beliefs from guiding decisions, thus enabling behavioral variation.

}, issn = {1097-4172}, doi = {10.1016/j.cell.2014.08.037}, author = {Tervo, Dougal G R and Proskurin, Mikhail and Manakov, Maxim and Kabra, Mayank and Vollmer, Alison and Branson, Kristin and Karpova, Alla Y} } @article {45052, title = {Distinguishing seemingly indistinguishable animals with computer vision.}, journal = {Nature Methods}, volume = {11}, year = {2014}, month = {2014 Jun 27}, pages = {721-2}, abstract = {

A general method to recognize and track unmarked animals within a population will enable new studies of social behavior and individuality.

}, issn = {1548-7105}, doi = {10.1038/nmeth.3004}, author = {Branson, Kristin} } @article {45058, title = {Fast, accurate reconstruction of cell lineages from large-scale fluorescence microscopy data.}, journal = {Nature Methods}, volume = {11}, year = {2014}, month = {2014 Jul 20}, pages = {951-8}, abstract = {

The comprehensive reconstruction of cell lineages in complex multicellular organisms is a central goal of developmental biology. We present an open-source computational framework for the segmentation and tracking of cell nuclei with high accuracy and speed. We demonstrate its (i) generality by reconstructing cell lineages in four-dimensional, terabyte-sized image data sets of fruit fly, zebrafish and mouse embryos acquired with three types of fluorescence microscopes, (ii) scalability by analyzing advanced stages of development with up to 20,000 cells per time point at 26,000 cells min(-1) on a single computer workstation and (iii) ease of use by adjusting only two parameters across all data sets and providing visualization and editing tools for efficient data curation. Our approach achieves on average 97.0\% linkage accuracy across all species and imaging modalities. Using our system, we performed the first cell lineage reconstruction of early Drosophila melanogaster nervous system development, revealing neuroblast dynamics throughout an entire embryo.

}, issn = {1548-7105}, doi = {10.1038/nmeth.3036}, author = {Amat, Fernando and Lemon, William and Mossing, Daniel P and McDole, Katie and Wan, Yinan and Branson, Kristin and Myers, Eugene W and Keller, Philipp J} } @article {5763, title = {JAABA: interactive machine learning for automatic annotation of animal behavior.}, journal = {Nature Methods}, volume = {10}, year = {2013}, month = {2012 Dec}, pages = {64-7}, abstract = {

We present a machine learning{\textendash}based system for automatically computing interpretable, quantitative measures of animal behavior. Through our interactive system, users encode their intuition about behavior by annotating a small set of video frames. These manual labels are converted into classifiers that can automatically annotate behaviors in screen-scale data sets. Our general-purpose system can create a variety of accurate individual and social behavior classifiers for different organisms, including mice and adult and larval Drosophila.

}, issn = {1548-7091}, author = {Kabra, Mayank and Robie, Alice A and Rivera-Alba, Marta and Branson, Steven and Branson, Kristin} } @proceedings {5767, title = {Learning animal social behavior from trajectory features.}, year = {2012}, month = {2012 Nov}, address = {Tsukuba, Japan}, author = {Eyjolfsdottir, Eyrun and Burgos-Artizzu, Xavier P. and Branson, Steven and Branson, Kristin and Anderson, David and Perona, Pietro} } @article {5036, title = {A simple strategy for detecting moving objects during locomotion revealed by animal-robot interactions.}, journal = {Current Biology}, volume = {22}, year = {2012}, month = {2012 Jun 20}, pages = {1344-50}, abstract = {

An important role of visual systems is to detect nearby predators, prey, and potential mates [1], which may be distinguished in part by their motion. When an animal is at rest, an object moving in any direction may easily be detected by motion-sensitive visual circuits [2, 3]. During locomotion,\ however, this strategy is compromised because the observer must detect a moving object within the pattern of\ optic flow created by its own motion through the stationary\ background. However, objects that move creating back-to-front (regressive) motion may be unambiguously distinguished from stationary objects because forward locomotion creates only front-to-back (progressive) optic flow. Thus, moving animals should exhibit an enhanced sensitivity to regressively moving objects. We explicitly tested this hypothesis by constructing a simple fly-sized robot that was programmed to interact with a real fly. Our measurements indicate that whereas walking female flies freeze in response to a regressively moving object, they ignore a progressively moving one. Regressive motion salience also explains observations of behaviors exhibited by pairs of walking flies. Because the assumptions underlying the regressive motion salience hypothesis are general, we suspect that the behavior we have observed in Drosophila may be widespread among eyed, motile organisms.

}, issn = {1879-0445}, doi = {10.1016/j.cub.2012.05.024}, author = {Zabala, Francisco and Polidoro, Peter and Robie, Alice A and Branson, Kristin and Perona, Pietro and Dickinson, Michael H} } @article {2161, title = {Multi-camera real-time three-dimensional tracking of multiple flying animals.}, journal = {Journal of the Royal Society, Interface}, volume = {8}, year = {2011}, month = {2011 Mar 6}, pages = {395-409}, abstract = {

Automated tracking of animal movement allows analyses that would not otherwise be possible by providing great quantities of data. The additional capability of tracking in real time{\textendash}with minimal latency{\textendash}opens up the experimental possibility of manipulating sensory feedback, thus allowing detailed explorations of the neural basis for control of behaviour. Here, we describe a system capable of tracking the three-dimensional position and body orientation of animals such as flies and birds. The system operates with less than 40 ms latency and can track multiple animals simultaneously. To achieve these results, a multi-target tracking algorithm was developed based on the extended Kalman filter and the nearest neighbour standard filter data association algorithm. In one implementation, an 11-camera system is capable of tracking three flies simultaneously at 60 frames per second using a gigabit network of nine standard Intel Pentium 4 and Core 2 Duo computers. This manuscript presents the rationale and details of the algorithms employed and shows three implementations of the system. An experiment was performed using the tracking system to measure the effect of visual contrast on the flight speed of Drosophila melanogaster. At low contrasts, speed is more variable and faster on average than at high contrasts. Thus, the system is already a useful tool to study the neurobiology and behaviour of freely flying animals. If combined with other techniques, such as {\textquoteright}virtual reality{\textquoteright}-type computer graphics or genetic manipulation, the tracking system would offer a powerful new way to investigate the biology of flying animals.

}, issn = {1742-5662}, doi = {10.1098/rsif.2010.0230}, author = {Straw, Andrew D and Branson, Kristin and Neumann, Titus R and Dickinson, Michael H} } @article {5758, title = {High-throughput ethomics in large groups of Drosophila.}, journal = {Nature Methods}, volume = {6}, year = {2009}, month = {2009 Jun}, pages = {451-7}, abstract = {

We present a camera-based method for automatically quantifying the individual and social behaviors of fruit flies, Drosophila melanogaster, interacting in a planar arena. Our system includes machine-vision algorithms that accurately track many individuals without swapping identities and classification algorithms that detect behaviors. The data may be represented as an ethogram that plots the time course of behaviors exhibited by each fly or as a vector that concisely captures the statistical properties of all behaviors displayed in a given period. We found that behavioral differences between individuals were consistent over time and were sufficient to accurately predict gender and genotype. In addition, we found that the relative positions of flies during social interactions vary according to gender, genotype and social environment. We expect that our software, which permits high-throughput screening, will complement existing molecular methods available in Drosophila, facilitating new investigations into the genetic and cellular basis of behavior.

}, keywords = {Animal, Animals, Artificial Intelligence, Automated, Behavior, Computer-Assisted, Drosophila melanogaster, Humans, Image Interpretation, Monitoring, Movement, Pattern Recognition, Physiologic, Posture, Social Behavior}, issn = {1548-7105}, doi = {10.1038/nmeth.1328}, author = {Branson, Kristin and Robie, Alice A and Bender, John and Perona, Pietro and Dickinson, Michael H} } @proceedings {5774, title = {Tracking multiple mouse contours (without too many samples).}, year = {2005}, month = {06/2005}, pages = {1039-46}, publisher = {IEEE}, address = {San Diego, CA}, abstract = {

We present a particle filtering algorithm for robustly tracking the contours of multiple deformable objects through severe occlusions. Our algorithm combines a multiple blob tracker with a contour tracker in a manner that keeps the required number of samples small. This is a natural combination because both algorithms have complementary strengths. The multiple blob tracker uses a natural multi-target model and searches a smaller and simpler space. On the other hand, contour tracking gives more fine-tuned results and relies on cues that are available during severe occlusions. Our choice of combination of these two algorithms accentuates the advantages of each. We demonstrate good performance on challenging video of three identical mice that contains multiple instances of severe occlusion.

}, author = {Branson, Kristin and Belongie, Serge} }