@inbook{455802cbbbd048e68b729d20440e2d8c,
title = "Measuring progress in robotics: benchmarking and the {\textquoteleft}measure-target confusion{\textquoteright}",
abstract = "While it is often said that in order to qualify as a true science robotics should aspire to reproducible and measurable results that allow benchmarking, I argue that a focus on benchmarking will be a hindrance for progress. Several academic disciplines that have been led into pursuing only reproducible and measurable {\textquoteleft}scientific{\textquoteright} results—robotics should be careful not to fall into that trap. Results that can be benchmarked must be specific and context-dependent, but robotics targets whole complex systems independently of a specific context—so working towards progress on the technical measure risks missing that target. It would constitute aiming for the measure rather than the target: what I call {\textquoteleft}measure-target confusion{\textquoteright}. The role of benchmarking in robotics shows that the more general problem to measure progress towards more intelligent machines will not be solved by technical benchmarks; we need a balanced approach with technical benchmarks, real-life testing and qualitative judgment.",
author = "M{\"u}ller, {Vincent C.}",
year = "2020",
month = jan,
day = "1",
doi = "10.1007/978-3-030-14126-4_9",
language = "English",
isbn = "978-3-030-14124-0",
series = "Cognitive Systems Monographs",
publisher = "Springer",
pages = "169--179",
editor = "Fabio Bonsignorio and Elena Messina and {del Pobil}, {Angel P.} and John Hallam",
booktitle = "Metrics of Sensory Motor Coordination and Integration in Robots and Animals",
address = "Germany",
}