+ {% if site.github.is_project_page %}
+ View on GitHub
+ {% endif %}
+ Read the Paper (arXiv)
+ {% if site.show_downloads %}
+ Download .zip
+ Download .tar.gz
+ {% endif %}
+
+
+
+ {{ content }}
+
+
+
+
+
diff --git a/paper/bcbench.bib b/paper/bcbench.bib
new file mode 100644
index 000000000..ef89a6b8e
--- /dev/null
+++ b/paper/bcbench.bib
@@ -0,0 +1,439 @@
+
+@misc{vijayaraghavan_vhdl-eval_2024,
+ title = {{VHDL}-{Eval}: {A} {Framework} for {Evaluating} {Large} {Language} {Models} in {VHDL} {Code} {Generation}},
+ shorttitle = {{VHDL}-{Eval}},
+ url = {http://arxiv.org/abs/2406.04379},
+ doi = {10.48550/arXiv.2406.04379},
+ abstract = {With the unprecedented advancements in Large Language Models (LLMs), their application domains have expanded to include code generation tasks across various programming languages. While significant progress has been made in enhancing LLMs for popular programming languages, there exists a notable gap in comprehensive evaluation frameworks tailored for Hardware Description Languages (HDLs), particularly VHDL. This paper addresses this gap by introducing a comprehensive evaluation framework designed specifically for assessing LLM performance in VHDL code generation task. We construct a dataset for evaluating LLMs on VHDL code generation task. This dataset is constructed by translating a collection of Verilog evaluation problems to VHDL and aggregating publicly available VHDL problems, resulting in a total of 202 problems. To assess the functional correctness of the generated VHDL code, we utilize a curated set of self-verifying testbenches specifically designed for those aggregated VHDL problem set. We conduct an initial evaluation of different LLMs and their variants, including zero-shot code generation, in-context learning (ICL), and Parameter-efficient fine-tuning (PEFT) methods. Our findings underscore the considerable challenges faced by existing LLMs in VHDL code generation, revealing significant scope for improvement. This study emphasizes the necessity of supervised fine-tuning code generation models specifically for VHDL, offering potential benefits to VHDL designers seeking efficient code generation solutions.},
+ urldate = {2026-05-14},
+ publisher = {arXiv},
+ author = {Vijayaraghavan, Prashanth and Shi, Luyao and Ambrogio, Stefano and Mackin, Charles and Nitsure, Apoorva and Beymer, David and Degan, Ehsan},
+ month = jun,
+ year = {2024},
+ note = {arXiv:2406.04379 [cs.SE]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Software Engineering},
+}
+
+@misc{jin_realbench_2025,
+ title = {{RealBench}: {Benchmarking} {Verilog} {Generation} {Models} with {Real}-{World} {IP} {Designs}},
+ copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International},
+ shorttitle = {{RealBench}},
+ url = {https://arxiv.org/abs/2507.16200},
+ doi = {10.48550/ARXIV.2507.16200},
+ abstract = {The automatic generation of Verilog code using Large Language Models (LLMs) has garnered significant interest in hardware design automation. However, existing benchmarks for evaluating LLMs in Verilog generation fall short in replicating real-world design workflows due to their designs’ simplicity, inadequate design specifications, and less rigorous verification environments. To address these limitations, we present RealBench, the first benchmark aiming at real-world IP-level Verilog generation tasks. RealBench features complex, structured, realworld open-source IP designs, multi-modal and formatted design specifications, and rigorous verification environments, including 100\% line coverage testbenches and a formal checker. It supports both module-level and system-level tasks, enabling comprehensive assessments of LLM capabilities. Evaluations on various LLMs and agents reveal that even one of the best-performing LLMs, o1-preview, achieves only a 13.3\% pass@1 on module-level tasks and 0\% on system-level tasks, highlighting the need for stronger Verilog generation models in the future.},
+ language = {en},
+ urldate = {2026-05-13},
+ publisher = {arXiv},
+ author = {Jin, Pengwei and Huang, Di and Li, Chongxiao and Cheng, Shuyao and Zhao, Yang and Zheng, Xinyao and Zhu, Jiaguo and Xing, Shuyi and Dou, Bohan and Zhang, Rui and Du, Zidong and Guo, Qi and Hu, Xing},
+ year = {2025},
+ note = {Version Number: 1},
+ keywords = {FOS: Computer and information sciences, Hardware Architecture (cs.AR), Machine Learning (cs.LG)},
+}
+
+@misc{pinckney_revisiting_2025,
+ title = {Revisiting {VerilogEval}: {A} {Year} of {Improvements} in {Large}-{Language} {Models} for {Hardware} {Code} {Generation}},
+ shorttitle = {Revisiting {VerilogEval}},
+ url = {http://arxiv.org/abs/2408.11053},
+ doi = {10.48550/arXiv.2408.11053},
+ abstract = {The application of large-language models (LLMs) to digital hardware code generation is an emerging field, with most LLMs primarily trained on natural language and software code. Hardware code like Verilog constitutes a small portion of training data, and few hardware benchmarks exist. The open-source VerilogEval benchmark, released in November 2023, provided a consistent evaluation framework for LLMs on code completion tasks. Since then, both commercial and open models have seen significant development. In this work, we evaluate new commercial and open models since VerilogEval's original release-including GPT-4o, GPT-4 Turbo, Llama3.1 (8B/70B/405B), Llama3 70B, Mistral Large, DeepSeek Coder (33B and 6.7B), CodeGemma 7B, and RTL-Coder-against an improved VerilogEval benchmark suite. We find measurable improvements in state-of-the-art models: GPT-4o achieves a 63\% pass rate on specification-to-RTL tasks. The recently released and open Llama3.1 405B achieves a 58\% pass rate, almost matching GPT-4o, while the smaller domain-specific RTL-Coder 6.7B models achieve an impressive 34\% pass rate. Additionally, we enhance VerilogEval's infrastructure by automatically classifying failures, introducing in-context learning support, and extending the tasks to specification-to-RTL translation. We find that prompt engineering remains crucial for achieving good pass rates and varies widely with model and task. A benchmark infrastructure that allows for prompt engineering and failure analysis is essential for continued model development and deployment.},
+ urldate = {2026-05-13},
+ publisher = {arXiv},
+ author = {Pinckney, Nathaniel and Batten, Christopher and Liu, Mingjie and Ren, Haoxing and Khailany, Brucek},
+ month = feb,
+ year = {2025},
+ note = {arXiv:2408.11053 [cs.AR]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Hardware Architecture},
+}
+
+@misc{liu_verilogeval_2023,
+ title = {{VerilogEval}: {Evaluating} {Large} {Language} {Models} for {Verilog} {Code} {Generation}},
+ shorttitle = {{VerilogEval}},
+ url = {http://arxiv.org/abs/2309.07544},
+ doi = {10.48550/arXiv.2309.07544},
+ abstract = {The increasing popularity of large language models (LLMs) has paved the way for their application in diverse domains. This paper proposes a benchmarking framework tailored specifically for evaluating LLM performance in the context of Verilog code generation for hardware design and verification. We present a comprehensive evaluation dataset consisting of 156 problems from the Verilog instructional website HDLBits. The evaluation set consists of a diverse set of Verilog code generation tasks, ranging from simple combinational circuits to complex finite state machines. The Verilog code completions can be automatically tested for functional correctness by comparing the transient simulation outputs of the generated design with a golden solution. We also demonstrate that the Verilog code generation capability of pretrained language models could be improved with supervised fine-tuning by bootstrapping with LLM generated synthetic problem-code pairs.},
+ urldate = {2026-05-13},
+ publisher = {arXiv},
+ author = {Liu, Mingjie and Pinckney, Nathaniel and Khailany, Brucek and Ren, Haoxing},
+ month = dec,
+ year = {2023},
+ note = {arXiv:2309.07544 [cs.LG]},
+ keywords = {Computer Science - Machine Learning, Computer Science - Software Engineering},
+}
+
+@misc{liang_swe-bench_2025,
+ title = {The {SWE}-{Bench} {Illusion}: {When} {State}-of-the-{Art} {LLMs} {Remember} {Instead} of {Reason}},
+ copyright = {Creative Commons Attribution Non Commercial No Derivatives 4.0 International},
+ shorttitle = {The {SWE}-{Bench} {Illusion}},
+ url = {https://arxiv.org/abs/2506.12286},
+ doi = {10.48550/ARXIV.2506.12286},
+ abstract = {As large language models (LLMs) become increasingly capable and widely adopted, benchmarks play a central role in assessing their practical utility. For example, SWE-Bench Verified has emerged as a critical benchmark for evaluating LLMs’ software engineering abilities, particularly their aptitude for resolving real-world GitHub issues. Recent LLMs show impressive performance on SWE-Bench Verified, leading to optimism about their capacity for complex coding tasks. However, current evaluation protocols may overstate these models’ true capabilities. It is crucial to distinguish LLMs’ generalizable problem-solving ability and other learned artifacts. In this work, we introduce two new diagnostic tasks: file path identification from issue descriptions alone and ground truth function reproduction with only the current file context and issue description to probe models’ underlying knowledge. We present empirical evidence that performance gains on SWE-Bench Verified may be partially driven by memorization rather than genuine problem-solving. We show that state-of-the-art (SoTA) models achieve up to 76\% accuracy in identifying buggy file paths using only issue descriptions, without access to repository structure. This performance is merely up to 53\% on tasks from repositories not included in SWE-Bench, pointing to possible data contamination or memorization. Similar patterns are also observed for the function reproduction task, where the verbatim similarity is much higher on SWE-Bench Verified than on other similar coding benchmarks (up to 35\% consecutive 5-gram overlap ratio on SWE-Bench Verified and Full, but only up to 18\% for tasks in other benchmarks). These findings raise concerns about the validity of existing results and underscore the need for more robust, contamination-resistant benchmarks to reliably evaluate LLMs’ coding abilities.},
+ language = {en},
+ urldate = {2026-05-04},
+ publisher = {arXiv},
+ author = {Liang, Shanchao and Garg, Spandan and Moghaddam, Roshanak Zilouchian},
+ year = {2025},
+ note = {Version Number: 4},
+ keywords = {Artificial Intelligence (cs.AI), FOS: Computer and information sciences, Software Engineering (cs.SE)},
+}
+
+@inproceedings{jimenez_swe-bench_2024,
+ title = {{SWE}-bench: Can Language Models Resolve Real-World {GitHub} Issues?},
+ author = {Jimenez, Carlos E. and Yang, John and Wettig, Alexander and Yao, Shunyu and Pei, Kexin and Press, Ofir and Narasimhan, Karthik},
+ booktitle = {The Twelfth International Conference on Learning Representations},
+ year = {2024},
+ note = {ICLR 2024 oral},
+ url = {https://openreview.net/forum?id=VTF8yNQM66},
+}
+
+@misc{biderman_lessons_2025,
+ title = {Lessons from the {Trenches} on {Reproducible} {Evaluation} of {Language} {Models}},
+ abstract = {Effective evaluation of language models remains an open challenge in NLP. Researchers and engineers face methodological issues such as the sensitivity of models to evaluation setup, difficulty of proper comparisons across methods, and the lack of reproducibility and transparency. In this paper we draw on three years of experience in evaluating large language models to provide guidance and lessons for researchers. First, we provide an overview of common challenges faced in language model evaluation. Second, we delineate best practices for addressing or lessening the impact of these challenges on research. Third, we present the Language Model Evaluation Harness (lm-eval): an open source library for independent, reproducible, and extensible evaluation of language models that seeks to address these issues. We describe the features of the library as well as case studies in which the library has been used to alleviate these methodological concerns.},
+ language = {en},
+ author = {Biderman, Stella and Schoelkopf, Hailey and Sutawika, Lintang and Gao, Leo and Tow, Jonathan and Abbasi, Baber and Aji, Alham Fikri and Sasanka, Pawan and Black, Sidney and Clive, Jordan and DiPofi, Anthony and Etxaniz, Julen and Forde, Jessica Zosa and Foster, Charles and Hsu, Jeffrey and Jaiswal, Mimansa and Li, Haonan and Lovering, Charles and Muennighoff, Niklas and Pavlick, Ellie and Skowron, Aviya and Tan, Samson and Tang, Xiangru and Wang, Kevin A and Indra, Genta and Yvon, Francois and Zou, Andy},
+ year = {2025},
+}
+
+@misc{bean_measuring_2025,
+ title = {Measuring what {Matters}: {Construct} {Validity} in {Large} {Language} {Model} {Benchmarks}},
+ abstract = {Evaluating large language models (LLMs) is crucial for both assessing their capabilities and identifying safety or robustness issues prior to deployment. Reliably measuring abstract and complex phenomena such as ‘safety’ and ‘robustness’ requires strong construct validity, that is, having measures that represent what matters to the phenomenon. With a team of 29 expert reviewers, we conduct a systematic review of 445 LLM benchmarks from leading conferences in natural language processing and machine learning. Across the reviewed articles, we find patterns related to the measured phenomena, tasks, and scoring metrics which undermine the validity of the resulting claims. To address these shortcomings, we provide eight key recommendations and detailed actionable guidance to researchers and practitioners in developing LLM benchmarks.},
+ language = {en},
+ author = {Bean, Andrew M and Kearns, Ryan Othniel and Romanou, Angelika and Hafner, Franziska Sofia and Mayne, Harry and Batzner, Jan and Foroutan, Negar and Schmitz, Chris and Korgul, Karolina and Batra, Hunar and Deb, Oishi and Beharry, Emma and Emde, Cornelius and Foster, Thomas and Gausen, Anna and Grandury, María and Han, Simeng and Hofmann, Valentin and Ibrahim, Lujain and Kim, Hazel and Kirk, Hannah Rose and Lin, Fangru and Liu, Gabrielle Kaili-May and Luettgau, Lennart and Magomere, Jabez and Rystrøm, Jonathan and Sotnikova, Anna and Yang, Yushi and Zhao, Yilun and Bibi, Adel and Bosselut, Antoine and Clark, Ronald and Cohan, Arman and Foerster, Jakob and Gal, Yarin and Hale, Scott A and Raji, Inioluwa Deborah and Summerfield, Christopher and Torr, Philip H S and Ududec, Cozmin and Rocher, Luc and Mahdi, Adam},
+ year = {2025},
+}
+
+@misc{barres_tau2-bench_2025,
+ title = {\$tau{\textasciicircum}2\$-{Bench}: {Evaluating} {Conversational} {Agents} in a {Dual}-{Control} {Environment}},
+ shorttitle = {\$tau{\textasciicircum}2\$-{Bench}},
+ url = {http://arxiv.org/abs/2506.07982},
+ doi = {10.48550/arXiv.2506.07982},
+ abstract = {Existing benchmarks for conversational AI agents simulate single-control environments, where only the AI agent can use tools to interact with the world, while the user remains a passive information provider. This differs from real-world scenarios like technical support, where users need to actively participate in modifying the state of the (shared) world. In order to address this gap, we introduce \$τ{\textasciicircum}2\$-bench, with four key contributions: 1) A novel Telecom dual-control domain modeled as a Dec-POMDP, where both agent and user make use of tools to act in a shared, dynamic environment that tests both agent coordination and communication, 2) A compositional task generator that programmatically creates diverse, verifiable tasks from atomic components, ensuring domain coverage and controlled complexity, 3) A reliable user simulator tightly coupled with the environment, whose behavior is constrained by tools and observable states, improving simulation fidelity, 4) Fine-grained analysis of agent performance through multiple ablations including separating errors arising from reasoning vs communication/coordination. In particular, our experiments show significant performance drops when agents shift from no-user to dual-control, highlighting the challenges of guiding users. Overall, \$τ{\textasciicircum}2\$-bench provides a controlled testbed for agents that must both reason effectively and guide user actions.},
+ urldate = {2026-02-19},
+ publisher = {arXiv},
+ author = {Barres, Victor and Dong, Honghua and Ray, Soham and Si, Xujie and Narasimhan, Karthik},
+ month = jun,
+ year = {2025},
+ note = {arXiv:2506.07982 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
+}
+
+@misc{yao_tau-bench_2024,
+ title = {tau-bench: {A} {Benchmark} for {Tool}-{Agent}-{User} {Interaction} in {Real}-{World} {Domains}},
+ shorttitle = {tau-bench},
+ url = {http://arxiv.org/abs/2406.12045},
+ doi = {10.48550/arXiv.2406.12045},
+ abstract = {Existing benchmarks do not test language agents on their interaction with human users or ability to follow domain-specific rules, both of which are vital for deploying them in real world applications. We propose tau-bench, a benchmark emulating dynamic conversations between a user (simulated by language models) and a language agent provided with domain-specific API tools and policy guidelines. We employ an efficient and faithful evaluation process that compares the database state at the end of a conversation with the annotated goal state. We also propose a new metric (pass{\textasciicircum}k) to evaluate the reliability of agent behavior over multiple trials. Our experiments show that even state-of-the-art function calling agents (like gpt-4o) succeed on {\textless}50\% of the tasks, and are quite inconsistent (pass{\textasciicircum}8 {\textless}25\% in retail). Our findings point to the need for methods that can improve the ability of agents to act consistently and follow rules reliably.},
+ urldate = {2026-02-23},
+ publisher = {arXiv},
+ author = {Yao, Shunyu and Shinn, Noah and Razavi, Pedram and Narasimhan, Karthik},
+ month = jun,
+ year = {2024},
+ note = {arXiv:2406.12045 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
+}
+
+@misc{wallraven_benchmarking_2026,
+ title = {Benchmarking {Large} {Language} {Models} for {ABAP} {Code} {Generation}: {An} {Empirical} {Study} on {Iterative} {Improvement} by {Compiler} {Feedback}},
+ shorttitle = {Benchmarking {Large} {Language} {Models} for {ABAP} {Code} {Generation}},
+ url = {http://arxiv.org/abs/2601.15188},
+ doi = {10.48550/arXiv.2601.15188},
+ abstract = {This work investigates the performance of Large Language Models (LLMs) in generating ABAP code. Despite successful applications of generative AI in many programming languages, there are hardly any systematic analyses of ABAP code generation to date. The aim of the study is to empirically analyze to what extent various LLMs can generate syntactically correct and functional ABAP code, how effectively they use compiler feedback for iterative improvement, and which task types pose special challenges. For this purpose, a benchmark with 180 tasks is conducted, consisting of adapted HumanEval tasks and practical SAP scenarios. The results show significant performance differences between the models: more powerful LLMs achieve success rates of around 75\% after several iterations and benefit greatly from compiler feedback, while smaller models perform significantly weaker. Overall, the study highlights the high potential of powerful LLMs for ABAP development processes, especially in iterative error correction.},
+ urldate = {2026-03-11},
+ publisher = {arXiv},
+ author = {Wallraven, Stephan and Köhne, Tim and Westenberger, Hartmut and Moser, Andreas},
+ month = jan,
+ year = {2026},
+ note = {arXiv:2601.15188 [cs]
+version: 1},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Programming Languages, Computer Science - Software Engineering},
+}
+
+@misc{liu_empirical_2025,
+ title = {An {Empirical} {Study} on {Failures} in {Automated} {Issue} {Solving}},
+ url = {http://arxiv.org/abs/2509.13941},
+ doi = {10.48550/arXiv.2509.13941},
+ abstract = {Automated issue solving seeks to autonomously identify and repair defective code snippets across an entire codebase. SWE-Bench has emerged as the most widely adopted benchmark for evaluating progress in this area. While LLM-based agentic tools show great promise, they still fail on a substantial portion of tasks. Moreover, current evaluations primarily report aggregate issue-solving rates, which obscure the underlying causes of success and failure, making it challenging to diagnose model weaknesses or guide targeted improvements. To bridge this gap, we first analyze the performance and efficiency of three SOTA tools, spanning both pipeline-based and agentic architectures, in automated issue solving tasks of SWE-Bench-Verified under varying task characteristics. Furthermore, to move from high-level performance metrics to underlying cause analysis, we conducted a systematic manual analysis of 150 failed instances. From this analysis, we developed a comprehensive taxonomy of failure modes comprising 3 primary phases, 9 main categories, and 25 fine-grained subcategories. Then we systematically analyze the distribution of the identified failure modes, the results reveal distinct failure fingerprints between the two architectural paradigms, with the majority of agentic failures stemming from flawed reasoning and cognitive deadlocks. Motivated by these insights, we propose a collaborative Expert-Executor framework. It introduces a supervisory Expert agent tasked with providing strategic oversight and course-correction for a primary Executor agent. This architecture is designed to correct flawed reasoning and break the cognitive deadlocks that frequently lead to failure. Experiments show that our framework solves 22.2\% of previously intractable issues for a leading single agent. These findings pave the way for building more robust agents through diagnostic evaluation and collaborative design.},
+ urldate = {2026-02-27},
+ publisher = {arXiv},
+ author = {Liu, Simiao and Liu, Fang and Li, Liehao and Tan, Xin and Zhu, Yinghao and Lian, Xiaoli and Zhang, Li},
+ month = sep,
+ year = {2025},
+ note = {arXiv:2509.13941 [cs]
+version: 1},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Software Engineering},
+}
+
+@misc{li_skillsbench_2026,
+ title = {{SkillsBench}: {Benchmarking} {How} {Well} {Agent} {Skills} {Work} {Across} {Diverse} {Tasks}},
+ shorttitle = {{SkillsBench}},
+ url = {http://arxiv.org/abs/2602.12670},
+ doi = {10.48550/arXiv.2602.12670},
+ abstract = {Agent Skills are structured packages of procedural knowledge that augment LLM agents at inference time. Despite rapid adoption, there is no standard way to measure whether they actually help. We present SkillsBench, a benchmark of 86 tasks across 11 domains paired with curated Skills and deterministic verifiers. Each task is evaluated under three conditions: no Skills, curated Skills, and self-generated Skills. We test 7 agent-model configurations over 7,308 trajectories. Curated Skills raise average pass rate by 16.2 percentage points(pp), but effects vary widely by domain (+4.5pp for Software Engineering to +51.9pp for Healthcare) and 16 of 84 tasks show negative deltas. Self-generated Skills provide no benefit on average, showing that models cannot reliably author the procedural knowledge they benefit from consuming. Focused Skills with 2--3 modules outperform comprehensive documentation, and smaller models with Skills can match larger models without them.},
+ urldate = {2026-02-27},
+ publisher = {arXiv},
+ author = {Li, Xiangyi and Chen, Wenbo and Liu, Yimin and Zheng, Shenghan and Chen, Xiaokun and He, Yifeng and Li, Yubo and You, Bingran and Shen, Haotian and Sun, Jiankai and Wang, Shuyi and Zeng, Qunhong and Wang, Di and Zhao, Xuandong and Wang, Yuanli and Chaim, Roey Ben and Di, Zonglin and Gao, Yipeng and He, Junwei and He, Yizhuo and Jing, Liqiang and Kong, Luyang and Lan, Xin and Li, Jiachen and Li, Songlin and Li, Yijiang and Lin, Yueqian and Liu, Xinyi and Liu, Xuanqing and Lyu, Haoran and Ma, Ze and Wang, Bowei and Wang, Runhui and Wang, Tianyu and Ye, Wengao and Zhang, Yue and Xing, Hanwen and Xue, Yiqi and Dillmann, Steven and Lee, Han-chung},
+ month = feb,
+ year = {2026},
+ note = {arXiv:2602.12670 [cs]},
+ keywords = {Computer Science - Artificial Intelligence},
+}
+
+@article{cemri_why_2025,
+ title = {{WHY} {DO} {MULTI}-{AGENT} {LLM} {SYSTEMS} {FAIL}?},
+ abstract = {Despite growing enthusiasm for Multi-Agent LLM Systems (MAS), their performance gains across popular benchmarks often remain minimal compared to single-agent frameworks. This gap highlights the need to systematically analyze the challenges hindering MAS effectiveness.},
+ language = {en},
+ author = {Cemri, Mert and Pan, Melissa Z and Yang, Shuyi and Agrawal, Lakshya A and Chopra, Bhavya and Tiwari, Rishabh and Keutzer, Kurt and Parameswaran, Aditya and Klein, Dan and Ramchandran, Kannan and Zaharia, Matei and Gonzalez, Joseph E and Stoica, Ion},
+ year = {2025},
+}
+
+@inproceedings{orlanski_measuring_2023,
+ title = {Measuring the {Impact} of {Programming} {Language} {Distribution}},
+ issn = {2640-3498},
+ url = {https://proceedings.mlr.press/v202/orlanski23a.html},
+ abstract = {Current benchmarks for evaluating neural code models focus on only a small subset of programming languages, excluding many popular languages such as Go or Rust. To ameliorate this issue, we present the BabelCode framework for execution-based evaluation of any benchmark in any language. BabelCode enables new investigations into the qualitative performance of models’ memory, runtime, and individual test case results. Additionally, we present a new code translation dataset called Translating Python Programming Puzzles (TP3) from the Python Programming Puzzles (Schuster et al., 2021) benchmark that involves translating expert-level python functions to any language. With both BabelCode and the TP3 benchmark, we investigate if balancing the distributions of 14 languages in a training dataset improves a large language model’s performance on low-resource languages. Training a model on a balanced corpus results in, on average, 12.34\% higher \$pass@k\$ across all tasks and languages compared to the baseline. We find that this strategy achieves 66.48\% better \$pass@k\$ on low-resource languages at the cost of only a 12.94\% decrease to high-resource languages. In our three translation tasks, this strategy yields, on average, 30.77\% better low-resource \$pass@k\$ while having 19.58\% worse high-resource \$pass@k\$.},
+ language = {en},
+ urldate = {2026-02-26},
+ booktitle = {Proceedings of the 40th {International} {Conference} on {Machine} {Learning}},
+ publisher = {PMLR},
+ author = {Orlanski, Gabriel and Xiao, Kefan and Garcia, Xavier and Hui, Jeffrey and Howland, Joshua and Malmaud, Jonathan and Austin, Jacob and Singh, Rishabh and Catasta, Michele},
+ month = jul,
+ year = {2023},
+ pages = {26619--26645},
+}
+
+@article{mundler_swt-bench_nodate,
+ title = {{SWT}-{Bench}: {Testing} and {Validating} {Real}-{World} {Bug}-{Fixes} with {Code} {Agents}},
+ abstract = {Rigorous software testing is crucial for developing and maintaining high-quality code, making automated test generation a promising avenue for both improving software quality and boosting the effectiveness of code generation methods. However, while code generation with Large Language Models (LLMs) is an extraordinarily active research area, test generation remains relatively unexplored. We address this gap and investigate the capability of LLM-based Code Agents to formalize user issues into test cases. To this end, we propose a novel benchmark based on popular GitHub repositories, containing real-world issues, ground-truth bug-fixes, and golden tests. We find that LLMs generally perform surprisingly well at generating relevant test cases, with Code Agents designed for code repair exceeding the performance of systems designed specifically for test generation. Further, as test generation is a similar but more structured task than code generation, it allows for a more fine-grained analysis using issue reproduction rate and coverage changes, providing a dual metric for analyzing systems designed for code repair. Finally, we find that generated tests are an effective filter for proposed code fixes, doubling the precision of SWE-AGENT. We release all data and code at github.com/logic-star-ai/SWT-Bench.},
+ language = {en},
+ author = {Mündler, Niels and Müller, Mark Niklas and He, Jingxuan and Vechev, Martin},
+}
+
+@misc{merrill_terminal-bench_2026,
+ title = {Terminal-{Bench}: {Benchmarking} {Agents} on {Hard}, {Realistic} {Tasks} in {Command} {Line} {Interfaces}},
+ shorttitle = {Terminal-{Bench}},
+ url = {http://arxiv.org/abs/2601.11868},
+ doi = {10.48550/arXiv.2601.11868},
+ abstract = {AI agents may soon become capable of autonomously completing valuable, long-horizon tasks in diverse domains. Current benchmarks either do not measure real-world tasks, or are not sufficiently difficult to meaningfully measure frontier models. To this end, we present Terminal-Bench 2.0: a carefully curated hard benchmark composed of 89 tasks in computer terminal environments inspired by problems from real workflows. Each task features a unique environment, human-written solution, and comprehensive tests for verification. We show that frontier models and agents score less than 65{\textbackslash}\% on the benchmark and conduct an error analysis to identify areas for model and agent improvement. We publish the dataset and evaluation harness to assist developers and researchers in future work at https://www.tbench.ai/ .},
+ urldate = {2026-02-26},
+ publisher = {arXiv},
+ author = {Merrill, Mike A. and Shaw, Alexander G. and Carlini, Nicholas and Li, Boxuan and Raj, Harsh and Bercovich, Ivan and Shi, Lin and Shin, Jeong Yeon and Walshe, Thomas and Buchanan, E. Kelly and Shen, Junhong and Ye, Guanghao and Lin, Haowei and Poulos, Jason and Wang, Maoyu and Nezhurina, Marianna and Jitsev, Jenia and Lu, Di and Mastromichalakis, Orfeas Menis and Xu, Zhiwei and Chen, Zizhao and Liu, Yue and Zhang, Robert and Chen, Leon Liangyu and Kashyap, Anurag and Uslu, Jan-Lucas and Li, Jeffrey and Wu, Jianbo and Yan, Minghao and Bian, Song and Sharma, Vedang and Sun, Ke and Dillmann, Steven and Anand, Akshay and Lanpouthakoun, Andrew and Koopah, Bardia and Hu, Changran and Guha, Etash and Dreiman, Gabriel H. S. and Zhu, Jiacheng and Krauth, Karl and Zhong, Li and Muennighoff, Niklas and Amanfu, Robert and Tan, Shangyin and Pimpalgaonkar, Shreyas and Aggarwal, Tushar and Lin, Xiangning and Lan, Xin and Zhao, Xuandong and Liang, Yiqing and Wang, Yuanli and Wang, Zilong and Zhou, Changzhi and Heineman, David and Liu, Hange and Trivedi, Harsh and Yang, John and Lin, Junhong and Shetty, Manish and Yang, Michael and Omi, Nabil and Raoof, Negin and Li, Shanda and Zhuo, Terry Yue and Lin, Wuwei and Dai, Yiwei and Wang, Yuxin and Chai, Wenhao and Zhou, Shang and Wahdany, Dariush and She, Ziyu and Hu, Jiaming and Dong, Zhikang and Zhu, Yuxuan and Cui, Sasha and Saiyed, Ahson and Kolbeinsson, Arinbjörn and Hu, Jesse and Rytting, Christopher Michael and Marten, Ryan and Wang, Yixin and Dimakis, Alex and Konwinski, Andy and Schmidt, Ludwig},
+ month = jan,
+ year = {2026},
+ note = {arXiv:2601.11868 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Software Engineering},
+}
+
+@misc{austin_program_2021,
+ title = {Program {Synthesis} with {Large} {Language} {Models}},
+ url = {http://arxiv.org/abs/2108.07732},
+ doi = {10.48550/arXiv.2108.07732},
+ abstract = {This paper explores the limits of the current generation of large language models for program synthesis in general purpose programming languages. We evaluate a collection of such models (with between 244M and 137B parameters) on two new benchmarks, MBPP and MathQA-Python, in both the few-shot and fine-tuning regimes. Our benchmarks are designed to measure the ability of these models to synthesize short Python programs from natural language descriptions. The Mostly Basic Programming Problems (MBPP) dataset contains 974 programming tasks, designed to be solvable by entry-level programmers. The MathQA-Python dataset, a Python version of the MathQA benchmark, contains 23914 problems that evaluate the ability of the models to synthesize code from more complex text. On both datasets, we find that synthesis performance scales log-linearly with model size. Our largest models, even without finetuning on a code dataset, can synthesize solutions to 59.6 percent of the problems from MBPP using few-shot learning with a well-designed prompt. Fine-tuning on a held-out portion of the dataset improves performance by about 10 percentage points across most model sizes. On the MathQA-Python dataset, the largest fine-tuned model achieves 83.8 percent accuracy. Going further, we study the model's ability to engage in dialog about code, incorporating human feedback to improve its solutions. We find that natural language feedback from a human halves the error rate compared to the model's initial prediction. Additionally, we conduct an error analysis to shed light on where these models fall short and what types of programs are most difficult to generate. Finally, we explore the semantic grounding of these models by fine-tuning them to predict the results of program execution. We find that even our best models are generally unable to predict the output of a program given a specific input.},
+ urldate = {2026-02-25},
+ publisher = {arXiv},
+ author = {Austin, Jacob and Odena, Augustus and Nye, Maxwell and Bosma, Maarten and Michalewski, Henryk and Dohan, David and Jiang, Ellen and Cai, Carrie and Terry, Michael and Le, Quoc and Sutton, Charles},
+ month = aug,
+ year = {2021},
+ note = {arXiv:2108.07732 [cs]},
+ keywords = {Computer Science - Machine Learning, Computer Science - Programming Languages},
+}
+
+@misc{cassano_multipl-e_2022,
+ title = {{MultiPL}-{E}: {A} {Scalable} and {Extensible} {Approach} to {Benchmarking} {Neural} {Code} {Generation}},
+ shorttitle = {{MultiPL}-{E}},
+ url = {http://arxiv.org/abs/2208.08227},
+ doi = {10.48550/arXiv.2208.08227},
+ abstract = {Large language models have demonstrated the ability to generate both natural language and programming language text. Such models open up the possibility of multi-language code generation: could code generation models generalize knowledge from one language to another? Although contemporary code generation models can generate semantically correct Python code, little is known about their abilities with other languages. We propose MultiPL-E, a system for translating unit test-driven code generation benchmarks to new languages. We create the first massively multilingual code generation benchmark by using MultiPL-E to translate two popular Python code generation benchmarks to 18 additional programming languages. We use MultiPL-E to extend the HumanEval benchmark and MBPP benchmark to 18 languages that encompass a range of programming paradigms and popularity. Using these new parallel benchmarks, we evaluate the multi-language performance of three state-of-the-art code generation models: Codex, CodeGen, and InCoder. We find that Codex matches or even exceeds its performance on Python for several other languages. The range of programming languages represented in MultiPL-E allow us to explore the impact of language frequency and language features on model performance. Finally, the MultiPL-E approach of compiling code generation benchmarks to new programming languages is both scalable and extensible, making it straightforward to evaluate new models, benchmarks, and languages.},
+ urldate = {2026-02-25},
+ publisher = {arXiv},
+ author = {Cassano, Federico and Gouwar, John and Nguyen, Daniel and Nguyen, Sydney and Phipps-Costin, Luna and Pinckney, Donald and Yee, Ming-Ho and Zi, Yangtian and Anderson, Carolyn Jane and Feldman, Molly Q. and Guha, Arjun and Greenberg, Michael and Jangda, Abhinav},
+ month = dec,
+ year = {2022},
+ note = {arXiv:2208.08227 [cs]},
+ keywords = {Computer Science - Machine Learning, Computer Science - Programming Languages},
+}
+
+@misc{cassano_knowledge_2024,
+ title = {Knowledge {Transfer} from {High}-{Resource} to {Low}-{Resource} {Programming} {Languages} for {Code} {LLMs}},
+ url = {http://arxiv.org/abs/2308.09895},
+ doi = {10.48550/arXiv.2308.09895},
+ abstract = {Over the past few years, Large Language Models of Code (Code LLMs) have started to have a significant impact on programming practice. Code LLMs are also emerging as building blocks for research in programming languages and software engineering. However, Code LLMs produce impressive results on programming languages that are well represented in their training data (e.g., Java, Python, or JavaScript), but struggle with low-resource languages that have limited training data available. Low resource languages include OCaml, Racket, and several others. This paper presents an effective approach for boosting the performance of Code LLMs on low-resource languages using semi-synthetic data. Our approach, MultiPL-T, translates training data from high-resource languages into training data for low-resource languages in the following way. 1) We use a Code LLM to synthesize tests for commented code from a high-resource language, filtering out faulty tests and code with low test coverage. 2) We use a Code LLM to translate Python code to a target low-resource language, and use tests to validate the translation. We apply this approach to generate tens of thousands of validated training items for Julia, Lua, OCaml, R, and Racket. Furthermore, we use an open model (StarCoderBase) with open training data (The Stack), which allows us to decontaminate benchmarks, train models without violating licenses, and run experiments that could not otherwise be done. With MultiPL-T generated data, we present fine-tuned versions of StarCoderBase and Code Llama for Julia, Lua, OCaml, R, and Racket. On established benchmarks (MultiPL-E), these models outperform other open Code LLMs. The MultiPL-T approach is easy to apply to new languages, and is significantly more efficient and effective than alternatives such as training longer.},
+ urldate = {2026-02-25},
+ publisher = {arXiv},
+ author = {Cassano, Federico and Gouwar, John and Lucchetti, Francesca and Schlesinger, Claire and Freeman, Anders and Anderson, Carolyn Jane and Feldman, Molly Q. and Greenberg, Michael and Jangda, Abhinav and Guha, Arjun},
+ month = sep,
+ year = {2024},
+ note = {arXiv:2308.09895 [cs]},
+ keywords = {Computer Science - Machine Learning, Computer Science - Programming Languages},
+}
+
+@misc{cavalin_improving_2025,
+ title = {Improving {Score} {Reliability} of {Multiple} {Choice} {Benchmarks} with {Consistency} {Evaluation} and {Altered} {Answer} {Choices}},
+ url = {http://arxiv.org/abs/2511.21860},
+ doi = {10.48550/arXiv.2511.21860},
+ abstract = {In this work we present the Consistency-Rebalanced Accuracy (CoRA) metric, improving the reliability of Large Language Model (LLM) scores computed on multiple choice (MC) benchmarks. Our metric explores the response consistency of the LLMs, taking advantage of synthetically-generated questions with altered answer choices. With two intermediate scores, i.e. Bare-Minimum-Consistency Accuracy (BMCA) and Consistency Index (CI), CoRA is computed by adjusting the multiple-choice question answering (MCQA) scores to better reflect the level of consistency of the LLM. We present evaluations in different benchmarks using diverse LLMs, and not only demonstrate that LLMs can present low response consistency even when they present high MCQA scores, but also that CoRA can successfully scale down the scores of inconsistent models.},
+ urldate = {2026-02-21},
+ publisher = {arXiv},
+ author = {Cavalin, Paulo and Sanctos, Cassia and Grave, Marcelo and Pinhanez, Claudio and Primerano, Yago},
+ month = nov,
+ year = {2025},
+ note = {arXiv:2511.21860 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
+}
+
+@misc{badertdinov_swe-rebench_2025,
+ title = {{SWE}-rebench: {An} {Automated} {Pipeline} for {Task} {Collection} and {Decontaminated} {Evaluation} of {Software} {Engineering} {Agents}},
+ shorttitle = {{SWE}-rebench},
+ url = {http://arxiv.org/abs/2505.20411},
+ doi = {10.48550/arXiv.2505.20411},
+ abstract = {LLM-based agents have shown promising capabilities in a growing range of software engineering (SWE) tasks. However, advancing this field faces two critical challenges. First, high-quality training data is scarce, especially data that reflects real-world SWE scenarios, where agents must interact with development environments, execute code and adapt behavior based on the outcomes of their actions. Existing datasets are either limited to one-shot code generation or comprise small, manually curated collections of interactive tasks, lacking both scale and diversity. Second, the lack of fresh interactive SWE tasks affects evaluation of rapidly improving models, as static benchmarks quickly become outdated due to contamination issues. To address these limitations, we introduce a novel, automated, and scalable pipeline to continuously extract real-world interactive SWE tasks from diverse GitHub repositories. Using this pipeline, we construct SWE-rebench, a public dataset comprising over 21,000 interactive Python-based SWE tasks, suitable for reinforcement learning of SWE agents at scale. Additionally, we use continuous supply of fresh tasks collected using SWE-rebench methodology to build a contamination-free benchmark for agentic software engineering. We compare results of various LLMs on this benchmark to results on SWE-bench Verified and show that performance of some language models might be inflated due to contamination issues.},
+ urldate = {2026-02-20},
+ publisher = {arXiv},
+ author = {Badertdinov, Ibragim and Golubev, Alexander and Nekrashevich, Maksim and Shevtsov, Anton and Karasik, Simon and Andriushchenko, Andrei and Trofimova, Maria and Litvintseva, Daria and Yangel, Boris},
+ month = nov,
+ year = {2025},
+ note = {arXiv:2505.20411 [cs]},
+ keywords = {Computer Science - Computation and Language, Computer Science - Software Engineering},
+}
+
+@misc{zhou_featurebench_2026,
+ title = {{FeatureBench}: {Benchmarking} {Agentic} {Coding} for {Complex} {Feature} {Development}},
+ shorttitle = {{FeatureBench}},
+ url = {http://arxiv.org/abs/2602.10975},
+ doi = {10.48550/arXiv.2602.10975},
+ abstract = {Agents powered by large language models (LLMs) are increasingly adopted in the software industry, contributing code as collaborators or even autonomous developers. As their presence grows, it becomes important to assess the current boundaries of their coding abilities. Existing agentic coding benchmarks, however, cover a limited task scope, e.g., bug fixing within a single pull request (PR), and often rely on non-executable evaluations or lack an automated approach for continually updating the evaluation coverage. To address such issues, we propose FeatureBench, a benchmark designed to evaluate agentic coding performance in end-to-end, feature-oriented software development. FeatureBench incorporates an execution-based evaluation protocol and a scalable test-driven method that automatically derives tasks from code repositories with minimal human effort. By tracing from unit tests along a dependency graph, our approach can identify feature-level coding tasks spanning multiple commits and PRs scattered across the development timeline, while ensuring the proper functioning of other features after the separation. Using this framework, we curated 200 challenging evaluation tasks and 3825 executable environments from 24 open-source repositories in the first version of our benchmark. Empirical evaluation reveals that the state-of-the-art agentic model, such as Claude 4.5 Opus, which achieves a 74.4\% resolved rate on SWE-bench, succeeds on only 11.0\% of tasks, opening new opportunities for advancing agentic coding. Moreover, benefiting from our automated task collection toolkit, FeatureBench can be easily scaled and updated over time to mitigate data leakage. The inherent verifiability of constructed environments also makes our method potentially valuable for agent training.},
+ urldate = {2026-02-20},
+ publisher = {arXiv},
+ author = {Zhou, Qixing and Zhang, Jiacheng and Wang, Haiyang and Hao, Rui and Wang, Jiahe and Han, Minghao and Yang, Yuxue and Wu, Shuzhe and Pan, Feiyang and Fan, Lue and Tu, Dandan and Zhang, Zhaoxiang},
+ month = feb,
+ year = {2026},
+ note = {arXiv:2602.10975 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Software Engineering},
+}
+
+@misc{wang_are_2025,
+ title = {Are "{Solved} {Issues}" in {SWE}-bench {Really} {Solved} {Correctly}? {An} {Empirical} {Study}},
+ shorttitle = {Are "{Solved} {Issues}" in {SWE}-bench {Really} {Solved} {Correctly}?},
+ url = {http://arxiv.org/abs/2503.15223},
+ doi = {10.1145/3744916.3764576},
+ abstract = {Automated issue solving aims to resolve real-world issues in software repositories. The most popular benchmarks for automated issue solving are SWE-bench and its human-filtered subset SWE-bench Verified. These benchmarks leverage testing to validate generated patches. However, because testing is rarely exhaustive, a patch may pass the tests but nevertheless fail to match the developers' expectations. Unfortunately, it is currently unclear to what extent evaluations performed with SWE-bench suffer from such plausible but incorrect patches. This paper presents an in-depth empirical study of the correctness of plausible patches generated by three state-of-the-art issue-solving tools evaluated on SWE-bench Verified. We extensively test and inspect generated patches, and compare them against human-written ground truth patches. The core of our methodology is a novel technique PatchDiff for differential patch testing, which automatically exposes behavioral discrepancies between two patches. Our findings reveal critical weaknesses in SWE-bench's patch validation mechanism, which causes 7.8\% of all patches to count as correct while failing the developer-written test suite. Moreover, our novel automated technique reveals that even more (29.6\%) plausible patches induce different behavior than the ground truth patches. These behavioral differences are often due to similar, but divergent implementations (46.8\%) and due to generated patches that adapt more behavior than the ground truth patches (27.3\%). Our manual inspection shows that 28.6\% of behaviorally divergent patches are certainly incorrect. Combined, the different weaknesses lead to an inflation of reported resolution rates by 6.2 absolute percent points. Our findings are a call to arms for more robust and reliable evaluation of issue-solving tools. We envision our automated differential patch testing technique to be useful for this purpose.},
+ urldate = {2026-02-20},
+ author = {Wang, You and Pradel, Michael and Liu, Zhongxin},
+ month = sep,
+ year = {2025},
+ note = {arXiv:2503.15223 [cs]},
+ keywords = {Computer Science - Software Engineering},
+}
+
+@misc{prathifkumar_does_2025,
+ title = {Does {SWE}-{Bench}-{Verified} {Test} {Agent} {Ability} or {Model} {Memory}?},
+ url = {http://arxiv.org/abs/2512.10218},
+ doi = {10.48550/arXiv.2512.10218},
+ abstract = {SWE-Bench-Verified, a dataset comprising 500 issues, serves as a de facto benchmark for evaluating various large language models (LLMs) on their ability to resolve GitHub issues. But this benchmark may overlap with model training data. If that is true, scores may reflect training recall, not issue-solving skill. To study this, we test two Claude models that frequently appear in top-performing agents submitted to the benchmark. We ask them to find relevant files using only issue text, and then issue text plus file paths. We then run the same setup on BeetleBox and SWE-rebench. Despite both benchmarks involving popular open-source Python projects, models performed 3 times better on SWE-Bench-Verified. They were also 6 times better at finding edited files, without any additional context about the projects themselves. This gap suggests the models may have seen many SWE-Bench-Verified tasks during training. As a result, scores on this benchmark may not reflect an agent's ability to handle real software issues, yet it continues to be used in ways that can misrepresent progress and lead to choices that favour agents that use certain models over strong agent design. Our setup tests the localization step with minimal context to the extent that the task should be logically impossible to solve. Our results show the risk of relying on older popular benchmarks and support the shift toward newer datasets built with contamination in mind.},
+ urldate = {2026-02-20},
+ publisher = {arXiv},
+ author = {Prathifkumar, Thanosan and Mathews, Noble Saji and Nagappan, Meiyappan},
+ month = dec,
+ year = {2025},
+ note = {arXiv:2512.10218 [cs]},
+ keywords = {Computer Science - Software Engineering},
+}
+
+@misc{joel_survey_2025,
+ title = {A {Survey} on {LLM}-based {Code} {Generation} for {Low}-{Resource} and {Domain}-{Specific} {Programming} {Languages}},
+ url = {http://arxiv.org/abs/2410.03981},
+ doi = {10.48550/arXiv.2410.03981},
+ abstract = {Large Language Models (LLMs) have shown impressive capabilities in code generation for popular programming languages. However, their performance on Low-Resource Programming Languages (LRPLs) and Domain-Specific Languages (DSLs) remains a significant challenge, affecting millions of developers-3.5 million users in Rust alone-who cannot fully utilize LLM capabilities. LRPLs and DSLs encounter unique obstacles, including data scarcity and, for DSLs, specialized syntax that is poorly represented in general-purpose datasets. Addressing these challenges is crucial, as LRPLs and DSLs enhance development efficiency in specialized domains, such as finance and science. While several surveys discuss LLMs in software engineering, none focus specifically on the challenges and opportunities associated with LRPLs and DSLs. Our survey fills this gap by systematically reviewing the current state, methodologies, and challenges in leveraging LLMs for code generation in these languages. We filtered 111 papers from over 27,000 published studies between 2020 and 2024 to evaluate the capabilities and limitations of LLMs in LRPLs and DSLs. We report the LLMs used, benchmarks, and metrics for evaluation, strategies for enhancing performance, and methods for dataset collection and curation. We identified four main evaluation techniques and several metrics for assessing code generation in LRPLs and DSLs. Our analysis categorizes improvement methods into six groups and summarizes novel architectures proposed by researchers. Despite various techniques and metrics, a standard approach and benchmark dataset for evaluating code generation in LRPLs and DSLs are lacking. This survey serves as a resource for researchers and practitioners at the intersection of LLMs, software engineering, and specialized programming languages, laying the groundwork for future advancements in code generation for LRPLs and DSLs.},
+ urldate = {2026-02-19},
+ publisher = {arXiv},
+ author = {Joel, Sathvik and Wu, Jie JW and Fard, Fatemeh H.},
+ month = sep,
+ year = {2025},
+ note = {arXiv:2410.03981 [cs]},
+ keywords = {Computer Science - Machine Learning, Computer Science - Software Engineering},
+}
+
+@misc{deng_swe-bench_2025,
+ title = {{SWE}-{Bench} {Pro}: {Can} {AI} {Agents} {Solve} {Long}-{Horizon} {Software} {Engineering} {Tasks}?},
+ shorttitle = {{SWE}-{Bench} {Pro}},
+ url = {http://arxiv.org/abs/2509.16941},
+ doi = {10.48550/arXiv.2509.16941},
+ abstract = {We introduce SWE-Bench Pro, a substantially more challenging benchmark that builds upon the best practices of SWE-BENCH [25], but is explicitly designed to capture realistic, complex, enterprise-level problems beyond the scope of SWE-BENCH. SWE-BENCH PRO contains 1,865 problems sourced from a diverse set of 41 actively maintained repositories spanning business applications, B2B services, and developer tools. The benchmark is partitioned into a public set with open access to problems sourced from 11 repositories, a held-out set of 12 repositories and a commercial set of 18 proprietary repositories where we have formal partnership agreements with early-stage startups. Problems in the held-out and the commercial set are not publicly accessible, but we release results on the commercial set. Our benchmark features long-horizon tasks that may require hours to days for a professional software engineer to complete, often involving patches across multiple files and substantial code modifications. All tasks are human-verified and augmented with sufficient context to ensure resolvability. To better understand these limitations, we cluster the failure modes observed in the collected agent trajectories for a clearer characterization of the error patterns exhibited by current models. Overall, SWE-BENCH PRO provides a contamination-resistant testbed that more faithfully captures the complexity and diversity of real-world software development, advancing the pursuit of truly autonomous software engineering agents at a professional level.},
+ urldate = {2026-02-19},
+ publisher = {arXiv},
+ author = {Deng, Xiang and Da, Jeff and Pan, Edwin and He, Yannis Yiming and Ide, Charles and Garg, Kanak and Lauffer, Niklas and Park, Andrew and Pasari, Nitin and Rane, Chetan and Sampath, Karmini and Krishnan, Maya and Kundurthy, Srivatsa and Hendryx, Sean and Wang, Zifan and Bharadwaj, Vijay and Holm, Jeff and Aluri, Raja and Zhang, Chen Bo Calvin and Jacobson, Noah and Liu, Bing and Kenstler, Brad},
+ month = nov,
+ year = {2025},
+ note = {arXiv:2509.16941 [cs]},
+ keywords = {Computer Science - Computation and Language, Computer Science - Software Engineering},
+}
+
+@misc{chen_evaluating_2021,
+ title = {Evaluating {Large} {Language} {Models} {Trained} on {Code}},
+ url = {http://arxiv.org/abs/2107.03374},
+ doi = {10.48550/arXiv.2107.03374},
+ abstract = {We introduce Codex, a GPT language model fine-tuned on publicly available code from GitHub, and study its Python code-writing capabilities. A distinct production version of Codex powers GitHub Copilot. On HumanEval, a new evaluation set we release to measure functional correctness for synthesizing programs from docstrings, our model solves 28.8\% of the problems, while GPT-3 solves 0\% and GPT-J solves 11.4\%. Furthermore, we find that repeated sampling from the model is a surprisingly effective strategy for producing working solutions to difficult prompts. Using this method, we solve 70.2\% of our problems with 100 samples per problem. Careful investigation of our model reveals its limitations, including difficulty with docstrings describing long chains of operations and with binding operations to variables. Finally, we discuss the potential broader impacts of deploying powerful code generation technologies, covering safety, security, and economics.},
+ urldate = {2026-02-19},
+ publisher = {arXiv},
+ author = {Chen, Mark and Tworek, Jerry and Jun, Heewoo and Yuan, Qiming and Pinto, Henrique Ponde de Oliveira and Kaplan, Jared and Edwards, Harri and Burda, Yuri and Joseph, Nicholas and Brockman, Greg and Ray, Alex and Puri, Raul and Krueger, Gretchen and Petrov, Michael and Khlaaf, Heidy and Sastry, Girish and Mishkin, Pamela and Chan, Brooke and Gray, Scott and Ryder, Nick and Pavlov, Mikhail and Power, Alethea and Kaiser, Lukasz and Bavarian, Mohammad and Winter, Clemens and Tillet, Philippe and Such, Felipe Petroski and Cummings, Dave and Plappert, Matthias and Chantzis, Fotios and Barnes, Elizabeth and Herbert-Voss, Ariel and Guss, William Hebgen and Nichol, Alex and Paino, Alex and Tezak, Nikolas and Tang, Jie and Babuschkin, Igor and Balaji, Suchir and Jain, Shantanu and Saunders, William and Hesse, Christopher and Carr, Andrew N. and Leike, Jan and Achiam, Josh and Misra, Vedant and Morikawa, Evan and Radford, Alec and Knight, Matthew and Brundage, Miles and Murati, Mira and Mayer, Katie and Welinder, Peter and McGrew, Bob and Amodei, Dario and McCandlish, Sam and Sutskever, Ilya and Zaremba, Wojciech},
+ month = jul,
+ year = {2021},
+ note = {arXiv:2107.03374 [cs]},
+ keywords = {Computer Science - Machine Learning},
+}
+
+@misc{gloaguen_evaluating_2026,
+ title = {Evaluating {AGENTS}.md: {Are} {Repository}-{Level} {Context} {Files} {Helpful} for {Coding} {Agents}?},
+ shorttitle = {Evaluating {AGENTS}.md},
+ url = {http://arxiv.org/abs/2602.11988},
+ doi = {10.48550/arXiv.2602.11988},
+ abstract = {A widespread practice in software development is to tailor coding agents to repositories using context files, such as AGENTS.md, by either manually or automatically generating them. Although this practice is strongly encouraged by agent developers, there is currently no rigorous investigation into whether such context files are actually effective for real-world tasks. In this work, we study this question and evaluate coding agents' task completion performance in two complementary settings: established SWE-bench tasks from popular repositories, with LLM-generated context files following agent-developer recommendations, and a novel collection of issues from repositories containing developer-committed context files. Across multiple coding agents and LLMs, we find that context files tend to reduce task success rates compared to providing no repository context, while also increasing inference cost by over 20\%. Behaviorally, both LLM-generated and developer-provided context files encourage broader exploration (e.g., more thorough testing and file traversal), and coding agents tend to respect their instructions. Ultimately, we conclude that unnecessary requirements from context files make tasks harder, and human-written context files should describe only minimal requirements.},
+ urldate = {2026-02-19},
+ publisher = {arXiv},
+ author = {Gloaguen, Thibaud and Mündler, Niels and Müller, Mark and Raychev, Veselin and Vechev, Martin},
+ month = feb,
+ year = {2026},
+ note = {arXiv:2602.11988 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Software Engineering},
+}
+
+@misc{zhang_swe-bench_2025,
+ title = {{SWE}-bench {Goes} {Live}!},
+ url = {http://arxiv.org/abs/2505.23419},
+ doi = {10.48550/arXiv.2505.23419},
+ abstract = {The issue-resolving task, where a model generates patches to fix real-world bugs, has emerged as a critical benchmark for evaluating the capabilities of large language models (LLMs). While SWE-bench and its variants have become standard in this domain, they suffer from key limitations: they have not been updated since their initial releases, cover a narrow set of repositories, and depend heavily on manual effort for instance construction and environment setup. These factors hinder scalability and introduce risks of overfitting and data contamination. In this work, we present SWE-bench-Live, a live-updatable benchmark designed to overcome these challenges. Our initial release consists of 1,319 tasks derived from real GitHub issues created since 2024, spanning 93 repositories. Each task is accompanied by a dedicated Docker image to ensure reproducible execution. Central to our benchmark is {\textbackslash}method, an automated curation pipeline that streamlines the entire process from instance creation to environment setup, removing manual bottlenecks and enabling scalability and continuous updates. We evaluate a range of state-of-the-art agent frameworks and LLMs on SWE-bench-Live, revealing a substantial performance gap compared to static benchmarks like SWE-bench, even under controlled evaluation conditions. To better understand this discrepancy, we perform detailed analyses across repository origin, issue recency, and task difficulty. By providing a fresh, diverse, and executable benchmark grounded in live repository activity, SWE-bench-Live facilitates rigorous, contamination-resistant evaluation of LLMs and agents in dynamic, real-world software development settings.},
+ urldate = {2026-02-19},
+ publisher = {arXiv},
+ author = {Zhang, Linghao and He, Shilin and Zhang, Chaoyun and Kang, Yu and Li, Bowen and Xie, Chengxing and Wang, Junhao and Wang, Maoquan and Huang, Yufan and Fu, Shengyu and Nallipogu, Elsie and Lin, Qingwei and Dang, Yingnong and Rajmohan, Saravan and Zhang, Dongmei},
+ month = jun,
+ year = {2025},
+ note = {arXiv:2505.23419 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Software Engineering},
+}
+
+@misc{bjarnason_randomness_2026,
+ title = {On {Randomness} in {Agentic} {Evals}},
+ url = {http://arxiv.org/abs/2602.07150},
+ doi = {10.48550/arXiv.2602.07150},
+ abstract = {Agentic systems are evaluated on benchmarks where agents interact with environments to solve tasks. Most papers report a pass@1 score computed from a single run per task, assuming this gives a reliable performance estimate. We test this assumption by collecting 60,000 agentic trajectories on SWE-Bench-Verified, spanning three models and two scaffolds. We find substantial variance: single-run pass@1 estimates vary by 2.2 to 6.0 percentage points depending on which run is selected, with standard deviations exceeding 1.5 percentage points even at temperature 0. This variance has critical implications: reported improvements of 2--3 percentage points may reflect evaluation noise rather than genuine algorithmic progress. Through token-level analysis, we show that trajectories diverge early, often within the first few percent of tokens, and that these small differences cascade into different solution strategies. To enable reliable evaluation of agentic systems, we recommend three concrete practices: (1) estimate pass@1 from multiple independent runs per task, especially when measuring small improvements, (2) use statistical power analysis to determine the number of runs needed to detect expected effect sizes, and (3) consider metrics like pass@k (optimistic bound) and pass{\textasciicircum}k (pessimistic bound) with k{\textgreater}1 to better characterize the full performance envelope. While these practices increase evaluation cost, they are essential for distinguishing genuine scientific progress from statistical noise.},
+ urldate = {2026-02-19},
+ publisher = {arXiv},
+ author = {Bjarnason, Bjarni Haukur and Silva, André and Monperrus, Martin},
+ month = feb,
+ year = {2026},
+ note = {arXiv:2602.07150 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Computer Science - Software Engineering},
+}
+
+@misc{yang_swe-bench_2024,
+ title = {{SWE}-bench {Multimodal}: {Do} {AI} {Systems} {Generalize} to {Visual} {Software} {Domains}?},
+ shorttitle = {{SWE}-bench {Multimodal}},
+ url = {http://arxiv.org/abs/2410.03859},
+ doi = {10.48550/arXiv.2410.03859},
+ abstract = {Autonomous systems for software engineering are now capable of fixing bugs and developing features. These systems are commonly evaluated on SWE-bench (Jimenez et al., 2024a), which assesses their ability to solve software issues from GitHub repositories. However, SWE-bench uses only Python repositories, with problem statements presented predominantly as text and lacking visual elements such as images. This limited coverage motivates our inquiry into how existing systems might perform on unrepresented software engineering domains (e.g., front-end, game development, DevOps), which use different programming languages and paradigms. Therefore, we propose SWE-bench Multimodal (SWE-bench M), to evaluate systems on their ability to fix bugs in visual, user-facing JavaScript software. SWE-bench M features 617 task instances collected from 17 JavaScript libraries used for web interface design, diagramming, data visualization, syntax highlighting, and interactive mapping. Each SWE-bench M task instance contains at least one image in its problem statement or unit tests. Our analysis finds that top-performing SWE-bench systems struggle with SWE-bench M, revealing limitations in visual problem-solving and cross-language generalization. Lastly, we show that SWE-agent's flexible language-agnostic features enable it to substantially outperform alternatives on SWE-bench M, resolving 12\% of task instances compared to 6\% for the next best system.},
+ urldate = {2026-02-18},
+ publisher = {arXiv},
+ author = {Yang, John and Jimenez, Carlos E. and Zhang, Alex L. and Lieret, Kilian and Yang, Joyce and Wu, Xindi and Press, Ori and Muennighoff, Niklas and Synnaeve, Gabriel and Narasimhan, Karthik R. and Yang, Diyi and Wang, Sida I. and Press, Ofir},
+ month = oct,
+ year = {2024},
+ note = {arXiv:2410.03859 [cs]},
+ keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Software Engineering},
+}
diff --git a/paper/fancyhdr.sty b/paper/fancyhdr.sty
new file mode 100644
index 000000000..93bce8b3e
--- /dev/null
+++ b/paper/fancyhdr.sty
@@ -0,0 +1,485 @@
+% fancyhdr.sty version 3.2
+% Fancy headers and footers for LaTeX.
+% Piet van Oostrum,
+% Dept of Computer and Information Sciences, University of Utrecht,
+% Padualaan 14, P.O. Box 80.089, 3508 TB Utrecht, The Netherlands
+% Telephone: +31 30 2532180. Email: piet@cs.uu.nl
+% ========================================================================
+% LICENCE:
+% This file may be distributed under the terms of the LaTeX Project Public
+% License, as described in lppl.txt in the base LaTeX distribution.
+% Either version 1 or, at your option, any later version.
+% ========================================================================
+% MODIFICATION HISTORY:
+% Sep 16, 1994
+% version 1.4: Correction for use with \reversemargin
+% Sep 29, 1994:
+% version 1.5: Added the \iftopfloat, \ifbotfloat and \iffloatpage commands
+% Oct 4, 1994:
+% version 1.6: Reset single spacing in headers/footers for use with
+% setspace.sty or doublespace.sty
+% Oct 4, 1994:
+% version 1.7: changed \let\@mkboth\markboth to
+% \def\@mkboth{\protect\markboth} to make it more robust
+% Dec 5, 1994:
+% version 1.8: corrections for amsbook/amsart: define \@chapapp and (more
+% importantly) use the \chapter/sectionmark definitions from ps@headings if
+% they exist (which should be true for all standard classes).
+% May 31, 1995:
+% version 1.9: The proposed \renewcommand{\headrulewidth}{\iffloatpage...
+% construction in the doc did not work properly with the fancyplain style.
+% June 1, 1995:
+% version 1.91: The definition of \@mkboth wasn't restored on subsequent
+% \pagestyle{fancy}'s.
+% June 1, 1995:
+% version 1.92: The sequence \pagestyle{fancyplain} \pagestyle{plain}
+% \pagestyle{fancy} would erroneously select the plain version.
+% June 1, 1995:
+% version 1.93: \fancypagestyle command added.
+% Dec 11, 1995:
+% version 1.94: suggested by Conrad Hughes
+% CJCH, Dec 11, 1995: added \footruleskip to allow control over footrule
+% position (old hardcoded value of .3\normalbaselineskip is far too high
+% when used with very small footer fonts).
+% Jan 31, 1996:
+% version 1.95: call \@normalsize in the reset code if that is defined,
+% otherwise \normalsize.
+% this is to solve a problem with ucthesis.cls, as this doesn't
+% define \@currsize. Unfortunately for latex209 calling \normalsize doesn't
+% work as this is optimized to do very little, so there \@normalsize should
+% be called. Hopefully this code works for all versions of LaTeX known to
+% mankind.
+% April 25, 1996:
+% version 1.96: initialize \headwidth to a magic (negative) value to catch
+% most common cases that people change it before calling \pagestyle{fancy}.
+% Note it can't be initialized when reading in this file, because
+% \textwidth could be changed afterwards. This is quite probable.
+% We also switch to \MakeUppercase rather than \uppercase and introduce a
+% \nouppercase command for use in headers. and footers.
+% May 3, 1996:
+% version 1.97: Two changes:
+% 1. Undo the change in version 1.8 (using the pagestyle{headings} defaults
+% for the chapter and section marks. The current version of amsbook and
+% amsart classes don't seem to need them anymore. Moreover the standard
+% latex classes don't use \markboth if twoside isn't selected, and this is
+% confusing as \leftmark doesn't work as expected.
+% 2. include a call to \ps@empty in ps@@fancy. This is to solve a problem
+% in the amsbook and amsart classes, that make global changes to \topskip,
+% which are reset in \ps@empty. Hopefully this doesn't break other things.
+% May 7, 1996:
+% version 1.98:
+% Added % after the line \def\nouppercase
+% May 7, 1996:
+% version 1.99: This is the alpha version of fancyhdr 2.0
+% Introduced the new commands \fancyhead, \fancyfoot, and \fancyhf.
+% Changed \headrulewidth, \footrulewidth, \footruleskip to
+% macros rather than length parameters, In this way they can be
+% conditionalized and they don't consume length registers. There is no need
+% to have them as length registers unless you want to do calculations with
+% them, which is unlikely. Note that this may make some uses of them
+% incompatible (i.e. if you have a file that uses \setlength or \xxxx=)
+% May 10, 1996:
+% version 1.99a:
+% Added a few more % signs
+% May 10, 1996:
+% version 1.99b:
+% Changed the syntax of \f@nfor to be resistent to catcode changes of :=
+% Removed the [1] from the defs of \lhead etc. because the parameter is
+% consumed by the \@[xy]lhead etc. macros.
+% June 24, 1997:
+% version 1.99c:
+% corrected \nouppercase to also include the protected form of \MakeUppercase
+% \global added to manipulation of \headwidth.
+% \iffootnote command added.
+% Some comments added about \@fancyhead and \@fancyfoot.
+% Aug 24, 1998
+% version 1.99d
+% Changed the default \ps@empty to \ps@@empty in order to allow
+% \fancypagestyle{empty} redefinition.
+% Oct 11, 2000
+% version 2.0
+% Added LPPL license clause.
+%
+% A check for \headheight is added. An errormessage is given (once) if the
+% header is too large. Empty headers don't generate the error even if
+% \headheight is very small or even 0pt.
+% Warning added for the use of 'E' option when twoside option is not used.
+% In this case the 'E' fields will never be used.
+%
+% Mar 10, 2002
+% version 2.1beta
+% New command: \fancyhfoffset[place]{length}
+% defines offsets to be applied to the header/footer to let it stick into
+% the margins (if length > 0).
+% place is like in fancyhead, except that only E,O,L,R can be used.
+% This replaces the old calculation based on \headwidth and the marginpar
+% area.
+% \headwidth will be dynamically calculated in the headers/footers when
+% this is used.
+%
+% Mar 26, 2002
+% version 2.1beta2
+% \fancyhfoffset now also takes h,f as possible letters in the argument to
+% allow the header and footer widths to be different.
+% New commands \fancyheadoffset and \fancyfootoffset added comparable to
+% \fancyhead and \fancyfoot.
+% Errormessages and warnings have been made more informative.
+%
+% Dec 9, 2002
+% version 2.1
+% The defaults for \footrulewidth, \plainheadrulewidth and
+% \plainfootrulewidth are changed from \z@skip to 0pt. In this way when
+% someone inadvertantly uses \setlength to change any of these, the value
+% of \z@skip will not be changed, rather an errormessage will be given.
+
+% March 3, 2004
+% Release of version 3.0
+
+% Oct 7, 2004
+% version 3.1
+% Added '\endlinechar=13' to \fancy@reset to prevent problems with
+% includegraphics in header when verbatiminput is active.
+
+% March 22, 2005
+% version 3.2
+% reset \everypar (the real one) in \fancy@reset because spanish.ldf does
+% strange things with \everypar between << and >>.
+
+\def\ifancy@mpty#1{\def\temp@a{#1}\ifx\temp@a\@empty}
+
+\def\fancy@def#1#2{\ifancy@mpty{#2}\fancy@gbl\def#1{\leavevmode}\else
+ \fancy@gbl\def#1{#2\strut}\fi}
+
+\let\fancy@gbl\global
+
+\def\@fancyerrmsg#1{%
+ \ifx\PackageError\undefined
+ \errmessage{#1}\else
+ \PackageError{Fancyhdr}{#1}{}\fi}
+\def\@fancywarning#1{%
+ \ifx\PackageWarning\undefined
+ \errmessage{#1}\else
+ \PackageWarning{Fancyhdr}{#1}{}\fi}
+
+% Usage: \@forc \var{charstring}{command to be executed for each char}
+% This is similar to LaTeX's \@tfor, but expands the charstring.
+
+\def\@forc#1#2#3{\expandafter\f@rc\expandafter#1\expandafter{#2}{#3}}
+\def\f@rc#1#2#3{\def\temp@ty{#2}\ifx\@empty\temp@ty\else
+ \f@@rc#1#2\f@@rc{#3}\fi}
+\def\f@@rc#1#2#3\f@@rc#4{\def#1{#2}#4\f@rc#1{#3}{#4}}
+
+% Usage: \f@nfor\name:=list\do{body}
+% Like LaTeX's \@for but an empty list is treated as a list with an empty
+% element
+
+\newcommand{\f@nfor}[3]{\edef\@fortmp{#2}%
+ \expandafter\@forloop#2,\@nil,\@nil\@@#1{#3}}
+
+% Usage: \def@ult \cs{defaults}{argument}
+% sets \cs to the characters from defaults appearing in argument
+% or defaults if it would be empty. All characters are lowercased.
+
+\newcommand\def@ult[3]{%
+ \edef\temp@a{\lowercase{\edef\noexpand\temp@a{#3}}}\temp@a
+ \def#1{}%
+ \@forc\tmpf@ra{#2}%
+ {\expandafter\if@in\tmpf@ra\temp@a{\edef#1{#1\tmpf@ra}}{}}%
+ \ifx\@empty#1\def#1{#2}\fi}
+%
+% \if@in
+%
+\newcommand{\if@in}[4]{%
+ \edef\temp@a{#2}\def\temp@b##1#1##2\temp@b{\def\temp@b{##1}}%
+ \expandafter\temp@b#2#1\temp@b\ifx\temp@a\temp@b #4\else #3\fi}
+
+\newcommand{\fancyhead}{\@ifnextchar[{\f@ncyhf\fancyhead h}%
+ {\f@ncyhf\fancyhead h[]}}
+\newcommand{\fancyfoot}{\@ifnextchar[{\f@ncyhf\fancyfoot f}%
+ {\f@ncyhf\fancyfoot f[]}}
+\newcommand{\fancyhf}{\@ifnextchar[{\f@ncyhf\fancyhf{}}%
+ {\f@ncyhf\fancyhf{}[]}}
+
+% New commands for offsets added
+
+\newcommand{\fancyheadoffset}{\@ifnextchar[{\f@ncyhfoffs\fancyheadoffset h}%
+ {\f@ncyhfoffs\fancyheadoffset h[]}}
+\newcommand{\fancyfootoffset}{\@ifnextchar[{\f@ncyhfoffs\fancyfootoffset f}%
+ {\f@ncyhfoffs\fancyfootoffset f[]}}
+\newcommand{\fancyhfoffset}{\@ifnextchar[{\f@ncyhfoffs\fancyhfoffset{}}%
+ {\f@ncyhfoffs\fancyhfoffset{}[]}}
+
+% The header and footer fields are stored in command sequences with
+% names of the form: \f@ncy with for [eo], from [lcr]
+% and from [hf].
+
+\def\f@ncyhf#1#2[#3]#4{%
+ \def\temp@c{}%
+ \@forc\tmpf@ra{#3}%
+ {\expandafter\if@in\tmpf@ra{eolcrhf,EOLCRHF}%
+ {}{\edef\temp@c{\temp@c\tmpf@ra}}}%
+ \ifx\@empty\temp@c\else
+ \@fancyerrmsg{Illegal char `\temp@c' in \string#1 argument:
+ [#3]}%
+ \fi
+ \f@nfor\temp@c{#3}%
+ {\def@ult\f@@@eo{eo}\temp@c
+ \if@twoside\else
+ \if\f@@@eo e\@fancywarning
+ {\string#1's `E' option without twoside option is useless}\fi\fi
+ \def@ult\f@@@lcr{lcr}\temp@c
+ \def@ult\f@@@hf{hf}{#2\temp@c}%
+ \@forc\f@@eo\f@@@eo
+ {\@forc\f@@lcr\f@@@lcr
+ {\@forc\f@@hf\f@@@hf
+ {\expandafter\fancy@def\csname
+ f@ncy\f@@eo\f@@lcr\f@@hf\endcsname
+ {#4}}}}}}
+
+\def\f@ncyhfoffs#1#2[#3]#4{%
+ \def\temp@c{}%
+ \@forc\tmpf@ra{#3}%
+ {\expandafter\if@in\tmpf@ra{eolrhf,EOLRHF}%
+ {}{\edef\temp@c{\temp@c\tmpf@ra}}}%
+ \ifx\@empty\temp@c\else
+ \@fancyerrmsg{Illegal char `\temp@c' in \string#1 argument:
+ [#3]}%
+ \fi
+ \f@nfor\temp@c{#3}%
+ {\def@ult\f@@@eo{eo}\temp@c
+ \if@twoside\else
+ \if\f@@@eo e\@fancywarning
+ {\string#1's `E' option without twoside option is useless}\fi\fi
+ \def@ult\f@@@lcr{lr}\temp@c
+ \def@ult\f@@@hf{hf}{#2\temp@c}%
+ \@forc\f@@eo\f@@@eo
+ {\@forc\f@@lcr\f@@@lcr
+ {\@forc\f@@hf\f@@@hf
+ {\expandafter\setlength\csname
+ f@ncyO@\f@@eo\f@@lcr\f@@hf\endcsname
+ {#4}}}}}%
+ \fancy@setoffs}
+
+% Fancyheadings version 1 commands. These are more or less deprecated,
+% but they continue to work.
+
+\newcommand{\lhead}{\@ifnextchar[{\@xlhead}{\@ylhead}}
+\def\@xlhead[#1]#2{\fancy@def\f@ncyelh{#1}\fancy@def\f@ncyolh{#2}}
+\def\@ylhead#1{\fancy@def\f@ncyelh{#1}\fancy@def\f@ncyolh{#1}}
+
+\newcommand{\chead}{\@ifnextchar[{\@xchead}{\@ychead}}
+\def\@xchead[#1]#2{\fancy@def\f@ncyech{#1}\fancy@def\f@ncyoch{#2}}
+\def\@ychead#1{\fancy@def\f@ncyech{#1}\fancy@def\f@ncyoch{#1}}
+
+\newcommand{\rhead}{\@ifnextchar[{\@xrhead}{\@yrhead}}
+\def\@xrhead[#1]#2{\fancy@def\f@ncyerh{#1}\fancy@def\f@ncyorh{#2}}
+\def\@yrhead#1{\fancy@def\f@ncyerh{#1}\fancy@def\f@ncyorh{#1}}
+
+\newcommand{\lfoot}{\@ifnextchar[{\@xlfoot}{\@ylfoot}}
+\def\@xlfoot[#1]#2{\fancy@def\f@ncyelf{#1}\fancy@def\f@ncyolf{#2}}
+\def\@ylfoot#1{\fancy@def\f@ncyelf{#1}\fancy@def\f@ncyolf{#1}}
+
+\newcommand{\cfoot}{\@ifnextchar[{\@xcfoot}{\@ycfoot}}
+\def\@xcfoot[#1]#2{\fancy@def\f@ncyecf{#1}\fancy@def\f@ncyocf{#2}}
+\def\@ycfoot#1{\fancy@def\f@ncyecf{#1}\fancy@def\f@ncyocf{#1}}
+
+\newcommand{\rfoot}{\@ifnextchar[{\@xrfoot}{\@yrfoot}}
+\def\@xrfoot[#1]#2{\fancy@def\f@ncyerf{#1}\fancy@def\f@ncyorf{#2}}
+\def\@yrfoot#1{\fancy@def\f@ncyerf{#1}\fancy@def\f@ncyorf{#1}}
+
+\newlength{\fancy@headwidth}
+\let\headwidth\fancy@headwidth
+\newlength{\f@ncyO@elh}
+\newlength{\f@ncyO@erh}
+\newlength{\f@ncyO@olh}
+\newlength{\f@ncyO@orh}
+\newlength{\f@ncyO@elf}
+\newlength{\f@ncyO@erf}
+\newlength{\f@ncyO@olf}
+\newlength{\f@ncyO@orf}
+\newcommand{\headrulewidth}{0.4pt}
+\newcommand{\footrulewidth}{0pt}
+\newcommand{\footruleskip}{.3\normalbaselineskip}
+
+% Fancyplain stuff shouldn't be used anymore (rather
+% \fancypagestyle{plain} should be used), but it must be present for
+% compatibility reasons.
+
+\newcommand{\plainheadrulewidth}{0pt}
+\newcommand{\plainfootrulewidth}{0pt}
+\newif\if@fancyplain \@fancyplainfalse
+\def\fancyplain#1#2{\if@fancyplain#1\else#2\fi}
+
+\headwidth=-123456789sp %magic constant
+
+% Command to reset various things in the headers:
+% a.o. single spacing (taken from setspace.sty)
+% and the catcode of ^^M (so that epsf files in the header work if a
+% verbatim crosses a page boundary)
+% It also defines a \nouppercase command that disables \uppercase and
+% \Makeuppercase. It can only be used in the headers and footers.
+\let\fnch@everypar\everypar% save real \everypar because of spanish.ldf
+\def\fancy@reset{\fnch@everypar{}\restorecr\endlinechar=13
+ \def\baselinestretch{1}%
+ \def\nouppercase##1{{\let\uppercase\relax\let\MakeUppercase\relax
+ \expandafter\let\csname MakeUppercase \endcsname\relax##1}}%
+ \ifx\undefined\@newbaseline% NFSS not present; 2.09 or 2e
+ \ifx\@normalsize\undefined \normalsize % for ucthesis.cls
+ \else \@normalsize \fi
+ \else% NFSS (2.09) present
+ \@newbaseline%
+ \fi}
+
+% Initialization of the head and foot text.
+
+% The default values still contain \fancyplain for compatibility.
+\fancyhf{} % clear all
+% lefthead empty on ``plain'' pages, \rightmark on even, \leftmark on odd pages
+% evenhead empty on ``plain'' pages, \leftmark on even, \rightmark on odd pages
+\if@twoside
+ \fancyhead[el,or]{\fancyplain{}{\sl\rightmark}}
+ \fancyhead[er,ol]{\fancyplain{}{\sl\leftmark}}
+\else
+ \fancyhead[l]{\fancyplain{}{\sl\rightmark}}
+ \fancyhead[r]{\fancyplain{}{\sl\leftmark}}
+\fi
+\fancyfoot[c]{\rm\thepage} % page number
+
+% Use box 0 as a temp box and dimen 0 as temp dimen.
+% This can be done, because this code will always
+% be used inside another box, and therefore the changes are local.
+
+\def\@fancyvbox#1#2{\setbox0\vbox{#2}\ifdim\ht0>#1\@fancywarning
+ {\string#1 is too small (\the#1): ^^J Make it at least \the\ht0.^^J
+ We now make it that large for the rest of the document.^^J
+ This may cause the page layout to be inconsistent, however\@gobble}%
+ \dimen0=#1\global\setlength{#1}{\ht0}\ht0=\dimen0\fi
+ \box0}
+
+% Put together a header or footer given the left, center and
+% right text, fillers at left and right and a rule.
+% The \lap commands put the text into an hbox of zero size,
+% so overlapping text does not generate an errormessage.
+% These macros have 5 parameters:
+% 1. LEFTSIDE BEARING % This determines at which side the header will stick
+% out. When \fancyhfoffset is used this calculates \headwidth, otherwise
+% it is \hss or \relax (after expansion).
+% 2. \f@ncyolh, \f@ncyelh, \f@ncyolf or \f@ncyelf. This is the left component.
+% 3. \f@ncyoch, \f@ncyech, \f@ncyocf or \f@ncyecf. This is the middle comp.
+% 4. \f@ncyorh, \f@ncyerh, \f@ncyorf or \f@ncyerf. This is the right component.
+% 5. RIGHTSIDE BEARING. This is always \relax or \hss (after expansion).
+
+\def\@fancyhead#1#2#3#4#5{#1\hbox to\headwidth{\fancy@reset
+ \@fancyvbox\headheight{\hbox
+ {\rlap{\parbox[b]{\headwidth}{\raggedright#2}}\hfill
+ \parbox[b]{\headwidth}{\centering#3}\hfill
+ \llap{\parbox[b]{\headwidth}{\raggedleft#4}}}\headrule}}#5}
+
+\def\@fancyfoot#1#2#3#4#5{#1\hbox to\headwidth{\fancy@reset
+ \@fancyvbox\footskip{\footrule
+ \hbox{\rlap{\parbox[t]{\headwidth}{\raggedright#2}}\hfill
+ \parbox[t]{\headwidth}{\centering#3}\hfill
+ \llap{\parbox[t]{\headwidth}{\raggedleft#4}}}}}#5}
+
+\def\headrule{{\if@fancyplain\let\headrulewidth\plainheadrulewidth\fi
+ \hrule\@height\headrulewidth\@width\headwidth \vskip-\headrulewidth}}
+
+\def\footrule{{\if@fancyplain\let\footrulewidth\plainfootrulewidth\fi
+ \vskip-\footruleskip\vskip-\footrulewidth
+ \hrule\@width\headwidth\@height\footrulewidth\vskip\footruleskip}}
+
+\def\ps@fancy{%
+\@ifundefined{@chapapp}{\let\@chapapp\chaptername}{}%for amsbook
+%
+% Define \MakeUppercase for old LaTeXen.
+% Note: we used \def rather than \let, so that \let\uppercase\relax (from
+% the version 1 documentation) will still work.
+%
+\@ifundefined{MakeUppercase}{\def\MakeUppercase{\uppercase}}{}%
+\@ifundefined{chapter}{\def\sectionmark##1{\markboth
+{\MakeUppercase{\ifnum \c@secnumdepth>\z@
+ \thesection\hskip 1em\relax \fi ##1}}{}}%
+\def\subsectionmark##1{\markright {\ifnum \c@secnumdepth >\@ne
+ \thesubsection\hskip 1em\relax \fi ##1}}}%
+{\def\chaptermark##1{\markboth {\MakeUppercase{\ifnum \c@secnumdepth>\m@ne
+ \@chapapp\ \thechapter. \ \fi ##1}}{}}%
+\def\sectionmark##1{\markright{\MakeUppercase{\ifnum \c@secnumdepth >\z@
+ \thesection. \ \fi ##1}}}}%
+%\csname ps@headings\endcsname % use \ps@headings defaults if they exist
+\ps@@fancy
+\gdef\ps@fancy{\@fancyplainfalse\ps@@fancy}%
+% Initialize \headwidth if the user didn't
+%
+\ifdim\headwidth<0sp
+%
+% This catches the case that \headwidth hasn't been initialized and the
+% case that the user added something to \headwidth in the expectation that
+% it was initialized to \textwidth. We compensate this now. This loses if
+% the user intended to multiply it by a factor. But that case is more
+% likely done by saying something like \headwidth=1.2\textwidth.
+% The doc says you have to change \headwidth after the first call to
+% \pagestyle{fancy}. This code is just to catch the most common cases were
+% that requirement is violated.
+%
+ \global\advance\headwidth123456789sp\global\advance\headwidth\textwidth
+\fi}
+\def\ps@fancyplain{\ps@fancy \let\ps@plain\ps@plain@fancy}
+\def\ps@plain@fancy{\@fancyplaintrue\ps@@fancy}
+\let\ps@@empty\ps@empty
+\def\ps@@fancy{%
+\ps@@empty % This is for amsbook/amsart, which do strange things with \topskip
+\def\@mkboth{\protect\markboth}%
+\def\@oddhead{\@fancyhead\fancy@Oolh\f@ncyolh\f@ncyoch\f@ncyorh\fancy@Oorh}%
+\def\@oddfoot{\@fancyfoot\fancy@Oolf\f@ncyolf\f@ncyocf\f@ncyorf\fancy@Oorf}%
+\def\@evenhead{\@fancyhead\fancy@Oelh\f@ncyelh\f@ncyech\f@ncyerh\fancy@Oerh}%
+\def\@evenfoot{\@fancyfoot\fancy@Oelf\f@ncyelf\f@ncyecf\f@ncyerf\fancy@Oerf}%
+}
+% Default definitions for compatibility mode:
+% These cause the header/footer to take the defined \headwidth as width
+% And to shift in the direction of the marginpar area
+
+\def\fancy@Oolh{\if@reversemargin\hss\else\relax\fi}
+\def\fancy@Oorh{\if@reversemargin\relax\else\hss\fi}
+\let\fancy@Oelh\fancy@Oorh
+\let\fancy@Oerh\fancy@Oolh
+
+\let\fancy@Oolf\fancy@Oolh
+\let\fancy@Oorf\fancy@Oorh
+\let\fancy@Oelf\fancy@Oelh
+\let\fancy@Oerf\fancy@Oerh
+
+% New definitions for the use of \fancyhfoffset
+% These calculate the \headwidth from \textwidth and the specified offsets.
+
+\def\fancy@offsolh{\headwidth=\textwidth\advance\headwidth\f@ncyO@olh
+ \advance\headwidth\f@ncyO@orh\hskip-\f@ncyO@olh}
+\def\fancy@offselh{\headwidth=\textwidth\advance\headwidth\f@ncyO@elh
+ \advance\headwidth\f@ncyO@erh\hskip-\f@ncyO@elh}
+
+\def\fancy@offsolf{\headwidth=\textwidth\advance\headwidth\f@ncyO@olf
+ \advance\headwidth\f@ncyO@orf\hskip-\f@ncyO@olf}
+\def\fancy@offself{\headwidth=\textwidth\advance\headwidth\f@ncyO@elf
+ \advance\headwidth\f@ncyO@erf\hskip-\f@ncyO@elf}
+
+\def\fancy@setoffs{%
+% Just in case \let\headwidth\textwidth was used
+ \fancy@gbl\let\headwidth\fancy@headwidth
+ \fancy@gbl\let\fancy@Oolh\fancy@offsolh
+ \fancy@gbl\let\fancy@Oelh\fancy@offselh
+ \fancy@gbl\let\fancy@Oorh\hss
+ \fancy@gbl\let\fancy@Oerh\hss
+ \fancy@gbl\let\fancy@Oolf\fancy@offsolf
+ \fancy@gbl\let\fancy@Oelf\fancy@offself
+ \fancy@gbl\let\fancy@Oorf\hss
+ \fancy@gbl\let\fancy@Oerf\hss}
+
+\newif\iffootnote
+\let\latex@makecol\@makecol
+\def\@makecol{\ifvoid\footins\footnotetrue\else\footnotefalse\fi
+\let\topfloat\@toplist\let\botfloat\@botlist\latex@makecol}
+\def\iftopfloat#1#2{\ifx\topfloat\empty #2\else #1\fi}
+\def\ifbotfloat#1#2{\ifx\botfloat\empty #2\else #1\fi}
+\def\iffloatpage#1#2{\if@fcolmade #1\else #2\fi}
+
+\newcommand{\fancypagestyle}[2]{%
+ \@namedef{ps@#1}{\let\fancy@gbl\relax#2\relax\ps@fancy}}
diff --git a/paper/icml2019.sty b/paper/icml2019.sty
new file mode 100644
index 000000000..c820b4c7b
--- /dev/null
+++ b/paper/icml2019.sty
@@ -0,0 +1,773 @@
+% File: icml2019.sty (LaTeX style file for ICML-2019, version of 2019-01-07)
+
+% This file contains the LaTeX formatting parameters for a two-column
+% conference proceedings that is 8.5 inches wide by 11 inches high.
+%
+% Modified by Iain Murray 2018: changed years, location. Remove affiliation notes when anonymous.
+% Move times dependency from .tex to .sty so fewer people delete it.
+%
+% Modified by Daniel Roy 2017: changed byline to use footnotes for affiliations, and removed emails
+%
+% Modified by Percy Liang 12/2/2013: changed the year, location from the previous template for ICML 2014
+
+% Modified by Fei Sha 9/2/2013: changed the year, location form the previous template for ICML 2013
+%
+% Modified by Fei Sha 4/24/2013: (1) remove the extra whitespace after the first author's email address (in %the camera-ready version) (2) change the Proceeding ... of ICML 2010 to 2014 so PDF's metadata will show up % correctly
+%
+% Modified by Sanjoy Dasgupta, 2013: changed years, location
+%
+% Modified by Francesco Figari, 2012: changed years, location
+%
+% Modified by Christoph Sawade and Tobias Scheffer, 2011: added line
+% numbers, changed years
+%
+% Modified by Hal Daume III, 2010: changed years, added hyperlinks
+%
+% Modified by Kiri Wagstaff, 2009: changed years
+%
+% Modified by Sam Roweis, 2008: changed years
+%
+% Modified by Ricardo Silva, 2007: update of the ifpdf verification
+%
+% Modified by Prasad Tadepalli and Andrew Moore, merely changing years.
+%
+% Modified by Kristian Kersting, 2005, based on Jennifer Dy's 2004 version
+% - running title. If the original title is to long or is breaking a line,
+% use \icmltitlerunning{...} in the preamble to supply a shorter form.
+% Added fancyhdr package to get a running head.
+% - Updated to store the page size because pdflatex does compile the
+% page size into the pdf.
+%
+% Hacked by Terran Lane, 2003:
+% - Updated to use LaTeX2e style file conventions (ProvidesPackage,
+% etc.)
+% - Added an ``appearing in'' block at the base of the first column
+% (thus keeping the ``appearing in'' note out of the bottom margin
+% where the printer should strip in the page numbers).
+% - Added a package option [accepted] that selects between the ``Under
+% review'' notice (default, when no option is specified) and the
+% ``Appearing in'' notice (for use when the paper has been accepted
+% and will appear).
+%
+% Originally created as: ml2k.sty (LaTeX style file for ICML-2000)
+% by P. Langley (12/23/99)
+
+%%%%%%%%%%%%%%%%%%%%
+%% This version of the style file supports both a ``review'' version
+%% and a ``final/accepted'' version. The difference is only in the
+%% text that appears in the note at the bottom of the first column of
+%% the first page. The default behavior is to print a note to the
+%% effect that the paper is under review and don't distribute it. The
+%% final/accepted version prints an ``Appearing in'' note. To get the
+%% latter behavior, in the calling file change the ``usepackage'' line
+%% from:
+%% \usepackage{icml2019}
+%% to
+%% \usepackage[accepted]{icml2019}
+%%%%%%%%%%%%%%%%%%%%
+
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesPackage{icml2019}[2019/01/07 v2.0 ICML Conference Style File]
+
+% Before 2018, \usepackage{times} was in the example TeX, but inevitably
+% not everybody did it.
+\RequirePackage{times}
+
+% Use fancyhdr package
+\RequirePackage{fancyhdr}
+\RequirePackage{color}
+\RequirePackage{algorithm}
+\RequirePackage{algorithmic}
+% \RequirePackage{natbib} % Removing NatBib, we'll use standard BibTex apastyle.
+ %Authors can chose to use what they prefer but as APA style
+\RequirePackage{eso-pic} % used by \AddToShipoutPicture
+\RequirePackage{forloop}
+
+%%%%%%%% Options
+\DeclareOption{accepted}{%
+ \renewcommand{\Notice@String}{\ICML@appearing}
+ \gdef\isaccepted{1}
+}
+\DeclareOption{nohyperref}{%
+ \gdef\nohyperref{1}
+}
+
+\ifdefined\nohyperref\else\ifdefined\hypersetup
+ \definecolor{mydarkblue}{rgb}{0,0.08,0.45}
+ \hypersetup{ %
+ pdftitle={},
+ pdfauthor={},
+ pdfsubject={Proceedings of the International Conference on Machine Learning 2019},
+ pdfkeywords={},
+ pdfborder=0 0 0,
+ pdfpagemode=UseNone,
+ colorlinks=true,
+ linkcolor=mydarkblue,
+ citecolor=mydarkblue,
+ filecolor=mydarkblue,
+ urlcolor=mydarkblue,
+ pdfview=FitH}
+
+ \ifdefined\isaccepted \else
+ \hypersetup{pdfauthor={Anonymous Submission}}
+ \fi
+\fi\fi
+
+%%%%%%%%%%%%%%%%%%%%
+% This string is printed at the bottom of the page for the
+% final/accepted version of the ``appearing in'' note. Modify it to
+% change that text.
+%%%%%%%%%%%%%%%%%%%%
+\newcommand{\ICML@appearing}
+
+%%%%%%%%%%%%%%%%%%%%
+% This string is printed at the bottom of the page for the draft/under
+% review version of the ``appearing in'' note. Modify it to change
+% that text.
+%%%%%%%%%%%%%%%%%%%%
+\newcommand{\Notice@String}{Preliminary work. Under review by MSJAR. Do not distribute.}
+
+% Cause the declared options to actually be parsed and activated
+\ProcessOptions\relax
+
+% Uncomment the following for debugging. It will cause LaTeX to dump
+% the version of the ``appearing in'' string that will actually appear
+% in the document.
+%\typeout{>> Notice string='\Notice@String'}
+
+% Change citation commands to be more like old ICML styles
+%\newcommand{\yrcite}[1]{\citeyearpar{#1}} % Removing NatBib, we'll use standard BibTex apastyle
+%\renewcommand{\cite}[1]{\citep{#1}} % Removing NatBib, we'll use standard BibTex apastyle
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% to ensure the letter format is used. pdflatex does compile the
+% page size into the pdf. This is done using \pdfpagewidth and
+% \pdfpageheight. As Latex does not know this directives, we first
+% check whether pdflatex or latex is used.
+%
+% Kristian Kersting 2005
+%
+% in order to account for the more recent use of pdfetex as the default
+% compiler, I have changed the pdf verification.
+%
+% Ricardo Silva 2007
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paperwidth=8.5in
+\paperheight=11in
+
+% old PDFLaTex verification, circa 2005
+%
+%\newif\ifpdf\ifx\pdfoutput\undefined
+% \pdffalse % we are not running PDFLaTeX
+%\else
+% \pdfoutput=1 % we are running PDFLaTeX
+% \pdftrue
+%\fi
+
+\newif\ifpdf %adapted from ifpdf.sty
+\ifx\pdfoutput\undefined
+\else
+ \ifx\pdfoutput\relax
+ \else
+ \ifcase\pdfoutput
+ \else
+ \pdftrue
+ \fi
+ \fi
+\fi
+
+\ifpdf
+% \pdfpagewidth=\paperwidth
+% \pdfpageheight=\paperheight
+ \setlength{\pdfpagewidth}{8.5in}
+ \setlength{\pdfpageheight}{11in}
+\fi
+
+% Physical page layout
+
+\evensidemargin -0.23in
+\oddsidemargin -0.23in
+\setlength\textheight{9.0in}
+\setlength\textwidth{6.75in}
+\setlength\columnsep{0.25in}
+\setlength\headheight{10pt}
+\setlength\headsep{10pt}
+\addtolength{\topmargin}{-20pt}
+\addtolength{\topmargin}{-0.29in}
+
+% Historically many authors tried to include packages like geometry or fullpage,
+% which change the page layout. It either makes the proceedings inconsistent, or
+% wastes organizers' time chasing authors. So let's nip these problems in the
+% bud here. -- Iain Murray 2018.
+%\RequirePackage{printlen}
+\AtBeginDocument{%
+% To get the numbers below, include printlen package above and see lengths like this:
+%\printlength\oddsidemargin\\
+%\printlength\headheight\\
+%\printlength\textheight\\
+%\printlength\marginparsep\\
+%\printlength\footskip\\
+%\printlength\hoffset\\
+%\printlength\paperwidth\\
+%\printlength\topmargin\\
+%\printlength\headsep\\
+%\printlength\textwidth\\
+%\printlength\marginparwidth\\
+%\printlength\marginparpush\\
+%\printlength\voffset\\
+%\printlength\paperheight\\
+%
+\newif\ifmarginsmessedwith
+\marginsmessedwithfalse
+\ifdim\oddsidemargin=-16.62178pt \else oddsidemargin has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\headheight=10.0pt \else headheight has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\textheight=650.43pt \else textheight has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\marginparsep=11.0pt \else marginparsep has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\footskip=0.0pt \else footskip has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\hoffset=0.0pt \else hoffset has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\paperwidth=614.295pt \else paperwidth has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\topmargin=-24.95781pt \else topmargin has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\headsep=10.0pt \else headsep has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\textwidth=487.8225pt \else textwidth has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\marginparwidth=65.0pt \else marginparwidth has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\marginparpush=5.0pt \else marginparpush has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\voffset=0.0pt \else voffset has been altered.\\ \marginsmessedwithtrue\fi
+\ifdim\paperheight=794.96999pt \else paperheight has been altered.\\ \marginsmessedwithtrue\fi
+\ifmarginsmessedwith
+
+\textbf{\large \em The page layout violates the ICML style.}
+
+Please do not change the page layout, or include packages like geometry,
+savetrees, or fullpage, which change it for you.
+
+We're not able to reliably undo arbitrary changes to the style. Please remove
+the offending package(s), or layout-changing commands and try again.
+
+\fi}
+
+
+%% The following is adapted from code in the acmconf.sty conference
+%% style file. The constants in it are somewhat magical, and appear
+%% to work well with the two-column format on US letter paper that
+%% ICML uses, but will break if you change that layout, or if you use
+%% a longer block of text for the copyright notice string. Fiddle with
+%% them if necessary to get the block to fit/look right.
+%%
+%% -- Terran Lane, 2003
+%%
+%% The following comments are included verbatim from acmconf.sty:
+%%
+%%% This section (written by KBT) handles the 1" box in the lower left
+%%% corner of the left column of the first page by creating a picture,
+%%% and inserting the predefined string at the bottom (with a negative
+%%% displacement to offset the space allocated for a non-existent
+%%% caption).
+%%%
+\def\ftype@copyrightbox{8}
+\def\@copyrightspace{
+% Create a float object positioned at the bottom of the column. Note
+% that because of the mystical nature of floats, this has to be called
+% before the first column is populated with text (e.g., from the title
+% or abstract blocks). Otherwise, the text will force the float to
+% the next column. -- TDRL.
+\@float{copyrightbox}[b]
+\begin{center}
+\setlength{\unitlength}{1pc}
+\begin{picture}(20,1.5)
+% Create a line separating the main text from the note block.
+% 4.818pc==0.8in.
+\put(0,2.5){\line(1,0){4.818}}
+% Insert the text string itself. Note that the string has to be
+% enclosed in a parbox -- the \put call needs a box object to
+% position. Without the parbox, the text gets splattered across the
+% bottom of the page semi-randomly. The 19.75pc distance seems to be
+% the width of the column, though I can't find an appropriate distance
+% variable to substitute here. -- TDRL.
+\put(0,0){\parbox[b]{19.75pc}{\small \Notice@String}}
+\end{picture}
+\end{center}
+\end@float}
+
+% Note: A few Latex versions need the next line instead of the former.
+% \addtolength{\topmargin}{0.3in}
+% \setlength\footheight{0pt}
+\setlength\footskip{0pt}
+%\pagestyle{empty}
+\flushbottom \twocolumn
+\sloppy
+
+% Clear out the addcontentsline command
+\def\addcontentsline#1#2#3{}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% commands for formatting paper title, author names, and addresses.
+
+%%start%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%% title as running head -- Kristian Kersting 2005 %%%%%%%%%%%%%
+
+
+%\makeatletter
+%\newtoks\mytoksa
+%\newtoks\mytoksb
+%\newcommand\addtomylist[2]{%
+% \mytoksa\expandafter{#1}%
+% \mytoksb{#2}%
+% \edef#1{\the\mytoksa\the\mytoksb}%
+%}
+%\makeatother
+
+% box to check the size of the running head
+\newbox\titrun
+
+% general page style
+\pagestyle{fancy}
+\fancyhf{}
+\fancyhead{}
+\fancyfoot{}
+% set the width of the head rule to 1 point
+\renewcommand{\headrulewidth}{1pt}
+
+% definition to set the head as running head in the preamble
+\def\icmltitlerunning#1{\gdef\@icmltitlerunning{#1}}
+
+% main definition adapting \icmltitle from 2004
+\long\def\icmltitle#1{%
+
+ %check whether @icmltitlerunning exists
+ % if not \icmltitle is used as running head
+ \ifx\undefined\@icmltitlerunning%
+ \gdef\@icmltitlerunning{#1}
+ \fi
+
+ %add it to pdf information
+ \ifdefined\nohyperref\else\ifdefined\hypersetup
+ \hypersetup{pdftitle={#1}}
+ \fi\fi
+
+ %get the dimension of the running title
+ \global\setbox\titrun=\vbox{\small\bf\@icmltitlerunning}
+
+ % error flag
+ \gdef\@runningtitleerror{0}
+
+ % running title too long
+ \ifdim\wd\titrun>\textwidth%
+ {\gdef\@runningtitleerror{1}}%
+ % running title breaks a line
+ \else\ifdim\ht\titrun>6.25pt
+ {\gdef\@runningtitleerror{2}}%
+ \fi
+ \fi
+
+ % if there is somthing wrong with the running title
+ \ifnum\@runningtitleerror>0
+ \typeout{}%
+ \typeout{}%
+ \typeout{*******************************************************}%
+ \typeout{Title exceeds size limitations for running head.}%
+ \typeout{Please supply a shorter form for the running head}
+ \typeout{with \string\icmltitlerunning{...}\space prior to \string\begin{document}}%
+ \typeout{*******************************************************}%
+ \typeout{}%
+ \typeout{}%
+ % set default running title
+ \chead{\small\bf Title Suppressed Due to Excessive Size}%
+ \else
+ % 'everything' fine, set provided running title
+ \chead{\small\bf\@icmltitlerunning}%
+ \fi
+
+ % no running title on the first page of the paper
+ \thispagestyle{empty}
+
+%%%%%%%%%%%%%%%%%%%% Kristian Kersting %%%%%%%%%%%%%%%%%%%%%%%%%
+%end%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+ {\center\baselineskip 18pt
+ \toptitlebar{\Large\bf #1}\bottomtitlebar}
+}
+
+
+\gdef\icmlfullauthorlist{}
+\newcommand\addstringtofullauthorlist{\g@addto@macro\icmlfullauthorlist}
+\newcommand\addtofullauthorlist[1]{%
+ \ifdefined\icmlanyauthors%
+ \addstringtofullauthorlist{, #1}%
+ \else%
+ \addstringtofullauthorlist{#1}%
+ \gdef\icmlanyauthors{1}%
+ \fi%
+ \ifdefined\nohyperref\else\ifdefined\hypersetup%
+ \hypersetup{pdfauthor=\icmlfullauthorlist}%
+ \fi\fi}
+
+
+\def\toptitlebar{\hrule height1pt \vskip .25in}
+\def\bottomtitlebar{\vskip .22in \hrule height1pt \vskip .3in}
+
+\newenvironment{icmlauthorlist}{%
+ \setlength\topsep{0pt}
+ \setlength\parskip{0pt}
+ \begin{center}
+}{%
+ \end{center}
+}
+
+\newcounter{@affiliationcounter}
+\newcommand{\@pa}[1]{%
+% ``#1''
+\ifcsname the@affil#1\endcsname
+ % do nothing
+\else
+ \ifcsname @icmlsymbol#1\endcsname
+ % nothing
+ \else
+ \stepcounter{@affiliationcounter}%
+ \newcounter{@affil#1}%
+ \setcounter{@affil#1}{\value{@affiliationcounter}}%
+ \fi
+\fi%
+\ifcsname @icmlsymbol#1\endcsname
+ \textsuperscript{\csname @icmlsymbol#1\endcsname\,}%
+\else
+ %\expandafter\footnotemark[\arabic{@affil#1}\,]%
+ \textsuperscript{\arabic{@affil#1}\,}%
+\fi
+}
+
+%\newcommand{\icmlauthor}[2]{%
+%\addtofullauthorlist{#1}%
+%#1\@for\theaffil:=#2\do{\pa{\theaffil}}%
+%}
+\newcommand{\icmlauthor}[2]{%
+ \ifdefined\isaccepted
+ \mbox{\bf #1}\,\@for\theaffil:=#2\do{\@pa{\theaffil}} \addtofullauthorlist{#1}%
+ \else
+ \ifdefined\@icmlfirsttime
+ \else
+ \gdef\@icmlfirsttime{1}
+ \mbox{\bf Anonymous Authors} \addtofullauthorlist{Anonymous Authors}
+ \fi
+ \fi
+}
+
+\newcommand{\icmlsetsymbol}[2]{%
+ \expandafter\gdef\csname @icmlsymbol#1\endcsname{#2}
+ }
+
+
+\newcommand{\icmlaffiliation}[2]{%
+\ifdefined\isaccepted
+\ifcsname the@affil#1\endcsname
+ \expandafter\gdef\csname @affilname\csname the@affil#1\endcsname\endcsname{#2}%
+\else
+ {\bf AUTHORERR: Error in use of \textbackslash{}icmlaffiliation command. Label ``#1'' not mentioned in some \textbackslash{}icmlauthor\{author name\}\{labels here\} command beforehand. }
+ \typeout{}%
+ \typeout{}%
+ \typeout{*******************************************************}%
+ \typeout{Affiliation label undefined. }%
+ \typeout{Make sure \string\icmlaffiliation\space follows }
+ \typeout{all of \string\icmlauthor\space commands}%
+ \typeout{*******************************************************}%
+ \typeout{}%
+ \typeout{}%
+\fi
+\else % \isaccepted
+ % can be called multiple times... it's idempotent
+ \expandafter\gdef\csname @affilname1\endcsname{Anonymous Institution, Anonymous City, Anonymous Region, Anonymous Country}
+\fi
+}
+
+\newcommand{\icmlcorrespondingauthor}[2]{
+\ifdefined\isaccepted
+ \ifdefined\icmlcorrespondingauthor@text
+ \g@addto@macro\icmlcorrespondingauthor@text{, #1 \textless{}#2\textgreater{}}
+ \else
+ \gdef\icmlcorrespondingauthor@text{#1 \textless{}#2\textgreater{}}
+ \fi
+\else
+\gdef\icmlcorrespondingauthor@text{Anonymous Author \textless{}anon.email@domain.com\textgreater{}}
+\fi
+}
+
+\newcommand{\icmlEqualContribution}{\textsuperscript{*}Equal contribution }
+
+\newcounter{@affilnum}
+\newcommand{\printAffiliationsAndNotice}[1]{%
+\stepcounter{@affiliationcounter}%
+{\let\thefootnote\relax\footnotetext{\hspace*{-\footnotesep}\ifdefined\isaccepted #1\fi%
+\forloop{@affilnum}{1}{\value{@affilnum} < \value{@affiliationcounter}}{
+\textsuperscript{\arabic{@affilnum}}\ifcsname @affilname\the@affilnum\endcsname%
+\csname @affilname\the@affilnum\endcsname%
+\else
+{\bf AUTHORERR: Missing \textbackslash{}icmlaffiliation.}
+\fi
+}.
+\ifdefined\icmlcorrespondingauthor@text
+Correspondence to: \icmlcorrespondingauthor@text.
+%Pedro \else
+%Pedro {\bf AUTHORERR: Missing \textbackslash{}icmlcorrespondingauthor.}
+\fi
+
+\ \\
+\Notice@String
+}
+}
+}
+
+%\makeatother
+
+\long\def\icmladdress#1{%
+ {\bf The \textbackslash{}icmladdress command is no longer used. See the example\_paper PDF .tex for usage of \textbackslash{}icmlauther and \textbackslash{}icmlaffiliation.}
+}
+
+%% keywords as first class citizens
+\def\icmlkeywords#1{%
+% \ifdefined\isaccepted \else
+% \par {\bf Keywords:} #1%
+% \fi
+% \ifdefined\nohyperref\else\ifdefined\hypersetup
+% \hypersetup{pdfkeywords={#1}}
+% \fi\fi
+% \ifdefined\isaccepted \else
+% \par {\bf Keywords:} #1%
+% \fi
+ \ifdefined\nohyperref\else\ifdefined\hypersetup
+ \hypersetup{pdfkeywords={#1}}
+ \fi\fi
+}
+
+% modification to natbib citations
+%\setcitestyle{authoryear,round,citesep={;},aysep={,},yysep={;}} % Removing NatBib, we'll use standard BibTex apastyle
+
+% Redefinition of the abstract environment.
+\renewenvironment{abstract}
+ {%
+% Insert the ``appearing in'' copyright notice.
+%\@copyrightspace
+\centerline{\large\bf Abstract}
+ \vspace{-0.12in}\begin{quote}}
+ {\par\end{quote}\vskip 0.12in}
+
+% numbered section headings with different treatment of numbers
+
+\def\@startsection#1#2#3#4#5#6{\if@noskipsec \leavevmode \fi
+ \par \@tempskipa #4\relax
+ \@afterindenttrue
+% Altered the following line to indent a section's first paragraph.
+% \ifdim \@tempskipa <\z@ \@tempskipa -\@tempskipa \@afterindentfalse\fi
+ \ifdim \@tempskipa <\z@ \@tempskipa -\@tempskipa \fi
+ \if@nobreak \everypar{}\else
+ \addpenalty{\@secpenalty}\addvspace{\@tempskipa}\fi \@ifstar
+ {\@ssect{#3}{#4}{#5}{#6}}{\@dblarg{\@sict{#1}{#2}{#3}{#4}{#5}{#6}}}}
+
+\def\@sict#1#2#3#4#5#6[#7]#8{\ifnum #2>\c@secnumdepth
+ \def\@svsec{}\else
+ \refstepcounter{#1}\edef\@svsec{\csname the#1\endcsname}\fi
+ \@tempskipa #5\relax
+ \ifdim \@tempskipa>\z@
+ \begingroup #6\relax
+ \@hangfrom{\hskip #3\relax\@svsec.~}{\interlinepenalty \@M #8\par}
+ \endgroup
+ \csname #1mark\endcsname{#7}\addcontentsline
+ {toc}{#1}{\ifnum #2>\c@secnumdepth \else
+ \protect\numberline{\csname the#1\endcsname}\fi
+ #7}\else
+ \def\@svsechd{#6\hskip #3\@svsec #8\csname #1mark\endcsname
+ {#7}\addcontentsline
+ {toc}{#1}{\ifnum #2>\c@secnumdepth \else
+ \protect\numberline{\csname the#1\endcsname}\fi
+ #7}}\fi
+ \@xsect{#5}}
+
+\def\@sect#1#2#3#4#5#6[#7]#8{\ifnum #2>\c@secnumdepth
+ \def\@svsec{}\else
+ \refstepcounter{#1}\edef\@svsec{\csname the#1\endcsname\hskip 0.4em }\fi
+ \@tempskipa #5\relax
+ \ifdim \@tempskipa>\z@
+ \begingroup #6\relax
+ \@hangfrom{\hskip #3\relax\@svsec}{\interlinepenalty \@M #8\par}
+ \endgroup
+ \csname #1mark\endcsname{#7}\addcontentsline
+ {toc}{#1}{\ifnum #2>\c@secnumdepth \else
+ \protect\numberline{\csname the#1\endcsname}\fi
+ #7}\else
+ \def\@svsechd{#6\hskip #3\@svsec #8\csname #1mark\endcsname
+ {#7}\addcontentsline
+ {toc}{#1}{\ifnum #2>\c@secnumdepth \else
+ \protect\numberline{\csname the#1\endcsname}\fi
+ #7}}\fi
+ \@xsect{#5}}
+
+% section headings with less space above and below them
+\def\thesection {\arabic{section}}
+\def\thesubsection {\thesection.\arabic{subsection}}
+\def\section{\@startsection{section}{1}{\z@}{-0.12in}{0.02in}
+ {\large\bf\raggedright}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-0.10in}{0.01in}
+ {\normalsize\bf\raggedright}}
+\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{-0.08in}{0.01in}
+ {\normalsize\sc\raggedright}}
+\def\paragraph{\@startsection{paragraph}{4}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+\def\subparagraph{\@startsection{subparagraph}{5}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+
+% Footnotes
+\footnotesep 6.65pt %
+\skip\footins 9pt
+\def\footnoterule{\kern-3pt \hrule width 0.8in \kern 2.6pt }
+\setcounter{footnote}{0}
+
+% Lists and paragraphs
+\parindent 0pt
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 2pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+\parskip 6pt
+
+\leftmargin 2em \leftmargini\leftmargin \leftmarginii 2em
+\leftmarginiii 1.5em \leftmarginiv 1.0em \leftmarginv .5em
+\leftmarginvi .5em
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+ \labelwidth\leftmarginii\advance\labelwidth-\labelsep
+ \topsep 2pt plus 1pt minus 0.5pt
+ \parsep 1pt plus 0.5pt minus 0.5pt
+ \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+ \labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+ \topsep 1pt plus 0.5pt minus 0.5pt
+ \parsep \z@ \partopsep 0.5pt plus 0pt minus 0.5pt
+ \itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+ \labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+ \labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+ \labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\@normalsize{\@setsize\normalsize{11pt}\xpt\@xpt}
+\def\small{\@setsize\small{10pt}\ixpt\@ixpt}
+\def\footnotesize{\@setsize\footnotesize{10pt}\ixpt\@ixpt}
+\def\scriptsize{\@setsize\scriptsize{8pt}\viipt\@viipt}
+\def\tiny{\@setsize\tiny{7pt}\vipt\@vipt}
+\def\large{\@setsize\large{14pt}\xiipt\@xiipt}
+\def\Large{\@setsize\Large{16pt}\xivpt\@xivpt}
+\def\LARGE{\@setsize\LARGE{20pt}\xviipt\@xviipt}
+\def\huge{\@setsize\huge{23pt}\xxpt\@xxpt}
+\def\Huge{\@setsize\Huge{28pt}\xxvpt\@xxvpt}
+
+% Revised formatting for figure captions and table titles.
+\newsavebox\newcaptionbox\newdimen\newcaptionboxwid
+
+\long\def\@makecaption#1#2{
+ \vskip 10pt
+ \baselineskip 11pt
+ \setbox\@tempboxa\hbox{#1. #2}
+ \ifdim \wd\@tempboxa >\hsize
+ \sbox{\newcaptionbox}{\small\sl #1.~}
+ \newcaptionboxwid=\wd\newcaptionbox
+ \usebox\newcaptionbox {\footnotesize #2}
+% \usebox\newcaptionbox {\small #2}
+ \else
+ \centerline{{\small\sl #1.} {\small #2}}
+ \fi}
+
+\def\fnum@figure{Figure \thefigure}
+\def\fnum@table{Table \thetable}
+
+% Strut macros for skipping spaces above and below text in tables.
+\def\abovestrut#1{\rule[0in]{0in}{#1}\ignorespaces}
+\def\belowstrut#1{\rule[-#1]{0in}{#1}\ignorespaces}
+
+\def\abovespace{\abovestrut{0.20in}}
+\def\aroundspace{\abovestrut{0.20in}\belowstrut{0.10in}}
+\def\belowspace{\belowstrut{0.10in}}
+
+% Various personal itemization commands.
+\def\texitem#1{\par\noindent\hangindent 12pt
+ \hbox to 12pt {\hss #1 ~}\ignorespaces}
+\def\icmlitem{\texitem{$\bullet$}}
+
+% To comment out multiple lines of text.
+\long\def\comment#1{}
+
+
+
+
+%% Line counter (not in final version). Adapted from NIPS style file by Christoph Sawade
+
+% Vertical Ruler
+% This code is, largely, from the CVPR 2010 conference style file
+% ----- define vruler
+\makeatletter
+\newbox\icmlrulerbox
+\newcount\icmlrulercount
+\newdimen\icmlruleroffset
+\newdimen\cv@lineheight
+\newdimen\cv@boxheight
+\newbox\cv@tmpbox
+\newcount\cv@refno
+\newcount\cv@tot
+% NUMBER with left flushed zeros \fillzeros[]
+\newcount\cv@tmpc@ \newcount\cv@tmpc
+\def\fillzeros[#1]#2{\cv@tmpc@=#2\relax\ifnum\cv@tmpc@<0\cv@tmpc@=-\cv@tmpc@\fi
+\cv@tmpc=1 %
+\loop\ifnum\cv@tmpc@<10 \else \divide\cv@tmpc@ by 10 \advance\cv@tmpc by 1 \fi
+ \ifnum\cv@tmpc@=10\relax\cv@tmpc@=11\relax\fi \ifnum\cv@tmpc@>10 \repeat
+\ifnum#2<0\advance\cv@tmpc1\relax-\fi
+\loop\ifnum\cv@tmpc<#1\relax0\advance\cv@tmpc1\relax\fi \ifnum\cv@tmpc<#1 \repeat
+\cv@tmpc@=#2\relax\ifnum\cv@tmpc@<0\cv@tmpc@=-\cv@tmpc@\fi \relax\the\cv@tmpc@}%
+% \makevruler[][][][][]
+\def\makevruler[#1][#2][#3][#4][#5]{
+ \begingroup\offinterlineskip
+ \textheight=#5\vbadness=10000\vfuzz=120ex\overfullrule=0pt%
+ \global\setbox\icmlrulerbox=\vbox to \textheight{%
+ {
+ \parskip=0pt\hfuzz=150em\cv@boxheight=\textheight
+ \cv@lineheight=#1\global\icmlrulercount=#2%
+ \cv@tot\cv@boxheight\divide\cv@tot\cv@lineheight\advance\cv@tot2%
+ \cv@refno1\vskip-\cv@lineheight\vskip1ex%
+ \loop\setbox\cv@tmpbox=\hbox to0cm{ % side margin
+ \hfil {\hfil\fillzeros[#4]\icmlrulercount}
+ }%
+ \ht\cv@tmpbox\cv@lineheight\dp\cv@tmpbox0pt\box\cv@tmpbox\break
+ \advance\cv@refno1\global\advance\icmlrulercount#3\relax
+ \ifnum\cv@refno<\cv@tot\repeat
+ }
+ }
+ \endgroup
+}%
+\makeatother
+% ----- end of vruler
+
+
+% \makevruler[][][][][]
+\def\icmlruler#1{\makevruler[12pt][#1][1][3][\textheight]\usebox{\icmlrulerbox}}
+\AddToShipoutPicture{%
+\icmlruleroffset=\textheight
+\advance\icmlruleroffset by 5.2pt % top margin
+ \color[rgb]{.7,.7,.7}
+ \ifdefined\isaccepted \else
+ \AtTextUpperLeft{%
+ \put(\LenToUnit{-35pt},\LenToUnit{-\icmlruleroffset}){%left ruler
+ \icmlruler{\icmlrulercount}}
+% \put(\LenToUnit{1.04\textwidth},\LenToUnit{-\icmlruleroffset}){%right ruler
+% \icmlruler{\icmlrulercount}}
+ }
+ \fi
+}
+\endinput
diff --git a/paper/main.tex b/paper/main.tex
new file mode 100644
index 000000000..430c5757c
--- /dev/null
+++ b/paper/main.tex
@@ -0,0 +1,780 @@
+%%%%%% MSJAR 2019 LATEX Template. Based on ICML 2019 submission template %%%%%%%
+
+\documentclass{article}
+
+% Recommended, but optional, packages for figures and better typesetting:
+\usepackage{microtype}
+\usepackage{graphicx}
+% \usepackage{subfigure}
+\usepackage{booktabs} % for professional tables
+\usepackage{amsmath} % for equations
+\usepackage{amssymb} % for \mathbb
+%\usepackage[semicolon]{natbib}
+
+% hyperref makes hyperlinks in the resulting PDF.
+% If your build breaks (sometimes temporarily if a hyperlink spans a page)
+% please comment out the following usepackage line and replace
+% \usepackage{icml2019} with \usepackage[nohyperref]{icml2019} above.
+\usepackage{hyperref}
+
+% Attempt to make hyperref and algorithmic work together better:
+\newcommand{\theHalgorithm}{\arabic{algorithm}}
+\newcommand{\passk}[1]{pass\textasciicircum #1}
+
+\usepackage[accepted]{icml2019}
+
+\begin{document}
+\raggedbottom
+
+
+\twocolumn[
+\title{BC-Bench: Evaluating Agentic Engineering in a Domain-Specific Language for ERP}
+\date{\vspace{-0.2in}}
+\maketitle
+
+%//////////////////// Author information ////////////////////////
+% It is OKAY to include author information, even for blind
+% submissions: the style file will automatically remove it for you
+% unless you've provided the [accepted] option to the icml2019
+% package.
+
+% Include the name, group and email of the author.
+% Use the standard \icmlauthor{Name, Group, email@email.com}{}\\
+% Don't remove the curly brackets {} at the end.
+
+% Equal contribution.
+% Adding "equal" to the second bracket will create an * meaning equal contribution.
+% If you are using equal contribution read further instructions after square bracket ].
+% Affiliations will be numbered in order of appearance
+\icmlsetsymbol{equal}{*}
+
+\begin{icmlauthorlist}
+\icmlauthor{Haoran Sun, Microsoft, haoransun@microsoft.com}{equal}
+\icmlauthor{Klaus Marius Hansen, Microsoft, klaus.marius.hansen@microsoft.com}{equal}
+\end{icmlauthorlist}
+
+\vspace{0.4in}
+\begin{abstract}
+Agentic engineering systems have shown strong performance on general-purpose benchmarks, yet their effectiveness in enterprise resource planning (ERP) domain-specific languages (DSLs) remains underexplored. We introduce BC-Bench, a benchmark designed to evaluate agentic engineering on real-world tasks in AL, the DSL for Microsoft Dynamics 365 Business Central. BC-Bench comprises 101 manually curated tasks extracted from two Microsoft-owned production repositories, reflecting authentic ERP development workflows. Adapting the SWE-Bench methodology, we address the unique constraints of the AL ecosystem---including limited public resources and complex environment provisioning. Beyond generating functional code, BC-Bench evaluates test generation and supports multimodal problem statements where visual context is commonly present. We evaluate multiple frontier models across two agent harnesses, utilizing multi-run metrics to account for nondeterminism. In the Bug Fixing category, under our evaluated settings, between-model differences in resolution rate are larger than differences between the two evaluated agent harnesses, and improvements reported on general-purpose benchmarks do not consistently transfer to AL. These results highlight the need for domain-specific evaluation.
+
+\textbf{Keywords:} DSL, Code Generation, Benchmark, Evaluation, Language Model, AL, ERP
+\end{abstract}
+\vspace{0.4in}
+]
+
+% This command creates the footnote for the equal contribution*
+% If you need to mention equal contribution uncomment the next line
+\printAffiliationsAndNotice{\icmlEqualContribution} % Uncomment for equal contribution
+\clearpage
+
+\section{Background}
+
+Despite rapid advancements in language model (LM)-based code generation, there is limited work on evaluating agentic engineering workflows for Domain-Specific Languages (DSLs). Existing benchmarks primarily focus on general-purpose programming languages like Python \cite{jimenez_swe-bench_2024, deng_swe-bench_2025}, while benchmarks targeting DSLs remain relatively limited. Results from general-purpose benchmarks do not necessarily transfer to DSL settings \cite{cassano_knowledge_2024}.
+
+Microsoft Dynamics 365 Business Central is ERP software for small and medium-sized businesses. Its business logic is implemented in a DSL called AL. AL is a statically typed language inspired by Pascal. An AL program is made up of objects including pages (UI), codeunits (business logic) and tables (backed by real SQL tables), which are compiled to .NET assemblies that are deployed into a Business Central environment and run against its database. AL tests are themselves codeunits, marked with \texttt{Subtype = Test}, and they require a live Business Central environment to run. This tight coupling between code and a running environment, together with AL conventions around object triggers, record validation, permissions, and domain-specific test libraries, gives AL characteristics not commonly seen in general-purpose languages. Figure~\ref{fig:al-snippets} shows two short examples of AL code.
+
+Evaluating LM-based coding agents on DSLs in the ERP domain presents several practical challenges. First, publicly available resources for dataset construction are limited, as industrial codebases are often proprietary. For example, at the time of writing (February 2026), GitHub search returned about 2.4M repositories for Python and 581K for C\#, compared to about 1.7K for ABAP (an ERP language used in SAP systems) and 338 for AL under the MIT license\footnote{GitHub repository search snapshot for AL with MIT license (accessed February 2026): \href{https://github.com/search?q=language\%3AAL+license\%3AMIT\&type=repositories}{language:AL license: MIT github.com}.} (these time-varying counts are included to illustrate relative scale rather than exact totals). The scarcity of resources necessitates significant manual effort to curate a realistic and diverse dataset. Second, DSL ecosystems rely on domain-specific tooling and infrastructure, making it difficult to adapt existing benchmarking frameworks \cite{joel_survey_2025,yang_swe-bench_2024}.
+
+\begin{figure}[t]
+\vskip 0.05in
+\begin{center}
+\begin{minipage}{0.98\columnwidth}
+{\scriptsize
+\noindent\textbf{(a) Table object with auto-assigned No.}
+\begin{verbatim}
+table 18 Customer
+{
+ fields
+ {
+ field(1; "No."; Code[20]) { }
+ field(107; "No. Series"; Code[20])
+ {
+ TableRelation = "No. Series";
+ }
+ }
+
+ trigger OnInsert()
+ var
+ NoSeries: Codeunit "No. Series";
+ begin
+ if "No." = '' then
+ "No." := NoSeries.GetNextNo("No. Series");
+ end;
+}
+\end{verbatim}
+
+\vspace{2pt}
+
+\noindent\textbf{(b) Test codeunit}
+\begin{verbatim}
+codeunit 134530 "No. Series Tests"
+{
+ Subtype = Test;
+
+ [Test]
+ procedure FailsWhenSeriesRunsOut()
+ var
+ NoSeries: Codeunit "No. Series";
+ NoSeriesCode: Code[20];
+ begin
+ Initialize();
+
+ asserterror NoSeries.GetNextNo(NoSeriesCode);
+ LibraryAssert.ExpectedError(CannotAssignNewErr);
+ end;
+}
+\end{verbatim}
+}
+\end{minipage}
+\end{center}
+\caption{Simplified AL examples: a \texttt{Customer} table that auto-assigns its primary key from a No. Series, and a test codeunit that exercises No. Series behavior.}
+\label{fig:al-snippets}
+\vskip -0.1in
+\end{figure}
+
+Business Central's ecosystem is highly extensible and customizable, with thousands of partner-developed applications available on Microsoft Marketplace. As of February 2026, Microsoft Marketplace listed about 8.4K partner-published apps for Business Central\footnote{Microsoft Marketplace partner-app search snapshot for Business Central (accessed February 2026): \href{https://marketplace.microsoft.com/en-us/search/products?product=dynamics-365-business-central\&page=1\&filters=partners}{marketplace.microsoft.com}.} and about 8.8K partner-published apps across all Microsoft 365 products\footnote{Microsoft Marketplace partner-app search snapshot for Microsoft 365 products (accessed February 2026): \href{https://marketplace.microsoft.com/en-us/search/products?page=1\&filters=partners\&product=excel\%3Bofficemetaos\%3Bonenote\%3Bpowerpoint\%3Boutlook\%3Bproject\%3Bsharepoint\%3Bteams\%3Bword\%3Bviva}{marketplace.microsoft.com}.}. While numbers vary with time, this highlights the scale and diversity of real-world AL development. Measuring and improving agentic engineering performance in AL is critical to the development and maintenance of this ecosystem.
+
+We introduce BC-Bench, a benchmark for evaluating coding agents on real-world Business Central/AL engineering tasks, including a dataset of 101 manually curated tasks from two production repositories: a private repository NAV and a public repository BCApps \footnote{\href{https://github.com/microsoft/BCApps}{https://github.com/microsoft/BCApps}}. BC-Bench is open source and publicly available on GitHub \footnote{\href{https://github.com/microsoft/BC-Bench}{https://github.com/microsoft/BC-Bench}}. BC-Bench uses real-world agent harnesses, GitHub Copilot and Claude Code, to closely approximate practical usage by AL engineers. It is designed to support experimentation with different agent configurations, such as custom instructions, tools, and skills. The benchmark runs directly in GitHub Actions without additional implementation, reducing reproducibility issues that arise from minor implementation differences \cite{biderman_lessons_2025}.
+
+This paper makes the following contributions: (1) a benchmark for evaluating coding agents on real-world AL tasks; (2) a manually curated dataset from production repositories; (3) an evaluation harness that enables reproducible experimentation with agent configurations in an AL-specific environment.
+
+\section{Related Work}
+
+Recent work has introduced benchmarks for evaluating LM-based coding agents on real-world software engineering tasks. SWE-Bench \cite{jimenez_swe-bench_2024} pioneered evaluation using real-world Python issues. Extensions include SWE-Bench Multimodal \cite{yang_swe-bench_2024} which incorporates visual elements and expands to additional languages, and SWE-Bench-Live \cite{zhang_swe-bench_2025} which introduced RepoLaunch to simplify the environment setup for different programming languages. Addressing some known limitations of SWE-Bench, SWE-Bench Pro \cite{deng_swe-bench_2025} was introduced to improve task realism and mitigate contamination risks. Terminal-Bench \cite{merrill_terminal-bench_2026} goes beyond academic agent harnesses and uses real-world agent harnesses like Claude Code to better reflect actual developer workflow.
+
+While these benchmarks provide robust evaluation for general-purpose programming languages, they do not capture the constraints and characteristics of DSLs. A survey of code generation for DSLs notes that evaluation in this area faces challenges including scarce data, limited tooling, and domain-specific execution workflows \cite{joel_survey_2025}. Hardware description languages such as Verilog and VHDL provide concrete examples: benchmarks in these languages use domain-specific specifications and verification workflows, and show that LMs still face performance gaps on hardware design tasks \cite{jin_realbench_2025, vijayaraghavan_vhdl-eval_2024}. These efforts underscore the need for DSL-specific evaluation and show that code generation performance depends on the language, tooling, and domain context.
+
+Within ERP systems, similar challenges have been observed in the SAP and ABAP ecosystem, where benchmarking is hindered by limited data availability and specialized infrastructure \cite{wallraven_benchmarking_2026}.
+
+Benchmarking efforts for the AL ecosystem remain limited. AL development involves proprietary codebases, domain-specific abstractions, and tightly coupled business logic that existing benchmarks do not adequately represent. Practitioner-driven efforts indicate growing interest in this space. Notably, CentralGauge \footnote{\href{https://github.com/SShadowS/CentralGauge}{https://github.com/SShadowS/CentralGauge}} is a community benchmark inspired by HumanEval \cite{chen_evaluating_2021} and MBPP \cite{austin_program_2021}, and evaluates LM proficiency across different aspects of the AL language, such as data modeling and event patterns. However, it does not target repository-level engineering tasks and real-world problems. These limitations motivate the need for benchmarks that capture real-world engineering workflows in AL environments, which we address with BC-Bench.
+
+\section{Methods}
+
+BC-Bench is inspired by SWE-Bench and adapted to the Business Central/AL ecosystem.
+
+\subsection{Agent Harnesses}
+
+An agent harness is the system that orchestrates model interaction with the task environment, including prompting, tool use, file edits, and submission handling. We evaluate two production harnesses: GitHub Copilot and Claude Code.
+
+Because agent harnesses evolve rapidly, often with multiple releases per week, we track harness-version differences explicitly, as described in Section~\ref{sec:version-strategy}.
+
+\subsection{Dataset Construction}
+
+The dataset construction consists of a five-stage pipeline.
+
+\textbf{Stage 1: Repository selection.} We manually review merged pull requests (PRs) from two Microsoft-maintained AL repositories that contain the core Business Central application logic. NAV is an internal repository hosted on Azure DevOps, while BCApps is a public repository hosted on GitHub. To the best of our knowledge, there are no actively maintained public AL repositories with comprehensive test suites outside of Microsoft-maintained projects.
+
+\textbf{Stage 2: Attribute-based filtering.} We filter PRs to include only those that fix a single bug and introduce at least one test. This ensures that generated patches can be evaluated by executing the associated test suite.
+
+\textbf{Stage 3: Manual filtering.} We manually review bug descriptions (i.e., problem statements) and exclude instances lacking sufficient context for resolution. We further inspect the associated tests to remove cases where tests depend directly on the implementation of the fix (e.g., tests verifying newly introduced error messages via exact string matching).
+
+\textbf{Stage 4: Bug-PR pair extraction.} We adopt the SWE-Bench collection pipeline to extract bug-PR pairs that satisfy the previous criteria, with several modifications to support the AL ecosystem: (1) \textit{Multi-project mapping}, which identifies the application paths relevant to a given task within a monolithic codebase; (2) \textit{Release-based environment anchoring}, which replaces ephemeral CI builds with stable minor release versions (e.g., BC 26.4; see Appendix~\ref{app:bc-release-lifecycle}) to ensure long-term reproducibility; and (3) \textit{Base-commit alignment}, which selects a release-compatible base commit to resolve inconsistencies between PRs and anchored environments. Following SWE-Bench terminology, we refer to the human-authored fix in the merged PR as the \textit{gold patch}. Additionally, following SWE-Bench Multimodal \cite{yang_swe-bench_2024}, we support screenshots in problem statements to capture the visual nature of ERP bug reports.
+
+\textbf{Stage 5: Execution filtering.} We execute an automated validation pipeline to ensure that each task satisfies correctness constraints: tests that fail prior to the fix must pass after applying the patch, while tests that pass prior to the fix must continue to pass. This pipeline is executed upon task creation and re-run weekly to detect potential regressions due to changes in underlying dependencies or release versions.
+
+Table~\ref{tab:dataset_stats} summarizes the key characteristics of the final BC-Bench dataset for both Bug Fixing and Test Generation categories, and Table~\ref{tab:area_distribution} shows the distribution of tasks across functional areas. Most tasks are from BaseApp, a monolithic application with more than 2 million lines of code (LoC) and broad functional coverage, making it one of the most challenging applications in the ecosystem. The remaining tasks originate from specialized applications, such as Shopify integration, that depend on BaseApp.
+
+The dataset spans a diverse set of functional areas, with a concentration in core business domains such as Inventory, Finance, and Sales. Taken together, these properties make BC-Bench a realistic testbed for AL engineering tasks.
+
+\begin{table}[b]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{lr}
+\toprule
+\textbf{Metric} & \textbf{Value} \\
+\midrule
+Total tasks & 101 \\
+BaseApp tasks & 85 \\
+Other App tasks & 16 \\
+Avg. files modified (gold patch) & 1.3 \\
+Avg. gold patch LoC & 18.9 \\
+Median files modified (gold patch) & 1.0 \\
+Median gold patch LoC & 9.0 \\
+Tasks with images & 67 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Key statistics of the BC-Bench dataset. Averages and medians are computed over all tasks.}
+\label{tab:dataset_stats}
+\vskip 0.1in
+\end{table}
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{lr}
+\toprule
+\textbf{Area} & \textbf{Count} \\
+\midrule
+Inventory & 21 \\
+Finance & 19 \\
+Sales & 12 \\
+Project & 8 \\
+Shopify & 7 \\
+Manufacturing & 5 \\
+Warehouse & 5 \\
+CRM & 3 \\
+Service & 3 \\
+Other & 18 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Distribution of BC-Bench tasks by area. Areas with count less than two are grouped into Other.}
+\label{tab:area_distribution}
+\vskip 0.1in
+\end{table}
+
+\subsection{Category}
+
+We define two evaluation categories using the same dataset.
+
+\textbf{Bug Fixing.} Following SWE-Bench, each trial provides the agent with a problem statement and a codebase snapshot at a specified base commit. The agent is tasked with generating a functional patch that resolves the issue. When present, images referenced in the problem statement are copied into the testbed and linked via relative paths.
+
+\textbf{Test Generation.} We extend the SWE-Bench setup by introducing a Test Generation category using the same dataset. In addition to the problem statement and codebase, the gold patch is applied as unstaged changes. The agent is then tasked with generating a new test that reproduces the issue. The created test should fail on the base commit and pass after applying the gold patch.
+
+\subsection{Evaluation Metrics}
+
+BC-Bench focuses on the following metrics, all aggregated over five independent runs per task to account for the stochasticity of language models. This choice balances statistical reliability with practical evaluation constraints (e.g.\ cost), informed by preliminary experiments with ten runs on an earlier version of the dataset containing 55 tasks.
+
+\textbf{Mean resolution rate with a 95\% confidence interval.} Because LM agents are stochastic, we run the full dataset five times independently. For Bug Fixing, a task is considered resolved if the generated patch builds successfully and all evaluation tests pass; for Test Generation, a task is considered resolved if the created test fails on the base commit and passes after applying the gold patch. For each run, we compute the resolution rate over all tasks, and we report the mean of these five run-level resolution rates. We estimate the 95\% confidence interval by bootstrapping the five run-level resolution rates with 10,000 resamples, using SciPy's BCa method; this interval reflects run-to-run variability due to agent stochasticity on a fixed benchmark task set.
+
+
+\textbf{\passk{k}}. To measure consistency across repeated runs, we report \passk{k} \cite{yao_tau-bench_2024}. For a task with $n$ total runs and $c$ successful runs, \passk{k} is the probability that a set of $k$ runs all succeed, averaged across tasks:
+\[
+\text{pass\string^k}
+=
+\mathbb{E}_{\mathrm{task}}
+\left[
+\frac{\binom{c}{k}}{\binom{n}{k}}
+\right].
+\]
+A task contributes $0$ when it has fewer than $k$ successful runs. We perform five runs per task and report \passk{5}. In this setting, \passk{5} equals $1$ for a task only if all five runs succeed, and $0$ otherwise, so the aggregate \passk{5} is the fraction of tasks solved successfully in all five runs.
+
+We do not report pass@k, because it emphasizes eventual success after repeated retries, which does not match our target usage setting. In practice, software engineers care more about whether an agent works reliably when asked than whether one out of many retries eventually succeeds.
+
+\textbf{Duration.} We report duration as wall-clock agent execution time, averaged across task runs and then across the five independent evaluation runs. It covers model calls and tool uses, but excludes post-submission benchmark validation. We include it because latency directly affects the usefulness of agent-based workflows in real-world software engineering.
+
+\subsection{Version Strategy}
+\label{sec:version-strategy}
+
+BC-Bench adopts a semantic versioning strategy to track changes that may affect evaluation results. Updates to components such as the dataset, execution environment, and agent harnesses are versioned to ensure reproducibility and comparability across experiments while avoiding the cost of re-running every model for every small update. This is particularly important in LM-based evaluation, where minor implementation differences can significantly affect performance \cite{biderman_lessons_2025}. A detailed version-strategy change log is provided in Appendix~\ref{app:version-strategy}.
+
+\subsection{Evaluation Environment}
+
+All evaluations are conducted using GitHub Actions on a standardized Microsoft internal self-hosted runner. The execution environment is provisioned using BcContainerHelper\footnote{\href{https://github.com/microsoft/navcontainerhelper}{https://github.com/microsoft/navcontainerhelper}}, with Docker containers corresponding to each task's specified release version.
+
+The execution environment is used to run tests and verify the outputs of agent harnesses, but is not directly accessible to agents during task solving. In the default setting, agents interact with the AL codebase as plain text, without access to the underlying development environment. This reflects the current tooling reality: in headless environments, AL tooling is not natively available to coding agents.
+
+In selected experiments, we provide agent harnesses with access to AL MCP, a tooling framework designed to address this limitation by exposing parts of the development environment (e.g., compilation) to agents. BC-Bench allows us to evaluate the impact of such tooling on agent performance.
+
+There is no explicit turn or cost limit imposed on agent harnesses. Each trial is instead subject to a wall-clock timeout of 30 minutes, increased to 60 minutes starting in version 0.4. Trials are evaluated only when they produce a valid submission. Runs that fail to produce a submission, including cases of missing output, non-termination, or infrastructure instability, are treated as workflow failures rather than recorded agent failures. These workflow failures are re-run until the workflow complete successfully. To prevent blocked executions from stalling the pipeline, we additionally enforce a workflow-level timeout that aborts runs that do not terminate cleanly.
+
+\section{Results}
+
+\begin{table*}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{llllll}
+\toprule
+\textbf{Agent} & \textbf{Model} & \textbf{Mean with 95\% CI} & \textbf{\passk{5}} & \textbf{Duration} & \textbf{Version} \\
+\midrule
+Claude Code & claude-opus-4.6 & 68.5\% (65.7--71.3\%) & 49.5\% & 284s & 0.2.0 \\
+GitHub Copilot & claude-opus-4.6 & 65.1\% (62.6--67.6\%) & 50.5\% & 314s & 0.2.0 \\
+GitHub Copilot & gpt-5.2-codex & 60.8\% (59.4--62.2\%) & 49.5\% & 196s & 0.2.2 \\
+GitHub Copilot & claude-opus-4.5 & 59.8\% (58.3--61.3\%) & 38.6\% & 172s & 0.2.0 \\
+GitHub Copilot & claude-opus-4.5 & 58.4\% (56.6--60.2\%) & 38.6\% & 165s & 0.1.0 \\
+Claude Code & claude-opus-4.5 & 57.4\% (55.4--59.4\%) & 31.7\% & 205s & 0.1.0 \\
+GitHub Copilot & gpt-5.3-codex & 55.8\% (54.3--57.3\%) & 37.6\% & 106s & 0.2.1 \\
+GitHub Copilot & gpt-5.1-codex-max & 53.7\% (51.7--56.8\%) & 36.6\% & 229s & 0.2.2 \\
+GitHub Copilot & gpt-4.1 & 16.6\% (15.6--17.2\%) & 5.0\% & 256s & 0.2.2 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Bug Fixing performance across agent and model configurations on BC-Bench.}
+\label{tab:bugfix_results}
+\vskip 0.1in
+\end{table*}
+
+\begin{table*}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{llllll}
+\toprule
+\textbf{Agent} & \textbf{Model} & \textbf{Mean with 95\% CI} & \textbf{\passk{5}} & \textbf{Duration} & \textbf{Version} \\
+\midrule
+GitHub Copilot & claude-opus-4.6 & 60.4\% (58.8--62.0\%) & 37.6\% & 469s & 0.2.0 \\
+GitHub Copilot & claude-opus-4.5 & 45.5\% (43.0--48.0\%) & 20.8\% & 169s & 0.1.0 \\
+GitHub Copilot & gpt-5.3-codex & 45.3\% (42.5--48.1\%) & 20.8\% & 155s & 0.2.2 \\
+GitHub Copilot & gpt-5.2-codex & 44.0\% (40.8--48.5\%) & 16.8\% & 291s & 0.2.2 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Test Generation performance across agent and model configurations on BC-Bench.}
+\label{tab:test_generation_results}
+\vskip 0.1in
+\end{table*}
+
+We present results on BC-Bench, with metrics aggregated over five independent runs per configuration. Because benchmark versions differ across configurations, not all comparisons are direct. Comparisons within the same version support stronger conclusions, while cross-version comparisons should be interpreted more cautiously.
+
+Under matched conditions (GitHub Copilot, version 0.2.0, Bug Fixing) in Table~\ref{tab:bugfix_results}, claude-opus-4.6 outperforms claude-opus-4.5, improving mean resolution rate by 5.3 percentage points and \passk{5} by 11.9 percentage points.
+
+Outside the controlled comparisons, claude-opus-4.6 has the highest \passk{5} and mean resolution rate in the reported runs. Because the benchmark version differs across rows, this should not be read as a definitive ranking. The model is also slower than competing systems, pointing to a trade-off between accuracy and latency. Similar patterns appear in Table~\ref{tab:test_generation_results}.
+
+The earlier model GPT-4.1 exhibits a substantial performance gap compared to recent SOTA models, with approximately 4 times lower mean resolution rate and 10 times lower \passk{5}, highlighting the rapid progress of language models over the past year.
+
+\subsection{Model vs Agent Harness Effect}
+
+For each matched comparison, we evaluate both configurations on the same 101 tasks with five runs per task. For each task, we compute the difference in success counts between the configurations, and use a two-sided exact sign-flip permutation test to assess whether the paired differences consistently favor one configuration over the other. Under the null hypothesis, neither configuration has a systematic advantage.
+
+For the Bug Fixing category, we compare Claude Code and GitHub Copilot under the same model and benchmark version in Table~\ref{tab:harness_comparison}; neither comparison reaches the 0.05 significance threshold. We then fix the harness and benchmark version while varying the model in Table~\ref{tab:model_comparison}; both model comparisons are statistically significant at this threshold.
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\resizebox{\columnwidth}{!}{%
+\begin{tabular}{llll}
+\toprule
+\textbf{Fixed model/version} & \textbf{Setting A} & \textbf{Setting B} & \textbf{$p$-value} \\
+\midrule
+claude-opus-4.5, v0.1.0 & GitHub Copilot 58.4\% & Claude Code 57.4\% & 0.728 \\
+claude-opus-4.6, v0.2.0 & Claude Code 68.5\% & GitHub Copilot 65.1\% & 0.075 \\
+\bottomrule
+\end{tabular}
+}
+\end{small}
+\end{center}
+\caption{Matched harness comparisons on Bug Fixing with model and benchmark version fixed. Values are mean resolution rates.}
+\label{tab:harness_comparison}
+\vskip 0.1in
+\end{table}
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\resizebox{\columnwidth}{!}{%
+\begin{tabular}{llll}
+\toprule
+\textbf{Fixed harness/version} & \textbf{Setting A} & \textbf{Setting B} & \textbf{$p$-value} \\
+\midrule
+GitHub Copilot, v0.2.0 & claude-opus-4.6 65.1\% & claude-opus-4.5 59.8\% & 0.026 \\
+GitHub Copilot, v0.2.2 & gpt-5.2-codex 60.8\% & gpt-5.1-codex-max 53.7\% & 0.019 \\
+\bottomrule
+\end{tabular}
+}
+\end{small}
+\end{center}
+\caption{Matched model comparisons on Bug Fixing with harness and benchmark version fixed. Values are mean resolution rates.}
+\label{tab:model_comparison}
+\vskip 0.1in
+\end{table}
+
+These results suggest that, in our evaluated Bug Fixing settings, model choice appears to matter more than the choice between these two agent harnesses. However, this is based on only four matched comparisons and may not generalize; future work should extend this analysis to additional agent harnesses (e.g., Codex CLI) and to other categories such as Test Generation.
+
+\subsection{Cross-Benchmark Comparison}
+
+Model performance patterns on BC-Bench are broadly consistent with those reported on general-purpose benchmarks such as SWE-Bench, though this comparison should be interpreted cautiously. One notable difference appears in Test Generation: with GitHub Copilot and benchmark version fixed, Table~\ref{tab:cross_benchmark_test_generation} shows no statistically significant advantage for GPT-5.3-codex over GPT-5.2-codex, despite GPT-5.3-codex showing improvements on SWE-Bench Pro.
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\resizebox{\columnwidth}{!}{%
+\begin{tabular}{llll}
+\toprule
+\textbf{Fixed setting} & \textbf{Setting A} & \textbf{Setting B} & \textbf{$p$-value} \\
+\midrule
+GitHub Copilot, v0.2.2 & gpt-5.3-codex 45.3\% & gpt-5.2-codex 44.0\% & 0.672 \\
+\bottomrule
+\end{tabular}
+}
+\end{small}
+\end{center}
+\caption{Matched comparisons on Test Generation with harness and benchmark version fixed. Values are mean resolution rates.}
+\label{tab:cross_benchmark_test_generation}
+\vskip 0.1in
+\end{table}
+
+This result highlights the importance of domain-specific benchmarks, as improvements observed on general-purpose programming tasks do not necessarily transfer to DSLs such as AL.
+
+\subsection{Factors Affecting Accuracy}
+
+In this section, we analyze how different task characteristics affect agent performance, using results from GitHub Copilot with two representative models.
+
+\textbf{Patch Complexity.} We use the number of files changed and LoC in the gold patch as proxies for task complexity. Table~\ref{tab:resolution_by_files} shows a substantial drop in accuracy when tasks require modifying more than one file, with decreases of over 20 percentage points across both models. A similar degradation is observed in Table~\ref{tab:resolution_by_loc}, where accuracy drops by more than 25 percentage points for patches exceeding 10 LoC. These results suggest that current SOTA models still struggle with more complex, multi-file changes.
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{llll}
+\toprule
+\textbf{Files Changed} & \textbf{gpt-5.2} & \textbf{opus-4.6} & \textbf{Tasks} \\
+\midrule
+1 & 65.1\% & 70.2\% & 82 \\
+2+ & 42.1\% & 43.2\% & 19 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Mean resolution rate by files changed in the gold patch.}
+\label{tab:resolution_by_files}
+\vskip 0.1in
+\end{table}
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{llll}
+\toprule
+\textbf{LoC Changed} & \textbf{gpt-5.2} & \textbf{opus-4.6} & \textbf{Tasks} \\
+\midrule
+1--10 & 75.6\% & 78.5\% & 54 \\
+11--25 & 38.3\% & 49.6\% & 23 \\
+26+ & 49.2\% & 50.0\% & 24 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Mean resolution rate by LoC changed in the gold patch.}
+\label{tab:resolution_by_loc}
+\vskip 0.1in
+\end{table}
+
+\textbf{Functional Area.} Table~\ref{tab:resolution_by_area} groups tasks by assigned functional area and shows that performance varies across areas. Shopify and Warehouse tasks have lower resolution rates for both representative models, while GPT-5.2 performs better on Manufacturing tasks. Because some areas contain few tasks, these results should be read as diagnostic rather than definitive. This suggests that domain familiarity may affect resolution rate, in addition to task complexity.
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{llll}
+\toprule
+\textbf{Area} & \textbf{gpt-5.2} & \textbf{opus-4.6} & \textbf{Tasks} \\
+\midrule
+Inventory & 65.7\% & 70.5\% & 21 \\
+Finance & 51.6\% & 62.1\% & 19 \\
+Sales & 65.0\% & 70.0\% & 12 \\
+Project & 62.5\% & 70.0\% & 8 \\
+Shopify & 48.6\% & 54.3\% & 7 \\
+Manufacturing & 88.0\% & 64.0\% & 5 \\
+Warehouse & 40.0\% & 56.0\% & 5 \\
+Other & 63.3\% & 64.2\% & 24 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Mean resolution rate by functional area. Areas with fewer than five tasks are grouped into Other.}
+\label{tab:resolution_by_area}
+\vskip 0.1in
+\end{table}
+
+\textbf{Tasks with images.} Visual elements in bug descriptions are common in the Business Central ecosystem, particularly in reproduction steps. Table~\ref{tab:resolution_by_image_count} shows that tasks with more images exhibit higher resolution rates. We hypothesize that tasks with more images also include more detailed descriptions, helping the agent regardless of whether the images are processed. We find that image count correlates with problem statement length, suggesting that textual richness, rather than visual content itself, may be the primary driver of improved performance. Future work should investigate whether images are necessary for solving these tasks \cite{yang_swe-bench_2024}.
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{lllll}
+\toprule
+\textbf{Image} & \textbf{gpt-5.2} & \textbf{opus-4.6} & \textbf{Tasks} & \textbf{Chars} \\
+\midrule
+0 & 53.5\% & 57.1\% & 34 & 1,514 \\
+1--5 & 56.7\% & 60.0\% & 30 & 1,720 \\
+6+ & 70.8\% & 76.8\% & 37 & 2,118 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Mean resolution rate by number of images in the problem statement. Chars denotes the average character count of the problem statement.}
+\label{tab:resolution_by_image_count}
+\vskip 0.1in
+\end{table}
+
+\subsection{Failure Mode Analysis}
+\label{sec:failure-modes}
+
+We classify failed trials from a representative strong configuration in Table~\ref{tab:bugfix_results}: GitHub Copilot with claude-opus-4.6 on Bug Fixing v0.2.0. This setting has 505 trials (5 runs $\times$ 101 tasks), including 176 failures across 50 unique tasks.
+
+We first identify \emph{Timeout}, where the agent does not produce a submission before the trial limit, and \emph{Build Failure}, where the submitted patch does not build. For the remaining failures, we use the overlap between the generated patch and the gold patch as a deterministic proxy for localization, meaning whether the agent found the right place to edit: \emph{Incorrect File} edits none of the gold-patch files, \emph{Incorrect Region} edits a gold-patch file but does not overlap any region changed by the gold patch, and \emph{Wrong Solution} overlaps a region changed by the gold patch but still fails.
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{lrr}
+\toprule
+\textbf{Failure Mode} & \textbf{Trials} & \textbf{\%} \\
+\midrule
+Timeout & 4 & 2.3 \\
+Build Failure & 10 & 5.7 \\
+Incorrect File & 52 & 29.5 \\
+Incorrect Region & 32 & 18.2 \\
+Wrong Solution & 78 & 44.3 \\
+\midrule
+Total & 176 & 100 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Failure modes for GitHub Copilot with claude-opus-4.6 on Bug Fixing v0.2.0 (176 failed trials of 505).}
+\label{tab:failure_modes}
+\vskip 0.1in
+\end{table}
+
+Table~\ref{tab:failure_modes} shows that build failures are uncommon (5.7\%). Most failed trials produce a buildable patch, so compiler feedback remains useful but would not address most failures on its own. Incorrect File and Incorrect Region account for 47.7\% of failures, indicating that finding the right place to edit is a major failure source. This motivates evaluating coding agents with Language Server Protocol (LSP)\footnote{\href{https://microsoft.github.io/language-server-protocol/}{https://microsoft.github.io/language-server-protocol/}} tools such as find references, go-to definition, and symbol search. The largest single bucket is Wrong Solution (44.3\%), where the patch reaches the right region but still implements the wrong fix.
+
+We manually labeled the 22 unique Wrong Solution tasks (78 failed trials). Table~\ref{tab:wrong_solution_manual_labels} summarizes the resulting labels, with task-level labels in Appendix~\ref{app:wrong-solution-audit}. The labels separate the type of semantic miss after the patch has reached the relevant code region: \emph{Incomplete Fix} means the patch leaves an edge case, validation path, or propagation step unresolved; \emph{Incorrect Business Logic} means the patch edits relevant AL code but implements the wrong functional behavior, such as an incorrect condition or calculation; and \emph{Wrong AL/API Usage} means the patch uses the wrong AL construct, trigger, or API. These labels suggest different improvement directions: test execution and edge-case exploration for incomplete fixes, better retrieval of related AL control and data flows for incorrect business logic, and stronger API guidance for AL/API misuse.
+
+\begin{table}[t]
+\vskip 0.05in
+\begin{center}
+\begin{small}
+\begin{sc}
+\begin{tabular}{lrr}
+\toprule
+\textbf{Manual Label} & \textbf{Tasks} & \textbf{Failed Trials} \\
+\midrule
+Incomplete Fix & 10 & 36 \\
+Incorrect Business Logic & 10 & 34 \\
+Wrong AL/API Usage & 2 & 8 \\
+\midrule
+Total & 22 & 78 \\
+\bottomrule
+\end{tabular}
+\end{sc}
+\end{small}
+\end{center}
+\caption{Task-level manual labels for Wrong Solution failures.}
+\label{tab:wrong_solution_manual_labels}
+\vskip 0.1in
+\end{table}
+
+\section{Discussion}
+
+We introduce BC-Bench, a benchmark for evaluating coding agents on real-world AL engineering tasks in the Business Central ecosystem. BC-Bench is intended as a practical testbed, not only a leaderboard, helping guide future work on tools, skills, and agent configurations. For example, Business Central engineering teams use BC-Bench to prioritize tooling investments for coding agents. Our work shows both the importance and the practical challenges of benchmarking DSLs.
+
+In the Bug Fixing category, we observe larger between-model differences than between the two evaluated agent harnesses, and the matched harness comparisons show no statistically significant effect. Model performance patterns on BC-Bench are broadly consistent with general-purpose benchmarks, but we observe meaningful deviations---most notably that GPT-5.3-codex does not outperform its predecessor on BC-Bench despite gains on SWE-Bench Pro. This suggests that improvements on general-purpose programming tasks do not reliably transfer to DSLs. Among task characteristics, gold patch complexity (measured by number of files changed and LoC) is a strong predictor of difficulty, and resolution rates also vary across functional areas.
+
+Our failure analysis shows that unsuccessful trials are dominated by localization errors and semantically wrong solutions rather than build failures, suggesting that improvements should target repository navigation, context retrieval, and test feedback. These observations yield concrete hypotheses for controlled BC-Bench ablations.
+
+\subsection{Iterative refinement}
+
+During BC-Bench development, we encountered issues that affect evaluation reliability. For example, manual analysis of agent failures on an earlier version revealed underspecified problem statements and unreachable links. These experiences show that real-world software engineering benchmarks require ongoing maintenance, not just one-time construction.
+
+\subsection{Limitations}
+
+\textbf{Limited Dataset Diversity.} The dataset consists of 101 tasks from Microsoft-owned repositories, which do not represent the full diversity of AL development. The included applications are broadly applicable across customers and the partner ecosystem, whereas many partner-developed AL applications target specific industries, regions, or integrations. All tasks are single bug fixes; scenarios such as feature development, upgrades, localization, and performance tuning remain unexplored. These gaps motivate the \hyperref[sec:dataset-evolution]{Dataset Evolution} direction discussed in Future Work.
+
+\textbf{Limited Representation of Real-Development Environment.} By default, agents interact with the codebase as plain text without AL-specific tooling such as compilation, unlike how engineers work in Visual Studio Code.
+
+\textbf{Dependency on Tests.} For the Bug Fixing category, BC-Bench uses test execution to verify the correctness of the coding agents' output; however, the collected tests do not necessarily guarantee the correctness of the solution.
+
+\subsection{Future Work}
+
+\textbf{Alternative Evaluation Metrics.} Functional correctness, as measured by passing tests, is not sufficient to characterize a high-quality fix. Future work should explore review-oriented evaluation, such as code review, to assess best practices, security risks, and variability in performance and cost metrics such as duration and token consumption.
+
+\textbf{Experiments.} Building on the failure-mode analysis, a useful next step is to run controlled experiments for AL-specific tooling, including AL MCP and AL LSP, both developed by Microsoft. These experiments can evaluate capabilities such as compilation, symbol search, and go-to definition. Whether these interventions yield meaningful gains is an empirical question that BC-Bench enables answering.
+
+\phantomsection\label{sec:dataset-evolution}\textbf{Dataset Evolution.} The current dataset consists of 101 manually curated, static single-bug tasks, which introduces a risk of contamination after release. Future work should explore continuously updated or dynamically generated benchmarks to mitigate this risk and improve long-term validity.
+
+\section{Acknowledgements}
+We thank Jiawen Sun for contributions to dataset collection; Angus Taylor for discussions on statistical methodology, including bootstrap confidence intervals; Thaddeus Loke, Esben Nyhuus Kristoffersen, and Darrick Joo for valuable technical guidance and feedback; Alexander Holstrup for infrastructure support; and Joost Bulsink for supporting the public release of BC-Bench. We also thank the anonymous reviewers for their constructive feedback and suggestions, which helped improve the quality and clarity of this paper.
+
+\clearpage
+
+\bibliographystyle{ieeetr}
+\bibliography{bcbench}
+
+\clearpage
+
+\appendix
+\section{Version Anchoring and Base-Commit Alignment}
+\label{app:bc-release-lifecycle}
+
+Business Central versions follow a \textit{major.minor} scheme (e.g., 26.4). Major versions are released biannually and introduce new features and platform-level changes, while minor releases provide incremental updates such as bug fixes and stability improvements.
+
+In BC-Bench, we anchor each task to a specific minor release rather than an ephemeral CI build, whose availability is short-lived. This design improves long-term reproducibility when benchmarks are re-executed.
+
+Anchoring to a minor release introduces a divergence from the original development history: the base commit used for evaluation may differ from the base commit of the corresponding PR. This occurs because the selected release may include changes that were not present at the time the PR was merged, potentially introducing incompatibilities. To address this, we perform \textit{base-commit alignment} by selecting an earlier, compatible commit within the release, excluding commits that introduce incompatibilities with the task.
+
+One caveat is that hotfix backports may introduce inconsistencies in the evaluation environment. To mitigate potential drift, our execution filtering pipeline is re-run weekly to ensure that all tasks remain valid under the anchored environments.
+
+\section{Version History}
+\label{app:version-strategy}
+
+\href{https://www.nuget.org/packages/Microsoft.Dynamics.BusinessCentral.Development.Tools}{altool} is a command-line tool used for compiling and packaging AL extensions for Business Central, designed for CI/CD pipelines. It contains the AL MCP server.
+
+\begin{itemize}
+\item \textbf{Version 0.1.0} \hfill (2026-01-26)\\
+\textbf{Summary:} Successfully curated 101 tasks for both Bug Fixing and Test Generation categories. All tasks passed the five stages of dataset construction.\\
+\textbf{Versions:}
+\begin{itemize}
+\item GitHub Copilot CLI: 0.0.382
+\item Claude Code: 2.0.76
+\item altool: 17.0.30.49729-beta
+\end{itemize}
+
+\item \textbf{Version 0.2.0} \hfill (2026-02-07)\\
+\textbf{Summary:} After manual review of the dataset and comparison of the gold patch against agent outputs, clarified problem statements, fixed unreachable links, and made the prompt more precise so that agents ignore localization changes.\\
+\textbf{Versions:}
+\begin{itemize}
+\item GitHub Copilot CLI: 0.0.406
+\item Claude Code: 2.1.37
+\item altool: 17.0.30.49729-beta
+\end{itemize}
+
+\item \textbf{Version 0.2.1} \hfill (2026-02-18)\\
+\textbf{Summary:} Updated GitHub Copilot CLI to include support for gpt-5.3-codex.\\
+\textbf{Versions:}
+\begin{itemize}
+\item GitHub Copilot CLI: 0.0.409
+\item Claude Code: 2.1.37
+\item altool: 17.0.30.49729-beta
+\end{itemize}
+
+\item \textbf{Version 0.2.2} \hfill (2026-02-19)\\
+\textbf{Summary:} Updated GitHub Copilot CLI and Claude Code to include support for claude-sonnet-4.6.\\
+\textbf{Versions:}
+\begin{itemize}
+\item GitHub Copilot CLI: 0.0.411
+\item Claude Code: 2.1.47
+\item altool: 17.0.30.49729-beta
+\end{itemize}
+
+\item \textbf{Version 0.3.0} \hfill (2026-02-27)\\
+\textbf{Summary:} Updated the Test Generation prompt to state more explicitly that the agent is expected to create a new test case.\\
+\textbf{Versions:}
+\begin{itemize}
+\item GitHub Copilot CLI: 0.0.411
+\item Claude Code: 2.1.47
+\item altool: 17.0.30.49729-beta
+\end{itemize}
+
+\item \textbf{Version 0.3.1} \hfill (2026-03-11)\\
+\textbf{Summary:} Updated GitHub Copilot CLI and refreshed the list of allowed models.\\
+\textbf{Versions:}
+\begin{itemize}
+\item GitHub Copilot CLI: 1.0.2
+\item Claude Code: 2.1.47
+\item altool: 17.0.30.49729-beta
+\end{itemize}
+
+\item \textbf{Version 0.3.2} \hfill (2026-03-14)\\
+\textbf{Summary:} Updated altool to a newer version that fixes several bugs and adds more tools.\\
+\textbf{Versions:}
+\begin{itemize}
+\item GitHub Copilot CLI: 1.0.2
+\item Claude Code: 2.1.47
+\item altool: 17.0.33.55542
+\end{itemize}
+
+\item \textbf{Version 0.4.0} \hfill (2026-03-19)\\
+\textbf{Summary:} Improved the options and settings for Claude Code and GitHub Copilot CLI to prevent internet access. Repository setup now sparse-checks out only app folders, improving clone performance. Fixed two PNG file conversions for a dataset entry. Extended the timeout to accommodate the altool compile tool, and fixed altool integration after the update.\\
+\textbf{Versions:}
+\begin{itemize}
+\item GitHub Copilot CLI: 1.0.2
+\item Claude Code: 2.1.47
+\item altool: 17.0.33.55542
+\end{itemize}
+
+\end{itemize}
+
+\clearpage
+\onecolumn
+\section{Manual Labeling of Wrong Solution Failures}
+\label{app:wrong-solution-audit}
+
+This appendix documents the manual labeling behind Table~\ref{tab:wrong_solution_manual_labels}. The label meanings are defined in Section~\ref{sec:failure-modes}; here we describe the assignment protocol and provide the task-level audit table.
+
+We labeled the 22 unique tasks in the Wrong Solution failure mode rather than each failed trial independently, because repeated runs for the same task often produced related or identical patches. For each task, we inspected the problem statement, gold patch, generated patch variants, and failing test output. The audit uses outputs for GitHub Copilot with claude-opus-4.6 on Bug Fixing v0.2.0 from GitHub Actions runs \texttt{21815911073}, \texttt{21822868049}, \texttt{21832838543}, \texttt{21855870368}, and \texttt{21864728490}. Table~\ref{tab:wrong_solution_task_labels} reports the assigned label, failed-trial count, number of unique generated-patch variants, and duplicate-trial count. Duplicate trials count repeated generated patches for the same task, i.e., failed trials minus unique generated patch variants.
+
+\begin{center}
+\refstepcounter{table}\label{tab:wrong_solution_task_labels}
+\vskip 0.05in
+\begingroup
+\setlength{\tabcolsep}{4pt}
+\begin{scriptsize}
+\begin{tabular}{llrrr}
+\toprule
+\textbf{Instance ID} & \textbf{Manual Label} & \textbf{Failed Trials} & \textbf{Patch Variants} & \textbf{Duplicate Trials} \\
+\midrule
+\texttt{microsoft\_\_BCApps-4766} & Incomplete Fix & 5 & 2 & 3 \\
+\texttt{microsoft\_\_BCApps-5633} & Incomplete Fix & 2 & 2 & 0 \\
+\texttt{microsoftInternal\_\_NAV-176426} & Incorrect Business Logic & 5 & 1 & 4 \\
+\texttt{microsoftInternal\_\_NAV-177750} & Incomplete Fix & 5 & 5 & 0 \\
+\texttt{microsoftInternal\_\_NAV-181900} & Incorrect Business Logic & 2 & 2 & 0 \\
+\texttt{microsoftInternal\_\_NAV-182354} & Incomplete Fix & 5 & 1 & 4 \\
+\texttt{microsoftInternal\_\_NAV-183399} & Wrong AL/API Usage & 5 & 2 & 3 \\
+\texttt{microsoftInternal\_\_NAV-201169} & Incomplete Fix & 5 & 3 & 2 \\
+\texttt{microsoftInternal\_\_NAV-205825} & Incorrect Business Logic & 2 & 2 & 0 \\
+\texttt{microsoftInternal\_\_NAV-206135} & Incomplete Fix & 3 & 2 & 1 \\
+\texttt{microsoftInternal\_\_NAV-206527} & Incorrect Business Logic & 4 & 4 & 0 \\
+\texttt{microsoftInternal\_\_NAV-207236} & Incomplete Fix & 1 & 1 & 0 \\
+\texttt{microsoftInternal\_\_NAV-208320} & Wrong AL/API Usage & 3 & 3 & 0 \\
+\texttt{microsoftInternal\_\_NAV-208649} & Incorrect Business Logic & 5 & 2 & 3 \\
+\texttt{microsoftInternal\_\_NAV-211521} & Incorrect Business Logic & 1 & 1 & 0 \\
+\texttt{microsoftInternal\_\_NAV-216918} & Incorrect Business Logic & 5 & 5 & 0 \\
+\texttt{microsoftInternal\_\_NAV-218323} & Incorrect Business Logic & 1 & 1 & 0 \\
+\texttt{microsoftInternal\_\_NAV-218856} & Incomplete Fix & 5 & 5 & 0 \\
+\texttt{microsoftInternal\_\_NAV-220036} & Incorrect Business Logic & 4 & 2 & 2 \\
+\texttt{microsoftInternal\_\_NAV-220452} & Incorrect Business Logic & 5 & 5 & 0 \\
+\texttt{microsoftInternal\_\_NAV-223493} & Incomplete Fix & 4 & 2 & 2 \\
+\texttt{microsoftInternal\_\_NAV-224447} & Incomplete Fix & 1 & 1 & 0 \\
+\bottomrule
+\end{tabular}
+\end{scriptsize}
+\endgroup
+\vskip 0.05in
+{\small \textbf{Table~\thetable.} Task-level manual labels and generated patch duplication for the 22 unique tasks in the Wrong Solution failure mode. Patch variants are exact generated-diff variants for a task.}
+\end{center}
+\vskip 0.1in
+
+\clearpage
+\twocolumn
+
+
+\end{document}
+
+
+% MSJAR template is based on ICML 2019 template in order to
+% make it easier for authors to submit to MSJAR.
+
+% This document was modified from the file originally made available by
+% Pat Langley and Andrea Danyluk for ICML-2K. This version was created
+% by Iain Murray in 2018, and modified by Alexandre Bouchard in
+% 2019. Previous contributors include Dan Roy, Lise Getoor and Tobias
+% Scheffer, which was slightly modified from the 2010 version by
+% Thorsten Joachims & Johannes Fuernkranz, slightly modified from the
+% 2009 version by Kiri Wagstaff and Sam Roweis's 2008 version, which is
+% slightly modified from Prasad Tadepalli's 2007 version which is a
+% lightly changed version of the previous year's version by Andrew
+% Moore, which was in turn edited from those of Kristian Kersting and
+% Codrina Lauth. Alex Smola contributed to the algorithmic style files.