Google Scholar Profile

@article{barney_improving_2012,
	title = {Improving Students With Rubric-Based Self-Assessment and Oral Feedback},
	volume = {55},
	issn = {0018-9359},
	doi = {10.1109/TE.2011.2172981},
	url = {https://www.lmsteiner.com/papers/barney_2012_te.pdf},
	abstr = {Rubrics and oral feedback are approaches to help students improve performance and meet learning outcomes. However, their effect on the actual improvement achieved is inconclusive. This paper evaluates the effect of rubrics and oral feedback on student learning outcomes.

	An experiment was conducted in a software engineering course on requirements engineering, using the two approaches in course assignments. Both approaches led to statistically significant improvements, though no material improvement (i.e., a change by more than one grade) was achieved. The rubrics led to a significant decrease in the number of complaints and questions regarding grades.},
	number = {3},
	journal = {{IEEE} Transactions on Education},
	author = {Barney, S. and Khurum, M. and Petersen, K. and Unterkalmsteiner, M. and Jabangwe, R.},
	month = aug,
	year = {2012},
	pages = {319 -- 325}
},

@article{bjarnason_challenges_2014,
	title = {Challenges and Practices in Aligning Requirements with Verification and Validation: A Case Study of Six Companies},
	abstr = {Weak alignment of requirements engineering (RE) with verification and validation (VV) may lead to problems in delivering the required products in time with the right quality. For example, weak communication of requirements changes to testers may result in lack of verification of new requirements and incorrect verification of old invalid requirements, leading to software quality problems, wasted effort and delays. However, despite the serious implications of weak alignment research and practice both tend to focus on one or the other of RE or VV rather than on the alignment of the two.

	We have performed a multi-unit case study to gain insight into issues around aligning RE and VV by interviewing 30 practitioners from 6 software developing companies, involving 10 researchers in a flexible research process for case studies.

	The results describe current industry challenges and practices in aligning RE with VV, ranging from quality of the individual RE and VV activities, through tracing and tools, to change control and sharing a common understanding at strategy, goal and design level. The study identified that human aspects are central, i.e. cooperation and communication, and that requirements engineering practices are a critical basis for alignment. Further, the size of an organisation and its motivation for applying alignment practices, e.g. external enforcement of traceability, are variation factors that play a key role in achieving alignment.

	Our results provide a strategic roadmap for practitioners improvement work to address alignment challenges. Furthermore, the study provides a foundation for continued research to improve the alignment of RE with VV.},
	journal = {Empirical Software Engineering},
	author = {Bjarnason, Elizabeth and Runeson, Per and Borg, Markus and Unterkalmsteiner, Michael and Engström, Emelie and Regnell, Björn and Sabaliauskaite, Giedre and Loconsole, Annabella and Gorschek, Tony and Feldt, Robert},
	url = {https://www.lmsteiner.com/papers/bjarnason_2014_esej.pdf},
	year = {2014},
	volume = {19},
	number = {6},
	pages = {1809-1855}
},

@inproceedings{sabaliauskaite_challenges_2010,
	address = {Essen, Germany},
	title = {Challenges in aligning requirements engineering and verification in a large-scale industrial context},
	url = {https://www.lmsteiner.com/papers/sabaliauskaite_2010_refsq.pdf},
	abstr = {[Context and motivation] When developing software, coordination between different organizational units is essential in order to develop a good quality product, on time and within budget. Particularly, the synchronization between requirements and verification processes is crucial in order to assure that the developed software product satisfies customer requirements.

	[Question/problem] Our research question is: what are the current challenges in aligning the requirements and verification processes?

	[Principal ideas/results] We conducted an interview study at a large software development company. This paper presents preliminary findings of these interviews that identify key challenges in aligning requirements and verification processes.

	[Contribution] The result of this study includes a range of challenges faced by the studied organization grouped into the categories: organization and processes, people, tools, requirements process, testing process, change management, traceability, and measurement. The findings of this study can be used by practitioners as a basis for investigating alignment in their organizations, and by scientists in developing approaches for more efficient and effective management of the alignment between requirements and verification.},
	urldate = {2010-09-08},
	booktitle = {16th International Working Conference on Requirements Engineering: Foundation for Software Quality ({REFSQ})},
	publisher = {Springer Verlag},
	author = {Sabaliauskaite, G. and Loconsole, A. and Engström, E. and Unterkalmsteiner, M. and Regnell, B. and Runeson, P. and Gorschek, T. and Feldt, R.},
	year = {2010},
	pages = {128--142}
},

@article{unterkalmsteiner_conceptual_2014,
	title = {A conceptual framework for {SPI} evaluation},
	volume = {26},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2014_jsep.pdf},
	abstr = {Software Process Improvement (SPI) encompasses the analysis and modification of the processes within software development, aimed at improving key areas that contribute to the organizations' goals. The task of evaluating whether the selected improvement path meets these goals is challenging.

Based on the results of a systematic literature review on SPI measurement and evaluation practices, we developed a framework (SPI-MEF) that supports the planning and implementation of SPI evaluations. SPI-MEF guides the practitioner in scoping the evaluation, determining measures and performing the assessment. SPI-MEF does not assume a specific approach to process improvement and can be integrated in existing measurement programs, refocusing the assessment on evaluating the improvements initiative's outcome.

Sixteen industry and academic experts evaluated the framework's usability and capability to support practitioners, providing additional insights that were integrated in the application guidelines of the framework.},
	number = {2},
	journal = {Journal of Software: Evolution and Process},
	author = {Unterkalmsteiner, M. and Gorschek, T. and Islam, A. and Cheng, C. and Permadi, R. and Feldt, R.},
	year = {2014},
	pages = {251--279}
},

@article{unterkalmsteiner_taxonomy_2014,
	title = {A Taxonomy for Requirements Engineering and Software Test Alignment},
	journal = {{ACM} Transactions on Software Engineering and Methodology},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2014_tosem.pdf},
	author = {Unterkalmsteiner, Michael and Feldt, Robert and Gorschek, Tony},
	abstr = {Requirements Engineering and Software Testing are mature areas and have seen a lot of research. Nevertheless, their interactions have been sparsely explored beyond the concept of traceability.

	To fill this gap we propose a definition of requirements engineering and software test (REST) alignment, a taxonomy that characterizes the methods linking the respective areas, and a process to assess alignment. The taxonomy can support researchers to identify new opportunities for investigation, as well as practitioners to compare alignment methods and evaluate alignment, or lack thereof.

	We constructed the REST taxonomy by analyzing alignment methods published in literature, iteratively validating the emerging dimensions. The resulting concept of an information dyad characterizes the exchange of information required for any alignment to take place.

	We demonstrate use of the taxonomy by applying it on five in-depth cases and illustrate angles of analysis on a set of thirteen alignment methods. In addition we developed an assessment framework (REST-bench), applied it in an industrial assessment, and showed that it, with a low effort, can identify opportunities to improve REST alignment.

	Although we expect that the taxonomy can be further refined, we believe that the information dyad is a valid and useful construct to understand alignment.},
	year = {2014},
	volume = {23},
	number = {2},
	pages = {1--38}
},

@article{unterkalmsteiner_evaluation_2012,
	title = {Evaluation and Measurement of Software Process Improvement - A Systematic Literature Review},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2012_tse.pdf},
	volume = {38},
	number = {2},
	journal = {{IEEE} Transactions on Software Engineering},
	author = {Unterkalmsteiner, Michael and Gorschek, Tony and Islam, A. K. M. Moinul and Cheng, Chow Kian and Permadi, Rahadian Bayu and Feldt, Robert},
	abstr = {BACKGROUND - Software Process Improvement (SPI) is a systematic approach to increase the efficiency and effectiveness of a software development organization and to enhance software products.

	OBJECTIVE - This paper aims to identify and characterize evaluation strategies and measurements used to assess the impact of different SPI initiatives.

	METHOD - The systematic literature review includes 148 papers published between 1991 and 2008. The selected papers were classified according to SPI initiative, applied evaluation strategies and measurement perspectives. Potential confounding factors interfering with the evaluation of the improvement effort were assessed.

	RESULTS - Seven distinct evaluation strategies were identified, whereas the most common one, "Pre-Post Comparison", was applied in 49% of the inspected papers. Quality was the most measured attribute (62%), followed by Cost (41%) and Schedule (18%). Looking at measurement perspectives, "Project" represents the majority with 66%.
 
	CONCLUSION - The evaluation validity of SPI initiatives is challenged by the scarce consideration of potential confounding factors, particularly given that "Pre-Post Comparison" was identified as the most common evaluation strategy, and the inaccurate descriptions of the evaluation context. Measurements to assess the short and mid-term impact of SPI initiatives prevail, whereas long-term measurements in terms of customer satisfaction and return on investment tend to be less used.},
	month = apr,
	year = {2012},
	pages = {398--424}
},


@inproceedings{bin_ali_use_2014,
	address = {Seeon Monastery, Germany},
	title = {Use and evaluation of simulation for software process education: a case  study},
	abstr = {Software Engineering is an applied discipline and concepts are difficult to grasp only at a theoretical level alone. In the context of a project management course, we introduced and evaluated the use of software process simulation ({SPS}) based games for improving students’ understanding of software development processes. The effects of the intervention were measured by evaluating the students’ arguments for
choosing a particular development process. The arguments were assessed with the Evidence-Based Reasoning framework, which was extended to assess the strength of an argument. The results indicate that students generally have difficulty providing strong arguments for their choice of process models. Nevertheless, the assessment indicates that the intervention of the {SPS} game had a positive impact on the students’ arguments. Even though the illustrated argument assessment approach can be used to provide formative feedback to students, its use is rather costly and cannot be considered a replacement for traditional assessments.},
	booktitle = {European Conference Software Engineering Education ({ECSEE})},
	author = {bin Ali, Nauman and Unterkalmsteiner, Michael},
	url = {https://www.lmsteiner.com/papers/ali_2014_ecsee.pdf},
	year = {2014},
	pages = {1--15}
},


@article{giardino_what_2014,
	title = {What do we know about software development in startups?},
	volume = {31},
	url = {https://www.lmsteiner.com/papers/giardino_2014_software.pdf},
	abstr = {An impressive number of new startups are launched every day as a result of growing new markets, accessible technologies, and venture capital. New ventures such as Facebook, Supercell, Linkedin, Spotify, {WhatsApp}, and Dropbox, to name a few, are good examples of startups that evolved into successful businesses. However, despite many successful stories, the great majority of them fail prematurely. Operating in a chaotic and rapidly evolving domain conveys new uncharted challenges for startuppers. In this study, the authors characterize their context and identify common software development startup practices.},
	number = {5},
	journal = {{IEEE} Software},
	author = {Giardino, Carmine and Unterkalmsteiner, Michael and Paternoster, Nicolo and Gorschek, Tony and Abrahamsson, Pekka},
	year = {2014},
	pages = {28--32}
},


@article{paternoster_software_2014,
	title = {Software Development in Startup Companies: A Systematic Mapping Study},
	volume = {56},
	url = {https://www.lmsteiner.com/papers/paternoster_2014_ist.pdf},
	abstr = {Context: Software startups are newly created companies with no operating history and fast in producing cutting-edge technologies. These companies develop software under highly uncertain conditions, tackling fast-growing markets under severe lack of resources. Therefore, software startups present an unique combination of characteristics which pose several challenges to software development activities.

Objective: This study aims to structure and analyze the literature on software development in startup companies, determining thereby the potential for technology transfer and identifying software development work practices reported by practitioners and researchers.

Method: We conducted a systematic mapping study, developing a classification schema, ranking the selected primary studies according their rigor and relevance, and analyzing reported software development work practices in startups.

Results: A total of 43 primary studies were identified and mapped, synthesizing the available evidence on software development in startups. Only 16 studies are entirely dedicated to software development in startups, of which 10 result in a weak contribution (advice and implications (6); lesson learned (3); tool (1)). Nineteen studies focus on managerial and organizational factors. Moreover, only 9 studies exhibit high scientific rigor and relevance. From the reviewed primary studies, 213 software engineering work practices were extracted, categorized and analyzed.

Conclusion: This mapping study provides the first systematic exploration of the state-of-art on software startup research. The existing body of knowledge is limited to a few high quality studies. Furthermore, the results indicate that software engineering work practices are chosen opportunistically, adapted and configured to provide value under the constrains imposed by the startup context.},
	number = {10},
	journal = {Information and Software Technology},
	author = {Paternoster, Nicolo and Giardino, Carmine and Unterkalmsteiner, Michael and Gorschek, Tony and Abrahamsson, Pekka},
	year = {2014},
	pages = {1200--1218}
},


@inproceedings{bjarnason_industrial_2015,
	address = {Helsinki, Finland},
	title = {An Industrial Case Study on the Use of Test Cases as Requirements},
	abstr = {It is a conundrum that agile projects can succeed 'without requirements' when weak requirements engineering is a known cause for project failures. While Agile development projects often manage well without extensive requirements documentation, test cases are commonly used as requirements. We have investigated this agile practice at three companies in order to understandhow test cases can fill the role of requirements. We performed a case study based on twelve interviews performed in a previous study.
The findings include a range of benefits and challenges in using test cases for eliciting, validating, verifying, tracing and managing requirements. In addition, we identified three scenarios for applying the practice, namely as a mature practice, as a de facto practice and as part of an agile transition. The findings provide insights into how the role of requirements may be met in agile development including challenges to consider.},
	booktitle = {Proceedings of the 16th International Conference on Agile Software Development},
	publisher = {Springer},
	author = {Bjarnason, Elizabeth and Unterkalmsteiner, Michael and Engström, Emelie and Borg, Markus},
	year = {2015},
	pages = {27--39},
	url = {https://www.lmsteiner.com/papers/bjarnason_2015_xp.pdf}
},

@inproceedings{klotins_software_2015,
	address = {Braga, Portugal},
	title = {Software {Engineering} {Practices} in {Start}-up {Companies}: {A} {Mapping} {Study}},
	abstr = {Background - Startup companies are becoming important suppliers of innovative and software intensive  products. The failure rate among startups is high due to lack of resources, immaturity, multiple influences and dynamic technologies. However, software product engineering is the core activity in startups, therefore inadequacies in applied engineering practices might be a significant contributing factor for high failure rates. Aim - This study identifies and categorizes software engineering knowledge areas utilized in startups to map out the state-of-art, identifying gaps for further research. Method - We perform a systematic literature mapping study, applying snowball sampling to identify relevant primary studies. Results - We have identified 54 practices from 14 studies. Although 11 of 15 main knowledge areas from SWEBOK are covered, a large part of categories is not. Conclusions - Existing research does not provide reliable support for software engineering in any phase of a startup life cycle. Transfer of results to other startups is difficult due to low rigor in current studies.},
	booktitle = {6th {International} {Conference} on {Software} {Business}},
	publisher = {Springer},
	author = {Klotins, Eriks and Unterkalmsteiner, Michael and Gorschek, Tony},
	year = {2015},
	pages = {245--257},
	url = {https://www.lmsteiner.com/papers/klotins_2015_icsob.pdf}
},


@article{unterkalmsteiner_assessing_2015,
	title = {Assessing {Requirements} {Engineering} and {Software} {Test} {Alignment} - {Five} {Case} {Studies}},
	abstr = {The development of large, software-intensive systems is a complex undertaking that we generally tackle by a divide and conquer strategy. Companies thereby face the challenge of coordinating individual aspects of software development, in particular between requirements engineering (RE) and software testing (ST). A lack of REST alignment can not only lead to wasted effort but also to defective software. However, before a company can improve the mechanisms of coordination they need to be understood first. With REST-bench we aim at providing an assessment tool that illustrates the coordination in software development projects and identify concrete improvement opportunities. We have developed REST-bench on the sound fundamentals of a taxonomy on REST alignment methods and validated the method in five case studies. Following the principles of technical action research, we collaborated with five companies, applying REST-bench and iteratively improving the method based on the lessons we learned. We applied REST-bench both in Agile and plan-driven environments, in projects lasting from weeks to years, and staffed as large as 1000 employees. The improvement opportunities we identified and the feedback we received indicate that the assessment was effective and efficient. Furthermore, participants confirmed that their understanding on the coordination between RE and ST improved.},
	journal = {Journal of Systems and Software},
	author = {Unterkalmsteiner, Michael and Gorschek, Tony and Feldt, Robert and Klotins, Eriks},
	year = {2015},
	pages = {62--77},
	volume = {109},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2015_jss.pdf}
},

@article{unterkalmsteiner_large-scale_2015,
	title = {Large-scale Information Retrieval in Software Engineering - An Experience Report from Industrial Application},
	abstr = {Background: Software Engineering activities are information intensive. Research proposes Information Retrieval ({IR}) techniques to support engineers in their daily tasks, such as establishing and maintaining traceability links, fault identification, and software maintenance. Objective: We describe an engineering task, test case selection, and illustrate our problem analysis and solution discovery process. The objective of the study is to gain an understanding of to what extent {IR} techniques (one potential solution) can be applied to test case selection and provide decision support in a large-scale, industrial setting. Method: We analyze, in the context of the studied company, how test case selection is performed and design a series of experiments evaluating the performance of different {IR} techniques. Each experiment provides lessons learned from implementation, execution, and results, feeding to its successor. Results: The three experiments led to the following observations: (1) there is a lack of research on scalable parameter optimization of {IR} techniques for software engineering problems; (2) scaling {IR} techniques to industry data is challenging, in particular for latent semantic analysis; (3) the {IR} context poses constraints on the empirical evaluation of {IR} techniques, requiring more research on developing valid statistical approaches. Conclusions: We believe that our experiences in conducting a series of {IR} experiments with industry grade data are valuable for peer researchers so that they can avoid the pitfalls that we have encountered. Furthermore, we identified challenges that need to be addressed in order to bridge the gap between laboratory {IR} experiments and real applications of {IR} in the industry.},
	journal = {Empirical Software Engineering},
	author = {Unterkalmsteiner, Michael and Gorschek, Tony and Feldt, Robert and Lavesson, Niklas},
	year = {2016},
	volume = {21},
	number = {6},
	pages = {2324-2365},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2015_esej.pdf}
},

@PhDThesis{unterkalmsteiner_coordinating_2015,
	location = {Karlskrona, Sweden},
	title = {Coordinating Requirements Engineering and Software Testing},
	abstr = {The development of large, software-intensive systems is a complex undertaking that is generally tackled by a divide and conquer strategy. Organizations face thereby the challenge of coordinating the resources which enable the individual aspects of software development, commonly solved by adopting a particular process model. The alignment between requirements engineering (RE) and software testing (ST) activities is of particular interest as those two aspects are intrinsically connected: requirements are an expression of user / customer needs while testing increases the likelihood that those needs are actually satisfied.
The work in this thesis is driven by empirical problem identification, analysis and solution development towards two main objectives. The first is to develop an understanding of RE and ST alignment challenges and characteristics. Building this foundation is a necessary step that facilitates the second objective, the development of solutions relevant and scalable to industry practice that improve REST alignment.
The research methods employed to work towards these objectives are primarily empirical. Case study research is used to elicit data from practitioners while technical action research and field experiments are conducted to validate the developed solutions in practice.
This thesis contains four main contributions: (1) An in-depth study on REST alignment challenges and practices encountered in industry. (2) A conceptual framework in the form of a taxonomy providing constructs that further our understanding of REST alignment. The taxonomy is operationalized in an assessment framework, REST-bench (3), that was designed to be lightweight and can be applied as a postmortem in closing development projects. (4) An extensive investigation into the potential of information retrieval techniques to improve test coverage, a common REST alignment challenge, resulting in a solution prototype, risk-based testing supported by topic models (RiTTM).
REST-bench has been validated in five cases and has shown to be efficient and effective in identifying improvement opportunities in the coordination of RE and ST. Most of the concepts operationalized from the REST taxonomy were found to be useful, validating the conceptual framework. RiTTM, on the other hand, was validated in a single case experiment where it has shown great potential, in particular by identifying test cases that were originally overlooked by expert test engineers, improving effectively test coverage.},
	pagetotal = {301},
	school = {Blekinge Institute of Technology},
	organization = {Department of Software Engineering},
	type = {PhD Thesis},
	author = {Unterkalmsteiner, Michael},
	year = {2015},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_thesis_final.pdf}
},

@techreport{bjarnason_summary_2015,
	title = {Summary of 2nd International Workshop on Requirements Engineering and Testing (RET 2015): Co-located with ICSE 2015},
	url = {https://www.lmsteiner.com/papers/bjarnason_2015_RET.pdf},
	abstr = {The RET (Requirements Engineering and Testing) workshop series provides a meeting point for researchers and practitioners from the two separate fields of Requirements Engineering (RE) and Testing. The goal is to improve the connection and alignment of these two areas through an exchange of ideas, challenges, practices, experiences and results. The long term aim is to build a community and a body of knowledge within the intersection of RE and Testing, i.e. RET. The 2nd workshop was held in co-location with ICSE 2015 in Florence, Italy. The workshop continued in the same interactive vein as the 1st one and included a keynote, paper presentations with ample time for discussions, and a group exercise. For true impact and relevance this cross-cutting area requires contribution from both RE and Testing, and from both researchers and practitioners. A range of papers were presented from short experience papers to full research papers that cover connections between the two fields. One of the main outputs of the 2nd workshop was a categorization of the presented workshop papers according to an initial definition of the area of RET which identifies the aspects RE, Testing and coordination effect.},
	pages = {39--42},
	number = {Volume 40 Issue 5},
	institution = {{SIGSOFT} Software Engineering Notes},
	type = {Workshop summary},
	author = {Bjarnason, Elizabeth and Borg, Markus and Morandini, Mirko and Unterkalmsteiner, Michael and Felderer, Michael and Staats, Matthew},
	year = {2015}
},

@techreport{felderer_workshop_2014,
	title = {Workshop Summary of the 1st International Workshop on Requirements and Testing (RET'14)},
	url = {https://www.lmsteiner.com/papers/felderer_2014_RET.pdf},
	abstr = {The main objective of the RET workshop was to explore the interaction of Requirements Engineering (RE) and Testing, i.e. RET, in research and industry, and the challenges that result from this interaction. While much work has been done in the respective fields of requirements engineering and testing, there exists much more than can be done to understand the connection between the processes of RE and of testing.},
	number = {1410.3401},
	institution = {{arXiv}.org},
	type = {Workshop summary},
	author = {Felderer, Michael and Bjarnason, Elizabeth and Borg, Markus and Unterkalmsteiner, Michael and Morandini, Mirko and Staats, Matt},
	year = {2014}
},


@article{giardino_software_2015,
	title = {Software Development in Startup Companies: The Greenfield Startup Model},
	url = {https://www.lmsteiner.com/papers/giardino_2015_tse.pdf},
	abstr = {Software startups are newly created companies with no operating history and oriented towards producing cutting-edge products. However, despite the increasing importance of startups in the economy, few scientific studies attempt to address software engineering issues, especially for early-stage startups. If anything, startups need engineering practices of the same level or better than those of larger companies, as their time and resources are more scarce, and one failed project can put them out of business. In this study we aim to improve understanding of the software development strategies employed by startups. We performed this state-
of-practice investigation using a grounded theory approach. We packaged the results in the Greenfield Startup Model (GSM), which explains the priority of startups to release the product as quickly as possible. This strategy allows startups to verify product and market fit, and to adjust the product trajectory according to early collected user feedback. The need to shorten time-to-market, by speeding up the development through low-precision engineering activities, is counterbalanced by the need to restructure the product before targeting further growth. The resulting implications of the GSM outline challenges and gaps, pointing out opportunities for future research to develop and validate engineering practices in the startup context.},
	pages = {585-604},
	number = {6},
	volume = {42},
	journal = {IEEE Transactions on Software Engineering},
	author = {Giardino, Carmine and Paternoster, Nicolò and Unterkalmsteiner, Michael and Gorschek, Tony and Abrahamsson, Pekka},
	year = {2016}
},


@article{bjarnason_multi-case_2016,
	title = {A {Multi}-{Case} {Study} of {Agile} {Requirements} {Engineering} and  {Using} {Test} {Cases} as {Requirements}},
	abstr = {It is an enigma that agile projects can succeed ‘without requirements’ when weak requirements engineering is a known cause for project failures. While agile development projects often manage well without extensive requirements test cases are commonly viewed as requirements and detailed requirements are documented as test cases.
We have investigated this agile practice of using test cases as requirements to understand how test cases can support the main requirements activities, and how this practice varies.
We performed an iterative case study at three companies and collected data through 14 interviews and 2 focus groups.
The use of test cases as requirements poses both benefits and challenges when eliciting, validating, verifying, and managing requirements, and when used as a documented agreement. We have identified five variants of the test-cases-as-requirements practice, namely de facto, behaviour-driven, story-test driven, stand-alone strict and stand-alone manual for which the application of the practice varies concerning the time frame of requirements documentation, the requirements format, the extent to which the test cases are a machine executable specification and the use of tools which provide specific support for the practice of using test cases as requirements.
	  
The findings provide empirical insight into how agile development projects manage and communicate requirements. The identified variants of the practice of using test cases as requirements can be used to perform in-depth investigations into agile requirements engineering. Practitioners can use the provided recommendations as a guide in designing and improving their agile requirements practices based on project characteristics such as number of stakeholders and rate of change.},
	journal = {Information and Software Technology},
	author = {Bjarnason, Elizabeth and Unterkalmsteiner, Michael and Engström, Emelie and Borg, Markus},
	year = {2016},
	pages = {61-79},
	number = {0},
	volume = {77},
	url = {https://www.lmsteiner.com/papers/bjarnason_2016_ist.pdf}
},


@techreport{unterkalmsteiner_summary_2016,
	type = {Workshop summary},
	title = {Summary of the 3rd {International} {Workshop} on {Requirements} {Engineering} and {Testing} ({RET} 2016): [{Co}-located with {REFSQ} 2016]},
	shorttitle = {Summary of the 3rd {International} {Workshop} on {Requirements} {Engineering} and {Testing} ({RET} 2016)},
	abstr = {The RET (Requirements Engineering and Testing) workshop series provides a meeting point for researchers and practitioners from the two separate fields of Requirements Engineering (RE) and Testing. The goal is to improve the connection and alignment of these two areas through an exchange of ideas, challenges, practices, experiences and results. The long term aim is to build a community and a body of knowledge within the intersection of RE and Testing, i.e. RET. The 3rd workshop was held in co-location with REFSQ 2016 in Gothenburg, Sweden. The workshop continued in the same interactive vein as the predecessors and included a keynote, paper presentations with ample time for discussions, and panels. In order to create an RET knowledge base, this crosscutting area elicits contributions from both RE and Testing, and from both researchers and practitioners. A range of papers were presented from short positions papers to full research papers that cover connections between the two fields.},
	number = {Volume 41 Issue 3},
	urldate = {2016-06-30},
	institution = {SIGSOFT Software Engineering Notes},
	author = {Unterkalmsteiner, Michael and Gay, Gregory and Felderer, Michael and Bjarnason, Elizabeth and Borg, Markus and Morandini, Mirko},
	month = jun,
	year = {2016},
	pages = {31-33},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2016_sen.pdf}
},

@article{unterkalmsteiner_software_2016,
	title = {Software {Startups} - {A} {Research} {Agenda}},
	abstr = {Software startup companies develop innovative, software-intensive products within limited time frames and with few resources, searching for sustainable and scalable business models. Software startups are quite distinct from traditional mature software companies, but also from micro-, small-, and medium-sized enterprises, introducing new challenges relevant for software engineering research. This paper's research agenda focuses on software engineering in startups, identifying, in particular, 70+ research questions in the areas of supporting startup engineering activities, startup evolution models and patterns, ecosystems and innovation hubs, human aspects in software startups, applying startup concepts in non-startup environments, and methodologies and theories for startup research. We connect and motivate this research agenda with past studies in software startup research, while pointing out possible future directions. While all authors of this research agenda have their main background in Software Engineering or Computer Science, their interest in software startups broadens the perspective to the challenges, but also to the opportunities that emerge from multi-disciplinary research. Our audience is therefore primarily software engineering researchers, even though we aim at stimulating collaborations and research that crosses disciplinary boundaries. We believe that with this research agenda we cover a wide spectrum of the software startup industry current needs.},
	journal = {e-Informatica Software Engineering Journal},
	author = {Unterkalmsteiner, Michael and Abrahamsson, Pekka and Wang, Xiaofeng and Nguyen-Duc, Anh and Shah, Syed and Bajwa, Sohaib Shahid and Baltes, Guido H. and Conboy, Kieran and Cullina, Eoin and Dennehy, Denis and Edison, Henry and Fernandez-Sanchez, Carlos and Garbajosa, Juan and Gorschek, Tony and Klotins, Eriks and Hokkanen, Laura and Kon, Fabio and Lunesu, Ilaria and Marchesi, Michele and Morgan, Lorraine and Oivo, Markku and Selig, Christoph and Seppänen, Pertti and Sweetman, Roger and Tyrväinen, Pasi and Ungerer, Christina and Yagüe, Augustin},
	year = {2016},
	pages = {89-124},
	number = {1},
	volume = {2016},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2016_informatica.pdf}
}



@inproceedings{unterkalmsteiner_requirements_2017,
	address = {Essen, Germany},
	title = {Requirements quality assurance in industry: why, what and how?},
	abstr = {[Context \& Motivation] Natural language is the most common form to specify requirements in industry. The quality of the specification depends on the capability of the writer to formulate requirements aimed at different stakeholders: they are an expression of the customer's needs that are used by analysts, designers and testers. Given this central role of requirements as a mean to communicate intention, assuring their quality is essential to reduce misunderstandings that lead to potential waste.
[Problem] Quality assurance of requirement specifications is largely a manual effort that requires expertise and domain knowledge. However, this demanding cognitive process is also congested by trivial quality issues that should not occur in the first place.
[Principal ideas] We propose a taxonomy of requirements quality assurance complexity that characterizes cognitive load of verifying a quality aspect from the human perspective, and automation complexity and accuracy from the machine perspective. [Contribution] Once this taxonomy is realized and validated, it can serve as the basis for a decision framework of automated requirements quality assurance support.},
	booktitle = {23rd {International} {Working} {Conference} on {Requirements} {Engineering}: {Foundation} for {Software} {Quality} ({REFSQ})},
	publisher = {Springer},
	author = {Unterkalmsteiner, Michael and Gorschek, Tony},
	year = {2017},
	pages = {77-84},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2017_refsq.pdf}
}


@inproceedings{gren_is_2017,
	address = {Buenos Aires, Argentina},
	title = {Is it {Possible} to {Disregard} {Obsolete} {Requirements}? — {An} {Initial} {Experiment} on a {Potentially} {New} {Bias} in {Software} {Effort} {Estimation}},
	abstr = {Effort estimation is a complex area in decision-making, and is influenced by a diversity of factors that could increase the estimation error. The effects on effort estimation accuracy of having obsolete requirements in specifications have not yet been studied. This study aims at filling that gap. A total of 150 students were asked to provide effort estimates for different amounts of requirements, and one group was explicitly told to disregard some of the given requirements. The results show that even the extra text instructing participants to exclude requirements in the estimation task, had the subjects give higher estimates. The effect of having obsolete requirements in requirements specifications and backlogs in software effort estimation is not taken into account enough today, and this study provides empirical evidence that it possibly should. We also suggest different psychological explanations to the found effect.},
	booktitle = {Proceedings 10th {International} {Workshop} on {Cooperative} and {Human} {Aspects} of {Software} {Engineering} ({CHASE})},
	publisher = {IEEE},
	author = {Gren, Lucas and Berntsson Svensson, Richard and Unterkalmsteiner, Michael},
	year = {2017},
	pages = {56-61},
	url = {https://www.lmsteiner.com/papers/gren_2017_chase.pdf}
}


@article{klotins_software-intensive_2017,
	title = {Software-intensive product engineering in start-ups: a taxonomy},
	abstr = {Software start-ups are new companies aiming to launch an innovative product to mass markets fast with minimal resources. However a majority of start-ups fail before realizing their potential. Poor software engineering, among other factors, could be a significant contributor to the challenges experienced by start-ups.
Very little is known about the engineering context in start-up companies. On the surface, start-ups are characterized by uncertainty, high risk and minimal resources. However, such characterization is not granular enough to support identification of specific engineering challenges and to devise start-up specific engineering practices.
The first step towards understanding on software engineering in start-ups is definition of the Start-up Context Map - a taxonomy of engineering practices, environment factors and goals influencing the engineering process. Goal of the Start-up Context Map is to support further research on the field and to serve as an engineering decision support tool for start-ups.},
	journal = {IEEE Software},
	author = {Klotins, Eriks and Unterkalmsteiner, Michael and Gorschek, Tony},
	year = {2018},
	number = {4},
	volume = {35},
	pages = {44-52},
	url = {https://www.lmsteiner.com/papers/klotins_2017_ieeesw.pdf}
}


@article{unterkalmsteiner_process_2017,
	title = {Process {Improvement} {Archaeology} – {What} led us here and what’s next?},
	abstr = {While in every organization corporate culture and history change over time, intentional efforts to identify performance problems are of particular interest when trying to understand the current state of an organization. The results of past improvement initiatives can shed light on the evolution of an organization, and represent, with the advantage of perfect hindsight, a learning opportunity for future process improvements. We encountered the opportunity to test this premise in an applied research collaboration with the Swedish Transport Administration (STA), the government agency responsible for the planning, implementation and maintenance of long-term rail, road, shipping and aviation infrastructure in Sweden.},
	journal = {IEEE Software},
	author = {Unterkalmsteiner, Michael and Gorschek, Tony},
	year = {2018},
	number = {4},
	volume = {35},
	pages = {53-61},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2017_ieeesw.pdf}
}


@inproceedings{femmer_which_2017,
	address = {Lisbon, Portugal},
	title = {Which requirements artifact quality defects are automatically detectable? {A} case study},
	abstr = {[Context:] The quality of requirements engineering artifacts, e.g. requirements specifications, is acknowledged to be an important success factor for projects. Therefore, many companies spend significant amounts of money to control the quality of their RE artifacts. To reduce spending and improve the RE artifact quality, methods were proposed that combine manual quality control, i.e. reviews, with automated approaches. [Problem:] So far, we have seen various approaches to automatically detect certain aspects in RE artifacts. However, we still lack an overview what can and cannot be automatically detected. [Approach:] Starting from an industry guideline for RE artifacts, we classify 166 existing rules for RE artifacts along various categories to discuss the share and the characteristics of those rules that can be automated. For those rules, that cannot be automated, we discuss the main reasons. [Contribution:] We estimate that 53\% of the 166 rules can be checked automatically either perfectly or with a good heuristic. Most rules need only simple techniques for checking. The main reason why some rules resist automation is due to imprecise definition. [Impact:] By giving first estimates and analyses of automatically detectable and not automatically detectable rule violations, we aim to provide an overview of the potential of automated methods in requirements quality control.},
	booktitle = {4th {International} {Workshop} on {Artificial} {Intelligence} for {Requirements} ({AIRE})},
	publisher = {IEEE},
	author = {Femmer, Henning and Unterkalmsteiner, Michael and Gorschek, Tony},
	year = {2017},
	pages = {400-406},
	url = {https://www.lmsteiner.com/papers/femmer_2017_aire.pdf}
	}


@techreport{borg_summary_2018,
	type = {Workshop summary},
	title = {Summary of the 4th {International} {Workshop} on {Requirements} {Engineering} and {Testing} ({RET} 2017): [{Co}-located with {RE} 2017]},
	abstr = {The RET (Requirements Engineering and Testing) workshop series provides a meeting point for researchers and practitioners from the two separate fields of Requirements Engineering (RE) and Testing. The long term aim is to build a community and a body of knowledge within the intersection of RE and Testing, i.e., RET. The 4th workshop was co-located with the 25th International Requirements Engineering Conference (RE'17) in Lisbon, Portugal and attracted about 20 participants. In line with the previous workshop instances, RET 2017 o ered an interactive setting with a keynote, an invited talk, paper presentations, and a concluding hands-on exercise.},
	number = {Volume 42 Issue 4},
	urldate = {2016-06-30},
	institution = {SIGSOFT Software Engineering Notes},
	author = {Borg, Markus and Bjarnason, Elizabeth and Unterkalmsteiner, Michael and Yu, Tingting and Gay, Gregory and Felderer, Michael},
	year = {2018},
	pages = {28--31},
	url = {https://www.lmsteiner.com/papers/borg_2018_sen.pdf}
	}


@inproceedings{klotins_exploration_2018,
	address = {Gothenburg, Sweden},
	title = {Exploration of {Technical} {Debt} in {Start}-ups},
	abstr = {Context: Software start-ups are young companies aiming to build and market oftware-intensive products fast with little resources. Aiming to accelerate time-to-market, start-ups often opt for ad-hoc engineering practices, make shortcuts in product engineering, and accumulate technical debt.
Objective: In this paper we explore to what extent recedents, dimensions and outcomes associated with technical debt are prevalent in start-ups. Method: We apply a case survey method to identify aspects of technical debt and contextual information characterizing the engineering context in start-ups.
Results: By analyzing responses from 86 start-up cases we found that start-ups accumulate most technical debt in the testing dimension, despite attempts to automate testing. Furthermore, we found that start-up team size and experience is a leading precedent for accumulating technical debt: larger teams face more challenges in keeping the debt under control.
Conclusions: This study highlights the necessity to monitor levels of technical debt and to preemptively introduce practices to keep the debt under control. Adding more people to an already difficult to maintain product could amplify other precedents, such as resource shortages, communication issues and negatively affect decisions pertaining to the use of good engineering practices.},
	booktitle = {40th {International} {Conference} on {Software} {Engineering} ({ICSE})},
	publisher = {IEEE},
	author = {Klotins, Eriks and Unterkalmsteiner, Michael and Chatzipetrou, Panagiota and Gorschek, Tony and Prikladnicki, Rafael and Tripathi, Nirnaya and Pompermaier, Leandro Bento},
	year = {2018},
	pages = {75--84},
	url = {https://www.lmsteiner.com/papers/klotins_2018_icse.pdf}
}


@article{klotins_software_2018,
	title = {Software {Engineering} in {Start}-up {Companies} - {Analysis} of 88 {Experience} reports},
	volume = {24},
	abstr = {Start-up companies have become an important supplier of innovation and software-intensive products. The flexibility and reactiveness of start-ups enables fast development and launch of innovative products. However, a majority of software start-up companies fail before achieving any success. Among other factors, poor software engineering could be a significant contributor to the challenges experienced by start-ups. However, the state-of-practice of software engineering in start-ups, as well as the utilisation of state-of-the-art is largely an unexplored area. In this study we investigate how software engineering is applied in start-up context with a focus to identify key knowledge areas and opportunities for further research. We perform a multi-vocal exploratory study of 88 start-up experience reports. We develop a custom taxonomy to categorize the reported software engineering practices and their interrelation with business aspects, and apply qualitative data analysis to explore influences and dependencies between the knowledge areas. We identify the most frequently reported software engineering (requirements engineering, software design and quality) and business aspect (vision and strategy development) knowledge areas, and illustrate their relationships. We also present a summary of how relevant software engineering knowledge areas are implemented in start-ups and identify potentially useful practices for adoption in start-ups.  The results enable a more focused research on engineering practices in start-ups. We conclude that most engineering challenges in start-ups stem from inadequacies in requirements engineering. Many promising practices to address specific engineering challenges exists, however more research on adaptation of established practices, and validation of new start-up specific practices is needed.},
	journal = {Empirical Software Engineering},
	author = {Klotins, Eriks and Unterkalmsteiner, Michael and Gorschek, Tony},
	year = {2019},
	pages = {68--102},
  url = {https://www.lmsteiner.com/papers/klotins_2018_emse.pdf}
}


@article{klotins_software_2018,
	title = {Software {Engineering} {Anti}-patterns in {Start}-ups},
	abstr = {Software start-up failures are often explained with a poor business model, market issues, insufficient funding, or simply a bad product idea. However, inadequacies in software engineering are relatively unexplored and could be a significant contributing factor to the high start-up failure rate. In this paper we present the analysis of 88 start-up experience reports, revealing three anti-patterns associated with start-up progression phases. The anti-patterns address challenges of releasing the first version of the product, attracting customers, and expanding the product into new markets. The anti-patterns show that challenges and failure scenarios that appear to be business or market related are, at least partially, rooted in engineering inadequacies.},
	journal = {IEEE Software},
  number = {2},
  volume = {36},
	author = {Klotins, Eriks and Unterkalmsteiner, Michael and Gorschek, Tony},
	year = {2019},
  pages = {118--126},
	url = {https://www.lmsteiner.com/papers/klotins_2018_ieeesw.pdf}
}


@inproceedings{silva_monitoring_2018,
	address = {Pułtusk, Poland},
	title = {Monitoring and {Maintenance} of {Telecommunication} {Systems}: {Challenges} and {Research} {Perspectives}},
	abstr = {In this paper, we present challenges associated with monitoring and maintaining a large telecom system at Ericsson that was developed with high degree of component reuse. The system constitutes of multiple services, composed of both legacy and modern systems that are constantly changing and need to be adapted to changing business needs. The paper is based on firsthand experience from architecting, developing and maintaining such a system, pointing out current challenges and potential avenues for future research that might contribute to addressing them.},
	booktitle = {{KKIO} {Software} {Engineering} {Conference}},
	publisher = {Springer},
	author = {Silva, Lakmal and Unterkalmsteiner, Michael and Wnuk, Krzysztof},
	year = {2018},
  pages = {166--172},
	url = {https://www.lmsteiner.com/papers/silva_2018_kkio.pdf}
}


@inproceedings{hyrynsalmi_what_2018,
	address = {Kuwait},
	title = {What is a {Minimum} {Viable} ({Video}) {Game}? - {Towards} a research agenda},
	abstr = {The concept of ‘Minimum Viable Product’ (MVP) is largely adapted in the software industry as well as in academia. Minimum viable products are used to test hypotheses regarding the target audience, save resources from unnecessary development work and guide a company towards a stable business model. As the game industry is becoming an important business domain, it is not surprise that the concept has been adopted also in the game development. This study surveys how a Minimum Viable Game (MVG) is defined, what is reported in extant literature as well as present results from a small case study survey done to nine game development companies. The study shows that despite popularity of minimum viable games in the industrial fora, the presented views on the concept are diverged and there is lack of practical guidelines and research supporting game companies. This study points out research gaps in the area as well as calls for actions to further develop the concept and to define guidelines.},
	booktitle = {17th {IFIP} {Conference} on e-{Business}, e-{Services} and e-{Society} ({I}3E 2018)},
	publisher = {Springer},
	author = {Hyrynsalmi, Sami and Klotins, Eriks and Unterkalmsteiner, Michael and Gorschek, Tony and Tripathi, Nirnaya and Pompermaier, Leandro Bento and Prikladinicki, Rafael},
	year = {2018},
  pages = {217--231},
	url = {https://www.lmsteiner.com/papers/hyrynsalmi_2018_i3e.pdf}
}

@article{tripathi_anatomy_2018,
	title = {An anatomy of requirements engineering in software startups using multi-vocal literature and case survey},
	volume = {146},
	issn = {0164-1212},
	doi = {10.1016/j.jss.2018.08.059},
	abstr = {Context: Software startups aim to develop innovative products, grow rapidly, and thus become important in the development of economy and jobs. Requirements engineering (RE) is a key process area in software development, but its effects on software startups are unclear. Objective: The main objective of this study was to explore how RE (elicitation, documentation, prioritization and validation) is used in software startups. Method: A multi-vocal literature review (MLR) was used to find scientific and gray literature. In addition, a case survey was employed to gather empirical data to reach this study’s objective. Results: In the MLR, 36 primary articles were selected out of 28,643 articles. In the case survey, 80 respondents provided information about software startup cases across the globe. Data analysis revealed that during RE processes, internal sources (e.g., for source), analyses of similar products (e.g., elicitation), uses of informal notes (e.g., for documentation), values to customers, products and stakeholders (e.g., for prioritization) and internal reviews/prototypes (e.g., for validation) were the most used techniques. Conclusion: After an analysis of primary literature, it was concluded that research on this topic is still in early stages and more systematic research is needed. Furthermore, few topics were suggested for future research.},
	journal = {Journal of Systems and Software},
	author = {Tripathi, Nirnaya and Klotins, Eriks and Prikladnicki, Rafael and Oivo, Markku and Pompermaier, Leandro Bento and Kudakacheril, Arun Sojan and Unterkalmsteiner, Michael and Liukkunen, Kari and Gorschek, Tony},
	month = dec,
	year = {2018},
	pages = {130--151},
	url = {https://www.lmsteiner.com/papers/tripathi_2018_jss.pdf}
}


@inproceedings{yates_replicating_2019,
	address = {Cologne, Germany},
	title = {Replicating {Relevance}-{Ranked} {Synonym} {Discovery} in a {New} {Language} and {Domain}},
	abstr = {Domain-specific synonyms occur in many specialized search tasks, such as searching medical documents, legal documents, and software engineering artifacts. We replicate prior work on ranking domain-specific synonyms in the consumer health domain by applying the approach to a new domain and language: identifying Swedish language synonyms in the building construction domain. We chose this replication setting because identifying synonyms in this domain is helpful for downstream systems, where different users may query for documents (e.g., requirements) using different terminology. We consider two new features inspired by the change in language and methodological advances since the prior work's publication. An evaluation using data from the building construction domain supports the finding from the prior work that synonym discovery is best approached as a learning to rank task in which a human editor views ranked synonym candidates in order to construct a domain-specific thesaurus. We additionally find that FastText embeddings alone provide a strong baseline, though they do not perform as well as the strongest learning to rank method. Finally, we analyze the performance of individual features and the differences in the domains.},
	booktitle = {Proceedings 41st {European} {Conference} on {Information} {Retrieval}},
	publisher = {Springer},
	author = {Yates, Andrew and Unterkalmsteiner, Michael},
	year = {2019},
	pages = {429--442},
  url = {https://www.lmsteiner.com/papers/yates_2019_ecir.pdf}
}


@inproceedings{unterkalmsteiner_expert-sourcing_2019,
	address = {Essen, Germany},
	title = {Expert-sourcing {Domain}-specific {Knowledge}: {The} {Case} of {Synonym} {Validation}},
	abstr = {One prerequisite for supervised machine learning is high quality labelled data. Acquiring such data is, particularly if expert knowledge is required, costly or even impossible if the task needs to be performed by a single expert. In this paper, we illustrate tool support that we adopted and extended to source domain-specific knowledge from experts. We provide insight in design decisions that aim at motivating experts to dedicate their time at performing the labelling task. We are currently using the approach to identify true synonyms from a list of candidate synonyms. The identification of synonyms is important in scenarios were stakeholders from different companies and background need to collaborate, for example when defining and negotiating requirements. We foresee that the approach of expert-sourcing is applicable to any data labelling task in software engineering. The discussed design decisions are an initial draft that can be extended, refined and validated with further application.},
	booktitle = {2nd {Workshop} on {Natural} {Language} {Processing} for {Requirements} {Engineering}},
	publisher = {CEUR},
	author = {Unterkalmsteiner, Michael and Yates, Andrew},
	year = {2019},
	pages = {0--0},
  url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2019_nlp4re.pdf}
}


@article{klotins_progression_2019,
	title = {A progression model of software engineering goals, challenges, and practices in start-ups},
	abstr = {Context: Software start-ups are emerging as suppliers of innovation and software-intensive products. However, traditional software engineering practices are not evaluated in the context, nor adopted to goals and challenges of start-ups. As a result, there is insufficient support for software engineering in the start-up context.
Objective: We aim to collect data related to engineering goals, challenges, and practices in start-up companies to ascertain trends and patterns characterizing engineering work in start-ups. Such data allows researchers to understand better how goals and challenges are related to practices. This understanding can then inform future studies aimed at designing solutions addressing those goals and challenges. Besides, these trends and patterns can be useful for practitioners to make more informed decisions in their engineering practice.
Method: We use a case survey method to gather first-hand, in-depth experiences from a large sample of software start-ups. We use open coding and cross-case analysis to describe and identify patterns, and corroborate the findings with statistical analysis.
Results: We analyze 84 start-up cases and identify 16 goals, 9 challenges, and 16 engineering practices that are common among startups. We have mapped these goals, challenges, and practices to start-up life-cycle stages (inception, stabilization, growth, and maturity). Thus, creating the progression model guiding software engineering efforts in start-ups.
Conclusions: We conclude that start-ups to a large extent face the same challenges and use the same practices as established companies. However, the primary software engineering challenge in start-ups is to evolve multiple process areas at once, with a little margin for serious errors.},
	journal = {IEEE Trans. Softw. Eng.},
	author = {Klotins, Eriks and Unterkalmsteiner, Michael and Chatzipetrou, Panagiota and Gorschek, Tony and Prikladniki, Rafael and Tripathi, Nirnaya and Pompermaier, Leandro},
	year = {2021},
  number = {3},
  volume = {47},
	pages = {498--521},
	url = {https://www.lmsteiner.com/papers/klotins_2019_tse.pdf}
}


@inproceedings{badampudi_modern_2019,
	address = {Copenhagen, Denmark},
	series = {{EASE} '19},
	title = {Modern {Code} {Reviews} - {Preliminary} {Results} of a {Systematic} {Mapping} {Study}},
	abstr = {Reviewing source code is a common practice in a modern and collaborative coding environment. In the past few years, the research on modern code reviews has gained interest among practitioners and researchers. The objective of our investigation is to observe the evolution of research related to modern code reviews, identify research gaps and serve as a basis for future research. We use a systematic mapping approach to identify and classify 177 research papers. As preliminary result of our investigation, we present in this paper a classification scheme of the main contributions of modern code review research between 2005 and 2018.},
	booktitle = {Proceedings of the {Evaluation} and {Assessment} on {Software} {Engineering} ({EASE})},
	publisher = {ACM},
	author = {Badampudi, Deepika and Britto, Ricardo and Unterkalmsteiner, Michael},
	year = {2019},
	pages = {340--345},
	url = {https://www.lmsteiner.com/papers/badampudi_2019_ease.pdf}
}


@inproceedings{chatzipetrou_requirements_2019,
	address = {Kallithea - Chalkidiki, Greece},
	title = {Requirements' {Characteristics}: {How} do they {Impact} on {Project} {Budget} in a {Systems} {Engineering} {Context}?},
	abstr = {Background: Requirements engineering is of a principal importance when starting a new project. However, the number of the requirements involved in only one project can reach up to thousands. Controlling and assuring the quality of natural language requirements (NLRs), in these quantities, is challenging.
Aims: In a field study, we investigated with the Swedish Transportation Agency (STA) to what extent the characteristics of requirements had an influence on change requests and budget changes in the project.
Method: We choose the following models to characterize system requirements formulated in natural language: Concern-based Model of Requirements (CMR), Requirements Abstractions Model (RAM) and Software-Hardware model (SHM). The classification of the NLRs was conducted by the three authors. The robust statistical measure Fleiss' Kappa was used to verify the reliability of the results. We use descriptive statistics, contingency tables, results from Chi-Square test of association along with post hoc tests. Finally, a multivariate statistical technique, Correspondence analysis was used in order to provide a means of displaying a set of requirements in two-dimensional graphical form.
Results: The results showed that software requirements are associated with less budget cost than hardware requirements. Moreover, software requirements tend to stay open for a longer period indicating that they are "harder" to handle. Finally, the more discussion or interaction on a change request can lower the actual estimated change request cost.
Conclusions: The results lead us to the need for investigate further the reasons why the software requirements are treated differently from the hardware requirements, interview the project managers, understand better the way those requirements are formulated and propose effective ways of Software management.},
	booktitle = {Euromicro {Conference} on {Software} {Engineering} and {Applications} ({SEAA})},
	publisher = {IEEE},
	author = {Chatzipetrou, Panagiota and Unterkalmsteiner, Michael and Gorschek, Tony},
	year = {2019},
	pages = {260-267},
	url = {https://www.lmsteiner.com/papers/chatzipetrou_2019_seaa.pdf}
}


@inproceedings{huynh_khanh_test-case_2019,
	address = {Barcelona, Spain},
	title = {Test-{Case} {Quality} – {Understanding} {Practitioners}’ {Perspectives}},
	abstr = {Background: Test-case quality has always been one of the major concerns in software testing. To improve test-case quality, it is important to better understand how practitioners perceive the quality of test-cases. Objective: Motivated by that need, we investigated how practitioners define test-case quality and which aspects of test-cases are important for quality assessment. Method: We conducted semi-structured interviews with professional developers, testers and test architects from a multinational software company in Sweden. Before the interviews, we asked participants for actual test cases (written in natural language) that they perceive as good, normal, and bad respectively together with rationales for their assessment. We also compared their opinions on shared test cases and contrasted their views with the relevant literature. Results: We present a quality model which consists of 11 test-case quality attributes. We also identify a misalignment in defining test-case quality among practitioners and between academia and industry, along with suggestions for improving test-case quality in industry. Conclusion: The results show that practitioners’ background, including roles and working experience, are critical dimensions of how test-case quality is defined and assessed.},
	booktitle = {20th {International} {Conference} on {Product}-{Focused} {Software} {Process} {Improvement} ({PROFES})},
	publisher = {Springer},
	author = {Huynh Khanh, Vi Tran and bin Ali, Nauman and Börstler, Jürgen and Unterkalmsteiner, Michael},
	month = nov,
	year = {2019},
	pages = {37--52},
	url = {https://www.lmsteiner.com/papers/huynh_2019_profes.pdf}
}


@techreport{unterkalmsteiner_summary_2019,
	type = {Workshop summary},
	title = {Summary of the 5th {International} {Workshop} on {Requirements} {Engineering} and {Testing} ({RET} 2018)},
	abstr = {The RET (Requirements Engineering and Testing) workshop series provides a meeting point for researchers and practitioners from the two separate elds of Requirements Engineering (RE) and Testing. The goal is to improve the connection and alignment of these two areas through an exchange of ideas, challenges, practices, experiences and results. The long term aim is to build a community and a body of knowledge within the intersection of RE and Testing, i.e. RET. The 5th workshop was held in colocation with ICSE 2018 in Gothenburg, Sweden. The workshop continued in the same interactive vein as the predecessors. We introduced a new format for the presentations in which the paper authors had the opportunity to interact extensively with the audience. Each author was supported by a member of the organization committee to prepare either an extensive demo, collect more data in form of a questionnaire or perform a hands-on tutorial. We named this new format {\textbackslash}X-ray session". In order to create an RET knowledge base, this cross-cutting area elicits contributions from both RE and Testing, and from both researchers and practitioners. A range of papers were presented from short positions papers to full research papers that cover connections between the two elds. The workshop attracted 27 participants and the positive feedback on the new format encourages us to organize the workshop the next year again.},
	number = {Volume 44 Issue 1},
	urldate = {2019-11-21},
	institution = {SIGSOFT Software Engineering Notes},
	author = {Unterkalmsteiner, Michael and Yu, Tingting and Gay, Gregory and Bjarnason, Elizabeth and Borg, Markus and Felderer, Michael},
	month = mar,
	year = {2019},
	pages = {31--34},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2019_sen.pdf}
}


@inproceedings{unterkalmsteiner_tt-recs_2020,
	address = {Zurich, Switzerland},
	title = {{TT}-{RecS}: {The} {Taxonomic} {Trace} {Recommender} {System}},
	abstr = {Traditional trace links are established directly between source and target artefacts. This requires that the target artefact exists when the trace is established. We introduce the concept of indirect trace links between a source artefact and a knowledge organization structure, e.g. a taxonomy. This allows the creation of links (we call them taxonomic traces) before target artefacts are created. To gauge the viability of this concept and approach, we developed a prototype, TT-RecS, that allows to create such trace links either manually or with the help of a recommender system.},
	booktitle = {7th {International} {Workshop} on {Artificial} {Intelligence} and {Requirements} {Engineering}},
	publisher = {IEEE},
	author = {Unterkalmsteiner, Michael},
	month = sep,
	year = {2020},
	pages = {18--21},
	url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2020_aire.pdf}
}


@inproceedings{unterkalmsteiner_early_2020,
	address = {Zurich, Switzerland},
	title = {Early {Requirements} {Traceability} with {Domain}-{Specific} {Taxonomies} - {A} {Pilot} {Experiment}},
	abstr = {Background: Establishing traceability from requirements documents to downstream artifacts early can be beneficial as it allows engineers to reason about requirements quality (e.g. completeness, consistency, redundancy). However, creating such early traces is difficult if downstream artifacts do not exist yet. Objective: We propose to use domain-specific taxonomies to establish early traceability, raising the value and perceived benefits of trace links so that they are also available at later development phases, e.g. in design, testing or maintenance. Method: We developed a recommender system that suggests trace links from requirements to a domain-specific taxonomy based on a series of heuristics. We designed a controlled experiment to compare industry practitioners' efficiency, accuracy, consistency and confidence with and without support from the recommender. Results: We have piloted the experimental material with seven practitioners. The analysis of self-reported confidence suggests that the trace task itself is very challenging as both control and treatment group report low confidence on correctness and completeness. Conclusions: As a pilot, the experiment was successful since it provided initial feedback on the performance of the recommender, insight on the experimental material and illustrated that the collected data can be meaningfully analysed.},
	booktitle = {28th {IEEE} {International} {Requirements} {Engineering} {Conference}},
	publisher = {IEEE},
	author = {Unterkalmsteiner, Michael},
	month = sep,
	year = {2020},
	pages = {323--327},
  url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2020_re.pdf}
}

@inproceedings{frattini_automatic_2020,
	address = {Melbourne, Australia},
	title = {Automatic {Extraction} of {Cause}-{Effect}-{Relations} from {Requirements} {Artifacts}},
	abstr = {Background: The detection and extraction of causality from natural language sentences have shown great potential in various fields of application. The field of requirements engineering is eligible for multiple reasons: (1) requirements artifacts are primarily written in natural language, (2) causal sentences convey essential context about the subject of requirements, and (3) extracted and formalized causality relations are usable for a (semi-)automatic translation into further artifacts, such as test cases. Objective: We aim at understanding the value of interactive causality extraction based on syntactic criteria for the context of requirements engineering. Method: We developed a prototype of a system for automatic causality extraction and evaluate it by applying it to a set of publicly avail-able requirements artifacts, determining whether the automatic extraction reduces the manual effort of requirements formalization. Result: During the evaluation, we analyzed 2373 natural language sentences from 13 requirements documents, 282 of which were causal (11.88\%). The best evaluation of a requirements document provided an automatic extraction of 7.2 of 14 cause-effect graphs on average (51.42\%), which demonstrates the feasibility of the approach. Limitation: The feasibility of the approach has been proven in theory but actual human interaction with the system has been disregarded so far. Evaluating the applicability of the automatic causality extraction for a requirements engineer is left for future research. Conclusion: A syntactic approach for causality extraction is viable for the context of requirements engineering and can aid a pipeline towards an automatic generation of further artifacts, like test cases, from requirements artifacts.},
	booktitle = {35th {International} {Conference} on {Automated} {Software} {Engineering}},
	publisher = {ACM},
	author = {Frattini, Julian and Junker, Maximilian and Unterkalmsteiner, Michael and Mendez, Daniel},
	month = sep,
  pages = {561--572},
	year = {2020},
  url = {https://www.lmsteiner.com/papers/frattini_2020_ase.pdf}
},


@inproceedings{muhammad_compliance_2020,
	address = {Turin, Italy},
	title = {Compliance {Requirements} in {Large}-{Scale} {Software} {Development}: {An} {Industrial} {Case} {Study}},
	abstr = {Regulatory compliance is a well-studied area, including research on how to model, check, analyse, enact, and verify compliance of software. However, while the theoretical body of knowledge is vast, empirical evidence on challenges with regulatory compliance, as faced by industrial practitioners particularly in the Software Engineering domain, is still lacking. In this paper, we report on an industrial case study which aims at providing insights into common practices and challenges with checking and analysing regulatory compliance, and we discuss our insights in direct relation to the state of reported evidence.
Our study is performed at Ericsson AB, a large telecommunications company, which must comply to both locally and internationally governing regulatory entities and standards such as GDPR. The main contributions of this work are empirical evidence on challenges experienced by Ericsson that complement the existing body of knowledge on regulatory compliance.},
	booktitle = {21st {International} {Conference} on {Product}-{Focused} {Software} {Process} {Improvement}},
	publisher = {Springer},
	author = {Muhammad, Usman and Felderer, Michael and Unterkalmsteiner, Michael and Klotins, Eriks and Mendez, Daniel and Alegroth, Emil},
	month = nov,
  pages = {385--401},
	year = {2020},
  url = {https://www.lmsteiner.com/papers/muhammad_2020_profes.pdf}
},


@article{klotins_use_2021,
	title = {Use of {Agile} {Practices} in {Start}-up {Companies}},
  abstr = {Context: Software start-ups have shown their ability to develop and launch innovative software products and services. Small, motivated teams and uncertain project scope makes start-ups good candidates for adopting Agile practices.

Objective: We explore how start-ups use Agile practices and what effects can be associated with the use of those practices.

Method: We use a case survey to analyze 84 start-up cases and 56 Agile practices. We apply statistical methods to test for statistically significant associations between the use of Agile practices, team, and product factors.

Results: Our results suggest that development of the backlog, use of version control, code refactoring, and development of user stories are the most frequently reported practices. We identify 22 associations between the use of Agile practices, team, and product factors. The use of Agile practices is associated with effects on source code and overall product quality. A teams’ positive or negative attitude towards best engineering practices is a significant indicator for either adoption or rejection of certain Agile practices. To explore the relationships in our findings, we set forth a number of propositions that can be investigated in future research.

Conclusions: We conclude that start-ups use Agile practices, however without following any specific methodology. We identify the opportunity for more fine-grained studies into the adoption and effects of individual Agile practices. Start-up practitioners could benefit from Agile practices in terms of better overall quality, tighter control over team performance, and resource utilization.},
	volume = {15},
  issue = {1},
  pages = {47--64},
	journal = {e-Informatica Software Engineering Journal},
	author = {Klotins, Eriks and Unterkalmsteiner, Michael and Chatzipetrou, Panagiota and Gorschek, Tony and Prikladnicki, Rafael and Tripathi, Nirnaya and Pompermaier, Leandro Bento},
	year = {2021},
	url = {https://www.lmsteiner.com/papers/klotins_2021_einformatica.pdf}
}


@inproceedings{fischbach_automatic_2021,
	address = {Essen, Germany},
	title = {Automatic {Detection} of {Causality} in {Requirement} {Artifacts}: the {CiRA} {Approach}},
	shorttitle = {Automatic {Detection} of {Causality} in {Requirement} {Artifacts}},
	abstr = {System behavior is often expressed by causal relations in requirements (e.g., If event 1, then event 2). Automatically extracting this embedded causal knowledge supports not only reasoning about requirements dependencies, but also various automated engineering tasks such as seamless derivation of test cases. However, causality extraction from natural language is still an open research challenge as existing approaches fail to extract causality with reasonable performance. We understand causality extraction from requirements as a two-step problem: First, we need to detect if requirements have causal properties or not. Second, we need to understand and extract their causal relations. At present, though, we lack knowledge about the form and complexity of causality in requirements, which is necessary to develop a suitable approach addressing these two problems. We conduct an exploratory case study with 14,983 sentences from 53 requirements documents originating from 18 different domains and shed light on the form and complexity of causality in requirements. Based on our findings, we develop a tool-supported approach for causality detection (CiRA). This constitutes a first step towards causality extraction from NL requirements. We report on a case study and the resulting tool-supported approach for causality detection in requirements. Our case study corroborates, among other things, that causality is, in fact, a widely used linguistic pattern to describe system behavior, as about a third of the analyzed sentences are causal. We further demonstrate that our tool CiRA achieves a macro-F1 score of 82 \% on real word data and that it outperforms related approaches with an average gain of 11.06 \% in macro-Recall and 11.43 \% in macro-Precision. Finally, we disclose our open data sets as well as our tool to foster the discourse on the automatic detection of causality in the RE community.},
	booktitle = {Proceedings 27th {International} {Working} {Conference} on {Requirements} {Engineering}: {Foundation} for {Software} {Quality}},
	publisher = {Springer},
	author = {Fischbach, Jannik and Frattini, Julian and Spaans, Arjen and Kummeth, Maximilian and Vogelsang, Andreas and Mendez, Daniel and Unterkalmsteiner, Michael},
	month = apr,
	year = {2021},
	pages = {19--36},
	url = {https://www.lmsteiner.com/papers/fischbach_2021_refsq.pdf}
}


@article{huynh_khanh_assessing_2021,
	title = {Assessing test artifact quality – {A} tertiary study},
	abstr = {Context: Modern software development increasingly relies on software testing for an ever more frequent delivery of high quality software. This puts high demands on the quality of the central artifacts in software testing, test suites and test cases. 
Objective: We aim to develop a comprehensive model for capturing the dimensions of test case/suite quality, which are relevant for a variety of perspectives.
Method: We have carried out a systematic literature review to identify and analyze existing secondary studies on quality aspects of software testing artifacts.
Results: We identified 49 relevant secondary studies. Of these 49 studies, less than half did some form of quality appraisal of the included primary studies and only 3 took into account the quality of the primary study when synthesizing the results. We present an aggregation of the context dimensions and factors that can be used to characterize the environment in which the test case/suite quality is investigated. We also provide a comprehensive model of test case/suite quality with definitions for the quality attributes and measurements based on findings in the literature and ISO/IEC 25010:2011.
Conclusion: The test artifact quality model presented in the paper can be used to support test artifact quality assessment and improvement initiatives in practice. Furthermore, the model can also be used as a framework for documenting context characteristics to make research results more accessible for research and practice.},
	journal = {Information and Software Technology},
	author = {Huynh Khanh, Vi Tran and Unterkalmsteiner, Michael and Börstler, Jürgen and bin Ali, Nauman},
	year = {2021},
  volume = {139},
  pages = {106620},
	url = {https://www.lmsteiner.com/papers/huynh_2021_ist.pdf}
}


@inproceedings{kosenkov_vision_2021,
	address = {Bari, Italy},
	series = {{ESEM} '21},
	title = {Vision for an {Artefact}-{Based} {Approach} to {Regulatory} {Requirements} {Engineering}},
	abstr = {Background: Nowadays, regulatory requirements engineering (regulatory RE) faces challenges of interdisciplinary nature that cannot be tackled due to existing research gaps. Aims: We envision an approach to solve some of the challenges related to the nature and complexity of regulatory requirements, the necessity for domain knowledge, and the involvement of legal experts in regulatory RE. Method: We suggest the qualitative analysis of regulatory texts combined with the further case study to develop an empirical foundation for our research. Results: We outline our vision for the application of extended artefact-based modeling for regulatory RE. Conclusions: Empirical methodology is an essential instrument to address interdisciplinarity and complexity in regulatory RE. Artefact-based modeling supported by empirical results can solve a particular set of problems while not limiting the application of other methods and tools and facilitating the interaction between different fields of practice and research.},
	booktitle = {Proceedings 15th {International} {Symposium} on {Empirical} {Software} {Engineering} and {Measurement} ({ESEM})},
	publisher = {ACM},
	author = {Kosenkov, Oleksandr and Unterkalmsteiner, Michael and Mendez, Daniel and Fucci, Davide},
	year = {2021},
  pages = {1--6},
	keywords = {regulatory compliance, regulatory requirements engineering, software compliance},
  url = {https://www.lmsteiner.com/papers/kosenkov_2021_esem.pdf}
}


@inproceedings{fischbach_how_2021,
	address = {Turin, Italy},
	title = {How {Do} {Practitioners} {Interpret} {Conditionals} in {Requirements}?},
	abstr = {Context: Conditional statements like "If A and B then C" are core elements for describing software requirements. However, there are many ways to express such conditionals in natural language and also many ways how they can be interpreted. We hypothesize that conditional statements in requirements are a source of ambiguity, potentially affecting downstream activities such as test case generation negatively. Objective: Our goal is to understand how specific conditionals are interpreted by readers who work with requirements. Method: We conduct a descriptive survey with 104 RE practitioners and ask how they interpret 12 different conditional clauses. We map their interpretations to logical formulas written in Propositional (Temporal) Logic and discuss the implications. Results: The conditionals in our tested requirements were interpreted ambiguously. We found that practitioners disagree on whether an antecedent is only sufficient or also necessary for the consequent. Interestingly, the disagreement persists even when the system behavior is known to the practitioners. We also found that certain cue phrases are associated with specific interpretations. Conclusion: Conditionals in requirements are a source of ambiguity and there is not just one way to interpret them formally. This affects any analysis that builds upon formalized requirements (e.g., inconsistency checking, test-case generation). Our results may also influence guidelines for writing requirements.},
	booktitle = {Proceedings 22nd {International} {Conference} of {Product}-{Focused} {Software} {Process} {Improvement}},
	publisher = {Springer},
	author = {Fischbach, Jannik and Frattini, Julian and Mendez, Daniel and Unterkalmsteiner, Michael and Femmer, Henning and Vogelsang, Andreas},
	year = {2021},
	pages = {85--102},
  url = {https://www.lmsteiner.com/papers/fischbach_2021_profes.pdf}
}


@article{fagerholm_cognition_2022,
	title = {Cognition in {Software} {Engineering}: {A} {Taxonomy} and {Survey} of a {Half}-{Century} of {Research}},
	shorttitle = {Cognition in {Software} {Engineering}},
	abstr = {Cognition plays a fundamental role in most software engineering activities. This article provides a taxonomy of cognitive concepts and a survey of the literature since the beginning of the Software Engineering discipline. The taxonomy comprises the top-level concepts of perception, attention, memory, cognitive load, reasoning, cognitive biases, knowledge, social cognition, cognitive control, and errors, and procedures to assess them both qualitatively and quantitatively. The taxonomy provides a useful tool to filter existing studies, classify new studies, and support researchers in getting familiar with a (sub) area. In the literature survey, we systematically collected and analysed 311 scientific papers spanning five decades and classified them using the cognitive concepts from the taxonomy. Our analysis shows that the most developed areas of research correspond to the four life-cycle stages, software requirements, design, construction, and maintenance. Most research is quantitative and focuses on knowledge, cognitive load, memory, and reasoning. Overall, the state of the art appears fragmented when viewed from the perspective of cognition. There is a lack of use of cognitive concepts that would represent a coherent picture of the cognitive processes active in specific tasks. Accordingly, we discuss the research gap in each cognitive concept and provide recommendations for future research.},
	journal = {ACM Computing Surveys},
	author = {Fagerholm, Fabian and Felderer, Michael and Fucci, Davide and Unterkalmsteiner, Michael and Marculescu, Bogdan and Martini, Markus and Tengberg, Lars Göran Wallgren and Feldt, Robert and Lehtelä, Bettina and Nagyváradi, Balázs and Khattak, Jehan},
	month = jan,
	year = {2022},
  volume = {0},
  issue = {0},
  pages = {0},
	url = {https://www.lmsteiner.com/papers/fagerholm_2022_csur.pdf}
}


@article{frattini_causality_2021,
	title = {Causality in {Requirements} {Artifacts}: {Prevalence}, {Detection}, and {Impact}},
	volume = {0},
	shorttitle = {Causality in {Requirements} {Artifacts}},
	abstr = {Background: Causal relations in natural language (NL) requirements convey strong, semantic information. Automatically extracting such causal information enables multiple use cases, such as test case generation, but it also requires to reliably detect causal relations in the first place. Currently, this is still a cumbersome task as causality in NL requirements is still barely understood and, thus, barely detectable. Objective: In our empirically informed research, we aim at better understanding the notion of causality and supporting the automatic extraction of causal relations in NL requirements. Method: In a first case study, we investigate 14.983 sentences from 53 requirements documents to understand the extent and form in which causality occurs. Second, we present and evaluate a tool-supported approach, called CiRA, for causality detection. We conclude with a second case study where we demonstrate the applicability of our tool and investigate the impact of causality on NL requirements. Results: The first case study shows that causality constitutes around 28\% of all NL requirements sentences. We then demonstrate that our detection tool achieves a macro-F1 score of 82\% on real-world data and that it outperforms related approaches with an average gain of 11.06\% in macro-Recall and 11.43\% in macro-Precision. Finally, our second case study corroborates the positive correlations of causality with features of NL requirements. Conclusion: The results strengthen our confidence in the eligibility of causal relations for downstream reuse, while our tool and publicly available data constitute a first step in the ongoing endeavors of utilizing causality in RE and beyond.},
	number = {0},
	journal = {Requirements Engineering},
	author = {Frattini, Julian and Fischbach, Jannik and Mendez, Daniel and Unterkalmsteiner, Michael and Vogelsang, Andreas and Wnuk, Krzystof},
	month = dec,
	year = {2021},
	pages = {0},
  url = {https://www.lmsteiner.com/papers/frattini_2021_re.pdf}
}


@article{tran_how_2022,
	title = {How good are my search strings? {Reflections} on using an existing review as a quasi-gold standard},
	volume = {16},
	shorttitle = {How good are my search strings?},
	abstr = {Background: Systematic literature studies (SLS) have become a core research methodology in Evidence-based Software Engineering (EBSE). Search completeness, i.e., finding all relevant papers on the topic of interest, has been recognized as one of the most commonly discussed validity issues of SLSs.
Aim: This study aims at raising awareness on the issues related to search string construction and on search validation using a quasi-gold standard (QGS). Furthermore, we aim at providing guidelines for search string validation.
Method: We use a recently completed tertiary study as a case and complement our findings with the observations from other researchers studying and advancing EBSE.
Results: We found that the issue of assessing QGS quality has not seen much attention in the literature, and the validation of automated searches in SLSs could be improved. Hence, we propose to extend the current search validation approach by the additional analysis step of the automated search validation results and provide recommendations for the QGS construction.
Conclusion: In this paper, we report on new issues which could affect search completeness in SLSs. Furthermore, the proposed guideline and recommendations could help researchers implement a more reliable search strategy in their SLSs.},
	journal = {e-Informatica Software Engineering Journal},
	author = {Tran, Huynh Khanh Vi and Börstler, Jürgen and Ali, Nauman bin and Unterkalmsteiner, Michael},
	year = {2022},
  pages = {0},
	url = {https://www.lmsteiner.com/papers/huynh_2022_e-informatica.pdf}
}


@article{abdeen_approach_2022,
	title = {An {Approach} for {Performance} {Requirements} {Verification} and {Test} {Environments} {Generation}},
	abstr = {Model-based testing (MBT) is a method that supports the design and execution of test cases by models that specify the intended behaviors of a system under test. While systematic literature reviews on MBT in general exist, the state of the art on modeling and testing performance requirements has seen much less attention. Therefore, we conducted a systematic mapping study on model-based performance testing. Then, we studied natural language software requirements specifications in order to understand which and how performance requirements are typically specified. Since none of the identified MBT techniques supported a major benefit of modeling, namely identifying faults in requirements specifications, we developed the Performance Requirements verificatiOn and Test EnvironmentS generaTion approach (PRO-TEST). Finally, we evaluated PRO-TEST on 149 requirements specifications. We found and analyzed 57 primary studies from the systematic mapping study and extracted 50 performance requirements models. However, those models don’t achieve the goals of MBT, which are validating requirements, ensuring their testability, and generating the minimum required test cases. We analyzed 77 Software Requirements Specification (SRS) documents, extracted 149 performance requirements from those SRS, and illustrate that with PRO-TEST we can model performance requirements, find issues in those requirements and detect missing ones. We detected three not-quantifiable requirements, 43 not-quantified requirements, and 180 underspecified parameters in the 149 modeled performance requirements. Furthermore, we generated 96 test environments from those models. By modeling performance requirements with PRO-TEST, we can identify issues in the requirements related to their ambiguity, measurability, and completeness. Additionally, it allows to generate parameters for test environments.},
	journal = {Requirements Engineering},
	author = {Abdeen, Waleed and Chen, Xingru and Unterkalmsteiner, Michael},
	year = {2022},
  volume = {0},
  issue = {0},
  pages = {0},
	note = {In print},
	url = {https://www.lmsteiner.com/papers/abdeen_2022_rej.pdf}
}


@inproceedings{silva_multidimer_2022,
	address = {Pennsylvania, United States},
	title = {{MultiDimEr} : {A} {Multi}-{Dimensional} bug {analyzEr}},
	abstr = {Background: Bugs and bug management consumes a significant amount of time and effort from software development organizations. A reduction in bugs can significantly improve the capacity for new feature development. Aims: We categorize and visualize dimensions of bug reports to identify accruing technical debt. This evidence can serve practitioners and decision makers not only as an argumentative basis for steering improvement efforts, but also as a starting point for root cause analysis, reducing overall bug inflow. Method: We implemented a tool, MultiDimEr, that analyzes and visualizes bug reports. The tool was implemented and evaluated at Ericsson AB. Results: We present our preliminary findings using the MultiDimEr for bug analysis, where we successfully identified components generating most of the bugs and bug trends within certain components.
Conclusions: By analyzing the dimensions provided by MultiDimEr, we show that classifying and visualizing bug reports in different dimensions can stimulate discussions around bug hot spots as well as validating the accuracy of manually entered bug report attributes used in technical debt measurements such as fault slip through.},
	booktitle = {Proceedings {International} {Conference} on {Technical} {Debt}},
	author = {Silva, Lakmal and Unterkalmsteiner, Michael and Wnuk, Krzysztof},
	month = may,
	year = {2022},
	pages = {0},
	url = {https://www.lmsteiner.com/papers/silva_2022_techdebt.pdf}
}


@article{unterkalmsteiner2022,
  title = {A Compendium and Evaluation of Taxonomy Quality Attributes},
  author = {Unterkalmsteiner, Michael and Abdeen, Waleed},
  year = {2022},
  volume = {40},
  issue = {1},
  pages = {e13098},
  journal = {Expert Systems},
  abstr = {Introduction: Taxonomies capture knowledge about a particular domain in a succinct manner and establish a common understanding among peers. Researchers use taxonomies to convey information about a particular knowledge area or to support automation tasks, and practitioners use them to enable communication beyond organizational boundaries. Aims: Despite this important role of taxonomies in software engineering, their quality is seldom evaluated. Our aim is to identify and define taxonomy quality attributes that provide practical measurements, helping researchers and practitioners to compare taxonomies and choose the one most adequate for the task at hand. Methods: We reviewed 324 publications from software engineering and information systems research and synthesized, when provided, the definitions of quality attributes and measurements. We evaluated the usefulness of the measurements on six taxonomies from three domains. Results: We propose the definition of seven quality attributes and suggest internal and external measurements that can be used to assess a taxonomy’s quality. For two measurements we provide implementations in Python. We found the measurements useful for deciding which taxonomy is best suited for a particular purpose. Conclusion: While there exist several guidelines for creating taxonomies, there is a lack of actionable criteria to compare taxonomies. In this paper, we fill this gap by synthesizing from a wealth of literature seven, non‐overlapping taxonomy quality attributes and corresponding measurements. Future work encompasses their further evaluation of usefulness and empirical validation.},
  url = {https://www.lmsteiner.com/papers/unterkalmsteiner_2022_es.pdf}
    }

@inproceedings{frattini2022,
  title = {A {Live Extensible Ontology} of {Quality Factors} for {Textual Requirements}},
  booktitle = {{IEEE} 30th {International Requirements Engineering Conference} ({RE})},
  author = {Frattini, Julian and Montgomery, Lloyd and Fischbach, Jannik and Unterkalmsteiner, Michael and Mendez, Daniel and Fucci, Davide},
  year = {2022},
  pages = {274--280},
  publisher = {{IEEE}},
  address = {{Melbourne, Australia}},
  abstr = {Quality factors like passive voice or sentence length are commonly used in research and practice to evaluate the quality of natural language requirements since they indicate defects in requirements artifacts that potentially propagate to later stages in the development life cycle. However, as a research community, we still lack a holistic perspective on quality factors. This inhibits not only a comprehensive understanding of the existing body of knowledge but also the effective use and evolution of these factors. To this end, we propose an ontology of quality factors for textual requirements, which includes (1) a structure framing quality factors and related elements and (2) a central repository and web interface making these factors publicly accessible and usable. We contribute the first version of both by applying a rigorous ontology development method to 105 eligible primary studies and construct a first version of the repository and interface. We illustrate the usability of the ontology and invite fellow researchers to a joint community effort to complete and maintain this knowledge repository. We envision our ontology to reflect the community's harmonized perception of requirements quality factors, guide reporting of new quality factors, and provide central access to the current body of knowledge.},
  url = {https://www.lmsteiner.com/papers/frattini_2022_renext.pdf}
}

@article{badampudi2023,
  title = {Modern {{Code Reviews}} - {{A Survey}} of {{Literature}} and {{Practice}}},
  author = {Badampudi, Deepika and Unterkalmsteiner, Michael and Britto, Ricardo},
    year = {2023},
    pages = {0},
    volume = {0},
    issue = {0},
  journal = {Transactions on Software Engineering and Methodology},
  abstr = {Background:  Modern Code Review (MCR) is a lightweight alternative to traditional code inspections. While secondary studies on MCR exist; it is unknown whether the research community has targeted themes that practitioners consider important. Objectives: The objectives are to provide an overview of MCR research, analyze the practitioners' opinions on the importance of MCR research, investigate the alignment between research and practice, and propose future MCR research avenues. Method: We conducted a systematic mapping study to survey state-of-the-art until and including 2021, employed the Q-Methodology to analyze the practitioners' perception of the relevance of MCR research, and analyzed the primary studies' research impact. Results: We analyzed 244 primary studies, resulting in five themes. As a result of the 1300 survey data points, we found that the respondents are positive about research investigating the impact of MCR on product quality and MCR process properties. In contrast, they are negative about human factor- and support systems-related research. Conclusion: These results indicate a misalignment between the state-of-the-art and the themes deemed important by most survey respondents. Researchers should focus on solutions that can improve the state of MCR practice. We provide an MCR research agenda, which can potentially increase the impact of MCR research.},
  annotation = {In print},
  url = {https://www.lmsteiner.com/papers/badampudi_2023_tosem.pdf}
    }

  @article{fischbach2023,
  title = {Automatic Creation of Acceptance Tests by Extracting Conditionals from Requirements: {{NLP}} Approach and Case Study},
  shorttitle = {Automatic Creation of Acceptance Tests by Extracting Conditionals from Requirements},
  author = {Fischbach, Jannik and Frattini, Julian and Vogelsang, Andreas and Mendez, Daniel and Unterkalmsteiner, Michael and Wehrle, Andreas and Henao, Pablo Restrepo and Yousefi, Parisa and Juricic, Tedi and Radduenz, Jeannette and Wiecher, Carsten},
  year = {2023},
  month = mar,
  journal = {Journal of Systems and Software},
  volume = {197},
  pages = {111549},
  issn = {0164-1212},
  doi = {10.1016/j.jss.2022.111549},
  abstr = {Acceptance testing is crucial to determine whether a system fulfills end-user requirements. However, the creation of acceptance tests is a laborious task entailing two major challenges: (1) practitioners need to determine the right set of test cases that fully covers a requirement, and (2) they need to create test cases manually due to insufficient tool support. Existing approaches for automatically deriving test cases require semi-formal or even formal notations of requirements, though unrestricted natural language is prevalent in practice. In this paper, we present our tool-supported approach CiRA (Conditionals in Requirements Artifacts) capable of creating the minimal set of required test cases from conditional statements in informal requirements. We demonstrate the feasibility of CiRA in a case study with three industry partners. In our study, out of 578 manually created test cases, 71.8 \% can be generated automatically. Additionally, CiRA discovered 80 relevant test cases that were missed in manual test case design. CiRA is publicly available at www.cira.bth.se/demo/.},
  langid = {english},
  url = {https://www.lmsteiner.com/papers/fischbach_2023_jss.pdf}
}
  
@inproceedings{silva2023,
  title = {Towards Identifying and Minimizing Customer-Facing Documentation Debt},
  booktitle = {International {{Conference}} on {{Technical Debt}}},
  author = {Silva, Lakmal and Unterkalmsteiner, Michael and Wnuk, Krzysztof},
    year = {2023},
    pages = {0},
  publisher = {{IEEE}},
  address = {{Melbourne, Australia}},
  abstr = {Background: Software documentation often struggles to catch up with the pace of software evolution. The lack of correct, complete, and up-to-date documentation results in an increasing number of documentation defects which could introduce delays in integrating software systems. In our previous study on a bug analysis tool called MultiDimEr, we provided evidence that documentation-related defects contribute to a significant number of bug reports. Aims: First, we want to identify documentation defect types contributing to documentation defects and thereby identifying documentation debt. Secondly, we aim to find pragmatic solutions to minimize most common documentation defects to pay off the documentation debt in the long run. Method: We investigated documentation defects related to an industrial software system. First, we looked at the types of different documentation and associated bug reports. We categorized the defects according to an existing documentation defect taxonomy. Results: Based on a sample of 101 defects, we found that a majority of defects are caused by documentation defects falling into the Information Content (What) category (86). Within this category, the documentation defect types Erroneous code examples (23), Missing documentation (35), and Outdated content (19) contributed to most of the documentation defects. We propose to adapt two solutions to mitigate these types of documentation defects. Conclusions: In practice, documentation debt can easily go undetected since a large share of resources and focus is dedicated to deliver high-quality software. This study provides evidence that documentation debt can contribute to increase in maintenance costs due to the number of documentation defects. We suggest to adapt two main solutions to tackle documentation debt by implementing (i) Dynamic Documentation Generation (DDG) and/or (ii) Automated Documentation Testing (ADT), which are both based on defining a single and robust information source for documentation.},
  url = {https://www.lmsteiner.com/papers/silva_2023_techdebt.pdf}
}