Versions Compared

Old Version 1

changes.mady.by.user Unknown User (sbgs)

Saved on

compared with

New Version 2

changes.mady.by.user Unknown User (nmms)

Saved on

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

...

  • Manually, using the Import Execution Results action accessible from the Test Execution issue screen
  • via via the REST API to integrate with Continuous Integration (CI) platforms or other external execution processes. Please refer to the Xray REST API.

...

Code Block
themeConfluence
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite tests="151" failures="0" name="ut.com.xpandit.raven.service.impl.IssueDataSetTest" time="0.163114" errors="0" skipped="0">
  <properties>
    ...
  </properties>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidLimitOverflowOption_returnsExpectedSubset" time="0.114"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withNullOptionsAndValidIssue_throwsIllegalArgumentException" time="0.002"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidEmptyOptions_returnsAllIssues" time="0.002"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidGlobalSearchOptions_returnsExpectedTests" time="0.016"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndInvalidColumnSearchOption_returnsAllTests" time="0.007"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidLimitUnderOption_returnsExpectedSubset" time="0.001"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidGlobalSearchOptionThatMachesIssueKey_returnsExpectedTestWithMatchedKey" time="0.006"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidSummaryColumnAscSortOption_returnsExpectedIssuesInAscOrder" time="0.006"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidSummaryColumnDescSortOption_returnsExpectedIssuesInDescOrder" time="0.002"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidGlobalSearchOptionThatMatchesAllElements_returnsAllTests" time="0.001"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidColumnSearchOptionThatMatchesOneElement_returnsOneTest" time="0.002"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidColumnSearchOptionThatMatchesNoIssue_returnsEmptyList" time="0.001"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidGlobalSearchOptionThatMachesNoIssue_returnsEmptyList" time="0.001"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidKeyColumnDescSortOption_returnsExpectedIssuesInDescOrder" time="0.001"/>
  <testcase classname="ut.com.xpandit.raven.service.impl.IssueDataSetTest" name="testApplyOptions_withValidIssueAndValidKeyColumnAscSortOption_returnsExpectedIssuesInAscOrder" time="0.001"/>
</testsuite>

NUnit v3.0 XML output format

Xray supports an NUnit v3.0 XML format for importing execution results.

</testsuite>

NUnit v3.0 XML output format

Xray supports an NUnit v3.0 XML format for importing execution results.

Code Block
themeConfluence
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<test-run id="0" testcasecount="3" total="3" passed="3" failed="0" inconclusive="0" skipped="0" asserts="3" result="Passed" portable-engine-version="3.3.0.0" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.015218">
  <test-suite type="Assembly" id="1021" name="x, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null" fullname="x, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null" runstate="Runnable" testcasecount="14" result="Failed" site="Child" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.015218" total="3" passed="3" failed="0" inconclusive="0" skipped="0" asserts="3">
    <settings>
      ...
    </settings>
    <test-suite type="TestSuite" id="1022" name="x" fullname="x" runstate="Runnable" testcasecount="12" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.015218" total="3" passed="3" failed="0" inconclusive="0" skipped="0" asserts="3">
    <test-suite type="TestFixture" id="1004" name="CalculatorTests" fullname="x.CalculatorTests" classname="x.CalculatorTests" runstate="Runnable" testcasecount="12" result="Passed"
Code Block
themeConfluence
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<test-run id="0" testcasecount="14" total="14" passed="13" failed="1" inconclusive="0" skipped="0" asserts="14" result="Failed" portable-engine-version="3.3.0.0" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.140400014979" total="3" passed="3" failed="0" inconclusive="0" skipped="0" asserts="3">
      <test-suite type="AssemblyParameterizedMethod" id="10211008" name="CanAddNumbers"x, Versionfullname=1"x.0.0.0, Culture=neutral, PublicKeyToken=null" fullnameCalculatorTests.CanAddNumbers" classname="x, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null.CalculatorTests" runstate="Runnable" testcasecount="143" result="Failed" site="ChildPassed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.110549004228" total="143" passed="133" failed="10" inconclusive="0" skipped="0" asserts="143">
    <settings>
	<setting    <properties>
              <property name="WorkDirectoryRequirement" value="C:\Users\Sergio\xDEV-771" />
  </settings>
  <failure>
	<message><![CDATA[One or more child tests had errors]]></message>
  </failure>properties>
    <test-suitecase typeid="TestFixture1005" idname="1000CanAddNumbers(1,1,2)" namefullname="TestClassx.CalculatorTests.CanAddNumbers(1,1,2)" fullnamemethodname="TestClassCanAddNumbers" classname="TestClassx.CalculatorTests" runstate="Runnable" testcasecountseed="21846389584" result="FailedPassed" site="Child" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.084668" total="2" passed="1" failed="1" inconclusive="0" skipped="0" asserts="2">
	<failure>
	  <message><![CDATA[One or more child tests had errors]]></message>
	</failure>
	<test-suite type="ParameterizedMethod" id="1003" name="SubtractTest" fullname="TestClass.SubtractTest" classname="TestClass" runstate="Runnable" testcasecount="2" result="Failed" site="Child001194" asserts="1" />
    <test-case id="1006" name="CanAddNumbers(-1,-1,-2)" fullname="x.CalculatorTests.CanAddNumbers(-1,-1,-2)" methodname="CanAddNumbers" classname="x.CalculatorTests" runstate="Runnable" seed="1113780989" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.080887000067" totalasserts="2" passed="1" failed="1" inconclusive="0" skipped="0" asserts="2">
		<properties>
			<property name="Requirement" value="DEV-771" />
		  </properties><failure>
		<message><![CDATA[One or more child tests had errors]]></message>
	  </failure>
	  <test-case id="1001" name="SubtractTest(1)" fullname="TestClass.SubtractTest(11" />
    <test-case id="1007" name="CanAddNumbers(100,5,105)" fullname="x.CalculatorTests.CanAddNumbers(100,5,105)" methodname="SubtractTestCanAddNumbers" classname="TestClassx.CalculatorTests" runstate="Runnable" seed="11668331381585332966" result="FailedPassed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.043525000103" asserts="1" />
		<failure>
		  <message><![CDATA[  Expected: 10
But was:  1
]]></message>
		  <stack-trace><![CDATA[at TestClass.SubtractTest(Int32 x) in C:\Users\Sergio\x\TestClass.cs:line 13
]]></stack-trace>
		</failure>
	  </test-case>
	  <test-case id="1002" name="SubtractTest(10)" fullname="TestClass.SubtractTest(10)" methodname="SubtractTest" classname="TestClass" runstate="Runnable" seed="1003146807" result="Failed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="12.000098" asserts="1" />
	</test-suite>
  </test-suite>
  <test-suite type="TestSuite" id="1022" name="x" fullname="x" runstate="Runnable" testcasecount="12" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.015218" total="12" passed="12" failed="0" inconclusive="0" skipped="0" asserts="12">
	<test-suite type="TestFixture" id="1004" name="CalculatorTests" fullname="x.CalculatorTests" classname="x.CalculatorTests" runstate="Runnable" testcasecount="12" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.014979" total="12" passed="12" failed="0" inconclusive="0" skipped="0" asserts="12">
	  <test-suite type="ParameterizedMethod" id="1008" name="CanAddNumbers" fullname="x.CalculatorTests.CanAddNumbers" classname="x.CalculatorTests" runstate="Runnable" testcasecount="3" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.004228" total="3" passed="3" failed="0" inconclusive="0" skipped="0" asserts="3">
		  <properties>
</test-suite>
  </test-suite>
  </test-suite>
</test-suite>
</test-run>

NUnit v2.6 XML output format

Xray supports an NUnit v2.6 XML format for importing execution results.

Code Block
themeConfluence
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<test-results name="MobilityServices.ShoppingLists.Tests\bin\Debug\MobilityServices.ShoppingLists.Tests.dll" total="2" errors="0" failures="0" not-run="0" inconclusive="0" ignored="0" skipped="0" invalid="0" date="2017-01-18" time="15:46:53">
  <environment nunit-version="2.6.4.14350" clr-version="2.0.50727.8745" os-version="Microsoft Windows NT 6.2.9200.0" platform="Win32NT" cwd="D:\Work\Apps\NUnit-2.6.4\bin" machine-name="LAPTOP-UNKNOWN" user="unknown" user-domain="LAPTOP-UNKNOWN" />
  <culture-info current-culture="pt-PT" current-uiculture="en-GB" />
  <test-suite type="Assembly" name="MobilityServices.ShoppingLists.Tests\bin\Debug\MobilityServices.ShoppingLists.Tests.dll" executed="True" result="Success" success="True" time="4.473" asserts="0">
    <results>
      <test-suite type="Namespace" name="MobilityServices" executed="True" result="Success" success="True" time="4.473" asserts="0">
        <results>
          <test-suite type="Namespace" name="ShoppingLists" executed="True" result="Success" success="True" time="4.473" asserts="0">
            <results>
              <test-suite type="Namespace" name="Tests" executed="True" result="Success" success="True" time="4.473" asserts="0">
               <property name="Requirement" value="DEV-771" />
 <results>
                  </properties>
		<test-casesuite idtype="1005Namespace" name="CanAddNumbers(1,1,2)Controllers" fullnameexecuted="x.CalculatorTests.CanAddNumbers(1,1,2)True" methodnameresult="CanAddNumbersSuccess" classnamesuccess="x.CalculatorTestsTrue" runstatetime="Runnable4.473" seedasserts="18463895840">
 result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.001194" asserts="1" />
		<test-case id="1006              <results>
                      <test-suite type="TestFixture" name="CanAddNumbers(-1,-1,-2)ShoppingListsControllerTests" fullnameexecuted="x.CalculatorTests.CanAddNumbers(-1,-1,-2)True" methodnameresult="CanAddNumbersSuccess" classnamesuccess="x.CalculatorTestsTrue" runstatetime="Runnable4.473" seedasserts="11137809890">
 result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000067" asserts="1" />
		<test-case id="1007" name="CanAddNumbers(100,5,105)" fullname="x.CalculatorTests.CanAddNumbers(100,5,105)" methodname="CanAddNumbers" classname="x.CalculatorTests" runstate="Runnable" seed="1585332966" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000103                     <results>
                          <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.RemoveShoppingListItem" executed="True" result="Success" success="True" time="4.118" asserts="2" />
                          <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.TestAuthorization" executed="True" result="Success" success="True" time="0.355" asserts="1" />
	   </test-suite>
	  <test-suite type="ParameterizedMethod" id="1020" name="CanDivide" fullname="x.CalculatorTests.CanDivide" classname="x.CalculatorTests" runstate="Runnable" testcasecount="3" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.004041" total="3" passed="3" failed="0" inconclusive="0" skipped="0" asserts="3">
		  <properties>
                    <properties>
                              <property name="Requirement" value="DEV-771" />
                            </properties>
		<test-case id="1017" name="CanDivide(1,1,1)" fullname="x.CalculatorTests.CanDivide(1,1,1)" methodname="CanDivide" classname="x.CalculatorTests" runstate="Runnable" seed="1285501252" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000354" asserts="1" />
		<test-case id="1018" name="CanDivide(-1,-1,1)" fullname="x.CalculatorTests.CanDivide(-1,-1,1)" methodname="CanDivide" classname="x.CalculatorTests" runstate="Runnable" seed="1436436719" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000073" asserts="1" />
		<test-case id="1019" name="CanDivide(100,5,20)" fullname="x.CalculatorTests.CanDivide(100,5,20)" methodname="CanDivide" classname="x.CalculatorTests" runstate="Runnable" seed="213310888" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000060" asserts="1" />
	                          </test-case>
                        </results>
                      </test-suite>
	   <test-suite type="ParameterizedMethod" id="1016" name="CanMultiply" fullname="x.CalculatorTests.CanMultiply" classname="x.CalculatorTests" runstate="Runnable" testcasecount="3" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.002759" total="3" passed="3" failed="0" inconclusive="0" skipped="0" asserts="3">
		<test-case id="1013" name="CanMultiply(1,1,1)" fullname="x.CalculatorTests.CanMultiply(1,1,1)" methodname="CanMultiply" classname="x.CalculatorTests" runstate="Runnable" seed="1192735127" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000331" asserts="1">
				  <properties>
 </results>
                  </test-suite>
              <property name="label" value="multiplication" />
 </results>
            </properties>
		  </test-case>
		<test-case id="1014" name="CanMultiply(-1,-1,1)" fullname="x.CalculatorTests.CanMultiply(-1,-1,1)" methodname="CanMultiply" classname="x.CalculatorTests" runstate="Runnable" seed="39988064" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000059" asserts="1">
				  <properties>
     suite>
            </results>
          </test-suite>
       <property name="label" value="multiplication" /> </results>
      </test-suite>
    </properties>results>
		  </test-case>
		<test-case id="1015" name="CanMultiply(100,5,500)" fullname="x.CalculatorTests.CanMultiply(100,5,500)" methodname="CanMultiplyAgain" classname="x.CalculatorTests" runstate="Runnable" seed="1462346243" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000052" asserts="1">
				  <properties>
            <property name="requirement" value="DEV-34" />
          </properties>
		  </test-case>
	  </test-suite>
	  <test-suite type="ParameterizedMethod" id="1012" name="CanSubtract" fullname="x.CalculatorTests.CanSubtract" classname="x.CalculatorTests" runstate="Runnable" testcasecount="3" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.002827" total="3" passed="3" failed="0" inconclusive="0" skipped="0" asserts="3">
		  <properties>
            <property name="requirement" value="DEV-328" />
          </properties>
		<test-case id="1009" name="CanSubtract(1,1,0)" fullname="x.CalculatorTests.CanSubtract(1,1,0)" methodname="CanSubtract" classname="x.CalculatorTests" runstate="Runnable" seed="1019357734" result="Failed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000303" asserts="1">
		  <failure>
			<message><![CDATA[Error subtracting]]></message>
		  </failure>
	  </test-case>
		<test-case id="1010" name="CanSubtract(-1,-1,0)" fullname="x.CalculatorTests.CanSubtract(-1,-1,0)" methodname="CanSubtract" classname="x.CalculatorTests" runstate="Runnable" seed="1322022615" result="Failed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000056" asserts="1" >
			  <failure>
		<message><![CDATA[Error subtracting]]></message>
	  </failure>
	  </test-case>
		<test-case id="1011" name="CanSubtract(100,5,95)" fullname="x.CalculatorTests.CanSubtract(100,5,95)" methodname="CanSubtract" classname="x.CalculatorTests" runstate="Runnable" seed="4493553" result="Passed" start-time="2016-12-26 14:36:03Z" end-time="2016-12-26 14:36:03Z" duration="0.000053" asserts="1" />
	  </test-suite>
	</test-suite>
  </test-suite>
</test-suite>
</test-run>

NUnit v2.6 XML output format

Xray supports an NUnit v2.6 XML format for importing execution results.

suite>
</test-results>

Xray JSON format

Xray also provides a proprietary JSON format for importing execution results.

Although Xray supports multiple report formats used by different testing frameworks/runners (e.g. JUnit, NUnit, xUnit, TestNG, Cucumber, Robot Framework), there are scenarios where using these formats is not an option like:

  • Importing results for manual Tests;
  • Using a testing framework report that is not supported by Xray;
  • Having your own testing framework;
  • Limited support of existing formats to import detailed execution results back into Jira.


Info
titleImporting Execution Evidences
This format supports importing execution evidence (attachments) embedded in the JSON file encoded in Base 64.
Info
titleCreating new Test Execution issues

The info object allows you to specify specific execution information when creating new Test Execution issues.

Info
titleImporting results to Manual Tests

You can import manual Test execution results using the steps element for specifying the Test step results.

testExecutionKeyThe test execution key where to import the execution results
infoThe info object for creating new Test Execution issues (link)
testsThe Test Run result details (link)

"info" object - Test Execution issue

You can specify which Test Execution issue to import the results by setting the test execution key on the testExecutionKey property. Alternatively, you can create a new Test Execution issue for the execution results and specify the issue fields within the info object.

projectThe project key where the test execution will be created
summaryThe summary for the test execution issue
descriptionThe description for the test execution issue
versionThe version name for the Fix Version field of the test execution issue
revisionA revision for the revision custom field
userThe username for the Jira user who executed the tests
startDateThe start date for the test execution issue
finishDateThe finish date for the test execution issue
testPlanKeyThe test plan key for associating the test execution issue
testEnvironmentsThe test environments for the test execution issue

"tests" object - Test Run details

The test run details object allows you to import any detail about the execution itself. All Xray test types are supported. 

It is possible to import a single result (the test object itself with the "steps" (Manual tests) or "examples" (BDD tests)) or multiple execution results into the same Test Run (data-driven testing) using the "iterations" array.

testKeyThe test issue key
testInfoThe testInfo element (link)
startThe start date for the test run
finishThe finish date for the test run
commentThe comment for the test run
executedByThe user id who executed the test run
assigneeThe user id for the assignee of the test run
statusThe test run status (PASSED, FAILED, EXECUTING, TODO, custom statuses ...)
stepsThe step results (link)
examplesThe example results for BDD tests (link)
iterationsThe iteration containing data-driven test results (link)
defectsAn array of defect issue keys to associate with the test run
evidenceAn array of evidence items of the test run (link)
customFieldsAn array of custom fields for the test run (link)

"testInfo" object - Creating Test issues

It is possible to create new test issues when importing execution results using the Xray JSON format. For this, a testInfo element must be provided in order for Xray to create the issues.

If it is the first time you are importing an execution with a testInfo, Xray will create the tests automatically. Subsequent executions will reuse the same test issues.

Xray will first try to match test issues by the testKey if present. Otherwise, Manual or BDD tests are matched by summary whilst Generic tests are matched using the generic definition field, within the same project.

Any changes to the testInfo element will update the test issue specification in Jira.
If the match field (summary or definition) is changed, Xray will search for another issue and will create a new test case, or update an existing test case if no one is found. If you need to change the summary or the definition, you can do it manually (go to Jira and change the field), or you can include the testKey within the test element.

projectKeyThe project key where the test issue will be created
summaryThe summary for the test issue
descriptionThe description of the test issue
testTypeThe test type (e.g. Manual, Cucumber, Generic)
requirementKeysAn array of requirement issue keys to associate with the test
labelsThe test issue labels
stepsAn array of test steps (for Manual tests) (link)
definitionThe generic test definition
scenarioThe BDD scenario
scenarioTypeThe BDD scenario type (Scenario or Scenario Outline)

"steps" object - step definition

This object allows you to define the step fields for manual tests.

actionThe step action - native field
dataThe step data - native field
resultThe step expected result - native field

"iterations" object - Data-driven test results

If you need to import data-driven test results you need to use the iterations object. Xray will store all iterations within the same Test Run object. 

It is also possible to import iteration results with parameters. Currently, this is only supported for manual tests.

In this case, Xray will create a dataset automatically within the Test Run object.

nameThe iteration name
parametersAn array of parameters along with their values (link)
statusThe status for the iteration (PASSED, FAILED, EXECUTING, TODO, custom statuses ...)

"parameters" object - parameters within iteration results

nameThe parameter name
valueThe parameter value

"evidence" object - embedded attachments

dataThe attachment data encoded in base64
filenameThe file name for the attachment
contentTypeThe Content-Type representation header is used to indicate the original media type of the resource

"customFields" object - store test run custom fields

It is possible to import test run custom field values into the Test Run object. Xray will use the "id" to find the existing test run custom field in the project settings.

idThe test run custom field ID
valueThe test run custom field value

Xray JSON Schema

The JSON results file must comply with the following JSON Schema:

Code Block
themeConfluence
collapsetrue
{
  "$id": "XraySchema",
  "type": "object",
  "properties": {
    "testExecutionKey": {
      "type": "string"
    },
      "info": {
      "type": "object",
      "properties": {
        "project": {
          "type": "string"
        },
        "summary": {
          "type": "string"
        },
        "description": {
          "type": "string"
        },
        "version": {
          "type": "string"
        },
        "revision": {
          "type": "string"
        },
        "user": {
          "type": "string"
        },
        "startDate": {
Code Block
themeConfluence
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<!--This file represents the results of running a test suite-->
<test-results name="D:\Work\XpandIT\Repo\jm.customermobility\JMMobilityServices\JMMobilityServices.ShoppingLists.Tests\bin\Debug\JMMobilityServices.ShoppingLists.Tests.dll" total="17" errors="0" failures="0" not-run="0" inconclusive="0" ignored="0" skipped="0" invalid="0" date="2017-01-18" time="15:46:53">
  <environment nunit-version="2.6.4.14350" clr-version="2.0.50727.8745" os-version="Microsoft Windows NT 6.2.9200.0" platform="Win32NT" cwd="D:\Work\Apps\NUnit-2.6.4\bin" machine-name="LAPTOP-XPRT" user="rui.tome" user-domain="LAPTOP-XPRT" />
  <culture-info current-culture="pt-PT" current-uiculture="en-GB" />
  <test-suite type="Assembly" name="D:\Work\XpandIT\Repo\jm.customermobility\JMMobilityServices\JMMobilityServices.ShoppingLists.Tests\bin\Debug\JMMobilityServices.ShoppingLists.Tests.dll" executed="True" result="Success" success="True" time="59.109" asserts="0">
    <results>
      <test-suite type="Namespace" name="JMMobilityServices" executed="True" result="Success" success="True" time="59.104" asserts="0">
        <results>
          <test-suite type="Namespace" name="ShoppingLists" executed="True" result="Success" success="True" time="59.104" asserts="0">
            <results>
              <test-suite type="Namespace" name="Tests" executed="True" result="Success" success="True" time="59.103" asserts="0">
                <results>
                  <test-suite type="Namespace" name="Controllers" executed="True" result="Success" success="True" time="59.103" asserts="0">
                    <results>
                      <test-suite type="TestFixture" name="ShoppingListsControllerTests" executed="True" result="Success" success="True" time="59.102" asserts="0">
                        <results>
                          <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.AddClient" executed="True" result="Success" success="True" time="6.428" asserts="4" />
                          <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.AddShoppingListItem" executed="True" result="Success" success="True" time="2.837" asserts="1" />
                          <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.CreateShoppingList" executed="True" result="Success" success="True" time="4.765" asserts="2" />
                          <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.DeleteShoppingList" executed="True" result="Success" success="True" time="4.657" asserts="2" />
                  "type": "string",
       <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.GetPublicShoppingLists" executed="True" result="Successformat": success="True" time="2.829" asserts="4" />"date-time"
        },
        "finishDate": {
         <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.GetShoppingList" executed="True" result="Success" success="True" time="1.734" asserts="1" />"type": "string",
          "format": "date-time"
        },
        "testPlanKey": {
         <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.GetShoppingLists" executed="True" result="Success" success="True" time="1.522" asserts="2" />"type": "string"
        },
        "testEnvironments": {
         <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.GetSuggestedProducts" executed="True" result="Success" success="True" time="0.398" asserts="0" />
    "type": "array",
          "items": {
            "type": "string"
          }
        }
      },
      "additionalProperties": false
    },
      "tests": {
     <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.RemoveAllShoppingListItems" executed="True" result="Success" success="True" time="4.373" asserts="4" />"type": "array",
      "items": {
          "$ref": "#/definitions/Test"
      },
      "minItems": 1,
  <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.RemoveBoughtShoppingListItems" executed="True" result="Success" success="True" time="4.575" asserts="5" />
           }
  },
  "additionalProperties": false,
 
  "definitions": {
 
    "Test": {
      "type": "object",
      "properties": {
  <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.RemoveClient" executed="True" result="Success" success="True" time="3.476" asserts="2testKey": />{
          "type": "string"
        },
       <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.RemoveShoppingListItem" executed="True" result="Success" success="True" time="4.118" asserts="2" />"testInfo": {
          "$ref": "#/definitions/TestInfo"
        },
        "start": {
         <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.TestAuthorization" executed="True" result="Success" success="True" time="0.355" asserts="1">"type": "string",
          "format": "date-time"
        },
        "finish": {
          "type": "string",
        <properties>
  "format": "date-time"
        },
          "comment": {
         <property name="Requirementtype": value="XPTO-15" />string"
        },
        "executedBy": {
           </properties>"type": "string"
        },
        "assignee": {
         </test-case>
   "type": "string"
        },
        "status": {
       <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.UnmarkBoughtShoppingListItems" executed="True" result="Success" success="True" time="4.145type": asserts="5string" />
        },
        "steps": {
         <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.UpdateShoppingListItemBoughtStatus" executed="True" result="Success" success="True" time="3.972" asserts="3" />
  "type": "array",
          "items": {
            "$ref": "#/definitions/ManualTestStepResult"
          }
        },
        "examples": {
          "type": "array",
     <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.UpdateShoppingListItemQuantity" executed="True" result="Success" success="True" time="3.560" asserts="3items": />{
               "type": "string",
           <test-case name="JMMobilityServices.ShoppingLists.Tests.Controllers.ShoppingListsControllerTests.UpdateShoppingListName" executed="True" result="Success" success="True" time="2.927" asserts="2" />"enum": ["TODO", "FAILED", "PASSED", "EXECUTING"]
            }
        },
    </results>
    "results": { // DEPRECATED
          "type": "array",
    </test-suite>
      "items": {
             </results>"$ref": "#/definitions/IterationResult"
          }
        </test-suite>},
        "iterations": {
       </results>
   "type": "array",
          </test-suite>"items": {
            </results>"$ref": "#/definitions/IterationResult"
          </test-suite>}
        </results>
      </test-suite>
},
     </results>
  </test-suite>
</test-results>

Xray JSON format

Xray also supports a custom and generic format for importing execution results. This allows you to import any execution result as long as it is in JSON format. This way, you can import any execution result to Jira, either from automated or manual Tests.

Info
titleImporting Execution Evidences
This format supports importing execution evidences (attachments) embedded in the JSON file encoded in Base 64.
Info
titleCreating new Test Execution issues

The info object allows you to specify specific execution information when creating new Test Execution issues.

Info
titleImporting results to Manual Tests

You can import manual Test execution results using the steps element for specifying the Test step results.

The JSON results file must follow this schema:

Code Block
themeConfluence
{
    "type": "object",
    "$schema": "http://json-schema.org/draft-03/schema",
    "required": false,
    "properties": {
   "defects": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "evidence": {
          "type": "array",
          "testExecutionKeyitems": {
            "type$ref": "string#/definitions/EvidenceItem",
 
          }
        },
        "requiredevidences": false{ // DEPRECATED
        }  "type": "array",
          "infoitems": {
            "type$ref": "object#/definitions/EvidenceItem",
          }
       "required": false},
            "propertiescustomFields": {
                "project$ref": {"#/definitions/CustomField"
        }
      },
      "typerequired": ["stringstatus"],
        "dependencies": {
            "requiredevidence": false{
          "not": {    "required": ["evidences"] },
        },
        "summaryevidences": {
          "not": { "required": ["evidence"] }
      "type": "string"  },
          "steps": {
          "requiredallOf": true[
                },{
              "not": { "descriptionrequired": { ["examples"] }
            },
        "type": "string",
   {
              "not":  { "required": false
     ["results"] }
            },
                "user": {
              "not":  {    "typerequired": ["stringiterations",] }
            }
         "required": false]
        },
        },"examples": {
          "allOf": [
     "version": {
      {
              "typenot": { "stringrequired",
        : ["steps"] }
            "required": false},
                },{
              "not": { "revisionrequired": { ["results"] }
            },
        "type": "string",
   {
              "not":  { "required": false["iterations"] }
            }
          },]
        },
        "startDateresults": {
          "allOf": [
         "type": "string",
  {
              "not":  {  "required": false
["steps"] }
            },
        },
    {
              "finishDatenot": {
 "required": ["examples"] }
            },
     "type": "string",
      {
              "requirednot": false
    { "required": ["iterations"] }
            }
            }]
        },
        "testsiterations": {
            "typeallOf": "array",[
            "required": true,
{
              "itemsnot": {
                "type"required": ["objectsteps",] }
                "required": false},
                "properties": {
                    "commentnot": {
 "required": ["examples"] }
            },
         "type": "string",
  {
                     "not": { "required": false ["results"] }
            }
          },]
        }
      },
      "finishadditionalProperties": {false
    },
 
         "IterationResult": {
          "type": "stringobject",
      "properties": {
        "name": {
          "requiredtype": false"string"
        },
        "parameters": {
   },
          "type": "array",
          "startitems": {
            "type": "object",
            "typeproperties": "string",{
              "name": {
         "required": false
       "type": "string"
              },
              "value": {
     "status": {
          "type": "string"
             "type": "string", }
            },
            "required": true["name"],
            "additionalProperties": false
       },
   }
         },
        "testKeylog": {
          "type": "string"
        },
        "typeduration": "string",
{
          "type": "string"
        },
        "requiredstatus": true{
          "type": "string"
         },
					"executedBy": {
             "steps": {
           "type": "stringarray",
          "items": {
             "required$ref": false"#/definitions/ManualTestStepResult"
            }
        },
      },
      "required": ["status"],
       "evidencesadditionalProperties": {false
    },
 
             "ManualTestStepResult": {
      "type": "arrayobject",
      "properties": {
          "status": {
          "requiredtype": false,"string"
        },
        "comment": {
          "itemstype": {"string"
        },
          "evidence": {
          "type": "objectarray",
          "items": {
                 "required$ref": false,"#/definitions/EvidenceItem"
          }
          },
        "propertiesdefects": {
          "type": "array",
          "items": {
            "datatype": {"string"
          }
        },
        "actualResult": {
          "type": "string",
               
        }
       },
      "required": true["status"],
      "additionalProperties": false
    },
 
    "TestInfo": {
      "type": "object",
        },"properties": {
        "summary": {
          "type": "string"
        },
        "filenameprojectKey": {
          "type": "string"
        },
        "requirementKeys":  {
          "type": "stringarray",
                        "items": {
            "requiredtype": true"string"
          }
          },
        "labels": {
   },
       "type": "array",
            "items": {
            "contentTypetype": {"string"
          }
        },
        "type": {
          "type": "string",
        },
        steps: {
          type: "array",
        "required"  items: false{
            type: "object",
            properties: {
      }
        action: {
                type: "string"
  }
            },
            }
  data: {
                 },type: "string"
              },
      "results": {
       result: {
                "type": "arraystring",
              }
           "required": false},
            // custom fields
            "items"patternProperties: {
              ".+": {}
             "type": "object"},
            required: ["action"],
               "required"additionalProperties: false,
          }
          },
        "propertiesscenario": {
          "type": "string"
        },
        "definition": {
          "nametype": {"string"
        }
      },
      "dependencies": {
        "steps": {
          "typeallOf": "string",[
            {
              "not": { "required": ["scenario"] }
         "required": true
  },
            {
              "not": { "required": ["definition"] },
            }
          ]
        },
  "duration      "scenario": {
          "allOf": [
            {
              "typenot": { "longrequired",
  : ["steps"] }
            },
                      "required": false{
              "not": {     "required": ["definition"] }
            },
          ]
              },
        "logdefinition": {
          "allOf": [
            {
              "not": { "typerequired": ["stringsteps",] }
            },
            {
              "not": { "required": false["scenario"] }
            }
            ]
        },
      },
      "required": ["summary", "projectKey", "type"],
      "additionalProperties": false
    },
  
    "statusEvidenceItem": {
      "type": "object",
         "properties": {
          "data": {
          "type": "string",
        },
        "filename": {
          "type": "string"
        "required": true},
        "contentType": {
          "type": "string"
            },
      },
      "required": ["data", "filename"],
      "additionalProperties": false
    },
   
    "examplesCustomField": {
      "type": "array",
      "items": {
              "type": "object",
        "typeproperties": "array",
   {
          "id": {
            "type": "string"
         "required": false},
          "value": {}
        },
        "required": ["id", "value"],
        "itemsadditionalProperties": {false
      }
    }
  }
}


Example 1: Importing gherkin and other test results

In this example, we are importing execution results for three existing test issues in Jira. The last issue DEMO-9 must be a BDD Test with a Gherkin definition because the results contain examples. The remaining issues can be of any test type.

Code Block
themeConfluence
{
    "tests" : [
           {
             "typetestKey" : "stringDEMO-7",
            "start"                        }: "2013-05-03T11:47:35+01:00",
            "finish"                    }
                            }: "2013-05-03T11:50:56+01:00",
            "comment" : "Test was OK but the performance is very poor",
  }
          "status" : "PASS"
        },
        {
            "examplestestKey" : {"DEMO-8",
            "start" : "2013-05-03T12:14:12+01:00",
            "typefinish" : "array2013-05-03T12:15:23+01:00",
                        "required": false,
            "comment" : "Performance was better this time, in the context of test set DEMO-10.",
            "itemsstatus" : {"PASS"
        },
        {
            "typetestKey" : "stringDEMO-9",
            "start" : "2013-05-03T12:19:23+01:00",
          }
  "finish" : "2013-05-03T12:20:01+01:00",
            "comment" : "Error  }decreasing space shuttle speed.",
            "status" : "FAIL",
			"examples"      "steps": {
                        "type": "array",
: [
				"PASS",
				"PASS",
				"PASS",
				"PASS",
				"PASS",
				"FAIL"
			]
        }
            ]
}

Example 2: Generic Test

This is a simple example of a JSON file with execution results for a generic test.

Code Block
themeConfluence
{
    "requiredtests" : false,
    [
        {
            "itemstestKey" : {"ABC-129",
            "start"    : "2014-08-30T11:47:35+01:00",
            "typefinish" : "object2014-08-30T11:50:56+01:00",
            "comment" : "Successful execution",
             "requiredstatus" : false,"PASS"
        }
    ]
}

Example 3: Importing manual test results with steps

This is a simple example of a JSON file with execution results for a manual test.

Code Block
themeConfluence
{
                "propertiestests" : {[
        {
            "testKey"            "status": {: "DEMO-57",
            "start"    : "2014-08-30T12:19:23+01:00",
                    "type""finish" : "string2014-08-30T12:20:01+01:00",
            "comment" : "Error executing step 2!",
                   "required": true"status" : "FAILED",
                          "steps": [
      },
            {
                    "commentstatus": {"PASSED",
                    "actualResult": "Step 1: OK"
             "type": "string"   },
                {
                    "requiredstatus": false"FAILED",
                    "actualResult": "Step 2 *Failed* with an unexpected error message",
     },
               "evidences" : [
               "evidences": {
        {
                            "typedata": "array",
     (... base 64 encoded ...)",
                               "requiredfilename": false,
        "screenshot1.jpg",
                            "itemscontentType": {
"image/jpeg"
                        }
                  "type": "object",  ]
                }
            ]
        }
    ]
}

Example 4: Importing data-driven manual test results with auto-provisioning of tests

This is an example of a JSON file with a single test result.

This is a data-driven manual test with two iterations. For each iteration, we provide the parameters and the step results.

Xray will also create or update the test in Jira with the specification contained on the "testInfo" object.

Code Block
themeConfluence
{
"required": false,
                                        "propertiestests": {[
        {
            "start" : "2021-08-30T11:47:35+01:00",
                      "data": {"finish" : "2021-08-30T11:50:56+01:00",
                                    "comment" : "Successful execution",
            "typestatus" : "stringPASSED",
            "evidence" : [
                {
                    "requireddata": true
  "iVBORw0KGgoAAAANSUhEUgAABkIAAAO9CAYAAADezXv6AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAEn(...base64 file enconding)",
                    "filename": "image21.jpg",
                     },"contentType": "image/jpeg"
                }
                ],
            "filenametestInfo": {
                "summary": "Strong password validation",
                "type": "Manual",
                "typeprojectKey": "stringSTORE",
                "steps": [
                    {
                 "required": true
      "action": "Open the Change Password screen by selecting option \"My Profile > Password\"",
                          }"data": "",
                        "result": ""
                   "contentType": { },
                      {
                          "typeaction": "string",
Fill the password fields with data",
                        "data": "Current Password: ${Password}\nNew Password: ${Password}\nConfirm New Password: ${Password}",
            "required": false
           "result": "The new password is: ${Valid}\nError:\n${Message}"
                    }
        }
        ]
            },
            "iterations": [
       }
       {
                "parameters": [
            }
        {
                        },
								"actualResult"name": {
									"type": "string",
									"required": false
								}
"Password",
                             }
                        }"value": "2635ftvu23v7t!09"
                    },
                }
    {
        }
        }
    }
}

Example 1: Automated Tests

This is an example of a JSON file with execution results that will update Automated Tests in Jira. The last item DEMO-9 must be a Cucumber Test because the results contain examples.

Code Block
themeConfluence
{
    "testsname" : ["Valid",
        {
                "testKeyvalue" : "DEMO-7Valid",
            "start" : "2013-05-03T11:47:35+01:00",
        },
                  "finish" : "2013-05-03T11:50:56+01:00",
 {
              "comment"  : "Test was OK but the performance is very poor"name": "Message",
              "status"  : "PASS"
        },"value": ""
        {
            "testKey"}
 : "DEMO-8",
            "start" : "2013-05-03T12:14:12+01:00" ],
                "finishstatus" : "2013-05-03T12:15:23+01:00PASSED",
                "comment" : "Performance was better this time, in the context of test set DEMO-10.",
steps": [
                    {
              "status" : "PASS"
        }"actualResult": "",
          {
              "testKeystatus" : "DEMO-9PASSED",
                  "start" : "2013-05-03T12:19:23+01:00" },
            "finish" : "2013-05-03T12:20:01+01:00",
      {
      "comment" : "Error decreasing space shuttle speed.",
            "statusactualResult" : "FAILPassword changed successfully",
			"examples" : [
				"PASS",
				"PASS",
				"PASS",
				"PASS",
				"PASS",
				"FAIL"
			]
             }
    ]
}

Example 2: Manual Tests

This is an example of a JSON file with execution results that will update a Manual Test in Jira.

Code Block
themeConfluence
{
       "testsstatus" : ["PASSED"
          {          }
            "testKey"   : "DEMO-57", ]
            "start" : "2014-08-30T12:19:23+01:00" },
            "finish" : "2014-08-30T12:20:01+01:00",
 {
                "commentparameters": :[
  "Error executing step 2!",
            "status" : "FAIL",
			"steps": [
				{
                        "statusname": "PASSPassword",
                        "commentvalue": "123123"Step
   1: OK"
                },
                    {
                        "statusname": "FAILValid",
                        "commentvalue": "Not "Step 2 *Failed* with an unexpected error message",
Valid"
                    },
                    {
   "evidences" : [                     "name": "Message",
                        {
       "value": "Password is too simple."
                    }
 "data": "(... base 64 encoded ...)",
            ],
                "filenamestatus": "screenshot1.jpgFAILED",
                "steps": [
                   "contentType": "image/jpeg" {
                        }
"actualResult": "",
                     ]
   "status": "PASSED"
            }
			]
        },
    ]
}

Example 3: Generic Test

This is an example of a JSON file with execution results that will update a Generic Test in Jira.

Code Block
themeConfluence
{
      "tests" : [
        {
                  "testKey"       "actualResult": "ABC-129Password too simple!",
                        "startstatus" : "2014-08-30T11:47:35+01:00",
FAILED"
                  "finish" : "2014-08-30T11:50:56+01:00", }
            "comment"  : "Successful execution",
  ]
              }
          "status" : "PASS"]
        }
    ]
}

Importing Multiple Execution results

...