mirror
/
gr-lora
kopia lustrzana https://github.com/rpp0/gr-lora
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Auto export test results as Markdown to docs/test-results

pull/61/head
Pieter Robyns 2017-09-06 16:49:31 +02:00
rodzic 2d0e9245b0
commit f20c7db6d6
5 zmienionych plików z 284 dodań i 9 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

3
.gitignore vendored
Wyświetl plik

@ -5,5 +5,6 @@ build/
*nogit*
*.cfile
dev/
examples/lora-samples
apps/test-suites
*.sigmf-data
*.sigmf-meta

4
apps/generate_test_suites.py 100644 → 100755
Wyświetl plik

@ -29,7 +29,7 @@ class TestSuite():
self.name += "_" + self.hw
self.path = os.path.join(args.data_out, self.name)
self.pre_delay = 0.150
self.post_delay = 0.150
self.post_delay = 1.0
self.intra_delay = 0.1
self.lc = RN2483Controller("/dev/lora")
self.test_count = 0
@ -196,4 +196,4 @@ if __name__ == '__main__':
Test(payload="88", times=1),
Test(payload="ffff", times=10),
]
TestSuite(name='short', args=args, config_set=short_config_set, test_set=short_test_set).run()
#TestSuite(name='short', args=args, config_set=short_config_set, test_set=short_test_set).run()

44
docs/test-results/decode_long_usrp.md 100644
Wyświetl plik

@ -0,0 +1,44 @@
# Test suite: 'decode_long_usrp'
*Results on 2017-09-06 14:44:23.827805*
### 868.1 MHz, SF 7, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`00010203040506..f8f9fafbfcfdfe` | 1 | 1 | 100.00%
### 868.1 MHz, SF 8, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`00010203040506..f8f9fafbfcfdfe` | 1 | 1 | 100.00%
### 868.1 MHz, SF 9, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`00010203040506..f8f9fafbfcfdfe` | 1 | 1 | 100.00%
### 868.1 MHz, SF 10, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`00010203040506..f8f9fafbfcfdfe` | 1 | 1 | 100.00%
### 868.1 MHz, SF 11, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`00010203040506..f8f9fafbfcfdfe` | 1 | 1 | 100.00%
### 868.1 MHz, SF 12, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`00010203040506..f8f9fafbfcfdfe` | 1 | 1 | 100.00%
### Summary for suite 'decode_long_usrp'
Total payloads passed: 6 out of 6 (100.00%)

200
docs/test-results/short_usrp.md 100644
Wyświetl plik

@ -0,0 +1,200 @@
# Test suite: 'short_usrp'
*Results on 2017-09-06 14:45:03.172136*
### 868.1 MHz, SF 7, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 0 | 5 | 0.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 7, CR 4/7, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 7, CR 4/6, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 7, CR 4/5, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 8, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 8, CR 4/7, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 8, CR 4/6, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 0 | 5 | 0.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 8, CR 4/5, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 0 | 5 | 0.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 9, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 9, CR 4/7, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 9, CR 4/6, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 9, CR 4/5, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 10, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 10, CR 4/7, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 10, CR 4/6, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 10, CR 4/5, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 11, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 11, CR 4/7, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 11, CR 4/6, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 11, CR 4/5, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 5 | 5 | 100.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 12, CR 4/8, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 2 | 5 | 40.00%
`88 ` | 0 | 1 | 0.00%
`ffff ` | 9 | 10 | 90.00%
### 868.1 MHz, SF 12, CR 4/7, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 2 | 5 | 40.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 12, CR 4/6, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 4 | 5 | 80.00%
`88 ` | 1 | 1 | 100.00%
`ffff ` | 10 | 10 | 100.00%
### 868.1 MHz, SF 12, CR 4/5, BW 125 kHz, prlen 8, crc on, implicit off
Transmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:
--- | --- | --- | ---
`deadbeef ` | 4 | 5 | 80.00%
`88 ` | 0 | 1 | 0.00%
`ffff ` | 8 | 10 | 80.00%
### Summary for suite 'short_usrp'
Total payloads passed: 356 out of 384 (92.71%)

42
python/qa_testsuite.py
Wyświetl plik

@ -39,7 +39,7 @@ class TestSummary():
self.suite = suite
self._summary = []
self._summary_text = "-------- Test suite '{:s}' results on {:s} ---------\n".format(suite, str(datetime.datetime.utcnow()))
self._summary_markdown = ""
self._summary_markdown = "# Test suite: '{:s}'\n\n*Results on {:s}*\n".format(suite, str(datetime.datetime.utcnow()))
self._num_total_correct_payloads = 0
self._num_total_payloads = 0
self._num_tests = 0
@ -52,7 +52,7 @@ class TestSummary():
else:
raise Exception("Test result must be of type TestResult")
def export_summary(self, print_output=True):
def export_summary(self, path, print_output=True):
self._summary_text += "\nRan a total of {:n} tests, together containing {:n} payloads.\n".format(
self._num_tests,
self._num_total_payloads
@ -63,9 +63,21 @@ class TestSummary():
float(self._num_total_correct_payloads) / self._num_total_payloads
)
self._summary_markdown += "\n### Summary for suite '{:s}'\n\n".format(self.suite)
self._summary_markdown += "Total payloads passed: {:n} out of {:n} ({:.2%})\n\n".format(
self._num_total_correct_payloads,
self._num_total_payloads,
float(self._num_total_correct_payloads) / self._num_total_payloads
)
if print_output:
print(self._summary_text)
if not os.path.exists(path):
os.makedirs(path)
with open(os.path.join(path, self.suite + '.md'), 'w') as f:
f.write(self._summary_markdown)
def _evaluate_result(self, test_result, print_intermediate):
"""
Given a test result, evaluate it and generate text / markdown for the report.
@ -82,7 +94,8 @@ class TestSummary():
# Don't reprint configuration if it is the same as before
if(self._last_config != vars(lora_config)):
evaluation_text += "Configuration {:s}:\n".format(lora_config.string_repr())
evaluation_text += "{:s}:\n".format(lora_config.string_repr())
evaluation_markdown += "\n### {:s}\n\nTransmitted payload | :heavy_check_mark: | :hash: | :heavy_division_sign:\n--- | --- | --- | ---\n".format(lora_config.string_repr())
self._last_config = vars(lora_config)
# Determine number of correct payloads
@ -106,9 +119,25 @@ class TestSummary():
self._num_total_correct_payloads += 1
# Append to text report
evaluation_text += "\tTest {:>3n}: {:<30s} * {:<3n} :: passed {:>3n} out of {:<3n} ({:.2%})\n".format(self._num_tests, trunc(test.payload), test.times, num_correct_payloads, num_payloads, float(num_correct_payloads)/num_payloads)
evaluation_text += "\tTest {:>3n}: {:<30s} * {:<3n} :: passed {:>3n} out of {:<3n} ({:.2%})\n".format(
self._num_tests,
trunc(test.payload),
test.times,
num_correct_payloads,
num_payloads,
float(num_correct_payloads)/num_payloads
)
self._summary_text += evaluation_text
# Append to markdown report
evaluation_markdown += "`{:<30s}` | {:>3n} | {:>3n} | {:>.2%}\n".format(
trunc(test.payload),
num_correct_payloads,
num_payloads,
float(num_correct_payloads)/num_payloads
)
self._summary_markdown += evaluation_markdown
if(print_intermediate):
print(evaluation_text)
@ -127,11 +156,12 @@ class qa_testsuite():
self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server.bind((self.host, self.port))
self.server.settimeout(10)
self.server.settimeout(3)
# Determine test suites directory
current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
self.test_suites_directory = os.path.abspath(current_dir + '../apps/test-suites')
self.reports_directory = os.path.abspath(current_dir + '../docs/test-results')
# List test suites
self.test_suites = []
@ -225,7 +255,7 @@ class qa_testsuite():
decoded_data = self.get_payloads(times) # Output from the flowgraph
summary.add(TestResult(decoded_data=decoded_data, lora_config=lora_config, test=test), print_intermediate=True)
# Finally, export the result for the suite
summary.export_summary()
summary.export_summary(path=self.reports_directory)
if __name__ == '__main__':
"""