kopia lustrzana https://github.com/espressif/esp-idf
tools: Introduce support for blank lines in config and value files for mfg utility
Some users have requested this feature. In order to avoid complete refactoring or introducing more code complexity, a design choice to create temporary files without blank lines is made. Additionally, an extension check is added and there are multiple smaller code style and structure improvements. Closes https://github.com/espressif/esp-idf/issues/8421pull/9408/head
Djordje Nedic
BOT
rodzic
9ec829f352
commit
0331d0aa63
|
|
@ -2997,6 +2997,87 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
|
|||
|
||||
}
|
||||
|
||||
TEST_CASE("check and read data from partition generated via manufacturing utility with blank lines in csv files and multipage blob support disabled", "[mfg_gen]")
|
||||
{
|
||||
int childpid = fork();
|
||||
int status;
|
||||
|
||||
if (childpid == 0) {
|
||||
exit(execlp("bash", "bash",
|
||||
"-c",
|
||||
"rm -rf ../../../tools/mass_mfg/host_test && \
|
||||
cp -rf ../../../tools/mass_mfg/testdata mfg_testdata && \
|
||||
cp -rf ../nvs_partition_generator/testdata . && \
|
||||
mkdir -p ../../../tools/mass_mfg/host_test", NULL));
|
||||
} else {
|
||||
CHECK(childpid > 0);
|
||||
waitpid(childpid, &status, 0);
|
||||
CHECK(WEXITSTATUS(status) == 0);
|
||||
|
||||
childpid = fork();
|
||||
if (childpid == 0) {
|
||||
exit(execlp("python", "python",
|
||||
"../../../tools/mass_mfg/mfg_gen.py",
|
||||
"generate",
|
||||
"../../../tools/mass_mfg/samples/sample_config_blank_lines.csv",
|
||||
"../../../tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv",
|
||||
"Test",
|
||||
"0x3000",
|
||||
"--outdir",
|
||||
"../../../tools/mass_mfg/host_test",
|
||||
"--version",
|
||||
"1",NULL));
|
||||
|
||||
} else {
|
||||
CHECK(childpid > 0);
|
||||
waitpid(childpid, &status, 0);
|
||||
CHECK(WEXITSTATUS(status) == 0);
|
||||
|
||||
childpid = fork();
|
||||
if (childpid == 0) {
|
||||
exit(execlp("python", "python",
|
||||
"../nvs_partition_generator/nvs_partition_gen.py",
|
||||
"generate",
|
||||
"../../../tools/mass_mfg/host_test/csv/Test-1.csv",
|
||||
"../nvs_partition_generator/Test-1-partition.bin",
|
||||
"0x3000",
|
||||
"--version",
|
||||
"1",NULL));
|
||||
|
||||
} else {
|
||||
CHECK(childpid > 0);
|
||||
waitpid(childpid, &status, 0);
|
||||
CHECK(WEXITSTATUS(status) == 0);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
|
||||
check_nvs_part_gen_args_mfg(&emu1, "test", 3, "mfg_testdata/sample_singlepage_blob.bin", false, NULL);
|
||||
|
||||
SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin");
|
||||
check_nvs_part_gen_args_mfg(&emu2, "test", 3, "testdata/sample_singlepage_blob.bin", false, NULL);
|
||||
|
||||
|
||||
childpid = fork();
|
||||
if (childpid == 0) {
|
||||
exit(execlp("bash", " bash",
|
||||
"-c",
|
||||
"rm -rf ../../../tools/mass_mfg/host_test | \
|
||||
rm -rf mfg_testdata | \
|
||||
rm -rf testdata",NULL));
|
||||
} else {
|
||||
CHECK(childpid > 0);
|
||||
waitpid(childpid, &status, 0);
|
||||
CHECK(WEXITSTATUS(status) == 0);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support enabled", "[mfg_gen]")
|
||||
{
|
||||
int childpid = fork();
|
||||
|
|
|
|||
|
|
@ -23,6 +23,23 @@ except Exception as e:
|
|||
sys.exit('Please check IDF_PATH')
|
||||
|
||||
|
||||
def create_temp_files(args):
|
||||
new_filenames = []
|
||||
for filename in [args.conf, args.values]:
|
||||
name, ext = os.path.splitext(filename)
|
||||
new_filename = name + '_tmp' + ext
|
||||
strip_blank_lines(filename, new_filename)
|
||||
new_filenames.append(new_filename)
|
||||
return new_filenames
|
||||
|
||||
|
||||
def strip_blank_lines(input_filename, output_filename):
|
||||
with open(input_filename, 'r') as read_from, open(output_filename,'w', newline='') as write_to:
|
||||
for line in read_from:
|
||||
if not line.isspace():
|
||||
write_to.write(line)
|
||||
|
||||
|
||||
def verify_values_exist(input_values_file, keys_in_values_file):
|
||||
""" Verify all keys have corresponding values in values file
|
||||
"""
|
||||
|
|
@ -237,10 +254,6 @@ def set_repeat_value(total_keys_repeat, keys, csv_file, target_filename):
|
|||
|
||||
def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=False):
|
||||
file_identifier_value = '0'
|
||||
csv_str = 'csv'
|
||||
bin_str = 'bin'
|
||||
set_output_keyfile = False
|
||||
|
||||
# Add config data per namespace to `config_data_to_write` list
|
||||
config_data_to_write = add_config_data_per_namespace(args.conf)
|
||||
|
||||
|
|
@ -261,11 +274,9 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
|
|||
next(values_file_reader)
|
||||
|
||||
# Create new directory(if doesn't exist) to store csv file generated
|
||||
output_csv_target_dir = create_dir(csv_str, args.outdir)
|
||||
output_csv_target_dir = create_dir('csv', args.outdir)
|
||||
# Create new directory(if doesn't exist) to store bin file generated
|
||||
output_bin_target_dir = create_dir(bin_str, args.outdir)
|
||||
if args.keygen:
|
||||
set_output_keyfile = True
|
||||
output_bin_target_dir = create_dir('bin', args.outdir)
|
||||
|
||||
for values_data_line in values_file_reader:
|
||||
key_value_data = list(zip_longest(keys_in_values_file, values_data_line))
|
||||
|
|
@ -276,7 +287,7 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
|
|||
key_value_pair = key_value_data[:]
|
||||
|
||||
# Verify if output csv file does not exist
|
||||
csv_filename = args.prefix + '-' + file_identifier_value + '.' + csv_str
|
||||
csv_filename = args.prefix + '-' + file_identifier_value + '.' + 'csv'
|
||||
output_csv_file = output_csv_target_dir + csv_filename
|
||||
if os.path.isfile(output_csv_file):
|
||||
raise SystemExit('Target csv file: %s already exists.`' % output_csv_file)
|
||||
|
|
@ -286,14 +297,14 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
|
|||
print('\nCreated CSV file: ===>', output_csv_file)
|
||||
|
||||
# Verify if output bin file does not exist
|
||||
bin_filename = args.prefix + '-' + file_identifier_value + '.' + bin_str
|
||||
bin_filename = args.prefix + '-' + file_identifier_value + '.' + 'bin'
|
||||
output_bin_file = output_bin_target_dir + bin_filename
|
||||
if os.path.isfile(output_bin_file):
|
||||
raise SystemExit('Target binary file: %s already exists.`' % output_bin_file)
|
||||
|
||||
args.input = output_csv_file
|
||||
args.output = os.path.join(bin_str, bin_filename)
|
||||
if set_output_keyfile:
|
||||
args.output = os.path.join('bin', bin_filename)
|
||||
if args.keygen:
|
||||
args.keyfile = 'keys-' + args.prefix + '-' + file_identifier_value
|
||||
|
||||
if is_encr:
|
||||
|
|
@ -308,57 +319,38 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
|
|||
exit(1)
|
||||
|
||||
|
||||
def verify_empty_lines_exist(args, input_file):
|
||||
input_file_reader = csv.reader(input_file, delimiter=',')
|
||||
for file_data in input_file_reader:
|
||||
for data in file_data:
|
||||
if len(data.strip()) == 0:
|
||||
raise SystemExit('Error: config file: %s cannot have empty lines. ' % args.conf)
|
||||
else:
|
||||
break
|
||||
if not file_data:
|
||||
raise SystemExit('Error: config file: %s cannot have empty lines.' % args.conf)
|
||||
|
||||
input_file.seek(0)
|
||||
return input_file_reader
|
||||
|
||||
|
||||
def verify_file_format(args):
|
||||
keys_in_config_file = []
|
||||
keys_in_values_file = []
|
||||
keys_repeat = []
|
||||
|
||||
# Verify config file is not empty
|
||||
# Verify files have csv extension
|
||||
conf_name, conf_extension = os.path.splitext(args.conf)
|
||||
if conf_extension != '.csv':
|
||||
raise SystemExit('Error: config file: %s does not have the .csv extension.' % args.conf)
|
||||
values_name, values_extension = os.path.splitext(args.values)
|
||||
if values_extension != '.csv':
|
||||
raise SystemExit('Error: values file: %s does not have the .csv extension.' % args.values)
|
||||
|
||||
# Verify files are not empty
|
||||
if os.stat(args.conf).st_size == 0:
|
||||
raise SystemExit('Error: config file: %s is empty.' % args.conf)
|
||||
|
||||
# Verify values file is not empty
|
||||
if os.stat(args.values).st_size == 0:
|
||||
raise SystemExit('Error: values file: %s is empty.' % args.values)
|
||||
|
||||
# Verify config file does not have empty lines
|
||||
with open(args.conf, 'r') as csv_config_file:
|
||||
try:
|
||||
config_file_reader = verify_empty_lines_exist(args, csv_config_file)
|
||||
# Extract keys from config file
|
||||
for config_data in config_file_reader:
|
||||
if 'namespace' not in config_data:
|
||||
keys_in_config_file.append(config_data[0])
|
||||
if 'REPEAT' in config_data:
|
||||
keys_repeat.append(config_data[0])
|
||||
# Extract keys from config file
|
||||
with open(args.conf, 'r') as config_file:
|
||||
config_file_reader = csv.reader(config_file, delimiter=',')
|
||||
for config_data in config_file_reader:
|
||||
if 'namespace' not in config_data:
|
||||
keys_in_config_file.append(config_data[0])
|
||||
if 'REPEAT' in config_data:
|
||||
keys_repeat.append(config_data[0])
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
# Verify values file does not have empty lines
|
||||
with open(args.values, 'r') as csv_values_file:
|
||||
try:
|
||||
values_file_reader = verify_empty_lines_exist(args, csv_values_file)
|
||||
# Extract keys from values file
|
||||
keys_in_values_file = next(values_file_reader)
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
# Extract keys from values file
|
||||
with open(args.values, 'r') as values_file:
|
||||
values_file_reader = csv.reader(values_file, delimiter=',')
|
||||
keys_in_values_file = next(values_file_reader)
|
||||
|
||||
# Verify file identifier exists in values file
|
||||
if args.fileid:
|
||||
|
|
@ -371,7 +363,9 @@ def verify_file_format(args):
|
|||
|
||||
|
||||
def generate(args):
|
||||
args.outdir = os.path.join(args.outdir, '')
|
||||
# Create work files with no blank lines
|
||||
args.conf, args.values = create_temp_files(args)
|
||||
|
||||
# Verify input config and values file format
|
||||
keys_in_config_file, keys_in_values_file, keys_repeat = verify_file_format(args)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
dummyNamespace,namespace,
|
||||
|
||||
|
||||
dummyU8Key,data,u8
|
||||
|
||||
dummyI8Key,data,i8
|
||||
|
||||
dummyU16Key,data,u16
|
||||
|
||||
|
||||
dummyU32Key,data,u32
|
||||
|
||||
dummyI32Key,data,i32,REPEAT
|
||||
|
||||
dummyStringKey,data,string
|
||||
|
||||
dummyHex2BinKey,data,hex2bin
|
||||
|
||||
dummyBase64Key,data,base64
|
||||
hexFileKey,file,hex2bin
|
||||
base64FileKey,file,base64
|
||||
|
||||
stringFileKey,file,string
|
||||
|
||||
blobFileAKey,file,binary
|
||||
|
||||
|
||||
blobFileBKey,file,binary
|
||||
binFileKey,file,binary
|
||||
|
Nie można renderować tego pliku, ponieważ ma nieprawidłową liczbę pól w wierszu 13.
|
|
|
@ -0,0 +1,7 @@
|
|||
id,dummyU8Key,dummyI8Key,dummyU16Key,dummyU32Key,dummyI32Key,dummyStringKey,dummyHex2BinKey,dummyBase64Key,hexFileKey,base64FileKey,stringFileKey,blobFileAKey,blobFileBKey,binFileKey
|
||||
|
||||
1,127,-128,32768,4294967295,-2147483648,0A:0B:0C:0D:0E:0F,010203abcdef,MTIzYWJj,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
|
||||
|
||||
|
||||
2,126,-127,32767,4294967294,,A0:B0:C0:D0:E0:F0,102030abcdef,MTIzYWFh,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
|
||||
3,125,-126,32766,4294967293,,00:B3:C4:BD:E2:0F,010203efcdab,MTIzYmJi,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
|
||||
|
Ładowanie…
Reference in New Issue