Test more samples!

Signed-off-by: Jacob Torrey <jacob@thinkst.com>
pull/6/head
Jacob Torrey 2023-05-24 15:19:13 -06:00
rodzic 9dba86a070
commit d1e689801b
1 zmienionych plików z 2 dodań i 3 usunięć

Wyświetl plik

@ -7,8 +7,8 @@ from lzma_detect import run_on_file_chunked, run_on_text_chunked, PRELUDE_STR, L
AI_SAMPLE_DIR = 'samples/llm-generated/'
HUMAN_SAMPLE_DIR = 'samples/human-generated/'
MIN_LEN = 150
NUM_JSONL_SAMPLES = 50
MIN_LEN = 1000
NUM_JSONL_SAMPLES = 250
ai_files = os.listdir(AI_SAMPLE_DIR)
human_files = os.listdir(HUMAN_SAMPLE_DIR)
@ -47,7 +47,6 @@ def test_llm_sample(f, record_property):
else:
warn("Unable to confidently classify: " + f)
HUMAN_JSONL_FILE = 'samples/webtext.test.jsonl'
human_samples = []
with jsonlines.open(HUMAN_JSONL_FILE) as reader: