diff --git a/ai_detect_roc.png b/ai_detect_roc.png index e721e8d..091dad1 100644 Binary files a/ai_detect_roc.png and b/ai_detect_roc.png differ diff --git a/contentatscale-report.xml b/contentatscale-report.xml index 2fe53d6..b47a34f 100644 --- a/contentatscale-report.xml +++ b/contentatscale-report.xml @@ -1,235 +1,1087 @@ -E AssertionError: samples/webtext.test.jsonl:255006 (len: 365) is a human-generated sample, misclassified as AI-generated with confidence 0.86 +E AssertionError: alumnifund1.txt is a human-generated file, misclassified as AI-generated with confidence 1.0 assert 'AI' == 'Human' - Human - + AIE AssertionError: samples/webtext.test.jsonl:255016 (len: 92) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + + AIE AssertionError: reuters_santos.txt is a human-generated file, misclassified as AI-generated with confidence 0.91 assert 'AI' == 'Human' - Human - + AIE AssertionError: samples/webtext.test.jsonl:255067 (len: 847) is a human-generated sample, misclassified as AI-generated with confidence 0.6 + + AIE AssertionError: cnn_santos.txt is a human-generated file, misclassified as AI-generated with confidence 1.0 assert 'AI' == 'Human' - Human - + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.56 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.99 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.88 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.54 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.6 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.92 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + + AIE AssertionError: 2111.00667_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 assert 'Human' == 'AI' - AI - + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + + HumanE AssertionError: 2110.13472_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 assert 'Human' == 'AI' - AI - + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + + HumanE AssertionError: 2111.03612_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 assert 'Human' == 'AI' - AI - + HumanE AssertionError: samples/ieee-init.jsonl:8600008 [885] (title: Vertical Handof) is a human-generated sample, misclassified as AI-generated with confidence 0.99 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600018 [642] (title: Efficient Detec) is a human-generated sample, misclassified as AI-generated with confidence 0.97 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600029 [1139] (title: A Social Bots D) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600048 [820] (title: Improved Data-A) is a human-generated sample, misclassified as AI-generated with confidence 0.98 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600060 [731] (title: Efficient Chann) is a human-generated sample, misclassified as AI-generated with confidence 0.99 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600062 [699] (title: A Novel Route R) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600074 [1015] (title: Human Activity ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600085 [855] (title: The Residual An) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600101 [470] (title: Closed-form App) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600111 [489] (title: A GMM-UBM Based) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600114 [590] (title: An Analysis of ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600150 [867] (title: User Associatio) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600154 [719] (title: Prediction of H) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600210 [1049] (title: A Fingerprint L) is a human-generated sample, misclassified as AI-generated with confidence 0.99 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600225 [969] (title: Quantum Key Int) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600228 [473] (title: TMVM: Truncated) is a human-generated sample, misclassified as AI-generated with confidence 0.99 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600233 [961] (title: Power System Fr) is a human-generated sample, misclassified as AI-generated with confidence 1.0 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-init.jsonl:8600234 [880] (title: Association Ana) is a human-generated sample, misclassified as AI-generated with confidence 0.86 - assert 'AI' == 'Human' - - Human - + AIE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600013 (title: Robust offline trained neural network for TDOA bas) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + + HumanE AssertionError: 2111.00310_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 assert 'Human' == 'AI' - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600014 (title: Gaussian MAC with Feedback and Strictly Causal Sta) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + + HumanE AssertionError: 2111.06230_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 assert 'Human' == 'AI' - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600045 (title: A Method Based on Frequent Pattern Mining to Predi) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + + HumanE AssertionError: 2111.06012_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 assert 'Human' == 'AI' - AI - + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + + HumanE AssertionError: 2111.01676_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2112.00405_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: gpt_patient_summary.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.05241_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.12645_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.07793_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: cleaned_2111.00086_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.10340_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.6 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.11115_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.15093_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.01515_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.12501_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02041_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.01023_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.03715_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02760_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02574_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15023_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.03320_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15317_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.06181_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.03294_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.15436_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00867_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.05754_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: dv_t1_aliens.txt is an LLM-generated file, misclassified as human-generated with confidence 0.67 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.12341_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00086_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02326_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00514_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.14532_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.07408_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00035_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02687_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.05204_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.10478_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02188_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.10575_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.07525_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.10577_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.15473_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.10778_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.13900_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.04416_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.03913_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.01706_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.01243_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.11879_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.75 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.10329_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.04507_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.03837_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00572_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.11984_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.06464_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02110_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15802_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.11589_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15724_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15130_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.06644_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.01322_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: about_me_short.txt is an LLM-generated file, misclassified as human-generated with confidence 0.79 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: bing_b_song.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00526_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.12010_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.13317_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.57 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02643_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15534_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.07611_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.13658_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15725_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.92 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.01231_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.10319_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.01340_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.78 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.07699_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15705_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02844_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.6 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.04130_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02259_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.75 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00180_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.04574_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00808_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.12765_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.13229_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00554_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.81 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.11207_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.00607_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.07267_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.11205_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 0.91 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.03945_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.12383_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2111.02362_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: 2110.15799_generated.txt is an LLM-generated file, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/webtext.test.jsonl:255006 (len: 365) is a human-generated sample, misclassified as AI-generated with confidence 0.94 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/webtext.test.jsonl:255016 (len: 92) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/webtext.test.jsonl:255067 (len: 847) is a human-generated sample, misclassified as AI-generated with confidence 0.6 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/webtext.test.jsonl:255108 (len: 358) is a human-generated sample, misclassified as AI-generated with confidence 0.8 + assert 'AI' == 'Human' + - Human + + AIE socket.gaierror: [Errno -3] Temporary failure in name resolution + +During handling of the above exception, another exception occurred: +E httpcore.ConnectError: [Errno -3] Temporary failure in name resolution The above exception was the direct cause of the following exception: -E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/ieee-chatgpt-polish.jsonl:8600080 (title: Identification of Coronary Artery Diseased Subject) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600088 (title: Weighted Nuclear Norm and TV Regularization based ) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> - -The above exception was the direct cause of the following exception: -E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/ieee-chatgpt-polish.jsonl:8600091 (title: Improved Epoch Extraction Using Variational Mode D) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600094 (title: NLOS Detection and Mitigation for UWB/IMU Fusion S) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600099 (title: Ranking-based Collaborative Clustering for Heterog) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600131 (title: Cell-Phone Identification from Recompressed Audio ) is an LLM-generated sample, misclassified as human-generated with confidence 0.95 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600132 (title: Spatial Image Steganalysis Based on ResNeXt) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600143 (title: Balanced Use of Battery Power in Ad-hoc Wireless S) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600153 (title: A New Combinatorial Design Based Data En-Route Fil) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600169 (title: LayerOS: Scheduling Apps between the Local System ) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600178 (title: Cost Effective Hash Chain Based Key Pre-Distributi) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600186 (title: Atmospheric Absorption Loss Estimation of Terahert) is an LLM-generated sample, misclassified as human-generated with confidence 0.63 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600190 (title: Manner of Articulation based Split Lattices for Ph) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600201 (title: Control and Management of Optical Networks Using O) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600202 (title: Mridangam Artist Identification from Taniavartanam) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600206 (title: A 3D Placement of Unmanned Aerial Vehicle Base Sta) is an LLM-generated sample, misclassified as human-generated with confidence 0.97 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600219 (title: Deep Learning Intrusion Detection Model Based on O) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600231 (title: Uplink Asynchronous Fractional Pilots Scheduling i) is an LLM-generated sample, misclassified as human-generated with confidence 0.84 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/ieee-vicuna-generation.jsonl:8600178 (title: Cost Effective Hash Chain Based Key Pre-Distributi) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 - assert 'Human' == 'AI' - - AI - + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> - -The above exception was the direct cause of the following exception: -E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/gptzero_eval.csv:4 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> - -The above exception was the direct cause of the following exception: -E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/gptzero_eval.csv:7 was misclassified with confidence 0.99 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:10 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> - -The above exception was the direct cause of the following exception: -E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/gptzero_eval.csv:14 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:19 was misclassified with confidence 0.59 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:21 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:22 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:24 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:25 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:26 was misclassified with confidence 0.97 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:27 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:30 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:31 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:32 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:34 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:35 was misclassified with confidence 0.99 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:36 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:40 was misclassified with confidence 1.0 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:41 was misclassified with confidence 0.93 - assert 'Human' == 'AI' - - AI - + HumanE AssertionError: samples/gptzero_eval.csv:51 was misclassified with confidence 0.99 +E httpx.ConnectError: [Errno -3] Temporary failure in name resolutionE TypeError: int() argument must be a string, a bytes-like object or a real number, not 'NoneType'E AssertionError: samples/webtext.test.jsonl:255233 (len: 74) is a human-generated sample, misclassified as AI-generated with confidence 0.57 assert 'AI' == 'Human' - Human - + AIE AssertionError: samples/gptzero_eval.csv:75 was misclassified with confidence 0.99 + + AIE AssertionError: samples/webtext.test.jsonl:255356 (len: 326) is a human-generated sample, misclassified as AI-generated with confidence 1.0 assert 'AI' == 'Human' - Human - + AI \ No newline at end of file + + AIE AssertionError: samples/webtext.test.jsonl:255413 (len: 395) is a human-generated sample, misclassified as AI-generated with confidence 0.87 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.56 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.88 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.54 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.92 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.52 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.59 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.66 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.93 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.98 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.94 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/news.jsonl is a AI-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-init.jsonl:8600008 [885] (title: Vertical Handof) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600018 [642] (title: Efficient Detec) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600029 [1139] (title: A Social Bots D) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600048 [820] (title: Improved Data-A) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600060 [731] (title: Efficient Chann) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600062 [699] (title: A Novel Route R) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600074 [1015] (title: Human Activity ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600085 [855] (title: The Residual An) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600101 [470] (title: Closed-form App) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600111 [489] (title: A GMM-UBM Based) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600114 [590] (title: An Analysis of ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600150 [867] (title: User Associatio) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600154 [719] (title: Prediction of H) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600210 [1049] (title: A Fingerprint L) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600225 [969] (title: Quantum Key Int) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600228 [473] (title: TMVM: Truncated) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600233 [961] (title: Power System Fr) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600234 [880] (title: Association Ana) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600252 [1114] (title: Multi-objective) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600263 [1009] (title: Two-Layer Resou) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600345 [993] (title: Precomputed Mul) is a human-generated sample, misclassified as AI-generated with confidence 0.97 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600363 [2064] (title: A Hierarchical ) is a human-generated sample, misclassified as AI-generated with confidence 0.51 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600369 [1149] (title: Local Discrimin) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600371 [1042] (title: Autonomous Navi) is a human-generated sample, misclassified as AI-generated with confidence 0.98 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600496 [952] (title: Reliability Eva) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600504 [1174] (title: An Object-Based) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600519 [1124] (title: Outage Performa) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600526 [800] (title: Research on Eme) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600528 [646] (title: Mobile Data Min) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600545 [1001] (title: A Search Space ) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600581 [518] (title: Short-Term Fore) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600595 [1239] (title: An Artificial N) is a human-generated sample, misclassified as AI-generated with confidence 0.98 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600596 [644] (title: ANDES: A Python) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600597 [545] (title: Synchrophasor A) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600612 [1034] (title: The Potential f) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600617 [1062] (title: Geometrically M) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600623 [907] (title: A Probabilistic) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600657 [1012] (title: Effect of Solar) is a human-generated sample, misclassified as AI-generated with confidence 0.98 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600661 [822] (title: Synchrophasor-B) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600737 [634] (title: Disturbance Obs) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600738 [1441] (title: Enhancing Cloud) is a human-generated sample, misclassified as AI-generated with confidence 0.73 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600750 [1117] (title: Security and Pr) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600751 [992] (title: Blockchain-Base) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600752 [1145] (title: Active Learning) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600755 [999] (title: Covert Timing C) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600757 [1062] (title: Covert Wireless) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600766 [777] (title: A Disturbance R) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600778 [1093] (title: Securing Consum) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600779 [1202] (title: Privacy-Preserv) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600783 [1017] (title: Privacy of Thin) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600789 [607] (title: Distributed maj) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600791 [972] (title: Event-triggered) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600836 [925] (title: A Soft Ring-sha) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600844 [940] (title: Digital Twins o) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600854 [753] (title: Smart Contracts) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600864 [678] (title: Unsupervised Vi) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600871 [468] (title: A flex-rigid so) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600885 [633] (title: Recognition of ) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600896 [439] (title: Application of ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600904 [1133] (title: Smart Air Quali) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600906 [762] (title: A preliminary v) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600914 [982] (title: Naive Bayes Mul) is a human-generated sample, misclassified as AI-generated with confidence 0.75 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600916 [757] (title: Modelling of a ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600925 [562] (title: Data Assimilati) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600932 [702] (title: An Experimentat) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600933 [1400] (title: A Cloud Based S) is a human-generated sample, misclassified as AI-generated with confidence 0.95 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600959 [598] (title: Fall detection ) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8600984 [963] (title: Photoplethysmog) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601011 [711] (title: Machine Learnin) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601032 [808] (title: Iris Recognitio) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601097 [930] (title: Call Data Recor) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601176 [353] (title: Known security ) is a human-generated sample, misclassified as AI-generated with confidence 0.97 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601183 [1347] (title: Detecting Seizu) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601205 [625] (title: Usability of Te) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601220 [934] (title: Cloud Robotics ) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601226 [846] (title: Wireless Commun) is a human-generated sample, misclassified as AI-generated with confidence 0.98 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601250 [1410] (title: A Framework to ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601258 [779] (title: Challenges and ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601271 [1462] (title: Employment Law ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601274 [885] (title: Prohibition Sig) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601280 [1292] (title: The Experimenta) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601291 [1168] (title: Improving Assoc) is a human-generated sample, misclassified as AI-generated with confidence 0.85 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601324 [1166] (title: GAN-Based Semi-) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601331 [1053] (title: A Novel Paralle) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601338 [821] (title: Top-Down Synthe) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601354 [943] (title: Compensation of) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601418 [408] (title: The Classificat) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601446 [797] (title: Development and) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601454 [342] (title: The Training Se) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601456 [813] (title: Distribution of) is a human-generated sample, misclassified as AI-generated with confidence 0.72 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601472 [1208] (title: Information Dec) is a human-generated sample, misclassified as AI-generated with confidence 0.97 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601478 [648] (title: Experimental St) is a human-generated sample, misclassified as AI-generated with confidence 0.54 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601485 [406] (title: Approaches to s) is a human-generated sample, misclassified as AI-generated with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601534 [623] (title: An Overview of ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601641 [807] (title: Analysis on the) is a human-generated sample, misclassified as AI-generated with confidence 0.94 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601669 [735] (title: New Dimensional) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-init.jsonl:8601681 [1756] (title: mHealth4Afrika ) is a human-generated sample, misclassified as AI-generated with confidence 1.0 + assert 'AI' == 'Human' + - Human + + AIE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600013 (title: Robust offline trained neural network for TDOA bas) is an LLM-generated sample, misclassified as human-generated with confidence 0.92 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600014 (title: Gaussian MAC with Feedback and Strictly Causal Sta) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600045 (title: A Method Based on Frequent Pattern Mining to Predi) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600080 (title: Identification of Coronary Artery Diseased Subject) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600088 (title: Weighted Nuclear Norm and TV Regularization based ) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600089 (title: Modeling MME Residence Time in LTE based Cellular ) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600091 (title: Improved Epoch Extraction Using Variational Mode D) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600094 (title: NLOS Detection and Mitigation for UWB/IMU Fusion S) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600099 (title: Ranking-based Collaborative Clustering for Heterog) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600131 (title: Cell-Phone Identification from Recompressed Audio ) is an LLM-generated sample, misclassified as human-generated with confidence 0.82 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600132 (title: Spatial Image Steganalysis Based on ResNeXt) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600143 (title: Balanced Use of Battery Power in Ad-hoc Wireless S) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600153 (title: A New Combinatorial Design Based Data En-Route Fil) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600169 (title: LayerOS: Scheduling Apps between the Local System ) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600178 (title: Cost Effective Hash Chain Based Key Pre-Distributi) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600186 (title: Atmospheric Absorption Loss Estimation of Terahert) is an LLM-generated sample, misclassified as human-generated with confidence 0.78 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600190 (title: Manner of Articulation based Split Lattices for Ph) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600198 (title: Cell Search Algorithms at Low SNR for WiMAX System) is an LLM-generated sample, misclassified as human-generated with confidence 0.81 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600201 (title: Control and Management of Optical Networks Using O) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600202 (title: Mridangam Artist Identification from Taniavartanam) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600206 (title: A 3D Placement of Unmanned Aerial Vehicle Base Sta) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600216 (title: Research on the Hidden Markov Model in Hot Topics ) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600219 (title: Deep Learning Intrusion Detection Model Based on O) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600231 (title: Uplink Asynchronous Fractional Pilots Scheduling i) is an LLM-generated sample, misclassified as human-generated with confidence 0.92 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600237 (title: The Adaptive Communication Network Architecture of) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600246 (title: Dynamic Spectrum Assignment Based on Quantum Harmo) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600333 (title: Action-Stage Emphasized Spatiotemporal VLAD for Vi) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600342 (title: Towards High-Efficient Transaction Commitment in a) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600373 (title: Lightweight Feature Fusion Network for Single Imag) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600375 (title: Navion: A 2-mW Fully Integrated Real-Time Visual-I) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600380 (title: Dynamic Texture Classification Using Unsupervised ) is an LLM-generated sample, misclassified as human-generated with confidence 0.5 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600384 (title: A Generative Discriminatory Classified Network for) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600393 (title: Prescribed-Time Observers for Linear Systems in Ob) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600424 (title: Audiovisual Synchrony Detection with Optimized Aud) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600431 (title: Rotation and Precession Periods Estimation of Unst) is an LLM-generated sample, misclassified as human-generated with confidence 0.97 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600437 (title: Visualization of Railway Scene Classification Mode) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600444 (title: Interferometric SAR Baseline Estimation by Partiti) is an LLM-generated sample, misclassified as human-generated with confidence 0.89 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600476 (title: An Improved MMSE-RISIC Equalization Algorithm Base) is an LLM-generated sample, misclassified as human-generated with confidence 0.75 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600484 (title: A Modeling Attack Resistant R-XOR APUF Based on FP) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600485 (title: Towards 3D Human Action Recognition Using a Distil) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600501 (title: Multi-frame Image Super Resolution with Natural Im) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600508 (title: A Raspberry Pi Based Event Driven Quasi Real Time ) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600517 (title: A Pre-Compensation Method for Digital-to-Analog Co) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600543 (title: Real-Time Simulation of Electric Vehicle Battery C) is an LLM-generated sample, misclassified as human-generated with confidence 0.97 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600547 (title: Visualization of Large-Scale Electric Grid Oscilla) is an LLM-generated sample, misclassified as human-generated with confidence 0.97 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600549 (title: Residential Load Forecasting Using Deep Neural Net) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600580 (title: Bilevel Programming-Based Unit Commitment for Loca) is an LLM-generated sample, misclassified as human-generated with confidence 0.65 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600584 (title: Minimization of Ohmic Losses in Power Networks by ) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600650 (title: Fast Generator Placement Algorithm using High Perf) is an LLM-generated sample, misclassified as human-generated with confidence 0.56 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600676 (title: Solar Photovoltaic output prediction using Jackkni) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600725 (title: Supporting the Momentum Training Algorithm Using a) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600730 (title: High-Performance Visual Tracking With Extreme Lear) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600736 (title: Forecasting Traffic Volume at a Designated Cross-S) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600754 (title: A Castle of Glass: Leaky IoT Appliances in Modern ) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600762 (title: Latent Factor-Based Recommenders Relying on Extend) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600788 (title: Communication scheduling and remote estimation wit) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600819 (title: Small Parts Classification with Flexible Machine V) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600827 (title: Towards Real-Time Capable Simulations with a Conta) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600849 (title: Analog Ultra Low-Power Acoustic Wake-Up System Bas) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600851 (title: Ultra-low-latency Video Coding Method for Autonomo) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600852 (title: Speaker Recognition For Digital Forensic Audio Ana) is an LLM-generated sample, misclassified as human-generated with confidence 0.62 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600860 (title: The Industrial IoT for Nusantara) is an LLM-generated sample, misclassified as human-generated with confidence 0.97 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600863 (title: A new method for fast detection and pose estimatio) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600869 (title: Flood Modelling and Prediction Using Artificial Ne) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600891 (title: Smart ECG Holter Monitoring System Using Smartphon) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600895 (title: Realization of IPv6 connectivity via RS485 fieldbu) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600936 (title: Reactive HLA-based Distributed Simulation Systems ) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600937 (title: Delay-based Distribution and Optimization of A Sim) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8600998 (title: Adaptive Event Driven Framework for Real Time Mult) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601025 (title: MRAS Based Sensorless Control of High Speed PMSMs ) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601141 (title: Energy Saving Transmission Technique for Poisson D) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601150 (title: Device Classification for NILM using FIT-PS compar) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601203 (title: Performance Analysis of Machine Learning Classifie) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601218 (title: Detecting and Removing the Impulsive Noise in OFDM) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601232 (title: A Training Utility for Estimating the Bowling Spee) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601242 (title: Retinal Vessel Tortuosity Characterisation Using B) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601243 (title: Preprocessed Faster RCNN for Vehicle Detection) is an LLM-generated sample, misclassified as human-generated with confidence 0.98 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601254 (title: Emulating Software Defined Network Using Mininet a) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601280 (title: The Experimental Comparison of Features for Hand D) is an LLM-generated sample, misclassified as human-generated with confidence 0.75 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601303 (title: A Controllable Deflection Routing and Wavelength A) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601343 (title: Automatically Evaluating Balance: A Machine Learni) is an LLM-generated sample, misclassified as human-generated with confidence 0.81 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601346 (title: BLUEs and Reliability Analysis for General Censore) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601351 (title: PolSAR Image Semantic Segmentation Based on Deep T) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601356 (title: Identification of Composite Demand Side Model With) is an LLM-generated sample, misclassified as human-generated with confidence 0.52 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601362 (title: Cofactor-Based Efficient Endmember Extraction for ) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601379 (title: Stiffness Control of Deformable Robots Using Finit) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601430 (title: Short-Term Forecast of Electricity Load for LLC &#) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601440 (title: Perspective Neural Network Algorithms for Dynamic ) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601497 (title: Estimation of Probability of Error on Degree Teleg) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601560 (title: Photovoltaic Modules Power Degradation and Lifetim) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601567 (title: A new PV generation power prediction model based o) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601579 (title: Correlation Mining for Reconstruction Measures and) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601588 (title: An Equivalent Modeling Method for Multi-port Area ) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601654 (title: Equivalent Modeling of Distributed Photovoltaic Po) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601668 (title: Decision Tree-based Real-time Emergency Control St) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-chatgpt-polish.jsonl:8601703 (title: Optimal allocation of distributed reactive power c) is an LLM-generated sample, misclassified as human-generated with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/ieee-vicuna-generation.jsonl:8600178 (title: Cost Effective Hash Chain Based Key Pre-Distributi) is an LLM-generated sample, misclassified as human-generated with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + +The above exception was the direct cause of the following exception: +E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/ieee-vicuna-generation.jsonl:8601669 (title: New Dimensionality Reduction Method of Wind Power ) is an LLM-generated sample, misclassified as human-generated with confidence 0.84 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:4 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:7 was misclassified with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:9 was misclassified with confidence 0.72 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:10 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:12 was misclassified with confidence 0.72 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:14 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:21 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:22 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:24 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:25 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:26 was misclassified with confidence 0.97 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:27 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:30 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:31 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:32 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:34 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:35 was misclassified with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:36 was misclassified with confidence 1.0 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:40 was misclassified with confidence 0.99 + assert 'Human' == 'AI' + - AI + + HumanE AssertionError: samples/gptzero_eval.csv:51 was misclassified with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AIE httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + +The above exception was the direct cause of the following exception: +E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E httpcore.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None> + +The above exception was the direct cause of the following exception: +E httpx.RemoteProtocolError: <ConnectionTerminated error_code:ErrorCodes.NO_ERROR, last_stream_id:1, additional_data:None>E AssertionError: samples/gptzero_eval.csv:75 was misclassified with confidence 0.99 + assert 'AI' == 'Human' + - Human + + AI \ No newline at end of file diff --git a/nlzmadetect/src/nlzmadetect.nim b/nlzmadetect/src/nlzmadetect.nim index 6e18e92..3acab82 100644 --- a/nlzmadetect/src/nlzmadetect.nim +++ b/nlzmadetect/src/nlzmadetect.nim @@ -203,5 +203,3 @@ when defined(js) and isMainModule: if opacity < 0.0: opacity = 0.0 return opacity - - #window.onload = on_load diff --git a/plot_rocs.py b/plot_rocs.py index e5ca647..1de10bc 100644 --- a/plot_rocs.py +++ b/plot_rocs.py @@ -6,7 +6,7 @@ from sklearn.metrics import roc_curve, auc import re from junitparser import JUnitXml -MODELS = ['zippy', 'roberta', 'gptzero', 'crossplag', 'contentatscale'] +MODELS = ['zippy-lzma', 'zippy-zlib', 'roberta', 'gptzero', 'crossplag', 'contentatscale'] SKIPCASES = ['gpt2', 'gpt3'] MAX_PER_CASE = 500 diff --git a/test_contentatscale_detect.py b/test_contentatscale_detect.py index 381ab85..cca7b41 100644 --- a/test_contentatscale_detect.py +++ b/test_contentatscale_detect.py @@ -8,7 +8,7 @@ AI_SAMPLE_DIR = 'samples/llm-generated/' HUMAN_SAMPLE_DIR = 'samples/human-generated/' MIN_LEN = 150 -NUM_JSONL_SAMPLES = 15#500 +NUM_JSONL_SAMPLES = 500 ai_files = os.listdir(AI_SAMPLE_DIR) human_files = os.listdir(HUMAN_SAMPLE_DIR) @@ -68,37 +68,37 @@ def test_human_jsonl(i, record_property): record_property("score", str(score)) assert classification == 'Human', HUMAN_JSONL_FILE + ':' + str(i.get('id')) + ' (len: ' + str(i.get('length', -1)) + ') is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8)) -AI_JSONL_FILE = 'samples/xl-1542M.test.jsonl' -ai_samples = [] -with jsonlines.open(AI_JSONL_FILE) as reader: - for obj in reader: - ai_samples.append(obj) +# AI_JSONL_FILE = 'samples/xl-1542M.test.jsonl' +# ai_samples = [] +# with jsonlines.open(AI_JSONL_FILE) as reader: +# for obj in reader: +# ai_samples.append(obj) -@pytest.mark.parametrize('i', ai_samples[0:NUM_JSONL_SAMPLES]) -def test_llm_jsonl(i, record_property): - res = run_on_text_chunked(i.get('text', '')) - if res is None: - pytest.skip('Unable to classify') - (classification, score) = res - record_property("score", str(score)) - assert classification == 'AI', AI_JSONL_FILE + ':' + str(i.get('id')) + ' (text: ' + i.get('text', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) +# @pytest.mark.parametrize('i', ai_samples[0:NUM_JSONL_SAMPLES]) +# def test_gpt2_jsonl(i, record_property): +# res = run_on_text_chunked(i.get('text', '')) +# if res is None: +# pytest.skip('Unable to classify') +# (classification, score) = res +# record_property("score", str(score)) +# assert classification == 'AI', AI_JSONL_FILE + ':' + str(i.get('id')) + ' (text: ' + i.get('text', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) -GPT3_JSONL_FILE = 'samples/GPT-3-175b_samples.jsonl' -gpt3_samples = [] -with jsonlines.open(GPT3_JSONL_FILE) as reader: - for o in reader: - for l in o.split('<|endoftext|>'): - if len(l) >= MIN_LEN: - gpt3_samples.append(l) +# GPT3_JSONL_FILE = 'samples/GPT-3-175b_samples.jsonl' +# gpt3_samples = [] +# with jsonlines.open(GPT3_JSONL_FILE) as reader: +# for o in reader: +# for l in o.split('<|endoftext|>'): +# if len(l) >= MIN_LEN: +# gpt3_samples.append(l) -@pytest.mark.parametrize('i', gpt3_samples[0:NUM_JSONL_SAMPLES]) -def test_gpt3_jsonl(i, record_property): - res = run_on_text_chunked(i) - if res is None: - pytest.skip('Unable to classify') - (classification, score) = res - record_property("score", str(score)) - assert classification == 'AI', GPT3_JSONL_FILE + ' is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) +# @pytest.mark.parametrize('i', gpt3_samples[0:NUM_JSONL_SAMPLES]) +# def test_gpt3_jsonl(i, record_property): +# res = run_on_text_chunked(i) +# if res is None: +# pytest.skip('Unable to classify') +# (classification, score) = res +# record_property("score", str(score)) +# assert classification == 'AI', GPT3_JSONL_FILE[0:250] + ' is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) NEWS_JSONL_FILE = 'samples/news.jsonl' news_samples = [] diff --git a/test_zippy_detect.py b/test_zippy_detect.py index b2a5b55..dc7a28e 100644 --- a/test_zippy_detect.py +++ b/test_zippy_detect.py @@ -2,21 +2,24 @@ import pytest, os, jsonlines, csv from warnings import warn -from zippy import run_on_file_chunked, run_on_text_chunked, PRELUDE_STR, LzmaLlmDetector +from zippy import run_on_file_chunked, run_on_text_chunked, PRELUDE_STR, LzmaLlmDetector, CompressionEngine, ZlibLlmDetector, ENGINE +import zippy AI_SAMPLE_DIR = 'samples/llm-generated/' HUMAN_SAMPLE_DIR = 'samples/human-generated/' -MIN_LEN = 50 +MIN_LEN = 150 NUM_JSONL_SAMPLES = 500 ai_files = os.listdir(AI_SAMPLE_DIR) human_files = os.listdir(HUMAN_SAMPLE_DIR) -FUZZINESS = 3 CONFIDENCE_THRESHOLD : float = 0.00 # What confidence to treat as error vs warning -PRELUDE_RATIO = LzmaLlmDetector(prelude_str=PRELUDE_STR).prelude_ratio +if ENGINE == CompressionEngine.LZMA: + PRELUDE_RATIO = LzmaLlmDetector(prelude_str=PRELUDE_STR).prelude_ratio +elif ENGINE == CompressionEngine.ZLIB: + PRELUDE_RATIO = ZlibLlmDetector(prelude_str=PRELUDE_STR).prelude_ratio def test_training_file(record_property): (classification, score) = run_on_file_chunked('ai-generated.txt') @@ -25,7 +28,7 @@ def test_training_file(record_property): @pytest.mark.parametrize('f', human_files) def test_human_samples(f, record_property): - (classification, score) = run_on_file_chunked(HUMAN_SAMPLE_DIR + f, fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_file_chunked(HUMAN_SAMPLE_DIR + f, prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) if score > CONFIDENCE_THRESHOLD: assert classification == 'Human', f + ' is a human-generated file, misclassified as AI-generated with confidence ' + str(round(score, 8)) @@ -37,7 +40,7 @@ def test_human_samples(f, record_property): @pytest.mark.parametrize('f', ai_files) def test_llm_sample(f, record_property): - (classification, score) = run_on_file_chunked(AI_SAMPLE_DIR + f, fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_file_chunked(AI_SAMPLE_DIR + f, prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) if score > CONFIDENCE_THRESHOLD: assert classification == 'AI', f + ' is an LLM-generated file, misclassified as human-generated with confidence ' + str(round(score, 8)) @@ -56,36 +59,36 @@ with jsonlines.open(HUMAN_JSONL_FILE) as reader: @pytest.mark.parametrize('i', human_samples[0:NUM_JSONL_SAMPLES]) def test_human_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i.get('text', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('text', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == 'Human', HUMAN_JSONL_FILE + ':' + str(i.get('id')) + ' (len: ' + str(i.get('length', -1)) + ') is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8)) -AI_JSONL_FILE = 'samples/xl-1542M.test.jsonl' -ai_samples = [] -with jsonlines.open(AI_JSONL_FILE) as reader: - for obj in reader: - if obj.get('length', 0) >= MIN_LEN: - ai_samples.append(obj) +# AI_JSONL_FILE = 'samples/xl-1542M.test.jsonl' +# ai_samples = [] +# with jsonlines.open(AI_JSONL_FILE) as reader: +# for obj in reader: +# if obj.get('length', 0) >= MIN_LEN: +# ai_samples.append(obj) -@pytest.mark.parametrize('i', ai_samples[0:NUM_JSONL_SAMPLES]) -def test_gpt2_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i.get('text', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) - record_property("score", str(score)) - assert classification == 'AI', AI_JSONL_FILE + ':' + str(i.get('id')) + ' (text: ' + i.get('text', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) +# @pytest.mark.parametrize('i', ai_samples[0:NUM_JSONL_SAMPLES]) +# def test_gpt2_jsonl(i, record_property): +# (classification, score) = run_on_text_chunked(i.get('text', ''), prelude_ratio=PRELUDE_RATIO) +# record_property("score", str(score)) +# assert classification == 'AI', AI_JSONL_FILE + ':' + str(i.get('id')) + ' (text: ' + i.get('text', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) -GPT3_JSONL_FILE = 'samples/GPT-3-175b_samples.jsonl' -gpt3_samples = [] -with jsonlines.open(GPT3_JSONL_FILE) as reader: - for o in reader: - for l in o.split('<|endoftext|>'): - if len(l) >= MIN_LEN: - gpt3_samples.append(l) +# GPT3_JSONL_FILE = 'samples/GPT-3-175b_samples.jsonl' +# gpt3_samples = [] +# with jsonlines.open(GPT3_JSONL_FILE) as reader: +# for o in reader: +# for l in o.split('<|endoftext|>'): +# if len(l) >= MIN_LEN: +# gpt3_samples.append(l) -@pytest.mark.parametrize('i', gpt3_samples[0:NUM_JSONL_SAMPLES]) -def test_gpt3_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i, fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) - record_property("score", str(score)) - assert classification == 'AI', GPT3_JSONL_FILE + ' is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) +# @pytest.mark.parametrize('i', gpt3_samples[0:NUM_JSONL_SAMPLES]) +# def test_gpt3_jsonl(i, record_property): +# (classification, score) = run_on_text_chunked(i, prelude_ratio=PRELUDE_RATIO) +# record_property("score", str(score)) +# assert classification == 'AI', GPT3_JSONL_FILE + ' is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) NEWS_JSONL_FILE = 'samples/news.jsonl' news_samples = [] @@ -95,13 +98,13 @@ with jsonlines.open(NEWS_JSONL_FILE) as reader: @pytest.mark.parametrize('i', news_samples[0:NUM_JSONL_SAMPLES]) def test_humannews_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i.get('human', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('human', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == 'Human', NEWS_JSONL_FILE + ' is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8)) @pytest.mark.parametrize('i', news_samples[0:NUM_JSONL_SAMPLES]) def test_chatgptnews_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i.get('chatgpt', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('chatgpt', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == 'AI', NEWS_JSONL_FILE + ' is a AI-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) @@ -114,7 +117,7 @@ with jsonlines.open(CHEAT_HUMAN_JSONL_FILE) as reader: @pytest.mark.parametrize('i', ch_samples[0:NUM_JSONL_SAMPLES]) def test_cheat_human_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i.get('abstract', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('abstract', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == 'Human', CHEAT_HUMAN_JSONL_FILE + ':' + str(i.get('id')) + ' [' + str(len(i.get('abstract', ''))) + '] (title: ' + i.get('title', "").replace('\n', ' ')[:15] + ') is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8)) @@ -127,7 +130,7 @@ with jsonlines.open(CHEAT_GEN_JSONL_FILE) as reader: @pytest.mark.parametrize('i', cg_samples[0:NUM_JSONL_SAMPLES]) def test_cheat_generation_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i.get('abstract', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('abstract', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == 'AI', CHEAT_GEN_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) @@ -140,7 +143,7 @@ with jsonlines.open(CHEAT_POLISH_JSONL_FILE) as reader: @pytest.mark.parametrize('i', cp_samples[0:NUM_JSONL_SAMPLES]) def test_cheat_polish_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i.get('abstract', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('abstract', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == 'AI', CHEAT_POLISH_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) @@ -153,7 +156,7 @@ with jsonlines.open(CHEAT_VICUNAGEN_JSONL_FILE) as reader: @pytest.mark.parametrize('i', vg_samples[0:NUM_JSONL_SAMPLES]) def test_vicuna_generation_jsonl(i, record_property): - (classification, score) = run_on_text_chunked(i.get('abstract', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('abstract', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == 'AI', CHEAT_VICUNAGEN_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8)) @@ -167,12 +170,12 @@ with open(GPTZERO_EVAL_FILE) as fp: @pytest.mark.parametrize('i', list(filter(lambda x: x.get('Label') == 'Human', ge_samples[0:NUM_JSONL_SAMPLES]))) def test_gptzero_eval_dataset_human(i, record_property): - (classification, score) = run_on_text_chunked(i.get('Document', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('Document', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == i.get('Label'), GPTZERO_EVAL_FILE + ':' + str(i.get('Index')) + ' was misclassified with confidence ' + str(round(score, 8)) @pytest.mark.parametrize('i', list(filter(lambda x: x.get('Label') == 'AI', ge_samples[0:NUM_JSONL_SAMPLES]))) def test_gptzero_eval_dataset_ai(i, record_property): - (classification, score) = run_on_text_chunked(i.get('Document', ''), fuzziness=FUZZINESS, prelude_ratio=PRELUDE_RATIO) + (classification, score) = run_on_text_chunked(i.get('Document', ''), prelude_ratio=PRELUDE_RATIO) record_property("score", str(score)) assert classification == i.get('Label'), GPTZERO_EVAL_FILE + ':' + str(i.get('Index')) + ' was misclassified with confidence ' + str(round(score, 8)) diff --git a/zippy.py b/zippy.py index d8d7933..3455818 100755 --- a/zippy.py +++ b/zippy.py @@ -5,12 +5,21 @@ # Author: Jacob Torrey import lzma, argparse, os, itertools +from zlib import compressobj, Z_FINISH import re, sys +from abc import ABC, abstractmethod +from enum import Enum from typing import List, Optional, Tuple, TypeAlias from multiprocessing import Pool, cpu_count Score : TypeAlias = tuple[str, float] +class CompressionEngine(Enum): + LZMA = 1 + ZLIB = 2 + +ENGINE : CompressionEngine = CompressionEngine.ZLIB + def clean_text(s : str) -> str: ''' Removes formatting and other non-content data that may skew compression ratios (e.g., duplicate spaces) @@ -32,9 +41,61 @@ PRELUDE_FILE : str = 'ai-generated.txt' with open(PRELUDE_FILE, 'r') as fp: PRELUDE_STR = clean_text(fp.read()) -class LzmaLlmDetector: +class AIDetector(ABC): + ''' + Base class for AI detection + ''' + @abstractmethod + def score_text(self, sample : str) -> Optional[Score]: + pass + +class ZlibLlmDetector(AIDetector): + '''Class providing functionality to attempt to detect LLM/generative AI generated text using the zlib compression algorithm''' + def __init__(self, prelude_file : Optional[str] = None, prelude_str : Optional[str] = None, prelude_ratio : Optional[float] = None): + self.PRESET = 9 + self.WBITS = -15 + self.prelude_ratio = 0.0 + if prelude_ratio != None: + self.prelude_ratio = prelude_ratio + + if prelude_file != None: + with open(prelude_file) as fp: + self.prelude_str = fp.read() + self.prelude_ratio = self._compress(self.prelude_str) + + if prelude_str != None: + self.prelude_str = prelude_str + self.prelude_ratio = self._compress(self.prelude_str) + + def _compress(self, s : str) -> float: + orig_len = len(s.encode()) + c = compressobj(level=self.PRESET, wbits=self.WBITS, memLevel=9) + bytes = c.compress(s.encode()) + bytes += c.flush(Z_FINISH) + c_len = len(bytes) + #c_len = len(compress(s.encode(), level=self.PRESET, wbits=self.WBITS)) + return c_len / orig_len + + def score_text(self, sample: str) -> Score | None: + ''' + Returns a tuple of a string (AI or Human) and a float confidence (higher is more confident) that the sample was generated + by either an AI or human. Returns None if it cannot make a determination + ''' + if self.prelude_ratio == 0.0: + return None + sample_score = self._compress(self.prelude_str + sample) + #print(str((self.prelude_ratio, sample_score))) + delta = self.prelude_ratio - sample_score + determination = 'AI' + if delta < 0: + determination = 'Human' + + return (determination, abs(delta * 100)) + + +class LzmaLlmDetector(AIDetector): '''Class providing functionality to attempt to detect LLM/generative AI generated text using the LZMA compression algorithm''' - def __init__(self, prelude_file : Optional[str] = None, fuzziness_digits : int = 3, prelude_str : Optional[str] = None, prelude_ratio : Optional[float] = None) -> None: + def __init__(self, prelude_file : Optional[str] = None, prelude_str : Optional[str] = None, prelude_ratio : Optional[float] = None) -> None: '''Initializes a compression with the passed prelude file, and optionally the number of digits to round to compare prelude vs. sample compression''' self.PRESET : int = 2 self.comp = lzma.LZMACompressor(preset=self.PRESET) @@ -43,7 +104,6 @@ class LzmaLlmDetector: self.prelude_ratio : float = 0.0 if prelude_ratio != None: self.prelude_ratio = prelude_ratio - self.FUZZINESS_THRESHOLD = fuzziness_digits self.SHORT_SAMPLE_THRESHOLD : int = 350 # What sample length is considered "short" if prelude_file != None: @@ -102,39 +162,39 @@ class LzmaLlmDetector: if self.prelude_ratio == 0.0: return None (prelude_score, sample_score) = self.get_compression_ratio(sample) - #print(str((prelude_score, sample_score))) + print(str((self.prelude_ratio, sample_score))) delta = prelude_score - sample_score determination = 'AI' if delta < 0: determination = 'Human' - # If the sample doesn't 'move the needle', it's very close - # if round(delta, self.FUZZINESS_THRESHOLD) == 0 and len(sample) >= self.SHORT_SAMPLE_THRESHOLD: - # #print('Sample len to default to AI: ' + str(len(sample))) - # determination = 'AI' - # if round(delta, self.FUZZINESS_THRESHOLD) == 0 and len(sample) < self.SHORT_SAMPLE_THRESHOLD: - # #print('Sample len to default to Human: ' + str(len(sample))) - # determination = 'Human' - #if abs(delta * 100) < .1 and determination == 'AI': - # print("Very low-confidence determination of: " + determination) return (determination, abs(delta * 100)) -def run_on_file(filename : str, fuzziness : int = 3) -> Optional[Score]: +def run_on_file(filename : str) -> Optional[Score]: '''Given a filename (and an optional number of decimal places to round to) returns the score for the contents of that file''' with open(filename, 'r') as fp: - l = LzmaLlmDetector(PRELUDE_FILE, fuzziness) + if ENGINE == CompressionEngine.LZMA: + l = LzmaLlmDetector(prelude_file=PRELUDE_FILE) + elif ENGINE == CompressionEngine.ZLIB: + l = ZlibLlmDetector(prelude_file=PRELUDE_FILE) txt = fp.read() #print('Calculating score for input of length ' + str(len(txt))) return l.score_text(txt) -def _score_chunk(c : str, fuzziness : int = 3, prelude_file : Optional[str] = None, prelude_ratio : Optional[float] = None) -> Score: +def _score_chunk(c : str, prelude_file : Optional[str] = None, prelude_ratio : Optional[float] = None) -> Score: if prelude_file != None: - l = LzmaLlmDetector(fuzziness_digits=fuzziness, prelude_file=prelude_file) + if ENGINE == CompressionEngine.LZMA: + l = LzmaLlmDetector(prelude_file=prelude_file) + if ENGINE == CompressionEngine.ZLIB: + l = ZlibLlmDetector(prelude_file=prelude_file) else: - l = LzmaLlmDetector(fuzziness_digits=fuzziness, prelude_str=PRELUDE_STR, prelude_ratio=prelude_ratio) + if ENGINE == CompressionEngine.LZMA: + l = LzmaLlmDetector(prelude_str=PRELUDE_STR, prelude_ratio=prelude_ratio) + if ENGINE == CompressionEngine.ZLIB: + l = ZlibLlmDetector(prelude_str=PRELUDE_STR, prelude_ratio=prelude_ratio) return l.score_text(c) -def run_on_file_chunked(filename : str, chunk_size : int = 1500, fuzziness : int = 3, prelude_ratio : Optional[float] = None) -> Optional[Score]: +def run_on_file_chunked(filename : str, chunk_size : int = 1500, prelude_ratio : Optional[float] = None) -> Optional[Score]: ''' Given a filename (and an optional chunk size and number of decimal places to round to) returns the score for the contents of that file. This function chunks the file into at most chunk_size parts to score separately, then returns an average. This prevents a very large input @@ -142,9 +202,9 @@ def run_on_file_chunked(filename : str, chunk_size : int = 1500, fuzziness : int ''' with open(filename, 'r') as fp: contents = fp.read() - return run_on_text_chunked(contents, chunk_size, fuzziness=fuzziness, prelude_ratio=prelude_ratio) + return run_on_text_chunked(contents, chunk_size, prelude_ratio=prelude_ratio) -def run_on_text_chunked(s : str, chunk_size : int = 1500, fuzziness : int = 3, prelude_file : Optional[str] = None, prelude_ratio : Optional[float] = None) -> Optional[Score]: +def run_on_text_chunked(s : str, chunk_size : int = 1500, prelude_file : Optional[str] = None, prelude_ratio : Optional[float] = None) -> Optional[Score]: ''' Given a string (and an optional chunk size and number of decimal places to round to) returns the score for the passed string. This function chunks the input into at most chunk_size parts to score separately, then returns an average. This prevents a very large input @@ -163,11 +223,11 @@ def run_on_text_chunked(s : str, chunk_size : int = 1500, fuzziness : int = 3, p scores = [] if len(chunks) > 2: with Pool(cpu_count()) as pool: - for r in pool.starmap(_score_chunk, zip(chunks, itertools.repeat(fuzziness), itertools.repeat(prelude_file), itertools.repeat(prelude_ratio))): + for r in pool.starmap(_score_chunk, zip(chunks, itertools.repeat(prelude_file), itertools.repeat(prelude_ratio))): scores.append(r) else: for c in chunks: - scores.append(_score_chunk(c, fuzziness=fuzziness, prelude_file=prelude_file, prelude_ratio=prelude_ratio)) + scores.append(_score_chunk(c, prelude_file=prelude_file, prelude_ratio=prelude_ratio)) ssum : float = 0.0 for i, s in enumerate(scores): if s[0] == 'AI': @@ -183,10 +243,16 @@ def run_on_text_chunked(s : str, chunk_size : int = 1500, fuzziness : int = 3, p if __name__ == '__main__': parser = argparse.ArgumentParser() + parser.add_argument("-e", choices=['zlib', 'lzma'], help='Which compression engine to use: lzma or zlib', default='lzma', required=False) group = parser.add_mutually_exclusive_group() group.add_argument("-s", help='Read from stdin until EOF is reached instead of from a file', required=False, action='store_true') group.add_argument("sample_files", nargs='*', help='Text file(s) containing the sample to classify', default="") args = parser.parse_args() + if args.e: + if args.e == 'lzma': + ENGINE = CompressionEngine.LZMA + elif args.e == 'zlib': + ENGINE = CompressionEngine.ZLIB if args.s: print(str(run_on_text_chunked(''.join(list(sys.stdin))))) elif len(args.sample_files) == 0: