Completed a 500/set test with CHEAT

Signed-off-by: Jacob Torrey <jacob@thinkst.com>

plot_rocs.py

@@ -3,13 +3,13 @@
 import numpy as np
 import matplotlib.pyplot as plt
 from sklearn.metrics import roc_curve, auc
-from lzma_detect import run_on_file_chunked, PRELUDE_STR, LzmaLlmDetector
-from pathlib import Path
-from itertools import chain
-from math import sqrt
+import re
 from junitparser import JUnitXml
 
 MODELS = ['lzma', 'roberta', 'gptzero', 'openai']
+SKIPCASES = ['gpt2', 'gpt3']
+
+MAX_PER_CASE = 500
 
 plt.figure()
 
@@ -22,30 +22,51 @@ for model in MODELS:
     
     truths = []
     scores = []
+    per_case = {}
+    fails_per_case = {}
     for c in cases:
-        score = float(c._elem.getchildren()[0].getchildren()[0].values()[1])
+        if c.name is None:
+            print("ERROR")
+            continue
+        cname = re.sub('\[.*$', '', c.name)
+        if any(sub in cname for sub in SKIPCASES):
+            continue
+        if cname in per_case.keys():
+            per_case[cname] += 1
+        else:
+            per_case[cname] = 1
+            fails_per_case[cname] = 0
+        if per_case[cname] > MAX_PER_CASE:
+            continue
+        try:
+            score = float(c._elem.getchildren()[0].getchildren()[0].values()[1])
+        except:
+            continue
         if 'human' in c.name:
             truths.append(1)
             if c.is_passed:
                 scores.append(score)
             else:
+                fails_per_case[cname] += 1
                 scores.append(score * -1.0)
         else:
             truths.append(-1)
             if c.is_passed:
                 scores.append(score * -1.0)
             else:
+                fails_per_case[cname] += 1
                 scores.append(score)
 
     y_true = np.array(truths)
     y_scores = np.array(scores) 
-
+    print("Failures per case for " + model)
+    print(fails_per_case)
     # Compute the false positive rate (FPR), true positive rate (TPR), and threshold values
     fpr, tpr, thresholds = roc_curve(y_true, y_scores)
     gmeans = np.sqrt(tpr * (1-fpr))
     ix = np.argmax(gmeans)
     print('Best Threshold=%f, G-Mean=%.3f' % (thresholds[ix], gmeans[ix]))
-    print(thresholds)
+    #print(thresholds)
     # calculate the g-mean for each threshold
     # locate the index of the largest g-mean
     # Calculate the area under the ROC curve (AUC)

test_gptzero_detect.py

@@ -8,7 +8,7 @@ AI_SAMPLE_DIR = 'samples/llm-generated/'
 HUMAN_SAMPLE_DIR = 'samples/human-generated/'
 
 MIN_LEN = 150
-NUM_JSONL_SAMPLES = 50
+NUM_JSONL_SAMPLES = 500
 
 ai_files = os.listdir(AI_SAMPLE_DIR)
 human_files = os.listdir(HUMAN_SAMPLE_DIR)
@@ -99,3 +99,42 @@ def test_chatgptnews_jsonl(i, record_property):
     (classification, score) = run_on_text_chunked(i.get('chatgpt', ''))
     record_property("score", str(score))
     assert classification == 'AI', NEWS_JSONL_FILE + ' is a AI-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
+
+CHEAT_HUMAN_JSONL_FILE = 'samples/ieee-init.jsonl'
+ch_samples = []
+with jsonlines.open(CHEAT_HUMAN_JSONL_FILE) as reader:
+    for obj in reader:
+        if len(obj.get('abstract', '')) >= MIN_LEN:
+            ch_samples.append(obj)
+
+@pytest.mark.parametrize('i', ch_samples[0:NUM_JSONL_SAMPLES])
+def test_cheat_human_jsonl(i, record_property):
+    (classification, score) = run_on_text_chunked(i.get('abstract', ''))
+    record_property("score", str(score))
+    assert classification == 'Human', CHEAT_HUMAN_JSONL_FILE + ':' + str(i.get('id')) + ' [' + str(len(i.get('abstract', ''))) + '] (title: ' + i.get('title', "").replace('\n', ' ')[:15] + ') is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8))
+
+CHEAT_GEN_JSONL_FILE = 'samples/ieee-chatgpt-generation.jsonl'
+cg_samples = []
+with jsonlines.open(CHEAT_GEN_JSONL_FILE) as reader:
+    for obj in reader:
+        if len(obj.get('abstract', '')) >= MIN_LEN:
+            cg_samples.append(obj)
+
+@pytest.mark.parametrize('i', cg_samples[0:NUM_JSONL_SAMPLES])
+def test_cheat_generation_jsonl(i, record_property):
+    (classification, score) = run_on_text_chunked(i.get('abstract', ''))
+    record_property("score", str(score))
+    assert classification == 'AI', CHEAT_GEN_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
+
+CHEAT_POLISH_JSONL_FILE = 'samples/ieee-chatgpt-polish.jsonl'
+cp_samples = []
+with jsonlines.open(CHEAT_POLISH_JSONL_FILE) as reader:
+    for obj in reader:
+        if len(obj.get('abstract', '')) >= MIN_LEN:
+            cp_samples.append(obj)
+
+@pytest.mark.parametrize('i', cp_samples[0:NUM_JSONL_SAMPLES])
+def test_cheat_polish_jsonl(i, record_property):
+    (classification, score) = run_on_text_chunked(i.get('abstract', ''))
+    record_property("score", str(score))
+    assert classification == 'AI', CHEAT_POLISH_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))

test_openai_detect.py

@@ -5,7 +5,7 @@ from warnings import warn
 from openai_detect import run_on_file_chunked, run_on_text_chunked
 
 MIN_LEN = 1000
-NUM_JSONL_SAMPLES = 50
+NUM_JSONL_SAMPLES = 500
 
 AI_SAMPLE_DIR = 'samples/llm-generated/'
 HUMAN_SAMPLE_DIR = 'samples/human-generated/'
@@ -108,3 +108,42 @@ def test_chatgptnews_jsonl(i, record_property):
     (classification, score) = run_on_text_chunked(i.get('chatgpt', ''))
     record_property("score", str(score))
     assert classification == 'AI', NEWS_JSONL_FILE + ' is a AI-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
+
+CHEAT_HUMAN_JSONL_FILE = 'samples/ieee-init.jsonl'
+ch_samples = []
+with jsonlines.open(CHEAT_HUMAN_JSONL_FILE) as reader:
+    for obj in reader:
+        if len(obj.get('abstract', '')) >= MIN_LEN:
+            ch_samples.append(obj)
+
+@pytest.mark.parametrize('i', ch_samples[0:NUM_JSONL_SAMPLES])
+def test_cheat_human_jsonl(i, record_property):
+    (classification, score) = run_on_text_chunked(i.get('abstract', ''))
+    record_property("score", str(score))
+    assert classification == 'Human', CHEAT_HUMAN_JSONL_FILE + ':' + str(i.get('id')) + ' [' + str(len(i.get('abstract', ''))) + '] (title: ' + i.get('title', "").replace('\n', ' ')[:15] + ') is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8))
+
+CHEAT_GEN_JSONL_FILE = 'samples/ieee-chatgpt-generation.jsonl'
+cg_samples = []
+with jsonlines.open(CHEAT_GEN_JSONL_FILE) as reader:
+    for obj in reader:
+        if len(obj.get('abstract', '')) >= MIN_LEN:
+            cg_samples.append(obj)
+
+@pytest.mark.parametrize('i', cg_samples[0:NUM_JSONL_SAMPLES])
+def test_cheat_generation_jsonl(i, record_property):
+    (classification, score) = run_on_text_chunked(i.get('abstract', ''))
+    record_property("score", str(score))
+    assert classification == 'AI', CHEAT_GEN_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
+
+CHEAT_POLISH_JSONL_FILE = 'samples/ieee-chatgpt-polish.jsonl'
+cp_samples = []
+with jsonlines.open(CHEAT_POLISH_JSONL_FILE) as reader:
+    for obj in reader:
+        if len(obj.get('abstract', '')) >= MIN_LEN:
+            cp_samples.append(obj)
+
+@pytest.mark.parametrize('i', cp_samples[0:NUM_JSONL_SAMPLES])
+def test_cheat_polish_jsonl(i, record_property):
+    (classification, score) = run_on_text_chunked(i.get('abstract', ''))
+    record_property("score", str(score))
+    assert classification == 'AI', CHEAT_POLISH_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))