kopia lustrzana https://github.com/thinkst/zippy
Completed a 500/set test with CHEAT
Signed-off-by: Jacob Torrey <jacob@thinkst.com>pull/6/head
rodzic
894a09c332
commit
217ef17c25
File diff suppressed because one or more lines are too long
Plik binarny nie jest wyświetlany.
Przed Szerokość: | Wysokość: | Rozmiar: 58 KiB Po Szerokość: | Wysokość: | Rozmiar: 66 KiB |
6480
gptzero-report.xml
6480
gptzero-report.xml
File diff suppressed because one or more lines are too long
4544
openai-report.xml
4544
openai-report.xml
File diff suppressed because one or more lines are too long
35
plot_rocs.py
35
plot_rocs.py
|
@ -3,13 +3,13 @@
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
from sklearn.metrics import roc_curve, auc
|
from sklearn.metrics import roc_curve, auc
|
||||||
from lzma_detect import run_on_file_chunked, PRELUDE_STR, LzmaLlmDetector
|
import re
|
||||||
from pathlib import Path
|
|
||||||
from itertools import chain
|
|
||||||
from math import sqrt
|
|
||||||
from junitparser import JUnitXml
|
from junitparser import JUnitXml
|
||||||
|
|
||||||
MODELS = ['lzma', 'roberta', 'gptzero', 'openai']
|
MODELS = ['lzma', 'roberta', 'gptzero', 'openai']
|
||||||
|
SKIPCASES = ['gpt2', 'gpt3']
|
||||||
|
|
||||||
|
MAX_PER_CASE = 500
|
||||||
|
|
||||||
plt.figure()
|
plt.figure()
|
||||||
|
|
||||||
|
@ -22,30 +22,51 @@ for model in MODELS:
|
||||||
|
|
||||||
truths = []
|
truths = []
|
||||||
scores = []
|
scores = []
|
||||||
|
per_case = {}
|
||||||
|
fails_per_case = {}
|
||||||
for c in cases:
|
for c in cases:
|
||||||
score = float(c._elem.getchildren()[0].getchildren()[0].values()[1])
|
if c.name is None:
|
||||||
|
print("ERROR")
|
||||||
|
continue
|
||||||
|
cname = re.sub('\[.*$', '', c.name)
|
||||||
|
if any(sub in cname for sub in SKIPCASES):
|
||||||
|
continue
|
||||||
|
if cname in per_case.keys():
|
||||||
|
per_case[cname] += 1
|
||||||
|
else:
|
||||||
|
per_case[cname] = 1
|
||||||
|
fails_per_case[cname] = 0
|
||||||
|
if per_case[cname] > MAX_PER_CASE:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
score = float(c._elem.getchildren()[0].getchildren()[0].values()[1])
|
||||||
|
except:
|
||||||
|
continue
|
||||||
if 'human' in c.name:
|
if 'human' in c.name:
|
||||||
truths.append(1)
|
truths.append(1)
|
||||||
if c.is_passed:
|
if c.is_passed:
|
||||||
scores.append(score)
|
scores.append(score)
|
||||||
else:
|
else:
|
||||||
|
fails_per_case[cname] += 1
|
||||||
scores.append(score * -1.0)
|
scores.append(score * -1.0)
|
||||||
else:
|
else:
|
||||||
truths.append(-1)
|
truths.append(-1)
|
||||||
if c.is_passed:
|
if c.is_passed:
|
||||||
scores.append(score * -1.0)
|
scores.append(score * -1.0)
|
||||||
else:
|
else:
|
||||||
|
fails_per_case[cname] += 1
|
||||||
scores.append(score)
|
scores.append(score)
|
||||||
|
|
||||||
y_true = np.array(truths)
|
y_true = np.array(truths)
|
||||||
y_scores = np.array(scores)
|
y_scores = np.array(scores)
|
||||||
|
print("Failures per case for " + model)
|
||||||
|
print(fails_per_case)
|
||||||
# Compute the false positive rate (FPR), true positive rate (TPR), and threshold values
|
# Compute the false positive rate (FPR), true positive rate (TPR), and threshold values
|
||||||
fpr, tpr, thresholds = roc_curve(y_true, y_scores)
|
fpr, tpr, thresholds = roc_curve(y_true, y_scores)
|
||||||
gmeans = np.sqrt(tpr * (1-fpr))
|
gmeans = np.sqrt(tpr * (1-fpr))
|
||||||
ix = np.argmax(gmeans)
|
ix = np.argmax(gmeans)
|
||||||
print('Best Threshold=%f, G-Mean=%.3f' % (thresholds[ix], gmeans[ix]))
|
print('Best Threshold=%f, G-Mean=%.3f' % (thresholds[ix], gmeans[ix]))
|
||||||
print(thresholds)
|
#print(thresholds)
|
||||||
# calculate the g-mean for each threshold
|
# calculate the g-mean for each threshold
|
||||||
# locate the index of the largest g-mean
|
# locate the index of the largest g-mean
|
||||||
# Calculate the area under the ROC curve (AUC)
|
# Calculate the area under the ROC curve (AUC)
|
||||||
|
|
|
@ -8,7 +8,7 @@ AI_SAMPLE_DIR = 'samples/llm-generated/'
|
||||||
HUMAN_SAMPLE_DIR = 'samples/human-generated/'
|
HUMAN_SAMPLE_DIR = 'samples/human-generated/'
|
||||||
|
|
||||||
MIN_LEN = 150
|
MIN_LEN = 150
|
||||||
NUM_JSONL_SAMPLES = 50
|
NUM_JSONL_SAMPLES = 500
|
||||||
|
|
||||||
ai_files = os.listdir(AI_SAMPLE_DIR)
|
ai_files = os.listdir(AI_SAMPLE_DIR)
|
||||||
human_files = os.listdir(HUMAN_SAMPLE_DIR)
|
human_files = os.listdir(HUMAN_SAMPLE_DIR)
|
||||||
|
@ -99,3 +99,42 @@ def test_chatgptnews_jsonl(i, record_property):
|
||||||
(classification, score) = run_on_text_chunked(i.get('chatgpt', ''))
|
(classification, score) = run_on_text_chunked(i.get('chatgpt', ''))
|
||||||
record_property("score", str(score))
|
record_property("score", str(score))
|
||||||
assert classification == 'AI', NEWS_JSONL_FILE + ' is a AI-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
|
assert classification == 'AI', NEWS_JSONL_FILE + ' is a AI-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
|
||||||
|
|
||||||
|
CHEAT_HUMAN_JSONL_FILE = 'samples/ieee-init.jsonl'
|
||||||
|
ch_samples = []
|
||||||
|
with jsonlines.open(CHEAT_HUMAN_JSONL_FILE) as reader:
|
||||||
|
for obj in reader:
|
||||||
|
if len(obj.get('abstract', '')) >= MIN_LEN:
|
||||||
|
ch_samples.append(obj)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('i', ch_samples[0:NUM_JSONL_SAMPLES])
|
||||||
|
def test_cheat_human_jsonl(i, record_property):
|
||||||
|
(classification, score) = run_on_text_chunked(i.get('abstract', ''))
|
||||||
|
record_property("score", str(score))
|
||||||
|
assert classification == 'Human', CHEAT_HUMAN_JSONL_FILE + ':' + str(i.get('id')) + ' [' + str(len(i.get('abstract', ''))) + '] (title: ' + i.get('title', "").replace('\n', ' ')[:15] + ') is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8))
|
||||||
|
|
||||||
|
CHEAT_GEN_JSONL_FILE = 'samples/ieee-chatgpt-generation.jsonl'
|
||||||
|
cg_samples = []
|
||||||
|
with jsonlines.open(CHEAT_GEN_JSONL_FILE) as reader:
|
||||||
|
for obj in reader:
|
||||||
|
if len(obj.get('abstract', '')) >= MIN_LEN:
|
||||||
|
cg_samples.append(obj)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('i', cg_samples[0:NUM_JSONL_SAMPLES])
|
||||||
|
def test_cheat_generation_jsonl(i, record_property):
|
||||||
|
(classification, score) = run_on_text_chunked(i.get('abstract', ''))
|
||||||
|
record_property("score", str(score))
|
||||||
|
assert classification == 'AI', CHEAT_GEN_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
|
||||||
|
|
||||||
|
CHEAT_POLISH_JSONL_FILE = 'samples/ieee-chatgpt-polish.jsonl'
|
||||||
|
cp_samples = []
|
||||||
|
with jsonlines.open(CHEAT_POLISH_JSONL_FILE) as reader:
|
||||||
|
for obj in reader:
|
||||||
|
if len(obj.get('abstract', '')) >= MIN_LEN:
|
||||||
|
cp_samples.append(obj)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('i', cp_samples[0:NUM_JSONL_SAMPLES])
|
||||||
|
def test_cheat_polish_jsonl(i, record_property):
|
||||||
|
(classification, score) = run_on_text_chunked(i.get('abstract', ''))
|
||||||
|
record_property("score", str(score))
|
||||||
|
assert classification == 'AI', CHEAT_POLISH_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
|
||||||
|
|
|
@ -5,7 +5,7 @@ from warnings import warn
|
||||||
from openai_detect import run_on_file_chunked, run_on_text_chunked
|
from openai_detect import run_on_file_chunked, run_on_text_chunked
|
||||||
|
|
||||||
MIN_LEN = 1000
|
MIN_LEN = 1000
|
||||||
NUM_JSONL_SAMPLES = 50
|
NUM_JSONL_SAMPLES = 500
|
||||||
|
|
||||||
AI_SAMPLE_DIR = 'samples/llm-generated/'
|
AI_SAMPLE_DIR = 'samples/llm-generated/'
|
||||||
HUMAN_SAMPLE_DIR = 'samples/human-generated/'
|
HUMAN_SAMPLE_DIR = 'samples/human-generated/'
|
||||||
|
@ -108,3 +108,42 @@ def test_chatgptnews_jsonl(i, record_property):
|
||||||
(classification, score) = run_on_text_chunked(i.get('chatgpt', ''))
|
(classification, score) = run_on_text_chunked(i.get('chatgpt', ''))
|
||||||
record_property("score", str(score))
|
record_property("score", str(score))
|
||||||
assert classification == 'AI', NEWS_JSONL_FILE + ' is a AI-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
|
assert classification == 'AI', NEWS_JSONL_FILE + ' is a AI-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
|
||||||
|
|
||||||
|
CHEAT_HUMAN_JSONL_FILE = 'samples/ieee-init.jsonl'
|
||||||
|
ch_samples = []
|
||||||
|
with jsonlines.open(CHEAT_HUMAN_JSONL_FILE) as reader:
|
||||||
|
for obj in reader:
|
||||||
|
if len(obj.get('abstract', '')) >= MIN_LEN:
|
||||||
|
ch_samples.append(obj)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('i', ch_samples[0:NUM_JSONL_SAMPLES])
|
||||||
|
def test_cheat_human_jsonl(i, record_property):
|
||||||
|
(classification, score) = run_on_text_chunked(i.get('abstract', ''))
|
||||||
|
record_property("score", str(score))
|
||||||
|
assert classification == 'Human', CHEAT_HUMAN_JSONL_FILE + ':' + str(i.get('id')) + ' [' + str(len(i.get('abstract', ''))) + '] (title: ' + i.get('title', "").replace('\n', ' ')[:15] + ') is a human-generated sample, misclassified as AI-generated with confidence ' + str(round(score, 8))
|
||||||
|
|
||||||
|
CHEAT_GEN_JSONL_FILE = 'samples/ieee-chatgpt-generation.jsonl'
|
||||||
|
cg_samples = []
|
||||||
|
with jsonlines.open(CHEAT_GEN_JSONL_FILE) as reader:
|
||||||
|
for obj in reader:
|
||||||
|
if len(obj.get('abstract', '')) >= MIN_LEN:
|
||||||
|
cg_samples.append(obj)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('i', cg_samples[0:NUM_JSONL_SAMPLES])
|
||||||
|
def test_cheat_generation_jsonl(i, record_property):
|
||||||
|
(classification, score) = run_on_text_chunked(i.get('abstract', ''))
|
||||||
|
record_property("score", str(score))
|
||||||
|
assert classification == 'AI', CHEAT_GEN_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
|
||||||
|
|
||||||
|
CHEAT_POLISH_JSONL_FILE = 'samples/ieee-chatgpt-polish.jsonl'
|
||||||
|
cp_samples = []
|
||||||
|
with jsonlines.open(CHEAT_POLISH_JSONL_FILE) as reader:
|
||||||
|
for obj in reader:
|
||||||
|
if len(obj.get('abstract', '')) >= MIN_LEN:
|
||||||
|
cp_samples.append(obj)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('i', cp_samples[0:NUM_JSONL_SAMPLES])
|
||||||
|
def test_cheat_polish_jsonl(i, record_property):
|
||||||
|
(classification, score) = run_on_text_chunked(i.get('abstract', ''))
|
||||||
|
record_property("score", str(score))
|
||||||
|
assert classification == 'AI', CHEAT_POLISH_JSONL_FILE + ':' + str(i.get('id')) + ' (title: ' + i.get('title', "").replace('\n', ' ')[:50] + ') is an LLM-generated sample, misclassified as human-generated with confidence ' + str(round(score, 8))
|
||||||
|
|
Ładowanie…
Reference in New Issue