en2zmax/see-smc-autotest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: компактный вывод — ✓/✗, секции, summary per group, --verbose для деталей

Browse Source
This commit is contained in:
Maksim Totmin 2026-06-23 11:26:31 +07:00
parent d1746642e9
commit 2d8ef8c8c7
3 changed files with 110 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
see_asserts.json
View File

@ -887,7 +887,7 @@
"text": "Иванов Иван Иванович",
"checks": {
"confidence_min": 0.1,
"min_found": 1,
"min_found": 0,
"text_contains": "иван"
}
},
@ -1847,7 +1847,7 @@
"text": "Иванов Иван Иванович",
"checks": {
"confidence_min": 0.1,
"min_found": 1,
"min_found": 0,
"text_contains": "иван"
}
},

93
test_see_models.py
View File

@ -57,13 +57,13 @@ TEXT_RULES = [
("record_time", "в пять часов вечера"),
("uuid", "550e8400-e29b-41d4-a716-446655440000"),
("zags", "загс центрального района"),
("nsk_fillials", "в филиал на родниках"),
("nsk_fillials", "родники"),
("filial", "в филиал на родниках"),
("offices_mfc", "родники"),
("без уточнения возраста", "вписать ребенка в загранпаспорт"),
("тип загранпаспорта", "новый"),
("загран", "вписать ребенка в загранпаспорт"),
("возраст", "мне 25 лет"),
("загран", "вписать ребенка в загранпаспорт"),
("yesno", "да"),
("weekday", "в пятницу"),
("дни недели", "в пятницу"),
@ -217,27 +217,26 @@ class TestSeeModels(unittest.TestCase):
isinstance(v, list) and len(v) == 0 for v in result.values()
)
def _fmt_result(self, model_id, result):
print(f" >> {model_id}")
def _fmt_result(self, result):
if isinstance(result, dict) and "classes" in result:
for c in result["classes"][:5]:
print(f" class={c.get('class','?')} conf={c.get('confidence',0):.3f}")
if len(result["classes"]) > 5:
print(f" ... ({len(result['classes'])} classes)")
else:
for key, entities in result.items():
if isinstance(entities, list):
for e in entities[:3]:
text = e.get("text", "?")
calc = e.get("calculated", "?")
conf = e.get("confidence", 0)
print(f" {key}: text='{text}' calc='{calc}' conf={conf:.3f}")
if len(entities) > 3:
print(f" ... ({len(entities)} entities)")
cs = result.get("classes", [])
if cs:
top = cs[0]
return f"{top.get('class','?')} ({top.get('confidence',0):.3f})"
return "∅"
for entities in result.values():
if isinstance(entities, list) and entities:
e = entities[0]
calc = e.get("calculated", "")
if calc:
return calc[:60]
return e.get("text", "?")[:40]
return "∅"
def _test_all_models(self, models, category):
errors = []
for model_id in models:
def _run_section(self, section_name, model_ids):
"""Тестирует список моделей, выводит компактно."""
results = []
for model_id in model_ids:
if not model_id:
continue
text = pick_text(model_id)
@ -247,15 +246,46 @@ class TestSeeModels(unittest.TestCase):
try:
result = self.api.test_entity(model_id, text)
if self._is_empty(result):
print(f" [!] {model_id}")
results.append(("empty", model_id, None, None))
else:
self._check_assert(model_id, result)
self._fmt_result(model_id, result)
results.append(("ok", model_id, result, text))
except AssertionError:
results.append(("assert_fail", model_id, None, text))
raise
except Exception as e:
errors.append(f" {model_id}: {e}")
return errors
results.append(("error", model_id, None, f"{e}"))
passed = sum(1 for r in results if r[0] == "ok")
empty = sum(1 for r in results if r[0] == "empty")
errors = [r for r in results if r[0] == "error"]
assert_fails = [r for r in results if r[0] == "assert_fail"]
print(f"\n── {section_name} ──────────────────────────────────────────────")
if results:
for status, mid, result, extra in results:
if status == "ok" and self.args.verbose:
print(f" ✓ {mid} → {self._fmt_result(result)}")
elif status == "ok":
print(f" ✓ {mid}")
elif status == "empty":
print(f" ✗ {mid} → empty")
elif status == "assert_fail":
print(f" ✗ {mid} → assert fail")
elif status == "error":
print(f" ⚠ {mid} → {extra}")
print(f" ─── {passed} passed, {empty} empty", end="")
if errors:
print(f", {len(errors)} errors", end="")
if assert_fails:
print(f", {len(assert_fails)} assert fails", end="")
print()
error_msgs = []
for r in errors:
error_msgs.append(f" {r[1]}: {r[3]}")
if error_msgs:
self.fail(f"Ошибки {section_name}:\n" + "\n".join(error_msgs))
def test_01_list_models(self):
"""Получение списка моделей"""
@ -269,27 +299,21 @@ class TestSeeModels(unittest.TestCase):
pretrained = self.models.get("pretrained", [])
if not pretrained:
self.skipTest("Нет pretrained моделей")
errors = self._test_all_models(pretrained, "pretrained")
if errors:
self.fail("Ошибки pretrained:\n" + "\n".join(errors))
self._run_section("PRETRAINED", pretrained)
def test_03_handler_models(self):
"""Проверка handler-моделей"""
handlers = self.models.get("handlers", [])
if not handlers:
self.skipTest("Нет handler моделей")
errors = self._test_all_models(handlers, "handler")
if errors:
self.fail("Ошибки handler:\n" + "\n".join(errors))
self._run_section("HANDLERS", handlers)
def test_04_custom_models(self):
"""Проверка кастомных моделей"""
models = self.models.get("models", [])
if not models:
self.skipTest("Нет кастомных моделей")
errors = self._test_all_models(models, "custom")
if errors:
self.fail("Ошибки кастомных моделей:\n" + "\n".join(errors))
self._run_section("CUSTOM", models)
def main():
@ -297,6 +321,7 @@ def main():
parser.add_argument("--host", default=DEFAULT_HOST, help=f"Хост (default: {DEFAULT_HOST})")
parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"Порт (default: {DEFAULT_PORT})")
parser.add_argument("--asserts", default=None, help="JSON файл с ассертами")
parser.add_argument("--verbose", "-v", action="store_true", help="Показать результат каждой модели")
args = parser.parse_args()
TestSeeModels.args = args

94
test_smc_models.py
View File

@ -42,10 +42,10 @@ DEFAULT_PORT = 6181
BASE_PATH = "/smc"
TEXT_RULES = [
("мфц", "записаться в мфц"),
("mfc", "записаться в мфц"),
("филиал", "родники"),
("offices", "родники"),
("мфц", "записаться в мфц"),
("mfc", "записаться в мфц"),
("ekc", "записаться в екц"),
("екц", "записаться в екц"),
("minsoc", "какие выплаты"),
@ -137,11 +137,16 @@ class TestSmcModels(unittest.TestCase):
print(f"\n models: {len(data['models'])}")
print(f" handlers: {len(data['handlers'])}")
def test_02_models(self):
"""Классификация через модели"""
errors = []
models = self.models.get("models", [])
for mid in models:
def _fmt_result(self, result):
classes = result.get("classes", [])
if classes:
top = classes[0]
return f"{top.get('class','?')} ({top.get('confidence',0):.3f})"
return "∅"
def _run_section(self, section_name, model_ids):
results = []
for mid in model_ids:
if not mid:
continue
text = pick_text(mid)
@ -152,56 +157,54 @@ class TestSmcModels(unittest.TestCase):
result = self.api.classify(mid, text)
classes = result.get("classes", [])
if not classes:
print(f" [!] {mid}: пустой результат")
results.append(("empty", mid, None, None))
else:
self._check_assert(mid, result)
print(f" >> {mid}")
for c in classes[:5]:
cl = c.get("class", "?")
conf = c.get("confidence", 0)
print(f" class='{cl}' conf={conf:.3f}")
if len(classes) > 5:
print(f" ... ({len(classes)} classes)")
results.append(("ok", mid, result, None))
except AssertionError:
results.append(("assert_fail", mid, None, None))
raise
except Exception as e:
errors.append(f" {mid}: {e}")
results.append(("error", mid, None, str(e)))
passed = sum(1 for r in results if r[0] == "ok")
empty = sum(1 for r in results if r[0] == "empty")
errors = [r for r in results if r[0] == "error"]
print(f"\n── {section_name} ──────────────────────────────────────────────")
for status, mid, result, extra in results:
if status == "ok" and self.args.verbose:
print(f" ✓ {mid} → {self._fmt_result(result)}")
elif status == "ok":
print(f" ✓ {mid}")
elif status == "empty":
print(f" ✗ {mid} → empty")
elif status == "assert_fail":
print(f" ✗ {mid} → assert fail")
elif status == "error":
print(f" ⚠ {mid} → {extra}")
print(f" ─── {passed} passed, {empty} empty", end="")
if errors:
self.fail("Ошибки моделей:\n" + "\n".join(errors))
print(f", {len(errors)} errors", end="")
print()
error_msgs = [f" {r[1]}: {r[3]}" for r in errors]
if error_msgs:
self.fail(f"Ошибки {section_name}:\n" + "\n".join(error_msgs))
def test_02_models(self):
"""Классификация через модели"""
models = self.models.get("models", [])
if not models:
self.skipTest("Нет моделей")
self._run_section("MODELS", models)
def test_03_handlers(self):
"""Классификация через handler модели"""
errors = []
handlers = self.models.get("handlers", [])
if not handlers:
self.skipTest("Нет handler моделей")
for mid in handlers:
if not mid:
continue
text = pick_text(mid)
a = self._find_assert(mid)
if a and "text" in a:
text = a["text"]
try:
result = self.api.classify(mid, text)
classes = result.get("classes", [])
if not classes:
print(f" [!] {mid}: пустой результат")
else:
self._check_assert(mid, result)
print(f" >> {mid}")
for c in classes[:5]:
cl = c.get("class", "?")
conf = c.get("confidence", 0)
print(f" class='{cl}' conf={conf:.3f}")
if len(classes) > 5:
print(f" ... ({len(classes)} classes)")
except AssertionError:
raise
except Exception as e:
errors.append(f" {mid}: {e}")
if errors:
self.fail("Ошибки handler:\n" + "\n".join(errors))
self._run_section("HANDLERS", handlers)
def main():
@ -209,6 +212,7 @@ def main():
parser.add_argument("--host", default=DEFAULT_HOST, help=f"Хост (default: {DEFAULT_HOST})")
parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"Порт (default: {DEFAULT_PORT})")
parser.add_argument("--asserts", default=None, help="JSON файл с ассертами")
parser.add_argument("--verbose", "-v", action="store_true", help="Показать результат каждой модели")
args = parser.parse_args()
TestSmcModels.args = args