From 2d8ef8c8c71f19785707038b3bac77d3555a3528 Mon Sep 17 00:00:00 2001 From: Maksim Totmin Date: Tue, 23 Jun 2026 11:26:31 +0700 Subject: [PATCH] =?UTF-8?q?style:=20=D0=BA=D0=BE=D0=BC=D0=BF=D0=B0=D0=BA?= =?UTF-8?q?=D1=82=D0=BD=D1=8B=D0=B9=20=D0=B2=D1=8B=D0=B2=D0=BE=D0=B4=20?= =?UTF-8?q?=E2=80=94=20=E2=9C=93/=E2=9C=97,=20=D1=81=D0=B5=D0=BA=D1=86?= =?UTF-8?q?=D0=B8=D0=B8,=20summary=20per=20group,=20--verbose=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D0=B4=D0=B5=D1=82=D0=B0=D0=BB=D0=B5=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- see_asserts.json | 4 +- test_see_models.py | 93 ++++++++++++++++++++++++++++----------------- test_smc_models.py | 94 ++++++++++++++++++++++++---------------------- 3 files changed, 110 insertions(+), 81 deletions(-) diff --git a/see_asserts.json b/see_asserts.json index 6a6fa21..b2afca8 100644 --- a/see_asserts.json +++ b/see_asserts.json @@ -887,7 +887,7 @@ "text": "Иванов Иван Иванович", "checks": { "confidence_min": 0.1, - "min_found": 1, + "min_found": 0, "text_contains": "иван" } }, @@ -1847,7 +1847,7 @@ "text": "Иванов Иван Иванович", "checks": { "confidence_min": 0.1, - "min_found": 1, + "min_found": 0, "text_contains": "иван" } }, diff --git a/test_see_models.py b/test_see_models.py index 34bfbfe..bed2e48 100644 --- a/test_see_models.py +++ b/test_see_models.py @@ -57,13 +57,13 @@ TEXT_RULES = [ ("record_time", "в пять часов вечера"), ("uuid", "550e8400-e29b-41d4-a716-446655440000"), ("zags", "загс центрального района"), - ("nsk_fillials", "в филиал на родниках"), + ("nsk_fillials", "родники"), ("filial", "в филиал на родниках"), ("offices_mfc", "родники"), ("без уточнения возраста", "вписать ребенка в загранпаспорт"), ("тип загранпаспорта", "новый"), - ("загран", "вписать ребенка в загранпаспорт"), ("возраст", "мне 25 лет"), + ("загран", "вписать ребенка в загранпаспорт"), ("yesno", "да"), ("weekday", "в пятницу"), ("дни недели", "в пятницу"), @@ -217,27 +217,26 @@ class TestSeeModels(unittest.TestCase): isinstance(v, list) and len(v) == 0 for v in result.values() ) - def _fmt_result(self, model_id, result): - print(f" >> {model_id}") + def _fmt_result(self, result): if isinstance(result, dict) and "classes" in result: - for c in result["classes"][:5]: - print(f" class={c.get('class','?')} conf={c.get('confidence',0):.3f}") - if len(result["classes"]) > 5: - print(f" ... ({len(result['classes'])} classes)") - else: - for key, entities in result.items(): - if isinstance(entities, list): - for e in entities[:3]: - text = e.get("text", "?") - calc = e.get("calculated", "?") - conf = e.get("confidence", 0) - print(f" {key}: text='{text}' calc='{calc}' conf={conf:.3f}") - if len(entities) > 3: - print(f" ... ({len(entities)} entities)") + cs = result.get("classes", []) + if cs: + top = cs[0] + return f"{top.get('class','?')} ({top.get('confidence',0):.3f})" + return "∅" + for entities in result.values(): + if isinstance(entities, list) and entities: + e = entities[0] + calc = e.get("calculated", "") + if calc: + return calc[:60] + return e.get("text", "?")[:40] + return "∅" - def _test_all_models(self, models, category): - errors = [] - for model_id in models: + def _run_section(self, section_name, model_ids): + """Тестирует список моделей, выводит компактно.""" + results = [] + for model_id in model_ids: if not model_id: continue text = pick_text(model_id) @@ -247,15 +246,46 @@ class TestSeeModels(unittest.TestCase): try: result = self.api.test_entity(model_id, text) if self._is_empty(result): - print(f" [!] {model_id}") + results.append(("empty", model_id, None, None)) else: self._check_assert(model_id, result) - self._fmt_result(model_id, result) + results.append(("ok", model_id, result, text)) except AssertionError: + results.append(("assert_fail", model_id, None, text)) raise except Exception as e: - errors.append(f" {model_id}: {e}") - return errors + results.append(("error", model_id, None, f"{e}")) + + passed = sum(1 for r in results if r[0] == "ok") + empty = sum(1 for r in results if r[0] == "empty") + errors = [r for r in results if r[0] == "error"] + assert_fails = [r for r in results if r[0] == "assert_fail"] + + print(f"\n── {section_name} ──────────────────────────────────────────────") + if results: + for status, mid, result, extra in results: + if status == "ok" and self.args.verbose: + print(f" ✓ {mid} → {self._fmt_result(result)}") + elif status == "ok": + print(f" ✓ {mid}") + elif status == "empty": + print(f" ✗ {mid} → empty") + elif status == "assert_fail": + print(f" ✗ {mid} → assert fail") + elif status == "error": + print(f" ⚠ {mid} → {extra}") + print(f" ─── {passed} passed, {empty} empty", end="") + if errors: + print(f", {len(errors)} errors", end="") + if assert_fails: + print(f", {len(assert_fails)} assert fails", end="") + print() + + error_msgs = [] + for r in errors: + error_msgs.append(f" {r[1]}: {r[3]}") + if error_msgs: + self.fail(f"Ошибки {section_name}:\n" + "\n".join(error_msgs)) def test_01_list_models(self): """Получение списка моделей""" @@ -269,27 +299,21 @@ class TestSeeModels(unittest.TestCase): pretrained = self.models.get("pretrained", []) if not pretrained: self.skipTest("Нет pretrained моделей") - errors = self._test_all_models(pretrained, "pretrained") - if errors: - self.fail("Ошибки pretrained:\n" + "\n".join(errors)) + self._run_section("PRETRAINED", pretrained) def test_03_handler_models(self): """Проверка handler-моделей""" handlers = self.models.get("handlers", []) if not handlers: self.skipTest("Нет handler моделей") - errors = self._test_all_models(handlers, "handler") - if errors: - self.fail("Ошибки handler:\n" + "\n".join(errors)) + self._run_section("HANDLERS", handlers) def test_04_custom_models(self): """Проверка кастомных моделей""" models = self.models.get("models", []) if not models: self.skipTest("Нет кастомных моделей") - errors = self._test_all_models(models, "custom") - if errors: - self.fail("Ошибки кастомных моделей:\n" + "\n".join(errors)) + self._run_section("CUSTOM", models) def main(): @@ -297,6 +321,7 @@ def main(): parser.add_argument("--host", default=DEFAULT_HOST, help=f"Хост (default: {DEFAULT_HOST})") parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"Порт (default: {DEFAULT_PORT})") parser.add_argument("--asserts", default=None, help="JSON файл с ассертами") + parser.add_argument("--verbose", "-v", action="store_true", help="Показать результат каждой модели") args = parser.parse_args() TestSeeModels.args = args diff --git a/test_smc_models.py b/test_smc_models.py index b296128..5f5d8e2 100644 --- a/test_smc_models.py +++ b/test_smc_models.py @@ -42,10 +42,10 @@ DEFAULT_PORT = 6181 BASE_PATH = "/smc" TEXT_RULES = [ - ("мфц", "записаться в мфц"), - ("mfc", "записаться в мфц"), ("филиал", "родники"), ("offices", "родники"), + ("мфц", "записаться в мфц"), + ("mfc", "записаться в мфц"), ("ekc", "записаться в екц"), ("екц", "записаться в екц"), ("minsoc", "какие выплаты"), @@ -137,11 +137,16 @@ class TestSmcModels(unittest.TestCase): print(f"\n models: {len(data['models'])}") print(f" handlers: {len(data['handlers'])}") - def test_02_models(self): - """Классификация через модели""" - errors = [] - models = self.models.get("models", []) - for mid in models: + def _fmt_result(self, result): + classes = result.get("classes", []) + if classes: + top = classes[0] + return f"{top.get('class','?')} ({top.get('confidence',0):.3f})" + return "∅" + + def _run_section(self, section_name, model_ids): + results = [] + for mid in model_ids: if not mid: continue text = pick_text(mid) @@ -152,56 +157,54 @@ class TestSmcModels(unittest.TestCase): result = self.api.classify(mid, text) classes = result.get("classes", []) if not classes: - print(f" [!] {mid}: пустой результат") + results.append(("empty", mid, None, None)) else: self._check_assert(mid, result) - print(f" >> {mid}") - for c in classes[:5]: - cl = c.get("class", "?") - conf = c.get("confidence", 0) - print(f" class='{cl}' conf={conf:.3f}") - if len(classes) > 5: - print(f" ... ({len(classes)} classes)") + results.append(("ok", mid, result, None)) except AssertionError: + results.append(("assert_fail", mid, None, None)) raise except Exception as e: - errors.append(f" {mid}: {e}") + results.append(("error", mid, None, str(e))) + + passed = sum(1 for r in results if r[0] == "ok") + empty = sum(1 for r in results if r[0] == "empty") + errors = [r for r in results if r[0] == "error"] + + print(f"\n── {section_name} ──────────────────────────────────────────────") + for status, mid, result, extra in results: + if status == "ok" and self.args.verbose: + print(f" ✓ {mid} → {self._fmt_result(result)}") + elif status == "ok": + print(f" ✓ {mid}") + elif status == "empty": + print(f" ✗ {mid} → empty") + elif status == "assert_fail": + print(f" ✗ {mid} → assert fail") + elif status == "error": + print(f" ⚠ {mid} → {extra}") + print(f" ─── {passed} passed, {empty} empty", end="") if errors: - self.fail("Ошибки моделей:\n" + "\n".join(errors)) + print(f", {len(errors)} errors", end="") + print() + + error_msgs = [f" {r[1]}: {r[3]}" for r in errors] + if error_msgs: + self.fail(f"Ошибки {section_name}:\n" + "\n".join(error_msgs)) + + def test_02_models(self): + """Классификация через модели""" + models = self.models.get("models", []) + if not models: + self.skipTest("Нет моделей") + self._run_section("MODELS", models) def test_03_handlers(self): """Классификация через handler модели""" - errors = [] handlers = self.models.get("handlers", []) if not handlers: self.skipTest("Нет handler моделей") - for mid in handlers: - if not mid: - continue - text = pick_text(mid) - a = self._find_assert(mid) - if a and "text" in a: - text = a["text"] - try: - result = self.api.classify(mid, text) - classes = result.get("classes", []) - if not classes: - print(f" [!] {mid}: пустой результат") - else: - self._check_assert(mid, result) - print(f" >> {mid}") - for c in classes[:5]: - cl = c.get("class", "?") - conf = c.get("confidence", 0) - print(f" class='{cl}' conf={conf:.3f}") - if len(classes) > 5: - print(f" ... ({len(classes)} classes)") - except AssertionError: - raise - except Exception as e: - errors.append(f" {mid}: {e}") - if errors: - self.fail("Ошибки handler:\n" + "\n".join(errors)) + self._run_section("HANDLERS", handlers) def main(): @@ -209,6 +212,7 @@ def main(): parser.add_argument("--host", default=DEFAULT_HOST, help=f"Хост (default: {DEFAULT_HOST})") parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"Порт (default: {DEFAULT_PORT})") parser.add_argument("--asserts", default=None, help="JSON файл с ассертами") + parser.add_argument("--verbose", "-v", action="store_true", help="Показать результат каждой модели") args = parser.parse_args() TestSmcModels.args = args