[{"data":1,"prerenderedAt":1112},["ShallowReactive",2],{"article-alternates":3,"article-\u002Fru\u002Fai\u002Fllm-ops-prompt-versioning-testing":13},{"i18nKey":4,"paths":5},"ai-004-2026-07",{"de":6,"en":7,"es":8,"fr":9,"it":10,"ru":11,"tr":12},"\u002Fde\u002Fai\u002Fprompt-versioning-ab-testing-llm-ops","\u002Fen\u002Fai\u002Fprompt-versioning-ab-testing-llm-ops-discipline","\u002Fes\u002Fai\u002Fversionamiento-prompts-ab-testing-llm","\u002Ffr\u002Fai\u002Fversionierung-und-ab-tests-von-prompts","\u002Fit\u002Fai\u002Fprompt-versiyonlama-ab-testi-llm-disiplini","\u002Fru\u002Fai\u002Fprompt-versioning-ab-testing-llm-ops","\u002Ftr\u002Fai\u002Fprompt-versiyonlama-ve-a-b-testi-llm-operasyonun-disiplini",{"_path":14,"_dir":15,"_draft":16,"_partial":16,"_locale":17,"title":18,"description":19,"publishedAt":20,"modifiedAt":20,"category":15,"i18nKey":4,"tags":21,"readingTime":27,"author":28,"body":29,"_type":812,"_id":1107,"_source":1108,"_file":1109,"_stem":1110,"_extension":1111},"\u002Fru\u002Fai\u002Fllm-ops-prompt-versioning-testing","ai",false,"","Версионирование и A\u002FB-тестирование промптов: дисциплина LLM-операций","Систематическое тестирование выходов LLM с помощью Promptfoo и LangSmith. Практика построения evaluation pipeline для production-grade AI приложений.","2026-07-30",[22,23,24,25,26],"llm-ops","prompt-engineering","evaluation","ab-testing","mlops",8,"Roibase",{"type":30,"children":31,"toc":1094},"root",[32,40,47,52,65,70,76,90,103,435,440,447,452,457,463,468,484,490,495,500,506,511,544,554,559,594,599,605,610,615,767,772,777,783,796,804,809,984,989,1002,1008,1013,1018,1023,1078,1083,1088],{"type":33,"tag":34,"props":35,"children":36},"element","p",{},[37],{"type":38,"value":39},"text","С того момента, когда вы начинаете использовать LLM в production, вы осознаёте необходимость дисциплины классической инженерии ПО — test suite. Что происходит с консистентностью выхода при изменении промпта? Как меняется соотношение стоимость-качество при обновлении версии модели? Как конвертировать ощущение \"Claude ответил лучше\" в численную метрику? В 2026 году, когда LLM-операции достигли зрелости, побеждают те, кто систематически, а не вручную отвечает на эти вопросы. Инструменты как Promptfoo и LangSmith, а также evaluation pipeline'ы — это страховка для поддержания LLM в production.",{"type":33,"tag":41,"props":42,"children":44},"h2",{"id":43},"изменение-промпта-изменение-кода",[45],{"type":38,"value":46},"Изменение промпта = Изменение кода",{"type":33,"tag":34,"props":48,"children":49},{},[50],{"type":38,"value":51},"Представьте workflow генерации маркетингового контента. Вы отправляете промпт Claude API, получаете черновик блога. В первой версии вы пишете \"напиши\", во второй добавляете в system prompt \"Пиши для Roibase, инженерный тон\", в третьей добавляете список \"ЗАПРЕЩЁННЫЕ СЛОВА\". Каждое изменение влияет на выход, но как вы измеряете это влияние?",{"type":33,"tag":34,"props":53,"children":54},{},[55,57,63],{"type":38,"value":56},"В классическом ПО есть unit-тесты — вход фиксирован, выход детерминирован. В LLM вход фиксирован, но выход стохастичен. Одного запуска недостаточно. Необходимо запустить один и тот же промпт 10 раз с разными seed'ами, посмотреть на среднее количество токенов, latency, оценку coherence. Поэтому ",{"type":33,"tag":58,"props":59,"children":60},"strong",{},[61],{"type":38,"value":62},"версионирование промпта",{"type":38,"value":64}," критично как версионирование кода. Вы отслеживаете изменения промпта в Git commit'ах, но можете не отслеживать изменения выхода. Здесь вступает в дело evaluation suite: при каждом commit'е автоматически запускаются тесты, вы видите регрессию метрик.",{"type":33,"tag":34,"props":66,"children":67},{},[68],{"type":38,"value":69},"Конкретный сценарий: в n8n workflow вы генерируете контент через Claude. Изменив в промпте \"1500 слов\" на \"1400-1600 слов\", средняя длина падает с 1520 до 1480 слов, стоимость токенов снижается на 3%, но оценка читаемости теряет 0.2 пункта. Чтобы увидеть этот трейдофф без ручных экспериментов, необходим автоматический evaluation pipeline.",{"type":33,"tag":41,"props":71,"children":73},{"id":72},"promptfoo-regression-test-suite-для-промптов",[74],{"type":38,"value":75},"Promptfoo: Regression Test Suite для промптов",{"type":33,"tag":34,"props":77,"children":78},{},[79,81,88],{"type":38,"value":80},"Promptfoo — это open-source CLI-инструмент, где вы определяете промпты в YAML-конфиге, задаёте test case'ы в CSV или JSON, пишете assertion'ы. Команда ",{"type":33,"tag":82,"props":83,"children":85},"code",{"className":84},[],[86],{"type":38,"value":87},"promptfoo eval",{"type":38,"value":89}," запускает все варианты, выдаёт таблицу успех\u002Fнеудач.",{"type":33,"tag":34,"props":91,"children":92},{},[93,95,101],{"type":38,"value":94},"Типичный ",{"type":33,"tag":82,"props":96,"children":98},{"className":97},[],[99],{"type":38,"value":100},"promptfoo.yaml",{"type":38,"value":102}," выглядит так:",{"type":33,"tag":104,"props":105,"children":109},"pre",{"code":106,"language":107,"meta":17,"className":108,"style":17},"prompts:\n  - id: baseline\n    text: \"Write a blog post about {{topic}}\"\n  - id: roibase-tone\n    text: \"Write a blog post about {{topic}}. Use engineering discipline tone. No hype words.\"\n\nproviders:\n  - anthropic:messages:claude-3-5-sonnet-20241022\n\ntests:\n  - vars:\n      topic: \"server-side GTM setup\"\n    assert:\n      - type: contains\n        value: \"first-party\"\n      - type: javascript\n        value: output.length > 1400 && output.length \u003C 1600\n      - type: cost\n        threshold: 0.05\n","yaml","language-yaml shiki shiki-themes github-dark",[110],{"type":33,"tag":82,"props":111,"children":112},{"__ignoreMap":17},[113,131,156,174,195,212,222,235,247,255,268,285,303,316,339,357,378,395,416],{"type":33,"tag":114,"props":115,"children":118},"span",{"class":116,"line":117},"line",1,[119,125],{"type":33,"tag":114,"props":120,"children":122},{"style":121},"--shiki-default:#85E89D",[123],{"type":38,"value":124},"prompts",{"type":33,"tag":114,"props":126,"children":128},{"style":127},"--shiki-default:#E1E4E8",[129],{"type":38,"value":130},":\n",{"type":33,"tag":114,"props":132,"children":134},{"class":116,"line":133},2,[135,140,145,150],{"type":33,"tag":114,"props":136,"children":137},{"style":127},[138],{"type":38,"value":139},"  - ",{"type":33,"tag":114,"props":141,"children":142},{"style":121},[143],{"type":38,"value":144},"id",{"type":33,"tag":114,"props":146,"children":147},{"style":127},[148],{"type":38,"value":149},": ",{"type":33,"tag":114,"props":151,"children":153},{"style":152},"--shiki-default:#9ECBFF",[154],{"type":38,"value":155},"baseline\n",{"type":33,"tag":114,"props":157,"children":159},{"class":116,"line":158},3,[160,165,169],{"type":33,"tag":114,"props":161,"children":162},{"style":121},[163],{"type":38,"value":164},"    text",{"type":33,"tag":114,"props":166,"children":167},{"style":127},[168],{"type":38,"value":149},{"type":33,"tag":114,"props":170,"children":171},{"style":152},[172],{"type":38,"value":173},"\"Write a blog post about {{topic}}\"\n",{"type":33,"tag":114,"props":175,"children":177},{"class":116,"line":176},4,[178,182,186,190],{"type":33,"tag":114,"props":179,"children":180},{"style":127},[181],{"type":38,"value":139},{"type":33,"tag":114,"props":183,"children":184},{"style":121},[185],{"type":38,"value":144},{"type":33,"tag":114,"props":187,"children":188},{"style":127},[189],{"type":38,"value":149},{"type":33,"tag":114,"props":191,"children":192},{"style":152},[193],{"type":38,"value":194},"roibase-tone\n",{"type":33,"tag":114,"props":196,"children":198},{"class":116,"line":197},5,[199,203,207],{"type":33,"tag":114,"props":200,"children":201},{"style":121},[202],{"type":38,"value":164},{"type":33,"tag":114,"props":204,"children":205},{"style":127},[206],{"type":38,"value":149},{"type":33,"tag":114,"props":208,"children":209},{"style":152},[210],{"type":38,"value":211},"\"Write a blog post about {{topic}}. Use engineering discipline tone. No hype words.\"\n",{"type":33,"tag":114,"props":213,"children":215},{"class":116,"line":214},6,[216],{"type":33,"tag":114,"props":217,"children":219},{"emptyLinePlaceholder":218},true,[220],{"type":38,"value":221},"\n",{"type":33,"tag":114,"props":223,"children":225},{"class":116,"line":224},7,[226,231],{"type":33,"tag":114,"props":227,"children":228},{"style":121},[229],{"type":38,"value":230},"providers",{"type":33,"tag":114,"props":232,"children":233},{"style":127},[234],{"type":38,"value":130},{"type":33,"tag":114,"props":236,"children":237},{"class":116,"line":27},[238,242],{"type":33,"tag":114,"props":239,"children":240},{"style":127},[241],{"type":38,"value":139},{"type":33,"tag":114,"props":243,"children":244},{"style":152},[245],{"type":38,"value":246},"anthropic:messages:claude-3-5-sonnet-20241022\n",{"type":33,"tag":114,"props":248,"children":250},{"class":116,"line":249},9,[251],{"type":33,"tag":114,"props":252,"children":253},{"emptyLinePlaceholder":218},[254],{"type":38,"value":221},{"type":33,"tag":114,"props":256,"children":258},{"class":116,"line":257},10,[259,264],{"type":33,"tag":114,"props":260,"children":261},{"style":121},[262],{"type":38,"value":263},"tests",{"type":33,"tag":114,"props":265,"children":266},{"style":127},[267],{"type":38,"value":130},{"type":33,"tag":114,"props":269,"children":271},{"class":116,"line":270},11,[272,276,281],{"type":33,"tag":114,"props":273,"children":274},{"style":127},[275],{"type":38,"value":139},{"type":33,"tag":114,"props":277,"children":278},{"style":121},[279],{"type":38,"value":280},"vars",{"type":33,"tag":114,"props":282,"children":283},{"style":127},[284],{"type":38,"value":130},{"type":33,"tag":114,"props":286,"children":288},{"class":116,"line":287},12,[289,294,298],{"type":33,"tag":114,"props":290,"children":291},{"style":121},[292],{"type":38,"value":293},"      topic",{"type":33,"tag":114,"props":295,"children":296},{"style":127},[297],{"type":38,"value":149},{"type":33,"tag":114,"props":299,"children":300},{"style":152},[301],{"type":38,"value":302},"\"server-side GTM setup\"\n",{"type":33,"tag":114,"props":304,"children":306},{"class":116,"line":305},13,[307,312],{"type":33,"tag":114,"props":308,"children":309},{"style":121},[310],{"type":38,"value":311},"    assert",{"type":33,"tag":114,"props":313,"children":314},{"style":127},[315],{"type":38,"value":130},{"type":33,"tag":114,"props":317,"children":319},{"class":116,"line":318},14,[320,325,330,334],{"type":33,"tag":114,"props":321,"children":322},{"style":127},[323],{"type":38,"value":324},"      - ",{"type":33,"tag":114,"props":326,"children":327},{"style":121},[328],{"type":38,"value":329},"type",{"type":33,"tag":114,"props":331,"children":332},{"style":127},[333],{"type":38,"value":149},{"type":33,"tag":114,"props":335,"children":336},{"style":152},[337],{"type":38,"value":338},"contains\n",{"type":33,"tag":114,"props":340,"children":342},{"class":116,"line":341},15,[343,348,352],{"type":33,"tag":114,"props":344,"children":345},{"style":121},[346],{"type":38,"value":347},"        value",{"type":33,"tag":114,"props":349,"children":350},{"style":127},[351],{"type":38,"value":149},{"type":33,"tag":114,"props":353,"children":354},{"style":152},[355],{"type":38,"value":356},"\"first-party\"\n",{"type":33,"tag":114,"props":358,"children":360},{"class":116,"line":359},16,[361,365,369,373],{"type":33,"tag":114,"props":362,"children":363},{"style":127},[364],{"type":38,"value":324},{"type":33,"tag":114,"props":366,"children":367},{"style":121},[368],{"type":38,"value":329},{"type":33,"tag":114,"props":370,"children":371},{"style":127},[372],{"type":38,"value":149},{"type":33,"tag":114,"props":374,"children":375},{"style":152},[376],{"type":38,"value":377},"javascript\n",{"type":33,"tag":114,"props":379,"children":381},{"class":116,"line":380},17,[382,386,390],{"type":33,"tag":114,"props":383,"children":384},{"style":121},[385],{"type":38,"value":347},{"type":33,"tag":114,"props":387,"children":388},{"style":127},[389],{"type":38,"value":149},{"type":33,"tag":114,"props":391,"children":392},{"style":152},[393],{"type":38,"value":394},"output.length > 1400 && output.length \u003C 1600\n",{"type":33,"tag":114,"props":396,"children":398},{"class":116,"line":397},18,[399,403,407,411],{"type":33,"tag":114,"props":400,"children":401},{"style":127},[402],{"type":38,"value":324},{"type":33,"tag":114,"props":404,"children":405},{"style":121},[406],{"type":38,"value":329},{"type":33,"tag":114,"props":408,"children":409},{"style":127},[410],{"type":38,"value":149},{"type":33,"tag":114,"props":412,"children":413},{"style":152},[414],{"type":38,"value":415},"cost\n",{"type":33,"tag":114,"props":417,"children":419},{"class":116,"line":418},19,[420,425,429],{"type":33,"tag":114,"props":421,"children":422},{"style":121},[423],{"type":38,"value":424},"        threshold",{"type":33,"tag":114,"props":426,"children":427},{"style":127},[428],{"type":38,"value":149},{"type":33,"tag":114,"props":430,"children":432},{"style":431},"--shiki-default:#79B8FF",[433],{"type":38,"value":434},"0.05\n",{"type":33,"tag":34,"props":436,"children":437},{},[438],{"type":38,"value":439},"Запустив этот конфиг, Promptfoo отправляет оба промпта в Claude, проверяет assertion'ы: появляется ли слово \"first-party\", находится ли объём в пределах 1400-1600 слов, стоит ли стоимость API ниже $0.05? При ошибке показывает, в каком промпте она произошла. Интегрировав в CI\u002FCD, вы получаете автоматическое тестирование изменений промпта в pull request'е — как классический unit-тест.",{"type":33,"tag":441,"props":442,"children":444},"h3",{"id":443},"почему-автоматизация-а-не-ручное-тестирование",[445],{"type":38,"value":446},"Почему автоматизация, а не ручное тестирование?",{"type":33,"tag":34,"props":448,"children":449},{},[450],{"type":38,"value":451},"Ручное тестирование: вы отправляете 5 разных тем в Claude, просматриваете выходы, даёте оценку \"хорошо\". На следующий день меняете промпт и снова тестируете вручную. К 10-й итерации вы забыли, какое изменение как повлияло на какую метрику.",{"type":33,"tag":34,"props":453,"children":454},{},[455],{"type":38,"value":456},"Автоматизация: у вас есть 50 test case'ов (реальные ключевые слова из GSC), при каждом изменении промпта они запускаются автоматически. Таблица регрессии: \"baseline prompt — в среднем 1520 слов, новый промпт — 1480 слов, снижение 2.6%\". Решение основано на метрике, не на чувстве.",{"type":33,"tag":41,"props":458,"children":460},{"id":459},"langsmith-production-observability",[461],{"type":38,"value":462},"LangSmith: Production Observability",{"type":33,"tag":34,"props":464,"children":465},{},[466],{"type":38,"value":467},"Promptfoo — это инструмент тестирования на этапе разработки. LangSmith (от команды LangChain) позволяет отслеживать, что происходит в production. Каждый вызов LLM логируется в LangSmith: вход, выход, latency, количество токенов, метаданные. В dashboard'е вы видите trace — retrieval, construction промпта, вызов LLM, post-processing цепь шаг за шагом.",{"type":33,"tag":34,"props":469,"children":470},{},[471,473,482],{"type":38,"value":472},"Пример: в работах Roibase по ",{"type":33,"tag":474,"props":475,"children":479},"a",{"href":476,"rel":477},"https:\u002F\u002Fwww.roibase.com.tr\u002Fru\u002Fgeo",[478],"nofollow",[480],{"type":38,"value":481},"оптимизации генеративных движков",{"type":38,"value":483}," мы строим LLM pipeline для отслеживания citation'ов ChatGPT. Pipeline: вопрос пользователя → embedding → Pinecone retrieval → injection контекста → Claude → extraction citation'ов. LangSmith записывает каждый шаг. Если rate citation'ов падает ниже 15%, приходит alert — сразу видна проблема с drift промпта или качеством retrieval.",{"type":33,"tag":441,"props":485,"children":487},{"id":486},"trace-vs-logging",[488],{"type":38,"value":489},"Trace vs Logging",{"type":33,"tag":34,"props":491,"children":492},{},[493],{"type":38,"value":494},"Классический logging: \"отправил этот промпт в Claude API, получил этот ответ\". Trace: \"Retrieval занял 120мс, вернул 5 документов, construction промпта — 15мс, Claude — 2.3 сек, total latency — 2.45 сек, SLA не нарушен\". Trace показывает весь end-to-end pipeline. В LLM цепях критично найти узкое место: если retrieval медленный — оптимизируйте индекс БД, если LLM медленный — изменяйте версию модели или уменьшайте токены промпта.",{"type":33,"tag":34,"props":496,"children":497},{},[498],{"type":38,"value":499},"При A\u002FB-тестировании в production LangSmith используется так: 50% трафика идёт на baseline промпт, 50% на новый — каждый вариант записывается отдельной trace-группе, метрики сравниваются в реальном времени. Baseline: средняя latency 2.1 сек, новый промпт: 1.9 сек, но оценка качества выхода падает с 0.85 до 0.80 — таблица трейдоффа видна live.",{"type":33,"tag":41,"props":501,"children":503},{"id":502},"evaluation-pipeline-автоматическая-оценка-качества",[504],{"type":38,"value":505},"Evaluation Pipeline: Автоматическая оценка качества",{"type":33,"tag":34,"props":507,"children":508},{},[509],{"type":38,"value":510},"Выход LLM субъективен — как автоматизировать вопрос \"хорошо ли это или плохо\"? Два подхода: rule-based assertion'ы и LLM-as-a-judge.",{"type":33,"tag":34,"props":512,"children":513},{},[514,519,521,527,529,535,536,542],{"type":33,"tag":58,"props":515,"children":516},{},[517],{"type":38,"value":518},"Rule-based:",{"type":38,"value":520}," assertion'ы как в Promptfoo — ",{"type":33,"tag":82,"props":522,"children":524},{"className":523},[],[525],{"type":38,"value":526},"contains",{"type":38,"value":528},", ",{"type":33,"tag":82,"props":530,"children":532},{"className":531},[],[533],{"type":38,"value":534},"length",{"type":38,"value":528},{"type":33,"tag":82,"props":537,"children":539},{"className":538},[],[540],{"type":38,"value":541},"regex-match",{"type":38,"value":543},". Правила типа \"1400-1600 слов\", \"ни одного восклицательного знака\", \"как минимум 1 внутренняя ссылка\". Быстро, детерминировано, но не захватывает семантическое качество.",{"type":33,"tag":34,"props":545,"children":546},{},[547,552],{"type":33,"tag":58,"props":548,"children":549},{},[550],{"type":38,"value":551},"LLM-as-a-judge:",{"type":38,"value":553}," вы отправляете выход другой LLM (обычно GPT-4 или Claude) на оценку. Пример: \"Соответствует ли этот пост инженерному тону? Оцени от 1 до 10\". Если judge выдаёт 7.5 — пройдено, если 6 — не пройдено. Этот метод захватывает семантическое качество, но недетерминирован — сама judge-модель стохастична. Решение: каждую eval запускать 3 раза и брать среднее.",{"type":33,"tag":34,"props":555,"children":556},{},[557],{"type":38,"value":558},"В content production workflow Roibase evaluation pipeline выглядит так:",{"type":33,"tag":560,"props":561,"children":562},"ol",{},[563,569,574,579,584,589],{"type":33,"tag":564,"props":565,"children":566},"li",{},[567],{"type":38,"value":568},"Claude генерирует черновик блога",{"type":33,"tag":564,"props":570,"children":571},{},[572],{"type":38,"value":573},"Отправляем черновик в Promptfoo",{"type":33,"tag":564,"props":575,"children":576},{},[577],{"type":38,"value":578},"Rule-based: проверяем количество слов, количество внутренних ссылок, запрещённые слова",{"type":33,"tag":564,"props":580,"children":581},{},[582],{"type":38,"value":583},"LLM-as-a-judge: GPT-4 оценивает \"соответствие тону\" от 1 до 10",{"type":33,"tag":564,"props":585,"children":586},{},[587],{"type":38,"value":588},"Все метрики записываются в Notion",{"type":33,"tag":564,"props":590,"children":591},{},[592],{"type":38,"value":593},"Если средняя оценка ниже 8 — alert в Slack",{"type":33,"tag":34,"props":595,"children":596},{},[597],{"type":38,"value":598},"Благодаря этому pipeline'у при генерации 1000 статей стандарты качества сохраняются. Вместо того чтобы manual QA-команда читала каждую статью, она смотрит только на ошибки eval — экономия 90% времени.",{"type":33,"tag":41,"props":600,"children":602},{"id":601},"ab-тест-два-промпта-два-трейдоффа-стоимость-качество",[603],{"type":38,"value":604},"A\u002FB-тест: Два промпта, два трейдоффа стоимость-качество",{"type":33,"tag":34,"props":606,"children":607},{},[608],{"type":38,"value":609},"A\u002FB-тест промптов в production работает как классический feature flag. Используете LaunchDarkly или custom flag service: 50% пользователей видят prompt_v1, 50% — prompt_v2. Для каждого варианта собираете метрики: среднее количество токенов, latency, downstream conversion (например, редактор утверждает ли черновик?).",{"type":33,"tag":34,"props":611,"children":612},{},[613],{"type":38,"value":614},"Конкретный пример: в Roibase тестируем новую версию промпта с category-specific guidance. Baseline — общий промпт, новый — содержит дополнительные инструкции по категориям. A\u002FB-тест идёт 2 недели:",{"type":33,"tag":616,"props":617,"children":618},"table",{},[619,648],{"type":33,"tag":620,"props":621,"children":622},"thead",{},[623],{"type":33,"tag":624,"props":625,"children":626},"tr",{},[627,633,638,643],{"type":33,"tag":628,"props":629,"children":630},"th",{},[631],{"type":38,"value":632},"Метрика",{"type":33,"tag":628,"props":634,"children":635},{},[636],{"type":38,"value":637},"Baseline",{"type":33,"tag":628,"props":639,"children":640},{},[641],{"type":38,"value":642},"Новый промпт",{"type":33,"tag":628,"props":644,"children":645},{},[646],{"type":38,"value":647},"Дельта",{"type":33,"tag":649,"props":650,"children":651},"tbody",{},[652,676,699,721,744],{"type":33,"tag":624,"props":653,"children":654},{},[655,661,666,671],{"type":33,"tag":656,"props":657,"children":658},"td",{},[659],{"type":38,"value":660},"Средний токен (input+output)",{"type":33,"tag":656,"props":662,"children":663},{},[664],{"type":38,"value":665},"3200",{"type":33,"tag":656,"props":667,"children":668},{},[669],{"type":38,"value":670},"3450",{"type":33,"tag":656,"props":672,"children":673},{},[674],{"type":38,"value":675},"+7.8%",{"type":33,"tag":624,"props":677,"children":678},{},[679,684,689,694],{"type":33,"tag":656,"props":680,"children":681},{},[682],{"type":38,"value":683},"Средняя latency (сек)",{"type":33,"tag":656,"props":685,"children":686},{},[687],{"type":38,"value":688},"2.1",{"type":33,"tag":656,"props":690,"children":691},{},[692],{"type":38,"value":693},"2.3",{"type":33,"tag":656,"props":695,"children":696},{},[697],{"type":38,"value":698},"+9.5%",{"type":33,"tag":624,"props":700,"children":701},{},[702,707,712,717],{"type":33,"tag":656,"props":703,"children":704},{},[705],{"type":38,"value":706},"Стоимость\u002Fстатья ($)",{"type":33,"tag":656,"props":708,"children":709},{},[710],{"type":38,"value":711},"0.042",{"type":33,"tag":656,"props":713,"children":714},{},[715],{"type":38,"value":716},"0.046",{"type":33,"tag":656,"props":718,"children":719},{},[720],{"type":38,"value":698},{"type":33,"tag":624,"props":722,"children":723},{},[724,729,734,739],{"type":33,"tag":656,"props":725,"children":726},{},[727],{"type":38,"value":728},"Процент одобрения редактором",{"type":33,"tag":656,"props":730,"children":731},{},[732],{"type":38,"value":733},"72%",{"type":33,"tag":656,"props":735,"children":736},{},[737],{"type":38,"value":738},"81%",{"type":33,"tag":656,"props":740,"children":741},{},[742],{"type":38,"value":743},"+12.5%",{"type":33,"tag":624,"props":745,"children":746},{},[747,752,757,762],{"type":33,"tag":656,"props":748,"children":749},{},[750],{"type":38,"value":751},"Точность внутренних ссылок",{"type":33,"tag":656,"props":753,"children":754},{},[755],{"type":38,"value":756},"65%",{"type":33,"tag":656,"props":758,"children":759},{},[760],{"type":38,"value":761},"89%",{"type":33,"tag":656,"props":763,"children":764},{},[765],{"type":38,"value":766},"+36.9%",{"type":33,"tag":34,"props":768,"children":769},{},[770],{"type":38,"value":771},"Новый промпт на 10% дороже, но процент одобрения редактором вырастает на 12.5% — стоимость revisions редактора ниже. Точность внутренних ссылок возрастает на 36.9% — SEO выигрыш покрывает стоимость. Решение: новый промпт побеждает, идёт в production.",{"type":33,"tag":34,"props":773,"children":774},{},[775],{"type":38,"value":776},"На время A\u002FB-теста в LangSmith создаются отдельные trace-группы для каждого варианта. Если заметите anomaly (например, у нового промпта 5% HTTP 429 rate limit ошибок), сразу это увидите.",{"type":33,"tag":41,"props":778,"children":780},{"id":779},"версионирование-git-метаданные",[781],{"type":38,"value":782},"Версионирование: Git + Метаданные",{"type":33,"tag":34,"props":784,"children":785},{},[786,788,794],{"type":38,"value":787},"Версию промпта храните в Git как код, но с отдельными метаданными. Папка ",{"type":33,"tag":82,"props":789,"children":791},{"className":790},[],[792],{"type":38,"value":793},"prompts\u002F",{"type":38,"value":795},":",{"type":33,"tag":104,"props":797,"children":799},{"code":798},"prompts\u002F\n  roibase-blog-v1.md\n  roibase-blog-v2.md\n  roibase-blog-v3.md\n",[800],{"type":33,"tag":82,"props":801,"children":802},{"__ignoreMap":17},[803],{"type":38,"value":798},{"type":33,"tag":34,"props":805,"children":806},{},[807],{"type":38,"value":808},"Каждый файл содержит frontmatter метаданные:",{"type":33,"tag":104,"props":810,"children":814},{"code":811,"language":812,"meta":17,"className":813,"style":17},"---\nversion: 3\nmodel: claude-3-5-sonnet-20241022\ntemperature: 0.7\nmax_tokens: 8000\ncreated: 2026-07-15\ndeprecated: false\ntest_suite: promptfoo-blog-eval.yaml\n---\n\n# РОЛЬ\nВы пишете для Roibase.\n...\n","markdown","language-markdown shiki shiki-themes github-dark",[815],{"type":33,"tag":82,"props":816,"children":817},{"__ignoreMap":17},[818,826,843,860,877,894,911,928,945,952,959,968,976],{"type":33,"tag":114,"props":819,"children":820},{"class":116,"line":117},[821],{"type":33,"tag":114,"props":822,"children":823},{"style":127},[824],{"type":38,"value":825},"---\n",{"type":33,"tag":114,"props":827,"children":828},{"class":116,"line":133},[829,834,838],{"type":33,"tag":114,"props":830,"children":831},{"style":121},[832],{"type":38,"value":833},"version",{"type":33,"tag":114,"props":835,"children":836},{"style":127},[837],{"type":38,"value":149},{"type":33,"tag":114,"props":839,"children":840},{"style":431},[841],{"type":38,"value":842},"3\n",{"type":33,"tag":114,"props":844,"children":845},{"class":116,"line":158},[846,851,855],{"type":33,"tag":114,"props":847,"children":848},{"style":121},[849],{"type":38,"value":850},"model",{"type":33,"tag":114,"props":852,"children":853},{"style":127},[854],{"type":38,"value":149},{"type":33,"tag":114,"props":856,"children":857},{"style":152},[858],{"type":38,"value":859},"claude-3-5-sonnet-20241022\n",{"type":33,"tag":114,"props":861,"children":862},{"class":116,"line":176},[863,868,872],{"type":33,"tag":114,"props":864,"children":865},{"style":121},[866],{"type":38,"value":867},"temperature",{"type":33,"tag":114,"props":869,"children":870},{"style":127},[871],{"type":38,"value":149},{"type":33,"tag":114,"props":873,"children":874},{"style":431},[875],{"type":38,"value":876},"0.7\n",{"type":33,"tag":114,"props":878,"children":879},{"class":116,"line":197},[880,885,889],{"type":33,"tag":114,"props":881,"children":882},{"style":121},[883],{"type":38,"value":884},"max_tokens",{"type":33,"tag":114,"props":886,"children":887},{"style":127},[888],{"type":38,"value":149},{"type":33,"tag":114,"props":890,"children":891},{"style":431},[892],{"type":38,"value":893},"8000\n",{"type":33,"tag":114,"props":895,"children":896},{"class":116,"line":214},[897,902,906],{"type":33,"tag":114,"props":898,"children":899},{"style":121},[900],{"type":38,"value":901},"created",{"type":33,"tag":114,"props":903,"children":904},{"style":127},[905],{"type":38,"value":149},{"type":33,"tag":114,"props":907,"children":908},{"style":431},[909],{"type":38,"value":910},"2026-07-15\n",{"type":33,"tag":114,"props":912,"children":913},{"class":116,"line":224},[914,919,923],{"type":33,"tag":114,"props":915,"children":916},{"style":121},[917],{"type":38,"value":918},"deprecated",{"type":33,"tag":114,"props":920,"children":921},{"style":127},[922],{"type":38,"value":149},{"type":33,"tag":114,"props":924,"children":925},{"style":431},[926],{"type":38,"value":927},"false\n",{"type":33,"tag":114,"props":929,"children":930},{"class":116,"line":27},[931,936,940],{"type":33,"tag":114,"props":932,"children":933},{"style":121},[934],{"type":38,"value":935},"test_suite",{"type":33,"tag":114,"props":937,"children":938},{"style":127},[939],{"type":38,"value":149},{"type":33,"tag":114,"props":941,"children":942},{"style":152},[943],{"type":38,"value":944},"promptfoo-blog-eval.yaml\n",{"type":33,"tag":114,"props":946,"children":947},{"class":116,"line":249},[948],{"type":33,"tag":114,"props":949,"children":950},{"style":127},[951],{"type":38,"value":825},{"type":33,"tag":114,"props":953,"children":954},{"class":116,"line":257},[955],{"type":33,"tag":114,"props":956,"children":957},{"emptyLinePlaceholder":218},[958],{"type":38,"value":221},{"type":33,"tag":114,"props":960,"children":961},{"class":116,"line":270},[962],{"type":33,"tag":114,"props":963,"children":965},{"style":964},"--shiki-default:#79B8FF;--shiki-default-font-weight:bold",[966],{"type":38,"value":967},"# РОЛЬ\n",{"type":33,"tag":114,"props":969,"children":970},{"class":116,"line":287},[971],{"type":33,"tag":114,"props":972,"children":973},{"style":127},[974],{"type":38,"value":975},"Вы пишете для Roibase.\n",{"type":33,"tag":114,"props":977,"children":978},{"class":116,"line":305},[979],{"type":33,"tag":114,"props":980,"children":981},{"style":127},[982],{"type":38,"value":983},"...\n",{"type":33,"tag":34,"props":985,"children":986},{},[987],{"type":38,"value":988},"Git commit message: \"prompt v3: добавлена category-specific guidance, расширен список запрещённых слов\". Когда CI\u002FCD видит commit, автоматически запускает Promptfoo test suite. Если тесты прошли — deploy на staging, 24 часа A\u002FB-теста, если успешно — идёт в production.",{"type":33,"tag":34,"props":990,"children":991},{},[992,994,1000],{"type":38,"value":993},"Благодаря версионированию rollback быстрый: если в production проблема, ",{"type":33,"tag":82,"props":995,"children":997},{"className":996},[],[998],{"type":38,"value":999},"git revert",{"type":38,"value":1001},", за 5 минут старый промпт активен.",{"type":33,"tag":41,"props":1003,"children":1005},{"id":1004},"оптимизация-стоимости-token-audit",[1006],{"type":38,"value":1007},"Оптимизация стоимости: Token Audit",{"type":33,"tag":34,"props":1009,"children":1010},{},[1011],{"type":38,"value":1012},"В LLM приложениях стоимость определяется формулой: input token + output token. Цена Claude Sonnet 3.5 API: $3 за 1M input token'ов, $15 за 1M output token'ов (цена 2026 года). Черновик блога из 1500 слов — ~2000 output token'ов, system prompt + user prompt — ~1200 input token'ов, итого ~$0.042 за статью.",{"type":33,"tag":34,"props":1014,"children":1015},{},[1016],{"type":38,"value":1017},"Если вы генерируете 1000 статей в месяц, это $42. Если оптимизировать промпт на 10% снижение output token'ов, сэкономите $6.3\u002Fмесяц — $75.6\u002Fгод. Кажется малым, но масштабируется. При 10,000 статей\u002Fмесяц это $756\u002Fгод.",{"type":33,"tag":34,"props":1019,"children":1020},{},[1021],{"type":38,"value":1022},"В Promptfoo eval suite добавляете cost assertion:",{"type":33,"tag":104,"props":1024,"children":1026},{"code":1025,"language":107,"meta":17,"className":108,"style":17},"assert:\n  - type: cost\n    threshold: 0.045\n",[1027],{"type":33,"tag":82,"props":1028,"children":1029},{"__ignoreMap":17},[1030,1042,1061],{"type":33,"tag":114,"props":1031,"children":1032},{"class":116,"line":117},[1033,1038],{"type":33,"tag":114,"props":1034,"children":1035},{"style":121},[1036],{"type":38,"value":1037},"assert",{"type":33,"tag":114,"props":1039,"children":1040},{"style":127},[1041],{"type":38,"value":130},{"type":33,"tag":114,"props":1043,"children":1044},{"class":116,"line":133},[1045,1049,1053,1057],{"type":33,"tag":114,"props":1046,"children":1047},{"style":127},[1048],{"type":38,"value":139},{"type":33,"tag":114,"props":1050,"children":1051},{"style":121},[1052],{"type":38,"value":329},{"type":33,"tag":114,"props":1054,"children":1055},{"style":127},[1056],{"type":38,"value":149},{"type":33,"tag":114,"props":1058,"children":1059},{"style":152},[1060],{"type":38,"value":415},{"type":33,"tag":114,"props":1062,"children":1063},{"class":116,"line":158},[1064,1069,1073],{"type":33,"tag":114,"props":1065,"children":1066},{"style":121},[1067],{"type":38,"value":1068},"    threshold",{"type":33,"tag":114,"props":1070,"children":1071},{"style":127},[1072],{"type":38,"value":149},{"type":33,"tag":114,"props":1074,"children":1075},{"style":431},[1076],{"type":38,"value":1077},"0.045\n",{"type":33,"tag":34,"props":1079,"children":1080},{},[1081],{"type":38,"value":1082},"Если после изменения промпта стоимость превышает $0.045, тест не пройдёт. Вы устанавливаете этот threshold'ы, связывая его с business метриками (процент одобрения редактором, conversion).",{"type":33,"tag":34,"props":1084,"children":1085},{},[1086],{"type":38,"value":1087},"Для token audit смотрите на LangSmith trace'ы: какой компонент промпта потребляет больше всего токенов? Например, раздел \"ЗАПРЕТЫ\" в system prompt занимает 300 токенов — действительно ли он нужен при каждом выз",{"type":33,"tag":1089,"props":1090,"children":1091},"style",{},[1092],{"type":38,"value":1093},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":17,"searchDepth":158,"depth":158,"links":1095},[1096,1097,1100,1103,1104,1105,1106],{"id":43,"depth":133,"text":46},{"id":72,"depth":133,"text":75,"children":1098},[1099],{"id":443,"depth":158,"text":446},{"id":459,"depth":133,"text":462,"children":1101},[1102],{"id":486,"depth":158,"text":489},{"id":502,"depth":133,"text":505},{"id":601,"depth":133,"text":604},{"id":779,"depth":133,"text":782},{"id":1004,"depth":133,"text":1007},"content:ru:ai:llm-ops-prompt-versioning-testing.md","content","ru\u002Fai\u002Fllm-ops-prompt-versioning-testing.md","ru\u002Fai\u002Fllm-ops-prompt-versioning-testing","md",1785967485891]