한국어 AI 콘텐츠 안전 필터링: 유니코드 정규화부터 키워드와 LLM 심사의 두 겹 판정까지

// backend/src/content-safety/content-safety.service.ts
private normalizeText(text: string): string {
  let normalized = text.normalize('NFC');
  normalized = normalized.toLowerCase();
  // Remove spaces (prevents "아 동" bypass)
  normalized = normalized.replace(/\s+/g, '');
  return this.combineKoreanJamo(normalized).text;
}

// backend/src/content-safety/content-safety.service.ts (발췌)
checkText(text: string, blacklist: BlacklistEntry[]): SafetyCheckResult {
  const tracked = this.normalizeTracked(text);
  const normalized = tracked.text;
  const normalizedKeepSpaces = this.normalizeTextKeepSpaces(text);
  // ...
  for (const entry of blacklist) {
    if (entry.matchType === 'REGEX') {
      // ...
    } else {
      const keywordKeepSpaces = this.normalizeTextKeepSpaces(entry.keyword);
      const normalizedKeyword = this.normalizeText(entry.keyword);

      // Phase 1: Check with spaces preserved
      if (this.hasWordBoundaryMatch(normalizedKeepSpaces, keywordKeepSpaces)) {
        matches.push({ keyword: entry.keyword, category: entry.category });
        continue;
      }

      // Phase 2: Check space-removed text (catches bypass like "자 위" → "자위")
      const isMultiToken = keywordKeepSpaces !== normalizedKeyword;
      const matchedPhase2 = isMultiToken
        ? normalized.includes(normalizedKeyword)
        : this.hasWordBoundaryMatch(normalized, normalizedKeyword, isSpacedBypass);
      if (matchedPhase2) {
        matches.push({ keyword: entry.keyword, category: entry.category });
      }
    }
  }
  return { flagged: matches.length > 0, matches };
}

// backend/src/content-safety/content-safety.service.spec.ts
it('should not flag a keyword straddling two real words (탁자 위에)', () => {
  const result = service.checkText('탁자 위에 올려둔 컵', [entry('자위')]);
  expect(result.flagged).toBe(false);
});

// backend/src/content-safety/content-safety.service.spec.ts
it('should detect space-bypass "자 위" → "자위"', () => {
  // ...
  const result = service.checkText('자 위를 했다', blacklist);
  // ...
});

코드포인트 = (초성 × 21 + 중성) × 28 + 종성 + 0xAC00

// backend/src/content-safety/content-safety.service.ts (발췌)
// Choseong: ㄱㄲㄴㄷㄸㄹㅁㅂㅃㅅㅆㅇㅈㅉㅊㅋㅌㅍㅎ
const CHOSEONG_MAP: Record<number, number> = {
  0x3131: 0, // ㄱ
  0x3132: 1, // ㄲ
  0x3134: 2, // ㄴ
  0x3137: 3, // ㄷ
  // ... 19개
  0x314e: 18, // ㅎ
};

// Jungseong: ㅏㅐㅑㅒㅓㅔㅕㅖㅗㅘㅙㅚㅛㅜㅝㅞㅟㅠㅡㅢㅣ
const JUNGSEONG_MAP: Record<number, number> = {
  0x314f: 0, // ㅏ
  0x3150: 1, // ㅐ
  0x3151: 2, // ㅑ
  // ... 21개
  0x3163: 20, // ㅣ
};

// Jongseong mapping (consonant code -> jongseong index)
const JONGSEONG_MAP: Record<number, number> = {
  0x3131: 1, // ㄱ
  0x3132: 2, // ㄲ
  0x3134: 4, // ㄴ
  // ... 16개
  0x314e: 27, // ㅎ
};

// backend/src/content-safety/content-safety.service.ts (발췌)
private combineKoreanJamo(text: string): { text: string; startIndex: number[] } {
  const result: string[] = [];
  const startIndex: number[] = [];
  let i = 0;

  while (i < text.length) {
    const start = i;
    const code = text.charCodeAt(i);

    // Check if current char is a choseong consonant
    if (this.isChoseong(code) && i + 1 < text.length) {
      const nextCode = text.charCodeAt(i + 1);

      // Check if next char is a jungseong vowel
      if (this.isJungseong(nextCode)) {
        const cho = CHOSEONG_MAP[code];
        const jung = JUNGSEONG_MAP[nextCode];
        let jong = 0;

        // Check for trailing consonant (jongseong)
        if (i + 2 < text.length) {
          const thirdCode = text.charCodeAt(i + 2);

          if (this.isChoseong(thirdCode)) {
            // Only use as jongseong if NOT followed by a vowel
            if (i + 3 >= text.length || !this.isJungseong(text.charCodeAt(i + 3))) {
              if (JONGSEONG_MAP[thirdCode] !== undefined) {
                jong = JONGSEONG_MAP[thirdCode];
                i += 3;
                result.push(String.fromCharCode((cho * 21 + jung) * 28 + jong + 0xac00));
                startIndex.push(start);
                continue;
              }
            }
          }
        }

        // Combine without jongseong
        result.push(String.fromCharCode((cho * 21 + jung) * 28 + jong + 0xac00));
        startIndex.push(start);
        i += 2;
        continue;
      }
    }

    result.push(text[i]);
    startIndex.push(start);
    i++;
  }

  return { text: result.join(''), startIndex };
}

// backend/src/content-safety/content-safety.service.spec.ts
it('should detect sexual keywords in jamo form (bypass attempt)', () => {
  const result = service.checkText('ㅅㅔㄱㅅㅡ', sexualKeywords);
  expect(result.flagged).toBe(true);
});

// backend/src/chat/chat-turn.service.ts (발췌)
const blacklist = await this.contentSafety.loadBlacklist();
const inputCheck = this.contentSafety.checkText(content, blacklist);
const inputCriticalMatch = inputCheck.matches.find((m) =>
  isCriticalCategory(m.category),
);
// ... 일치한 키워드는 검토 큐에 action: 'warned' 로 남긴다

// 19+ 세션은 CRITICAL(아동·자해) 적중만 붙잡는다. 성적 키워드로 매 턴 묶이면 19+ 를 여는 의미가 없다
const holdForKeyword =
  pipelineContext.ageRating === AgeRating.NINETEEN_PLUS
    ? Boolean(inputCriticalMatch)
    : inputCheck.flagged;
pipelineContext.awaitInputSafety = holdForKeyword;
pipelineContext.keywordFallbackCategory = inputCriticalMatch?.category;

// backend/src/ai-pipeline/safety-policy.ts
export function resolveSafetyAction(v: SafetyVerdictInput): SafetyAction {
  if (v.decision !== 'REJECT') return 'ALLOW';
  const { intervene } = safetyPolicyFor(
    v.category,
    v.ageRating ?? AgeRating.ALL,
  );
  return intervene === 'NONE' ? 'ALLOW' : intervene;
}

// backend/src/chat/chat-turn.service.ts (발췌): 되돌림 경로
markTurnBlocked(BLOCKED_REASONS.INPUT_FILTER);
await this.chatService
  .markMessageSafetyRejected(savedMsg.id)
  .catch((err) => this.logger.warn('Failed to mark rejected message', err));

const { gemRefunded, error: refundError } =
  await this.issueRefundOnce(state);
if (refundError) {
  this.logger.error('Safety refund failed:', refundError);
}

endBlocked(sink, {
  message: SAFETY_BLOCK_MESSAGE,
  code: SSE_ERROR_CODES.INPUT_FILTER_BLOCKED,
  category: verdictCategory,
  gemRefunded,
});

// backend/src/chat/chat-turn.service.ts (발췌): streamItem()
const outputCheck = this.contentSafety.checkText(textToCheck, prepared.blacklist);
const outputCriticalMatch = outputCheck.matches.find((m) =>
  isCriticalCategory(m.category),
);
if (outputCheck.flagged) {
  // ... 검토 큐 기록 (CRITICAL 이면 'replaced', 아니면 'warned')
  if (outputCriticalMatch) {
    send(sseFrame(SSE_EVENTS.NARRATION, { text: OUTPUT_REPLACEMENT_TEXT }));
    send(
      sseFrame(SSE_EVENTS.ERROR, {
        code: SSE_ERROR_CODES.OUTPUT_FILTER_BLOCKED,
        category: primaryCategory,
      }),
      false,
    );
    return;
  }
}

// backend/src/content-safety/content-safety.service.ts (발췌)
const BLACKLIST_MEMO_TTL_SEC = 300;

async loadBlacklist(): Promise<BlacklistEntry[]> {
  const memoHit = this.memo.get();
  if (memoHit) return memoHit;
  return this.reloadBlacklistFromDb();
}

private async reloadBlacklistFromDb(): Promise<BlacklistEntry[]> {
  // 배포 중 새 enum 값 행이 먼저 들어와도 이 빌드가 읽다 죽지 않게 아는 값만 읽는다
  const keywords = await this.prisma.blacklistKeyword.findMany({
    where: {
      isActive: true,
      category: { in: Object.values(SafetyCategory) },
      matchType: { in: Object.values(SafetyMatchType) },
    },
    select: { keyword: true, category: true, matchType: true },
  });
  // ...
}

// backend/prisma/schema.prisma (발췌)
model ContentSafetyLog {
  id              String            @id @default(uuid())
  sessionId       String?           @map("session_id")
  userId          String?           @map("user_id")
  messageId       String?           @map("message_id")
  /// 판정이 난 턴의 Langfuse trace id. Langfuse 미설정 환경에서는 null.
  traceId         String?           @map("trace_id")
  filterStage     String            @map("filter_stage")
  originalContent String            @map("original_content") @db.Text
  matchedKeywords String[]          @map("matched_keywords")
  category        SafetyCategory?
  severity        String?
  action          String
  status          SafetyAlertStatus @default(PENDING)
  // ... reviewedBy, reviewedAt, notes, createdAt, isTestSession

  @@map("content_safety_logs")
  @@schema("safety")
}