New RelicとOpenAIを活用したエラーログ調査の自動化

SELECT count(*) FROM Log 
FACET aws.logGroup, messageId 
WHERE message LIKE '[ERROR]%'

cloudwatch-logs-message

SELECT message FROM Log
WHERE aws.logGroup IS NOT NULL 
AND messageId = {{ accumulations.tag.messageId }}

{
    "id": {{ json issueId }},
    "issueUrl": {{ json issuePageUrl }},
    "priority": {{ json priority }},
    "state": {{ json state }},
    "trigger": {{ json triggerEvent }},
    "isCorrelated": {{ json isCorrelated }},
    "createdAt": {{ createdAt }},
    "updatedAt": {{ updatedAt }},
    "enrichData" : {{ json cloudwatch-logs-message }},
    "accumulations" : {{ json accumulations }}  
}

mkdir newrelic-openai
cd newrelic-openai

FROM python:3.9

RUN apt-get update && \
    apt-get install -y nodejs npm && \
    rm -rf /var/lib/apt/lists/*

version: "3.8"
services:
  app:
    build: .
    container_name: app
    working_dir: /usr/src/app
    tty: true
    volumes:
      - ./:/usr/src/app

docker compose up -d
docker compose exec app bash

pip install --upgrade pip
pip install openai==0.27.0
pip freeze > requirements.txt

npm install -g serverless

service: newrelic-openai
frameworkVersion: "3"

provider:
  name: aws
  runtime: python3.9
  stage: dev
  region: ap-northeast-1
  iamRoleStatements:
    - Effect: Allow
      Action:
        - logs:CreateLogStream
        - logs:PutLogEvents
      Resource:
        - "*"

plugins:
  # Python依存関係の管理
  - serverless-python-requirements
  # ログサブスクリプションプラグイン
  - serverless-plugin-log-subscription

custom:
  # New Relicのログ収集先ARN
  logSubscription:
    destinationArn: ${env:NEW_RELIC_LOG_INGESTION}
  # Pythonの依存関係をLambdaレイヤーとして使用
  pythonRequirements:
    layer: true

functions:
  # ログレポーター関数
  reporter:
    name: ai-log-reporter
    handler: ai-log-reporter/handler.lambda_handler
    # Python依存関係レイヤーの参照
    layers:
      - Ref: PythonRequirementsLambdaLayer
    # 環境変数
    environment:
      OPENAI_API_KEY: ${env:OPENAI_API_KEY}
      SLACK_WEBHOOK_URL: ${env:SLACK_WEBHOOK_URL}
    # イベントトリガー
    events:
      - eventBridge:
          # イベントバスの設定
          eventBus: ${env:EVENT_BUS}
          pattern:
            source:
              - prefix: "aws.partner/newrelic.com"
  # エラーログエミッター関数
  emitter:
    name: error-log-emitter
    handler: error-log-emitter/handler.lambda_handler
    # ログサブスクリプションの有効化
    logSubscription: true

sls plugin install -n serverless-python-requirements
sls plugin install -n serverless-plugin-log-subscription

mkdir error-log-emitter

def lambda_handler(event, context):
    try:
        # 存在しないインデックスにアクセスしエラーを発生させる
        data = [1, 2, 3]
        fourth_element = data[3]
        print(fourth_element)

    except Exception as e:
        raise

mkdir ai-log-reporter

import json
import openai
import os
import logging
import urllib.request

logger = logging.getLogger()
logger.setLevel(logging.INFO)

SLACK_WEBHOOK_URL = os.environ['SLACK_WEBHOOK_URL']
OPENAI_API_KEY = os.environ['OPENAI_API_KEY']
OPENAI_MODEL = "text-davinci-003"


def send_to_slack(payload: dict) -> None:
    """Slack通知の送信"""
    try:
        data = json.dumps(payload).encode('utf-8')
        req = urllib.request.Request(SLACK_WEBHOOK_URL, data=data, headers={
                                     'Content-Type': 'application/json'})
        with urllib.request.urlopen(req) as response:
            if response.status == 200:
                logger.info("Slack通知が送信されました")
            else:
                logger.error("Slack通知の送信に失敗しました: ステータスコード %s", response.status)
    except Exception as e:
        logger.exception("Slack通知の送信中にエラーが発生しました: %s", e)
        raise


def create_slack_payload(result: str,
                         message: str,
                         issue_url: str,
                         condition_name: str,
                         log_group: str) -> dict:
    """Slack通知用のペイロードを作成する"""
    return {
        "text": "*<!channel> New Relic アラート通知*",
        "attachments": [
            {
                "color": "#E01E5A",
                "blocks": [
                    {
                        "type": "section",
                        "text": {
                         "type": "mrkdwn",
                         "text": f"*アラート名*\n<{issue_url}|{condition_name}>"}
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": f"*ロググループ名*\n {log_group}"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": f"*エラー内容*\n {message}"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": f"*エラー対応方法*\n {result}"
                        }
                    },
                ]
            }
        ]
    }


def lambda_handler(event, context):
    # OpenAI APIキーの設定
    openai.api_key = OPENAI_API_KEY
    try:
        # EventBriteのイベント情報を取得
        message = event['detail']['enrichData']['result'][0]['message']
        issue_url = event['detail']['issueUrl']
        condition_name = event['detail']['accumulations']['conditionName'][0]
        log_group = event['detail']['accumulations']['tag']['aws']['logGroup'][0]
        # OpenAI APIの呼び出し
        response = openai.Completion.create(
            model=OPENAI_MODEL,
            prompt=f"このエラーに対してどういった対応が必要ですか？: '{message}'",
            max_tokens=1000,
        )
        # OpenAI API 回答結果の取得
        result = response.choices[0].text.strip()
        # Slack通知の送信
        payload = create_slack_payload(
            result, message, issue_url, condition_name, log_group)
        send_to_slack(payload)
    except Exception as e:
        logger.exception("Lambda関数の実行中にエラーが発生しました: %s", e)
        raise

export AWS_ACCESS_KEY_ID=xxxx
export AWS_SECRET_ACCESS_KEY=xxxx
export AWS_DEFAULT_REGION=ap-northeast-1

export EVENT_BUS=<EventBridge イベントバス ARN>

export NEW_RELIC_LOG_INGESTION=<Lambda newrelic-log-ingestion ARN>

export OPENAI_API_KEY=<OpenAI API キー>

export SLACK_WEBHOOK_URL=<Slack Incoming Webhook URL>

sls deploy --verbose

sls invoke --function emitter

sls remove --verbose

docker compose down --rmi all --volumes --remove-orphans