Athena를 사용하여 AWS 로그 집계

Athena 개요

아직 버지니아와 오레곤에서만 사용할 수 있습니다.

해보자

데이터베이스 만들기

CREATE DATABASE aws_logs;

테이블 만들기

ELB

CREATE EXTERNAL TABLE IF NOT EXISTS aws_logs.elb_log (
  request_protocol string,
  request_timestamp string,
  elb_name string,
  request_ip string,
  request_port int,
  backend_ip string,
  backend_port int,
  request_processing_time double,
  backend_processing_time double,
  client_response_time double,
  elb_response_code string,
  backend_response_code string,
  received_bytes bigint,
  sent_bytes bigint,
  request_verb string,
  url string,
  protocol string,
  user_agent string,
  ssl_cipher string,
  ssl_protocol string,
  arn string,
  extra string )
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe'
WITH SERDEPROPERTIES (
    'serialization.format' = '1','input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*):([0-9]*) (-1|[.0-9]*) (-1|[.0-9]*) (-1|[.0-9]*) (-|[0-9]*) (-|[0-9]*) ([-0-9]*) ([-0-9]*) \\\"([^ ]*) ([^ ]*) (- |[^ ]*)\\\" (\"[^\"]*\") ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*)$' )
LOCATION 's3://*******/********/'

CloudFront

CREATE EXTERNAL TABLE IF NOT EXISTS aws_logs.cloudfront_log (
  date STRING,
  time STRING,
  xEdgeLocation STRING,
  scBytes INT,
  cIp STRING,
  csMethod STRING,
  csHost STRING,
  csUriStem STRING,
  scStatus INT,
  csReferer STRING,
  csUserAgent STRING,
  csUriQuery STRING,
  csCookie STRING,
  xEdgeResultType STRING,
  xEdgeRequestId STRING,
  xHostHeader STRING,
  csProtocol STRING,
  csBytes INT,
  timeTaken INT,
  xForwardedFor STRING,
  sslProtocol STRING,
  sslCipher STRING,
  xEdgeResponseResultType STRING,
  csProtocolVersion STRING
)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe'
WITH SERDEPROPERTIES (
  'serialization.format' = '1','input.regex' = '([^ ]*)\t([^ ]*)\t([^ ]*)\t([0-9]*)\t([.0-9]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([0-9]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([0-9]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)\t([^ ]*)$'
) LOCATION 's3://****/******/'

CloudTrail

CREATE EXTERNAL TABLE IF NOT EXISTS sample.records (
  records ARRAY<STRUCT<eventTime:STRING,eventSource:STRING,eventName:STRING>>
)
ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
WITH SERDEPROPERTIES (
  'serialization.format' = '1'
) LOCATION 's3://********/***/'

SELECT하다

CloudFront의 SELECT 참고 사항

SELECT count(*) 
FROM aws_logs.cloudfront_log 
WHERE date NOT LIKE '#%'

CloudTrail의 SELECT 참고 사항

SELECT record.eventTime
FROM aws_logs.traillog
CROSS JOIN unnest(records) AS t(record)
WHERE record.eventname='ConsoleLogin' limit 10

SELECT record
FROM aws_logs.traillog
CROSS JOIN unnest(records) AS t(record)
WHERE record.eventtime LIKE '2017-02-08%' limit 10

요약