katie/http-syslog-transformer

Katie KlossLucy!4d2d49f2026-09-15

import gleam/bit_array
import gleam/int
import gleam/list
import gleam/option.{Some}
import gleam/order
import gleam/regexp
import gleam/result
import gleam/string
import gleam/time/calendar
import gleam/time/duration
import gleam/time/timestamp

const all_months = [
  #("Jan", calendar.January),
  #("Feb", calendar.February),
  #("Mar", calendar.March),
  #("Apr", calendar.April),
  #("May", calendar.May),
  #("Jun", calendar.June),
  #("Jul", calendar.July),
  #("Aug", calendar.August),
  #("Sep", calendar.September),
  #("Oct", calendar.October),
  #("Nov", calendar.November),
  #("Dec", calendar.December),
]

pub type SyslogLine {
  SyslogLine(host: String, tag: String, body: String)
}

pub type NginxLine {
  NginxLine(
    tag: String,
    hostname: String,
    remote_addr: String,
    remote_user: String,
    time: timestamp.Timestamp,
    method: String,
    uri: String,
    version: String,
    response_code: Int,
    size: Int,
    referer: String,
    user_agent: String,
  )
}

pub type ParseError {
  NotSyslogData
  NotNginxData
}

pub opaque type Parser {
  Parser(syslog_regex: regexp.Regexp, nginx_regex: regexp.Regexp)
}

pub fn new() -> Parser {
  let assert Ok(syslog_regex) =
    regexp.from_string(
      "^<\\d+>([A-Z][a-z][a-z]\\s{1,2}\\d{1,2}\\s\\d{2}[:]\\d{2}[:]\\d{2})\\s([\\w][\\w\\d\\.@-]*)\\s(\\w+)\\W\\s(.*)",
    )

  let assert Ok(nginx_regex) =
    regexp.from_string(
      "^([a-f0-9:.]+) - ([^ ]+) \\[([^\\]]+)\\] \"([^ ]+) ([^ ]+) HTTP\\/([\\d.]+)\" (\\d{3}) (\\d+) \"([^\"]+)\" \"([^\"]*)\"$",
    )

  Parser(syslog_regex, nginx_regex)
}

pub fn parse_data(
  parser: Parser,
  data: BitArray,
) -> Result(NginxLine, ParseError) {
  data
  |> bit_array.to_string
  |> result.replace_error(NotSyslogData)
  |> result.try(fn(s) { parse_line(parser, s) })
}

fn parse_line(parser: Parser, line: String) -> Result(NginxLine, ParseError) {
  use syslog_line <- result.try(get_syslog(parser, line))
  get_nginx(parser, syslog_line)
}

fn get_syslog(parser: Parser, data: String) -> Result(SyslogLine, ParseError) {
  case regexp.scan(parser.syslog_regex, data) {
    [regexp.Match(submatches: [_, Some(host), Some(tag), Some(body)], ..), ..] ->
      Ok(SyslogLine(host, tag, body))
    _ -> Error(NotSyslogData)
  }
}

fn get_nginx(
  parser: Parser,
  data: SyslogLine,
) -> Result(NginxLine, ParseError) {
  case regexp.scan(parser.nginx_regex, data.body) {
    [
      regexp.Match(
        submatches: [
          Some(remote_addr),
          Some(remote_user),
          Some(time),
          Some(method),
          Some(uri),
          Some(version),
          Some(code),
          Some(size),
          Some(referer),
          Some(user_agent),
        ],
        ..,
      ),
      ..
    ] -> {
      use time <- result.try(
        parse_time(time) |> result.replace_error(NotNginxData),
      )

      Ok(NginxLine(
        data.tag,
        data.host,
        remote_addr,
        remote_user,
        time,
        method,
        uri,
        version,
        int.parse(code) |> result.unwrap(0),
        int.parse(size) |> result.unwrap(0),
        referer,
        user_agent,
      ))
    }
    _ -> Error(NotNginxData)
  }
}

pub fn parse_time(time: String) -> Result(timestamp.Timestamp, Nil) {
  // this is slow make it 🚀
  case time |> string.to_graphemes {
    [] -> Error(Nil)
    [
      day_a,
      day_b,
      _,
      month_a,
      month_b,
      month_c,
      _,
      year_a,
      year_b,
      year_c,
      year_d,
      _,
      hour_a,
      hour_b,
      _,
      minute_a,
      minute_b,
      _,
      second_a,
      second_b,
      _,
      offset_dir,
      offset_a,
      offset_b,
      offset_c,
      offset_d,
      ..
    ] -> {
      let month_from_abbr = fn(abbr) {
        all_months
        |> list.find(fn(m) { string.compare(abbr, m.0) == order.Eq })
      }

      let month = month_a <> month_b <> month_c
      use month <- result.try(month_from_abbr(month))
      let month = month.1

      let offset_sign = fn(offset: duration.Duration, offset_dir: String) -> Result(
        duration.Duration,
        Nil,
      ) {
        case offset_dir {
          "+" -> Ok(offset)
          "-" -> Ok(duration.subtract(duration.seconds(0), offset))
          _ -> Error(Nil)
        }
      }

      use year <- result.try(int.parse(year_a <> year_b <> year_c <> year_d))
      use day <- result.try(int.parse(day_a <> day_b))

      use hour <- result.try(int.parse(hour_a <> hour_b))
      use minute <- result.try(int.parse(minute_a <> minute_b))
      use second <- result.try(int.parse(second_a <> second_b))
      use offset <- result.try(int.parse(
        offset_a <> offset_b <> offset_c <> offset_d,
      ))
      let offset =
        duration.add(
          duration.hours(offset / 100),
          duration.minutes(offset % 100),
        )
      use offset <- result.try(offset_sign(offset, offset_dir))

      Ok(timestamp.from_calendar(
        date: calendar.Date(year, month, day),
        time: calendar.TimeOfDay(hour, minute, second, 0),
        offset: offset,
      ))
    }
    _ -> Error(Nil)
  }
}