output-eve-stream.c

Go to the documentation of this file.

/* Copyright (C) 2023-2024 Open Information Security Foundation
 *
 * You can copy, redistribute or modify this Program under the terms of
 * the GNU General Public License version 2 as published by the Free
 * Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 */
 
/**
 * \file
 */
 
#include "suricata-common.h"
#include "packet.h"
#include "detect.h"
#include "flow.h"
#include "conf.h"
 
#include "threads.h"
#include "tm-threads.h"
#include "threadvars.h"
#include "util-debug.h"
 
#include "decode-ipv4.h"
#include "detect-parse.h"
#include "detect-engine.h"
#include "detect-engine-mpm.h"
#include "detect-reference.h"
 
#include "output.h"
#include "output-json.h"
#include "output-json-flow.h"
#include "output-eve-stream.h"
 
#include "stream-tcp.h"
 
#include "util-unittest.h"
#include "util-unittest-helper.h"
#include "util-classification-config.h"
#include "util-privs.h"
#include "util-print.h"
#include "util-proto-name.h"
#include "util-logopenfile.h"
#include "util-time.h"
#include "util-buffer.h"
 
#include "action-globals.h"
 
#define MODULE_NAME "EveStreamLog"
 
typedef struct EveStreamOutputCtx_ {
    uint16_t trigger_flags; /**< presence of flags in packet trigger logging. 0xffff for all. */
    OutputJsonCtx *eve_ctx;
} EveStreamOutputCtx;
 
typedef struct EveStreamLogThread_ {
    EveStreamOutputCtx *stream_ctx;
    OutputJsonThreadCtx *ctx;
} EveStreamLogThread;
 
static TmEcode EveStreamLogThreadInit(ThreadVars *t, const void *initdata, void **data)
{
    EveStreamLogThread *aft = SCCalloc(1, sizeof(EveStreamLogThread));
    if (unlikely(aft == NULL))
        return TM_ECODE_FAILED;
 
    if (initdata == NULL) {
        SCLogDebug("Error getting context for EveLogDrop.  \"initdata\" argument NULL");
        goto error_exit;
    }
 
    /** Use the Output Context (file pointer and mutex) */
    aft->stream_ctx = ((OutputCtx *)initdata)->data;
    aft->ctx = CreateEveThreadCtx(t, aft->stream_ctx->eve_ctx);
    if (!aft->ctx) {
        goto error_exit;
    }
 
    *data = (void *)aft;
    return TM_ECODE_OK;
 
error_exit:
    SCFree(aft);
    return TM_ECODE_FAILED;
}
 
static TmEcode EveStreamLogThreadDeinit(ThreadVars *t, void *data)
{
    EveStreamLogThread *aft = (EveStreamLogThread *)data;
    if (aft == NULL) {
        return TM_ECODE_OK;
    }
 
    FreeEveThreadCtx(aft->ctx);
 
    /* clear memory */
    memset(aft, 0, sizeof(*aft));
 
    SCFree(aft);
    return TM_ECODE_OK;
}
 
static void EveStreamOutputCtxFree(EveStreamOutputCtx *ctx)
{
    if (ctx != NULL) {
        SCFree(ctx);
    }
}
 
static void EveStreamLogDeInitCtxSub(OutputCtx *output_ctx)
{
    OutputDropLoggerDisable();
 
    EveStreamOutputCtx *ctx = output_ctx->data;
    SCFree(ctx);
    SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
    SCFree(output_ctx);
}
 
static uint16_t SetFlag(ConfNode *conf, const char *opt, const uint16_t inflag)
{
    const char *v = ConfNodeLookupChildValue(conf, opt);
    if (v != NULL && ConfValIsTrue(v)) {
        return inflag;
    }
    return 0;
}
 
static OutputInitResult EveStreamLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
{
    OutputInitResult result = { NULL, false };
    OutputJsonCtx *ajt = parent_ctx->data;
 
    EveStreamOutputCtx *ctx = SCCalloc(1, sizeof(*ctx));
    if (ctx == NULL)
        return result;
 
    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
    if (unlikely(output_ctx == NULL)) {
        EveStreamOutputCtxFree(ctx);
        return result;
    }
 
    if (conf) {
        // TODO add all flags
 
        ctx->trigger_flags |= SetFlag(conf, "event-set", STREAM_PKT_FLAG_EVENTSET);
        ctx->trigger_flags |= SetFlag(conf, "state-update", STREAM_PKT_FLAG_STATE_UPDATE);
        ctx->trigger_flags |=
                SetFlag(conf, "spurious-retransmission", STREAM_PKT_FLAG_SPURIOUS_RETRANSMISSION);
        ctx->trigger_flags |= SetFlag(conf, "tcp-session-reuse", STREAM_PKT_FLAG_TCP_SESSION_REUSE);
 
        ctx->trigger_flags |= SetFlag(conf, "all", 0xFFFF);
        SCLogDebug("trigger_flags %04x", ctx->trigger_flags);
    }
    ctx->eve_ctx = ajt;
 
    output_ctx->data = ctx;
    output_ctx->DeInit = EveStreamLogDeInitCtxSub;
 
    result.ctx = output_ctx;
    result.ok = true;
 
    SCLogWarning("eve.stream facility is EXPERIMENTAL and can change w/o notice");
    return result;
}
 
void EveAddFlowTcpStreamFlags(const TcpStream *stream, const char *name, JsonBuilder *jb)
{
    jb_open_array(jb, name);
    if (stream->flags & STREAMTCP_STREAM_FLAG_HAS_GAP)
        jb_append_string(jb, "has_gap");
    if (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)
        jb_append_string(jb, "noreassembly");
    if (stream->flags & STREAMTCP_STREAM_FLAG_KEEPALIVE)
        jb_append_string(jb, "keepalive");
    if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED)
        jb_append_string(jb, "depth_reached");
    if (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW)
        jb_append_string(jb, "trigger_raw");
    if (stream->flags & STREAMTCP_STREAM_FLAG_TIMESTAMP)
        jb_append_string(jb, "timestamp");
    if (stream->flags & STREAMTCP_STREAM_FLAG_ZERO_TIMESTAMP)
        jb_append_string(jb, "zero_timestamp");
    if (stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED)
        jb_append_string(jb, "appproto_detection_completed");
    if (stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED)
        jb_append_string(jb, "appproto_detection_skipped");
    if (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
        jb_append_string(jb, "new_raw_disabled");
    if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW)
        jb_append_string(jb, "disable_raw");
    if (stream->flags & STREAMTCP_STREAM_FLAG_RST_RECV)
        jb_append_string(jb, "rst_recv");
    jb_close(jb);
}
 
void EveAddFlowTcpFlags(const TcpSession *ssn, const char *name, JsonBuilder *jb)
{
    jb_open_object(jb, "flags");
 
    jb_open_array(jb, name);
    if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
        jb_append_string(jb, "midstream");
    }
    if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM_ESTABLISHED) {
        jb_append_string(jb, "midstream_established");
    }
    if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM_SYNACK) {
        jb_append_string(jb, "midstream_synack");
    }
    if (ssn->flags & STREAMTCP_FLAG_TIMESTAMP) {
        jb_append_string(jb, "timestamp");
    }
    if (ssn->flags & STREAMTCP_FLAG_SERVER_WSCALE) {
        jb_append_string(jb, "server_wscale");
    }
    if (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) {
        jb_append_string(jb, "closed_by_rst");
    }
    if (ssn->flags & STREAMTCP_FLAG_4WHS) {
        jb_append_string(jb, "4whs");
    }
    if (ssn->flags & STREAMTCP_FLAG_DETECTION_EVASION_ATTEMPT) {
        jb_append_string(jb, "detect_evasion_attempt");
    }
    if (ssn->flags & STREAMTCP_FLAG_CLIENT_SACKOK) {
        jb_append_string(jb, "client_sackok");
    }
    if (ssn->flags & STREAMTCP_FLAG_CLIENT_SACKOK) {
        jb_append_string(jb, "sackok");
    }
    if (ssn->flags & STREAMTCP_FLAG_3WHS_CONFIRMED) {
        jb_append_string(jb, "3whs_confirmed");
    }
    if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) {
        jb_append_string(jb, "app_layer_disabled");
    }
    if (ssn->flags & STREAMTCP_FLAG_BYPASS) {
        jb_append_string(jb, "bypass");
    }
    if (ssn->flags & STREAMTCP_FLAG_TCP_FAST_OPEN) {
        jb_append_string(jb, "tcp_fast_open");
    }
    if (ssn->flags & STREAMTCP_FLAG_TFO_DATA_IGNORED) {
        jb_append_string(jb, "tfo_data_ignored");
    }
    jb_close(jb);
    jb_close(jb);
}
 
static void LogStreamSB(const StreamingBuffer *sb, JsonBuilder *js)
{
    jb_set_uint(js, "sb_region_size", sb->region.buf_size);
}
 
static void LogStream(const TcpStream *stream, JsonBuilder *js)
{
    jb_set_uint(js, "isn", stream->isn);
    jb_set_uint(js, "next_seq", stream->next_seq);
    jb_set_uint(js, "last_ack", stream->last_ack);
    jb_set_uint(js, "next_win", stream->next_win);
    if (!(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
        jb_set_uint(js, "base_seq", stream->base_seq);
        jb_set_uint(js, "segs_right_edge", stream->segs_right_edge);
    }
    jb_set_uint(js, "window", stream->window);
    jb_set_uint(js, "wscale", stream->wscale);
 
    EveAddFlowTcpStreamFlags(stream, "flags", js);
 
    TcpSegment *s;
    uint32_t segs = 0;
    RB_FOREACH(s, TCPSEG, (struct TCPSEG *)&stream->seg_tree)
    {
        segs++;
    }
    jb_set_uint(js, "seg_cnt", segs);
    LogStreamSB(&stream->sb, js);
}
 
/**
 * \brief   Log the stream packets
 *
 * \param tv    Pointer the current thread variables
 * \param data  Pointer to the EveStreamLogThread struct
 * \param p     Pointer the packet which is being logged
 *
 * \retval 0 on success
 */
static int EveStreamLogger(ThreadVars *tv, void *thread_data, const Packet *p)
{
    EveStreamLogThread *td = thread_data;
    EveStreamOutputCtx *ctx = td->stream_ctx;
 
    JsonAddrInfo addr = json_addr_info_zero;
    JsonAddrInfoInit(p, LOG_DIR_PACKET, &addr);
 
    JsonBuilder *js = CreateEveHeader(p, LOG_DIR_PACKET, "stream_tcp", &addr, ctx->eve_ctx);
    if (unlikely(js == NULL))
        return TM_ECODE_OK;
 
    if (p->flow != NULL) {
        if (p->flowflags & FLOW_PKT_TOSERVER) {
            jb_set_string(js, "direction", "to_server");
        } else {
            jb_set_string(js, "direction", "to_client");
        }
    }
 
    jb_open_object(js, "stream_tcp");
    jb_open_object(js, "packet");
 
    if (PacketIsIPv4(p)) {
        const IPV4Hdr *ip4h = PacketGetIPv4(p);
        jb_set_uint(js, "len", IPV4_GET_RAW_IPLEN(ip4h));
        jb_set_uint(js, "tos", IPV4_GET_RAW_IPTOS(ip4h));
        jb_set_uint(js, "ttl", IPV4_GET_RAW_IPTTL(ip4h));
        jb_set_uint(js, "ipid", IPV4_GET_RAW_IPID(ip4h));
    } else if (PacketIsIPv6(p)) {
        const IPV6Hdr *ip6h = PacketGetIPv6(p);
        jb_set_uint(js, "len", IPV6_GET_RAW_PLEN(ip6h));
        jb_set_uint(js, "tc", IPV6_GET_RAW_CLASS(ip6h));
        jb_set_uint(js, "hoplimit", IPV6_GET_RAW_HLIM(ip6h));
        jb_set_uint(js, "flowlbl", IPV6_GET_RAW_FLOW(ip6h));
    }
    if (PacketIsTCP(p)) {
        const TCPHdr *tcph = PacketGetTCP(p);
        jb_set_uint(js, "tcpseq", TCP_GET_RAW_SEQ(tcph));
        jb_set_uint(js, "tcpack", TCP_GET_RAW_ACK(tcph));
        jb_set_uint(js, "tcpwin", TCP_GET_RAW_WINDOW(tcph));
        jb_set_bool(js, "syn", TCP_ISSET_FLAG_RAW_SYN(tcph) ? true : false);
        jb_set_bool(js, "ack", TCP_ISSET_FLAG_RAW_ACK(tcph) ? true : false);
        jb_set_bool(js, "psh", TCP_ISSET_FLAG_RAW_PUSH(tcph) ? true : false);
        jb_set_bool(js, "rst", TCP_ISSET_FLAG_RAW_RST(tcph) ? true : false);
        jb_set_bool(js, "urg", TCP_ISSET_FLAG_RAW_URG(tcph) ? true : false);
        jb_set_bool(js, "fin", TCP_ISSET_FLAG_RAW_FIN(tcph) ? true : false);
        jb_set_uint(js, "tcpres", TCP_GET_RAW_X2(tcph));
        jb_set_uint(js, "tcpurgp", TCP_GET_RAW_URG_POINTER(tcph));
 
        jb_open_array(js, "flags");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_RETRANSMISSION)
            jb_append_string(js, "retransmission");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_SPURIOUS_RETRANSMISSION)
            jb_append_string(js, "spurious_retransmission");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_KEEPALIVE)
            jb_append_string(js, "keepalive");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_KEEPALIVEACK)
            jb_append_string(js, "keepalive_ack");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_WINDOWUPDATE)
            jb_append_string(js, "window_update");
 
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_EVENTSET)
            jb_append_string(js, "event_set");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_STATE_UPDATE)
            jb_append_string(js, "state_update");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_DUP_ACK)
            jb_append_string(js, "dup_ack");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_DSACK)
            jb_append_string(js, "dsack");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_ACK_UNSEEN_DATA)
            jb_append_string(js, "ack_unseen_data");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_TCP_SESSION_REUSE)
            jb_append_string(js, "tcp_session_reuse");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_TCP_ZERO_WIN_PROBE)
            jb_append_string(js, "zero_window_probe");
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_TCP_ZERO_WIN_PROBE_ACK)
            jb_append_string(js, "zero_window_probe_ack");
        jb_close(js);
    }
    jb_close(js);
 
    jb_open_object(js, "session");
    if (p->flow != NULL && p->flow->protoctx != NULL) {
        const TcpSession *ssn = p->flow->protoctx;
        const char *tcp_state = StreamTcpStateAsString(ssn->state);
        if (tcp_state != NULL)
            jb_set_string(js, "state", tcp_state);
        if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_STATE_UPDATE) {
            const char *tcp_pstate = StreamTcpStateAsString(ssn->pstate);
            if (tcp_pstate != NULL)
                jb_set_string(js, "pstate", tcp_pstate);
        }
        EveAddFlowTcpFlags(ssn, "flags", js);
 
        jb_open_object(js, "client");
        LogStream(&ssn->client, js);
        jb_close(js);
        jb_open_object(js, "server");
        LogStream(&ssn->server, js);
        jb_close(js);
    }
    jb_close(js);
 
    if (p->l4.vars.tcp.stream_pkt_flags & STREAM_PKT_FLAG_EVENTSET) {
        jb_open_array(js, "events");
        for (int i = 0; i < p->events.cnt; i++) {
            uint8_t event_code = p->events.events[i];
            bool is_decode = EVENT_IS_DECODER_PACKET_ERROR(event_code);
            if (is_decode)
                continue;
            if (event_code >= DECODE_EVENT_MAX)
                continue;
            const char *event = DEvents[event_code].event_name;
            if (event == NULL)
                continue;
            jb_append_string(js, event);
        }
        jb_close(js);
    }
 
    if (p->drop_reason != 0) {
        const char *str = PacketDropReasonToString(p->drop_reason);
        jb_set_string(js, "reason", str);
    }
 
    /* Close stream. */
    jb_close(js);
 
    OutputJsonBuilderBuffer(tv, p, p->flow, js, td->ctx);
    jb_free(js);
 
    return TM_ECODE_OK;
}
 
/**
 * \brief Check if we need to log this packet
 *
 * \param tv    Pointer the current thread variables
 * \param p     Pointer the packet which is tested
 *
 * \retval bool true or false
 */
static bool EveStreamLogCondition(ThreadVars *tv, void *data, const Packet *p)
{
    EveStreamLogThread *td = data;
    EveStreamOutputCtx *ctx = td->stream_ctx;
 
    return (p->proto == IPPROTO_TCP &&
            (ctx->trigger_flags == 0xffff ||
                    (p->l4.vars.tcp.stream_pkt_flags & ctx->trigger_flags) != 0));
}
 
void EveStreamLogRegister(void)
{
    OutputPacketLoggerFunctions output_logger_functions = {
        .LogFunc = EveStreamLogger,
        .FlushFunc = OutputJsonLogFlush,
        .ConditionFunc = EveStreamLogCondition,
        .ThreadInitFunc = EveStreamLogThreadInit,
        .ThreadDeinitFunc = EveStreamLogThreadDeinit,
        .ThreadExitPrintStatsFunc = NULL,
    };
 
    OutputRegisterPacketSubModule(LOGGER_JSON_STREAM, "eve-log", MODULE_NAME, "eve-log.stream",
            EveStreamLogInitCtxSub, &output_logger_functions);
}