From 3c183af134f393c48d51f26fa5c4292ac0617a40 Mon Sep 17 00:00:00 2001 From: Thomas Chevalier Date: Tue, 21 Jun 2022 19:07:24 +0200 Subject: [PATCH] Modify ulog configuration to use pgsql --- pgsql-schema.sql | 174 +++++++++++++++++++++++++++++++++++++++++++++++ ulogd.conf | 64 +++++++++-------- 2 files changed, 205 insertions(+), 33 deletions(-) create mode 100644 pgsql-schema.sql diff --git a/pgsql-schema.sql b/pgsql-schema.sql new file mode 100644 index 0000000..649bda1 --- /dev/null +++ b/pgsql-schema.sql @@ -0,0 +1,174 @@ +-- vi: et ai ts=2 +-- +-- Warning: postgresql >= 8.2 is required for the 'DROP .. IF EXISTS' +-- Warning: this script DESTROYS EVERYTHING ! +-- +-- NOTE : - we could / should use types cidr / inet / macaddr for IP ? (see http://www.postgresql.org/docs/8.2/static/datatype-net-types.html) +-- - ON UPDATE is not supported ? +-- - type 'integer' is used (we have to check for overflows ..) +-- - type 'datetime' has been replaced by 'timestamp' + + +DROP TABLE IF EXISTS ulog2_ct CASCADE; + +-- +-- conntrack +-- +CREATE TABLE ulog2_ct ( + ct_id bigint PRIMARY KEY UNIQUE NOT NULL, + oob_family smallint default NULL, + orig_ip_saddr_str inet default NULL, + orig_ip_daddr_str inet default NULL, + orig_ip_protocol smallint default NULL, + orig_l4_sport integer default NULL, + orig_l4_dport integer default NULL, + orig_raw_pktlen bigint default 0, + orig_raw_pktcount bigint default 0, + reply_ip_saddr_str inet default NULL, + reply_ip_daddr_str inet default NULL, + reply_ip_protocol smallint default NULL, + reply_l4_sport integer default NULL, + reply_l4_dport integer default NULL, + reply_raw_pktlen bigint default 0, + reply_raw_pktcount bigint default 0, + icmp_code smallint default NULL, + icmp_type smallint default NULL, + ct_mark bigint default 0, + flow_start_sec bigint default 0, + flow_start_usec bigint default 0, + flow_end_sec bigint default 0, + flow_end_usec bigint default 0, + ct_event smallint default 0 +); + +-- +-- Additional INDEX +-- + +-- CREATE INDEX ulog2_ct_oob_family ON ulog2_ct(oob_family); +-- CREATE INDEX ulog2_ct_orig_ip_saddr ON ulog2_ct(orig_ip_saddr_str); +-- CREATE INDEX ulog2_ct_orig_ip_daddr ON ulog2_ct(orig_ip_daddr_str); +-- CREATE INDEX ulog2_ct_reply_ip_saddr ON ulog2_ct(reply_ip_saddr_str); +-- CREATE INDEX ulog2_ct_reply_ip_daddr ON ulog2_ct(reply_ip_daddr_str); +-- CREATE INDEX ulog2_ct_orig_l4_sport ON ulog2_ct(orig_l4_sport); +-- CREATE INDEX ulog2_ct_orig_l4_dport ON ulog2_ct(orig_l4_dport); +-- CREATE INDEX ulog2_ct_reply_l4_sport ON ulog2_ct(reply_l4_sport); +-- CREATE INDEX ulog2_ct_reply_l4_dport ON ulog2_ct(reply_l4_dport); +-- CREATE INDEX ulog2_ct_event ON ulog2_ct(ct_event); + +-- +-- Procedures +-- + +CREATE OR REPLACE FUNCTION INSERT_CT( + IN _ct_id integer, + IN _oob_family integer, + IN _orig_ip_saddr inet, + IN _orig_ip_daddr inet, + IN _orig_ip_protocol integer, + IN _orig_l4_sport integer, + IN _orig_l4_dport integer, + IN _orig_raw_pktlen bigint, + IN _orig_raw_pktcount bigint, + IN _reply_ip_saddr inet, + IN _reply_ip_daddr inet, + IN _reply_ip_protocol integer, + IN _reply_l4_sport integer, + IN _reply_l4_dport integer, + IN _reply_raw_pktlen bigint, + IN _reply_raw_pktcount bigint, + IN _icmp_code integer, + IN _icmp_type integer, + IN _ct_mark bigint, + IN _flow_start_sec bigint, + IN _flow_start_usec bigint, + IN _flow_end_sec bigint, + IN _flow_end_usec bigint, + IN _ct_event integer + ) +RETURNS bigint AS $$ + INSERT INTO ulog2_ct (ct_id, oob_family, orig_ip_saddr_str, orig_ip_daddr_str, orig_ip_protocol, + orig_l4_sport, orig_l4_dport, orig_raw_pktlen, orig_raw_pktcount, + reply_ip_saddr_str, reply_ip_daddr_str, reply_ip_protocol, + reply_l4_sport, reply_l4_dport, reply_raw_pktlen, reply_raw_pktcount, + icmp_code, icmp_type, ct_mark, + flow_start_sec, flow_start_usec, + flow_end_sec, flow_end_usec, ct_event) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24); + RETURN _ct_id; +$$ LANGUAGE SQL SECURITY INVOKER; + +CREATE OR REPLACE FUNCTION INSERT_OR_REPLACE_CT( + IN _ct_id integer, + IN _oob_family integer, + IN _orig_ip_saddr inet, + IN _orig_ip_daddr inet, + IN _orig_ip_protocol integer, + IN _orig_l4_sport integer, + IN _orig_l4_dport integer, + IN _orig_raw_pktlen bigint, + IN _orig_raw_pktcount bigint, + IN _reply_ip_saddr inet, + IN _reply_ip_daddr inet, + IN _reply_ip_protocol integer, + IN _reply_l4_sport integer, + IN _reply_l4_dport integer, + IN _reply_raw_pktlen bigint, + IN _reply_raw_pktcount bigint, + IN _icmp_code integer, + IN _icmp_type integer, + IN _ct_mark bigint, + IN _flow_start_sec bigint, + IN _flow_start_usec bigint, + IN _flow_end_sec bigint, + IN _flow_end_usec bigint, + IN _ct_event integer + ) +RETURNS bigint AS $$ +DECLARE + _id bigint; +BEGIN + IF (_ct_event = 4) THEN + if (_orig_ip_protocol = 1) THEN + UPDATE ulog2_ct SET (orig_raw_pktlen, orig_raw_pktcount, + reply_raw_pktlen, reply_raw_pktcount, + ct_mark, flow_end_sec, flow_end_usec, ct_event) + = ($8,$9,$15,$16,$19,$22,$23,$24) + WHERE ct_id=$1 AND oob_family=$2 AND orig_ip_saddr_str = $3 + AND orig_ip_daddr_str = $4 AND orig_ip_protocol = $5 + AND reply_ip_saddr_str = $10 AND reply_ip_daddr_str = $11 + AND reply_ip_protocol = $12 + AND icmp_code = $17 AND icmp_type = $18 + AND ct_event < 4; + ELSE + UPDATE ulog2_ct SET (orig_raw_pktlen, orig_raw_pktcount, + reply_raw_pktlen, reply_raw_pktcount, + ct_mark, flow_end_sec, flow_end_usec, ct_event) + = ($8,$9,$15,$16,$19,$22,$23,$24) + WHERE ct_id=$1 AND oob_family=$2 AND orig_ip_saddr_str = $3 + AND orig_ip_daddr_str = $4 AND orig_ip_protocol = $5 + AND orig_l4_sport = $6 AND orig_l4_dport = $7 + AND reply_ip_saddr_str = $10 AND reply_ip_daddr_str = $11 + AND reply_ip_protocol = $12 AND reply_l4_sport = $13 + AND reply_l4_dport = $14 + AND ct_event < 4; + END IF; + ELSE + _id := INSERT_CT($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24); + END IF; + RETURN _id; +END +$$ LANGUAGE plpgsql SECURITY INVOKER; + + +CREATE OR REPLACE FUNCTION DELETE_CT_FLOW( + IN _ct_packet_id bigint + ) +RETURNS void AS $$ + -- remember : table with most constraints first + DELETE FROM ulog2_ct WHERE ulog2_ct._ct_id = $1; +$$ LANGUAGE SQL SECURITY INVOKER; + + +-- Created by Pierre Chifflier +-- Adapted by Thomas Chevalier \ No newline at end of file diff --git a/ulogd.conf b/ulogd.conf index 23b2ea9..58a2462 100644 --- a/ulogd.conf +++ b/ulogd.conf @@ -2,14 +2,8 @@ # https://connect.ed-diamond.com/GNU-Linux-Magazine/glmfhs-041/ulogd2-journalisation-avancee-avec-netfilter [global] -###################################################################### -# GLOBAL OPTIONS -###################################################################### - - # logfile for status messages logfile="syslog" - # loglevel: debug(1), info(3), notice(5), error(7) or fatal(8) (default 5) loglevel=3 @@ -58,44 +52,48 @@ stack=log1:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,print1:PRINTPKT,emu1:LOG # Packet logging stack=log2:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,mac2str1:HWHDR,print1:PRINTPKT,json1:JSON +# Conntrack logging +stack=ct1:NFCT,ip2str1:IP2STR,printflow1:PRINTFLOW,json2:JSON +stack=ct1:NFCT,ip2bin1:IP2BIN,pgsql1:PGSQL + + # Logging of system packet through NFLOG [log1] # netlink multicast group (the same as the iptables --nflog-group param) # Group O is used by the kernel to log connection tracking invalid message group=0 -#netlink_socket_buffer_size=217088 -#netlink_socket_buffer_maxsize=1085440 -# set number of packet to queue inside kernel -#netlink_qthreshold=1 -# set the delay before flushing packet in the queue inside kernel (in 10ms) -#netlink_qtimeout=100 -# packet logging through NFLOG for group 1 +# General packet logging [log2] -# netlink multicast group (the same as the iptables --nflog-group param) -group=1 # Group has to be different from the one use in log1 -#netlink_socket_buffer_size=217088 -#netlink_socket_buffer_maxsize=1085440 -# If your kernel is older than 2.6.29 and if a NFLOG input plugin with -# group 0 is not used by any stack, you need to have at least one NFLOG -# input plugin with bind set to 1. If you don't do that you may not -# receive any message from the kernel. -#bind=1 +# Group has to be different from the one use in log1 +group=1 + +[ct1] +# NEW = 1 +# UPDATE = 2 +# DESTROY = 4 +event_mask=0x0000005 +# If hash_enable=1 (the default), the kernel will automatically +# match NEW and DESTROY events and only report DESTROY events. +hash_enable=0 +# reliable=1 # enable reliable flow-based logging (may drop packets) [emu1] -file="/var/log/ulog/syslogemu.log" +file="/var/log/ulog/kernel.log" sync=1 [json1] sync=1 file="/var/log/ulog/ulogd.json" -#timestamp=0 -# device name to be used in JSON message -#device="My awesome Netfilter firewall" -# If boolean_label is set to 1 then the numeric_label put on packet -# by the input plugin is coding the action on packet: if 0, then -# packet has been blocked and if non null it has been accepted. -#boolean_label=1 -# Uncomment the following line to use JSON v1 event format that -# can provide better compatility with some JSON file reader. -#eventv1=1 + +[json2] +sync=1 +file="/var/log/ulog/ct.json" + +[pgsql1] +db="ulog2" +host="replace-with-cloud-sql-ip-address" +user="ulog" +table="ulog2_ct" +pass="replace-with-database-password" +procedure="INSERT_OR_REPLACE_CT"