Matthias Vallentin avatar Matthias Vallentin committed 555fbaa

Add Bro lexer.

Comments (0)

Files changed (3)

pygments/lexers/_mapping.py

     'BlitzMaxLexer': ('pygments.lexers.compiled', 'BlitzMax', ('blitzmax', 'bmax'), ('*.bmx',), ('text/x-bmx',)),
     'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)),
     'BrainfuckLexer': ('pygments.lexers.other', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)),
+    'BroLexer': ('pygments.lexers.agile', 'Bro', ('bro'), ('*.bro'), ()),
     'CLexer': ('pygments.lexers.compiled', 'C', ('c',), ('*.c', '*.h', '*.idc'), ('text/x-chdr', 'text/x-csrc')),
     'CMakeLexer': ('pygments.lexers.text', 'CMake', ('cmake',), ('*.cmake', 'CMakeLists.txt'), ('text/x-cmake',)),
     'CObjdumpLexer': ('pygments.lexers.asm', 'c-objdump', ('c-objdump',), ('*.c-objdump',), ('text/x-c-objdump',)),

pygments/lexers/agile.py

             (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop')
         ],
     }
+
+
+class BroLexer(RegexLexer):
+    name = 'Bro'
+    aliases = ['bro']
+    filenames = ['*.bro']
+
+    _hex = r'[0-9a-fA-F_]+'
+    _float = r'((\d*\.?\d+)|(\d+\.?\d*))([eE][-+]?\d+)?'
+    _h = r'[A-Za-z0-9][-A-Za-z0-9]*'
+
+    tokens = {
+        'root': [
+            # Whitespace
+            ('^@.*?\n', Comment.Preproc),
+            (r'#.*?\n', Comment.Single),
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'\\\n', Text),
+            # Keywords
+            (r'(add|alarm|break|case|const|continue|delete|do|else|enum|event'
+             r'|export|for|function|if|global|local|module|next'
+             r'|of|print|redef|return|schedule|type|when|while)\b', Keyword),
+            (r'(addr|any|bool|count|counter|double|file|int|interval|net'
+             r'|pattern|port|record|set|string|subnet|table|time|timer'
+             r'|vector)\b', Keyword.Type),
+            (r'(T|F)\b', Keyword.Constant),
+            (r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire'
+             r'|default|disable_print_hook|raw_output|encrypt|group|log'
+             r'|mergeable|optional|persistent|priority|redef'
+             r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation,
+                 Keyword)),
+            (r'\s+module\b', Keyword.Namespace),
+            # Addresses, ports and networks
+            (r'\d+/(tcp|udp|icmp|unknown)\b', Number),
+            (r'(\d+\.){3}\d+', Number),
+            (r'(' + _hex + r'){7}' + _hex, Number),
+            (r'0x' + _hex + r'(' + _hex + r'|:)*::(' + _hex + r'|:)*', Number),
+            (r'((\d+|:)(' + _hex + r'|:)*)?::(' + _hex + r'|:)*', Number),
+            (r'(\d+\.\d+\.|(\d+\.){2}\d+)', Number),
+            # Hostnames
+            (_h + r'(\.' + _h + r')+', String),
+            # Numeric
+            (_float + r'\s+(day|hr|min|sec|msec|usec)s?\b', Literal.Date),
+            (r'0[xX]' + _hex, Number.Hex),
+            (_float, Number.Float),
+            (r'\d+', Number.Integer),
+            (r'/', String.Regex, 'regex'),
+            (r'"', String, 'string'),
+            # Operators
+            (r'[!%*/+-:<=>?~|]', Operator),
+            (r'([-+=&|]{2}|[+-=!><]=)', Operator),
+            (r'(in|match)\b', Operator.Word),
+            (r'[{}()\[\]$.,;]', Punctuation),
+            # Identfier
+            (r'([_a-zA-Z]\w*)(::)', bygroups(Name, Name.Namespace)),
+            (r'[a-zA-Z_][a-zA-Z_0-9]*', Name)
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+            (r'[^\\"\n]+', String),
+            (r'\\\n', String),
+            (r'\\', String)
+        ],
+        'regex': [
+            (r'/', String.Regex, '#pop'),
+            (r'\\[\\nt/]', String.Regex), # String.Escape is too intense here.
+            (r'[^\\/\n]+', String.Regex),
+            (r'\\\n', String.Regex),
+            (r'\\', String.Regex)
+        ]
+    }

tests/examplefiles/test.bro

+@load notice
+@load utils/thresholds
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	redef enum Notice::Type += {
+		Login,
+		Password_Guessing,
+		Login_By_Password_Guesser,
+		Login_From_Interesting_Hostname,
+		Bytecount_Inconsistency,
+	};
+
+	type Info: record {
+		ts:              time         &log;
+		uid:             string       &log;
+		id:              conn_id      &log;
+		status:          string       &log &optional;
+		direction:       string       &log &optional;
+		remote_location: geo_location &log &optional;
+		client:          string       &log &optional;
+		server:          string       &log &optional;
+		resp_size:       count        &log &default=0;
+		
+		## Indicate if the SSH session is done being watched.
+		done:            bool         &default=F;
+	};
+
+	const password_guesses_limit = 30 &redef;
+	
+	# The size in bytes at which the SSH connection is presumed to be
+	# successful.
+	const authentication_data_size = 5500 &redef;
+	
+	# The amount of time to remember presumed non-successful logins to build
+	# model of a password guesser.
+	const guessing_timeout = 30 mins &redef;
+
+	# The set of countries for which you'd like to throw notices upon successful login
+	#   requires Bro compiled with libGeoIP support
+	const watched_countries: set[string] = {"RO"} &redef;
+
+	# Strange/bad host names to originate successful SSH logins
+	const interesting_hostnames =
+			/^d?ns[0-9]*\./ |
+			/^smtp[0-9]*\./ |
+			/^mail[0-9]*\./ |
+			/^pop[0-9]*\./  |
+			/^imap[0-9]*\./ |
+			/^www[0-9]*\./  |
+			/^ftp[0-9]*\./  &redef;
+
+	# This is a table with orig subnet as the key, and subnet as the value.
+	const ignore_guessers: table[subnet] of subnet &redef;
+	
+	# If true, we tell the event engine to not look at further data
+	# packets after the initial SSH handshake. Helps with performance
+	# (especially with large file transfers) but precludes some
+	# kinds of analyses (e.g., tracking connection size).
+	const skip_processing_after_detection = F &redef;
+	
+	# Keeps count of how many rejections a host has had
+	global password_rejections: table[addr] of TrackCount 
+		&write_expire=guessing_timeout
+		&synchronized;
+
+	# Keeps track of hosts identified as guessing passwords
+	# TODO: guessing_timeout doesn't work correctly here.  If a user redefs
+	#       the variable, it won't take effect.
+	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized;
+	
+	global log_ssh: event(rec: Info);
+}
+
+# Configure DPD and the packet filter
+redef capture_filters += { ["ssh"] = "tcp port 22" };
+redef dpd_config += { [ANALYZER_SSH] = [$ports = set(22/tcp)] };
+
+redef record connection += {
+	ssh: Info &optional;
+};
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Info, $ev=log_ssh]);
+}
+
+function set_session(c: connection)
+	{
+	if ( ! c?$ssh )
+		{
+		local info: Info;
+		info$ts=network_time();
+		info$uid=c$uid;
+		info$id=c$id;
+		c$ssh = info;
+		}
+	}
+
+function check_ssh_connection(c: connection, done: bool)
+	{
+	# If done watching this connection, just return.
+	if ( c$ssh$done )
+		return;
+	
+	# If this is still a live connection and the byte count has not
+	# crossed the threshold, just return and let the resheduled check happen later.
+	if ( !done && c$resp$size < authentication_data_size )
+		return;
+
+	# Make sure the server has sent back more than 50 bytes to filter out
+	# hosts that are just port scanning.  Nothing is ever logged if the server
+	# doesn't send back at least 50 bytes.
+	if ( c$resp$size < 50 )
+		return;
+
+	local status = "failure";
+	local direction = Site::is_local_addr(c$id$orig_h) ? "to" : "from";
+	local location: geo_location;
+	location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
+	
+	if ( done && c$resp$size < authentication_data_size )
+		{
+		# presumed failure
+		if ( c$id$orig_h !in password_rejections )
+			password_rejections[c$id$orig_h] = new_track_count();
+			
+		# Track the number of rejections
+		if ( !(c$id$orig_h in ignore_guessers &&
+		       c$id$resp_h in ignore_guessers[c$id$orig_h]) )
+			++password_rejections[c$id$orig_h]$n;
+			
+		if ( default_check_threshold(password_rejections[c$id$orig_h]) )
+			{
+			add password_guessers[c$id$orig_h];
+			NOTICE([$note=Password_Guessing,
+			        $conn=c,
+			        $msg=fmt("SSH password guessing by %s", c$id$orig_h),
+			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n),
+			        $n=password_rejections[c$id$orig_h]$n]);
+			}
+		} 
+	# TODO: This is to work around a quasi-bug in Bro which occasionally 
+	#       causes the byte count to be oversized.
+	#   Watch for Gregors work that adds an actual counter of bytes transferred.
+	else if ( c$resp$size < 20000000 ) 
+		{ 
+		# presumed successful login
+		status = "success";
+		c$ssh$done = T;
+
+		if ( c$id$orig_h in password_rejections &&
+		     password_rejections[c$id$orig_h]$n > password_guesses_limit &&
+		     c$id$orig_h !in password_guessers )
+			{
+			add password_guessers[c$id$orig_h];
+			NOTICE([$note=Login_By_Password_Guesser,
+			        $conn=c,
+			        $n=password_rejections[c$id$orig_h]$n,
+			        $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h),
+			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n)]);
+			}
+		
+		local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)",
+		              direction, location$country_code, location$region, location$city,
+		              location$latitude, location$longitude,
+		              id_string(c$id), c$resp$size);
+		NOTICE([$note=Login,
+		        $conn=c,
+		        $msg=message,
+		        $sub=location$country_code]);
+		
+		# Check to see if this login came from an interesting hostname
+		when ( local hostname = lookup_addr(c$id$orig_h) )
+			{
+			if ( interesting_hostnames in hostname )
+				{
+				NOTICE([$note=Login_From_Interesting_Hostname,
+				        $conn=c,
+				        $msg=fmt("Strange login from %s", hostname),
+				        $sub=hostname]);
+				}
+			}
+			
+		if ( location$country_code in watched_countries )
+			{
+			
+			}
+			
+		}
+	else if ( c$resp$size >= 200000000 ) 
+		{
+		NOTICE([$note=Bytecount_Inconsistency,
+		        $conn=c,
+		        $msg="During byte counting in SSH analysis, an overly large value was seen.",
+		        $sub=fmt("%d",c$resp$size)]);
+		}
+
+	c$ssh$remote_location = location;
+	c$ssh$status = status;
+	c$ssh$direction = direction;
+	c$ssh$resp_size = c$resp$size;
+	
+	Log::write(SSH, c$ssh);
+	
+	# Set the "done" flag to prevent the watching event from rescheduling
+	# after detection is done.
+	c$ssh$done;
+	
+	# Stop watching this connection, we don't care about it anymore.
+	if ( skip_processing_after_detection )
+		{
+		skip_further_processing(c$id);
+		set_record_packets(c$id, F);
+		}
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$ssh )
+		check_ssh_connection(c, T);
+	}
+
+event ssh_watcher(c: connection)
+	{
+	local id = c$id;
+	# don't go any further if this connection is gone already!
+	if ( !connection_exists(id) )
+		return;
+
+	check_ssh_connection(c, F);
+	if ( ! c$ssh$done )
+		schedule +15secs { ssh_watcher(c) };
+	}
+
+event ssh_server_version(c: connection, version: string) &priority=5
+	{
+	set_session(c);
+	c$ssh$server = version;
+	}
+	
+event ssh_client_version(c: connection, version: string) &priority=5
+	{
+	set_session(c);
+	c$ssh$client = version;
+	schedule +15secs { ssh_watcher(c) };
+	}
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.