feat: Add some basic support for Apache logs

dimonomid · dimonomid · commit 19a442e5b3ee · 2025-06-03T10:23:53.000+03:00
Just add a couple predefined time formats commonly seen in Apache logs.
No context field parsing yet, or anything else.
diff --git a/core/core_testdata/input_logfiles/apache_jun/error.log b/core/core_testdata/input_logfiles/apache_jun/error.log
@@ -0,0 +1,5 @@
+[Tue Jun 03 13:45:27.123456 2025] [core:notice] [pid 1234] AH00094: Command line: '/usr/sbin/httpd -D FOREGROUND'
+[Tue Jun 03 13:46:12.789012 2025] [mpm_prefork:notice] [pid 1234] AH00163: Apache/2.4.57 (Fedora) configured -- resuming normal operations
+[Tue Jun 03 13:47:01.987654 2025] [authz_core:error] [pid 1256] [client 192.168.1.101:49721] AH01630: client denied by server configuration: /var/www/html/private
+[Tue Jun 03 13:48:45.543210 2025] [php:error] [pid 1278] [client 192.168.1.101:49802] PHP Fatal error:  Uncaught Error: Call to undefined function mysql_connect() in /var/www/html/index.php:12
+[Tue Jun 03 13:50:00.000123 2025] [ssl:warn] [pid 1234] AH01906: RSA server certificate is a CA certificate (BasicConstraints: CA == TRUE !?)
diff --git a/core/core_testdata/test_cases_core/04_apache/test_scenario.yaml b/core/core_testdata/test_cases_core/04_apache/test_scenario.yaml
@@ -0,0 +1,24 @@
+descr: "Get logs across May and Jun, which means that lexicographically the timestamps reduce with the traditional syslog format"
+current_time: "2025-03-12T10:58:00Z"
+manager_params:
+  config_log_streams:
+    testhost-3:
+      log_files:
+        kind: all_from_dir
+        dir: ../../input_logfiles/apache_jun
+      options:
+        shell_init:
+          - 'export TZ=UTC'
+  initial_lstreams: "testhost-*"
+  client_id: "core-test-runner"
+test_steps:
+
+  - descr: "initial query spanning across may and jun, which makes"
+    query:
+      params:
+        max_num_lines: 30
+        to:   "2025-06-03T13:00:00Z"
+        to:   "2025-06-03T14:00:00Z"
+        pattern: ""
+        load_earlier: false
+      want: want_log_resp_01_initial.txt
diff --git a/core/core_testdata/test_cases_core/04_apache/want_log_resp_01_initial.txt b/core/core_testdata/test_cases_core/04_apache/want_log_resp_01_initial.txt
@@ -0,0 +1,43 @@
+NumMsgsTotal: 5
+LoadedEarlier: false
+Num errors: 0
+
+Num MinuteStats: 5
+- 2025-06-03-13-45: 1
+- 2025-06-03-13-46: 1
+- 2025-06-03-13-47: 1
+- 2025-06-03-13-48: 1
+- 2025-06-03-13-50: 1
+
+Num Logs: 5
+- 2025-06-03T13:45:27.123456000Z,F,/tmp/nerdlog_core_test_output/04_apache/lstreams/testhost-3/logfile,000001,000001,----,[core:notice] [pid 1234] AH00094: Command line: '/usr/sbin/httpd -D FOREGROUND'
+  context: {"lstream":"testhost-3"}
+  orig: [Tue Jun 03 13:45:27.123456 2025] [core:notice] [pid 1234] AH00094: Command line: '/usr/sbin/httpd -D FOREGROUND'
+- 2025-06-03T13:46:12.789012000Z,F,/tmp/nerdlog_core_test_output/04_apache/lstreams/testhost-3/logfile,000002,000002,----,[mpm_prefork:notice] [pid 1234] AH00163: Apache/2.4.57 (Fedora) configured -- resuming normal operations
+  context: {"lstream":"testhost-3"}
+  orig: [Tue Jun 03 13:46:12.789012 2025] [mpm_prefork:notice] [pid 1234] AH00163: Apache/2.4.57 (Fedora) configured -- resuming normal operations
+- 2025-06-03T13:47:01.987654000Z,F,/tmp/nerdlog_core_test_output/04_apache/lstreams/testhost-3/logfile,000003,000003,erro,[authz_core:error] [pid 1256] [client 192.168.1.101:49721] AH01630: client denied by server configuration: /var/www/html/private
+  context: {"lstream":"testhost-3"}
+  orig: [Tue Jun 03 13:47:01.987654 2025] [authz_core:error] [pid 1256] [client 192.168.1.101:49721] AH01630: client denied by server configuration: /var/www/html/private
+- 2025-06-03T13:48:45.543210000Z,F,/tmp/nerdlog_core_test_output/04_apache/lstreams/testhost-3/logfile,000004,000004,erro,[php:error] [pid 1278] [client 192.168.1.101:49802] PHP Fatal error:  Uncaught Error: Call to undefined function mysql_connect() in /var/www/html/index.php:12
+  context: {"lstream":"testhost-3"}
+  orig: [Tue Jun 03 13:48:45.543210 2025] [php:error] [pid 1278] [client 192.168.1.101:49802] PHP Fatal error:  Uncaught Error: Call to undefined function mysql_connect() in /var/www/html/index.php:12
+- 2025-06-03T13:50:00.000123000Z,F,/tmp/nerdlog_core_test_output/04_apache/lstreams/testhost-3/logfile,000005,000005,warn,[ssl:warn] [pid 1234] AH01906: RSA server certificate is a CA certificate (BasicConstraints: CA == TRUE !?)
+  context: {"lstream":"testhost-3"}
+  orig: [Tue Jun 03 13:50:00.000123 2025] [ssl:warn] [pid 1234] AH01906: RSA server certificate is a CA certificate (BasicConstraints: CA == TRUE !?)
+
+DebugInfo:
+{
+  "testhost-3": {
+    "AgentStdout": null,
+    "AgentStderr": [
+      "debug:prev logfile /tmp/nerdlog_core_test_output/04_apache/lstreams/testhost-3/logfile.1 doesn't exist, using a dummy empty file /tmp/nerdlog-empty-file",
+      "debug:index file doesn't exist or is empty, gonna refresh it",
+      "debug:the to 2025-06-03-14:00 isn't found, will use the end",
+      "debug:Getting logs from the very beginning in prev /tmp/nerdlog-empty-file until the end of latest /tmp/nerdlog_core_test_output/04_apache/lstreams/testhost-3/logfile",
+      "debug:Command to filter logs by time range:",
+      "debug: bash -c 'cat /tmp/nerdlog-empty-file \u0026\u0026 cat /tmp/nerdlog_core_test_output/04_apache/lstreams/testhost-3/logfile'",
+      "debug:Filtered out 0 from 5 lines"
+    ]
+  }
+}
diff --git a/core/parsing_time.go b/core/parsing_time.go
@@ -149,6 +149,10 @@ func DetectTimeLayout(logLine string) string {
 		"Mon Jan 2 15:04:05 2006",
 		"02-Jan-2006 15:04:05",
 		"Jan 02 15:04:05",
+
+		// Some variations of Apache HTTP Server logs
+		"[Mon Jan 02 15:04:05.999999 2006]",
+		"[Mon Jan 02 15:04:05 2006]",
 	}
 
 	for _, layout := range knownFormats {
