From b08d5a3ed809d967eb51db30d719af9eb916e546 Mon Sep 17 00:00:00 2001 From: Daniel Steger Date: Tue, 31 Dec 2019 10:25:08 +0100 Subject: [PATCH 01/11] sample for signing --- README.md | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 README.md diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 57b2251..78dd45c --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # logstash-forwarder-java + ## What is this ? Logstash-forwarder-java is a log shipper program written in java. This is in fact a java version of [logstash-forwarder](https://github.com/elasticsearch/logstash-forwarder) by jordansissel. From cbc9ad664f0a25ab105d59ec2f318faf609cc658 Mon Sep 17 00:00:00 2001 From: toliman Date: Tue, 31 Dec 2019 10:29:56 +0100 Subject: [PATCH 02/11] sample for signing removed --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 78dd45c..57b2251 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # logstash-forwarder-java - ## What is this ? Logstash-forwarder-java is a log shipper program written in java. This is in fact a java version of [logstash-forwarder](https://github.com/elasticsearch/logstash-forwarder) by jordansissel. From 1462050da47450f286f29096c951ab4a1a18c4ea Mon Sep 17 00:00:00 2001 From: toliman Date: Mon, 17 May 2021 22:04:40 +0200 Subject: [PATCH 03/11] Fixed correct spool counter on multiline events. --- README.md | 0 pom.xml | 7 +- .../fetter/logstashforwarder/FileReader.java | 429 +++++++++--------- .../fetter/logstashforwarder/Multiline.java | 121 ++--- .../logstashforwarder/FileReaderTest.java | 201 ++++---- 5 files changed, 381 insertions(+), 377 deletions(-) mode change 100755 => 100644 README.md diff --git a/README.md b/README.md old mode 100755 new mode 100644 diff --git a/pom.xml b/pom.xml index 7809709..b3ec7de 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 logstash-forwarder-java logstash-forwarder-java - 0.2.4 + 0.2.5 logstash-forwarder-java Java version of logstash forwarder https://github.com/didfet/logstash-forwarder-java @@ -81,6 +81,9 @@ 2.5.3 src/assembly/tarball.xml + + jar-with-dependencies + @@ -167,4 +170,4 @@ - \ No newline at end of file + diff --git a/src/main/java/info/fetter/logstashforwarder/FileReader.java b/src/main/java/info/fetter/logstashforwarder/FileReader.java index 41f2d89..e1c5552 100644 --- a/src/main/java/info/fetter/logstashforwarder/FileReader.java +++ b/src/main/java/info/fetter/logstashforwarder/FileReader.java @@ -1,217 +1,212 @@ -package info.fetter.logstashforwarder; - -/* - * Copyright 2015 Didier Fetter - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import info.fetter.logstashforwarder.util.AdapterException; -import info.fetter.logstashforwarder.util.RandomAccessFile; - -import java.io.File; -import java.io.IOException; -//import java.io.RandomAccessFile; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import org.apache.commons.lang.ArrayUtils; - -import org.apache.log4j.Logger; - - -public class FileReader extends Reader { - private static Logger logger = Logger.getLogger(FileReader.class); - private static final byte[] ZIP_MAGIC = new byte[] {(byte) 0x50, (byte) 0x4b, (byte) 0x03, (byte) 0x04}; - private static final byte[] LZW_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x9d}; - private static final byte[] LZH_MAGIC = new byte[] {(byte) 0x1f, (byte) 0xa0}; - private static final byte[] GZ_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x8b, (byte) 0x08}; - private static final byte[][] MAGICS = new byte[][] {ZIP_MAGIC, LZW_MAGIC, LZH_MAGIC, GZ_MAGIC}; - private Map pointerMap; - - public FileReader(int spoolSize) { - super(spoolSize); - } - - public int readFiles(Collection fileList) throws AdapterException { - int eventCount = 0; - if(logger.isTraceEnabled()) { - logger.trace("Reading " + fileList.size() + " file(s)"); - } - pointerMap = new HashMap(fileList.size(),1); - for(FileState state : fileList) { - eventCount += readFile(state, spoolSize - eventCount); - } - if(eventCount > 0) { - try { - adapter.sendEvents(eventList); - } catch(AdapterException e) { - eventList.clear(); // Be sure no events will be sent twice after reconnect - throw e; - } - } - for(FileState state : fileList) { - state.setPointer(pointerMap.get(state.getFile())); - } - eventList.clear(); - return eventCount; // Return number of events sent to adapter - } - - private int readFile(FileState state, int spaceLeftInSpool) { - File file = state.getFile(); - long pointer = state.getPointer(); - int numberOfEvents = 0; - try { - if(state.isDeleted() || state.getRandomAccessFile() == null) { // Don't try to read this file - if(logger.isTraceEnabled()) { - logger.trace("File : " + file + " has been deleted"); - } - } else if(state.getRandomAccessFile().length() == 0) { - if(logger.isTraceEnabled()) { - logger.trace("File : " + file + " is empty"); - } - } else { - int eventListSizeBefore = eventList.size(); - if(logger.isTraceEnabled()) { - logger.trace("File : " + file + " pointer : " + pointer); - logger.trace("Space left in spool : " + spaceLeftInSpool); - } - if(isCompressedFile(state)) { - pointer = file.length(); - } else { - pointer = readLines(state, spaceLeftInSpool); - } - numberOfEvents = eventList.size() - eventListSizeBefore; - } - } catch(IOException e) { - logger.warn("Exception raised while reading file : " + state.getFile(), e); - } - pointerMap.put(file, pointer); - return numberOfEvents; // Return number of events read - } - - private boolean isCompressedFile(FileState state) { - RandomAccessFile reader = state.getRandomAccessFile(); - try { - for(byte[] magic : MAGICS) { - byte[] fileBytes = new byte[magic.length]; - reader.seek(0); - int read = reader.read(fileBytes); - if (read != magic.length) { - continue; - } - if(Arrays.equals(magic, fileBytes)) { - logger.debug("Compressed file detected : " + state.getFile()); - return true; - } - } - } catch(IOException e) { - logger.warn("Exception raised while reading file : " + state.getFile(), e); - } - return false; - } - - private long readLines(FileState state, int spaceLeftInSpool) { - RandomAccessFile reader = state.getRandomAccessFile(); - long pos = state.getPointer(); - Multiline multiline = state.getMultiline(); - try { - reader.seek(pos); - byte[] line = readLine(reader); - byte[] bufferedLines = null; - while (line != null && spaceLeftInSpool > 0) { - if(logger.isTraceEnabled()) { - logger.trace("-- Read line : " + new String(line)); - logger.trace("-- Space left in spool : " + spaceLeftInSpool); - } - pos = reader.getFilePointer(); - if (multiline == null) { - addEvent(state, pos, line); - } - else { - if (logger.isTraceEnabled()) { - logger.trace("-- Multiline : " + multiline); - logger.trace("-- Multiline : matches " + multiline.isPatternFound(line)); - } - if (multiline.isPatternFound(line)) - { - // buffer the line - if (bufferedLines != null) - { - bufferedLines = ArrayUtils.addAll(bufferedLines, line); - } - else - { - bufferedLines = line; - } - } - else { - if (multiline.isPrevious()) { - // did not match, so new event started - if (bufferedLines != null) { - addEvent(state, pos, bufferedLines); - } - bufferedLines = line; - } - else { - // did not match, add the current line - if (bufferedLines != null) { - addEvent(state, pos, ArrayUtils.addAll(bufferedLines, line)); - bufferedLines = null; - } - else - addEvent(state, pos, line); - } - } - } - line = readLine(reader); - spaceLeftInSpool--; - } - if (bufferedLines != null) { - addEvent(state, pos, bufferedLines); // send any buffered lines left - } - reader.seek(pos); // Ensure we can re-read if necessary - } catch(IOException e) { - logger.warn("Exception raised while reading file : " + state.getFile(), e); - } - return pos; - } - - private byte[] readLine(RandomAccessFile reader) throws IOException { - byteBuffer.clear(); - int ch; - boolean seenCR = false; - while((ch=reader.read()) != -1) { - switch(ch) { - case '\n': - byte[] line = new byte[byteBuffer.position()]; - byteBuffer.rewind(); - byteBuffer.get(line); - return line; - case '\r': - seenCR = true; - break; - default: - if (seenCR) { - byteBuffer.put((byte) '\r'); - seenCR = false; - } - byteBuffer.put((byte)ch); - } - } - return null; - } - -} +package info.fetter.logstashforwarder; + +/* + * Copyright 2015 Didier Fetter + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import info.fetter.logstashforwarder.util.AdapterException; +import info.fetter.logstashforwarder.util.RandomAccessFile; + +import java.io.File; +import java.io.IOException; +//import java.io.RandomAccessFile; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.ArrayUtils; + +import org.apache.log4j.Logger; + + +public class FileReader extends Reader { + private static Logger logger = Logger.getLogger(FileReader.class); + private static final byte[] ZIP_MAGIC = new byte[]{(byte) 0x50, (byte) 0x4b, (byte) 0x03, (byte) 0x04}; + private static final byte[] LZW_MAGIC = new byte[]{(byte) 0x1f, (byte) 0x9d}; + private static final byte[] LZH_MAGIC = new byte[]{(byte) 0x1f, (byte) 0xa0}; + private static final byte[] GZ_MAGIC = new byte[]{(byte) 0x1f, (byte) 0x8b, (byte) 0x08}; + private static final byte[][] MAGICS = new byte[][]{ZIP_MAGIC, LZW_MAGIC, LZH_MAGIC, GZ_MAGIC}; + private Map pointerMap; + + public FileReader(int spoolSize) { + super(spoolSize); + } + + public int readFiles(Collection fileList) throws AdapterException { + int eventCount = 0; + if (logger.isTraceEnabled()) { + logger.trace("Reading " + fileList.size() + " file(s)"); + } + pointerMap = new HashMap(fileList.size(), 1); + for (FileState state : fileList) { + eventCount += readFile(state, spoolSize - eventCount); + } + if (eventCount > 0) { + try { + adapter.sendEvents(eventList); + } catch (AdapterException e) { + eventList.clear(); // Be sure no events will be sent twice after reconnect + throw e; + } + } + for (FileState state : fileList) { + state.setPointer(pointerMap.get(state.getFile())); + } + eventList.clear(); + return eventCount; // Return number of events sent to adapter + } + + private int readFile(FileState state, int spaceLeftInSpool) { + File file = state.getFile(); + long pointer = state.getPointer(); + int numberOfEvents = 0; + try { + if (state.isDeleted() || state.getRandomAccessFile() == null) { // Don't try to read this file + if (logger.isTraceEnabled()) { + logger.trace("File : " + file + " has been deleted"); + } + } else if (state.getRandomAccessFile().length() == 0) { + if (logger.isTraceEnabled()) { + logger.trace("File : " + file + " is empty"); + } + } else { + int eventListSizeBefore = eventList.size(); + if (logger.isTraceEnabled()) { + logger.trace("File : " + file + " pointer : " + pointer); + logger.trace("Space left in spool : " + spaceLeftInSpool); + } + if (isCompressedFile(state)) { + pointer = file.length(); + } else { + pointer = readLines(state, spaceLeftInSpool); + } + numberOfEvents = eventList.size() - eventListSizeBefore; + } + } catch (IOException e) { + logger.warn("Exception raised while reading file : " + state.getFile(), e); + } + pointerMap.put(file, pointer); + return numberOfEvents; // Return number of events read + } + + private boolean isCompressedFile(FileState state) { + RandomAccessFile reader = state.getRandomAccessFile(); + try { + for (byte[] magic : MAGICS) { + byte[] fileBytes = new byte[magic.length]; + reader.seek(0); + int read = reader.read(fileBytes); + if (read != magic.length) { + continue; + } + if (Arrays.equals(magic, fileBytes)) { + logger.debug("Compressed file detected : " + state.getFile()); + return true; + } + } + } catch (IOException e) { + logger.warn("Exception raised while reading file : " + state.getFile(), e); + } + return false; + } + + private long readLines(FileState state, int spaceLeftInSpool) { + RandomAccessFile reader = state.getRandomAccessFile(); + long pos = state.getPointer(); + Multiline multiline = state.getMultiline(); + try { + reader.seek(pos); + byte[] line = readLine(reader); + byte[] bufferedLines = null; + while (line != null && spaceLeftInSpool > 0) { + if (logger.isTraceEnabled()) { + logger.trace("-- Read line : " + new String(line)); + logger.trace("-- Space left in spool : " + spaceLeftInSpool); + } + pos = reader.getFilePointer(); + if (multiline == null) { + addEvent(state, pos, line); + spaceLeftInSpool--; + } else { + if (logger.isTraceEnabled()) { + logger.trace("-- Multiline : " + multiline); + logger.trace("-- Multiline : matches " + multiline.isPatternFound(line)); + } + if (multiline.isPatternFound(line)) { + // buffer the line + if (bufferedLines != null) { + bufferedLines = ArrayUtils.addAll(bufferedLines, line); + } else { + bufferedLines = line; + } + } else { + if (multiline.isPrevious()) { + // did not match, so new event started + if (bufferedLines != null) { + addEvent(state, pos, bufferedLines); + } + bufferedLines = line; + } else { + // did not match, add the current line + if (bufferedLines != null) { + addEvent(state, pos, ArrayUtils.addAll(bufferedLines, line)); + bufferedLines = null; + } else { + addEvent(state, pos, line); + } + } + spaceLeftInSpool--; + } + } + line = readLine(reader); + } + if (bufferedLines != null) { + addEvent(state, pos, bufferedLines); // send any buffered lines left + } + reader.seek(pos); // Ensure we can re-read if necessary + } catch (IOException e) { + logger.warn("Exception raised while reading file : " + state.getFile(), e); + } + return pos; + } + + private byte[] readLine(RandomAccessFile reader) throws IOException { + byteBuffer.clear(); + int ch; + boolean seenCR = false; + while ((ch = reader.read()) != -1) { + switch (ch) { + case '\n': + byte[] line = new byte[byteBuffer.position()]; + byteBuffer.rewind(); + byteBuffer.get(line); + return line; + case '\r': + seenCR = true; + break; + default: + if (seenCR) { + byteBuffer.put((byte) '\r'); + seenCR = false; + } + byteBuffer.put((byte) ch); + } + } + return null; + } + +} diff --git a/src/main/java/info/fetter/logstashforwarder/Multiline.java b/src/main/java/info/fetter/logstashforwarder/Multiline.java index b972c5c..f472d0f 100644 --- a/src/main/java/info/fetter/logstashforwarder/Multiline.java +++ b/src/main/java/info/fetter/logstashforwarder/Multiline.java @@ -20,70 +20,73 @@ import java.io.UnsupportedEncodingException; import java.util.Map; import java.util.regex.Pattern; + import org.apache.commons.lang.builder.ToStringBuilder; public class Multiline { - public enum WhatType { Previous, Next }; - - private Pattern pattern = null; - private boolean negate = false; - private WhatType what = WhatType.Previous; - - public Multiline() { - } - - public Multiline(Multiline event) { - if(event != null) { - this.negate = event.negate; - this.pattern = event.pattern; - this.what = event.what; - } - } - - public Multiline(Map fields) throws UnsupportedEncodingException { - String strPattern = ""; - for(String key : fields.keySet()) { - if ("pattern".equals(key)) - strPattern = fields.get(key); - else if ("negate".equals(key)) - negate = Boolean.parseBoolean(fields.get(key)); - else if ("what".equals(key)) - what = WhatType.valueOf(fields.get(key)); - else - throw new UnsupportedEncodingException(key + " not supported"); - } - pattern = Pattern.compile(strPattern); - - } - - public Pattern getPattern() { - return pattern; - } + public enum WhatType {Previous, Next} - public boolean isNegate() { - return negate; - } + ; - public WhatType getWhat() { - return what; - } - - public boolean isPrevious() { - return what == WhatType.Previous; - } - - public boolean isPatternFound (byte[] line) { - boolean result = pattern.matcher(new String(line)).find(); - if (negate) return !result; - return result; + private Pattern pattern = null; + private boolean negate = false; + private WhatType what = WhatType.Previous; + + public Multiline() { + } + + public Multiline(Multiline event) { + if (event != null) { + this.negate = event.negate; + this.pattern = event.pattern; + this.what = event.what; } - - @Override - public String toString() { - return new ToStringBuilder(this). - append("pattern", pattern). - append("negate", negate). - append("what", what). - toString(); + } + + public Multiline(Map fields) throws UnsupportedEncodingException { + String strPattern = ""; + for (String key : fields.keySet()) { + if ("pattern".equals(key)) + strPattern = fields.get(key); + else if ("negate".equals(key)) + negate = Boolean.parseBoolean(fields.get(key)); + else if ("what".equals(key)) + what = WhatType.valueOf(fields.get(key)); + else + throw new UnsupportedEncodingException(key + " not supported"); } + pattern = Pattern.compile(strPattern); + + } + + public Pattern getPattern() { + return pattern; + } + + public boolean isNegate() { + return negate; + } + + public WhatType getWhat() { + return what; + } + + public boolean isPrevious() { + return what == WhatType.Previous; + } + + public boolean isPatternFound(byte[] line) { + boolean result = pattern.matcher(new String(line)).find(); + if (negate) return !result; + return result; + } + + @Override + public String toString() { + return new ToStringBuilder(this). + append("pattern", pattern). + append("negate", negate). + append("what", what). + toString(); + } } diff --git a/src/test/java/info/fetter/logstashforwarder/FileReaderTest.java b/src/test/java/info/fetter/logstashforwarder/FileReaderTest.java index b3fad80..d6ef714 100644 --- a/src/test/java/info/fetter/logstashforwarder/FileReaderTest.java +++ b/src/test/java/info/fetter/logstashforwarder/FileReaderTest.java @@ -1,99 +1,102 @@ -package info.fetter.logstashforwarder; - -/* - * Copyright 2015 Didier Fetter - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import static org.apache.log4j.Level.*; -import info.fetter.logstashforwarder.util.AdapterException; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.io.FileUtils; -import org.apache.log4j.BasicConfigurator; -import org.apache.log4j.Logger; -import org.apache.log4j.spi.RootLogger; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class FileReaderTest { - Logger logger = Logger.getLogger(FileReaderTest.class); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - BasicConfigurator.configure(); - RootLogger.getRootLogger().setLevel(TRACE); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - BasicConfigurator.resetConfiguration(); - } - - @Test - public void testFileReader1() throws IOException, InterruptedException, AdapterException { - FileReader reader = new FileReader(2); - reader.setAdapter(new MockProtocolAdapter()); - List fileList = new ArrayList(1); - File file1 = new File("testFileReader1.txt"); - FileUtils.write(file1, "testFileReader1 line1\n"); - FileUtils.write(file1, " nl line12\n", true); - FileUtils.write(file1, "testFileReader1 line2\n", true); - FileUtils.write(file1, "testFileReader1 line3\n", true); - Thread.sleep(500); - FileState state = new FileState(file1); - fileList.add(state); - state.setFields(new Event().addField("testFileReader1", "testFileReader1")); - Map m = new HashMap(); - m.put("pattern", " nl"); - m.put("negate", "false"); - state.setMultiline(new Multiline(m)); - reader.readFiles(fileList); - reader.readFiles(fileList); - reader.readFiles(fileList); - //FileUtils.forceDelete(file1); - } - - @Test - public void testFileReader2() throws IOException, InterruptedException, AdapterException { - FileReader reader = new FileReader(2); - reader.setAdapter(new MockProtocolAdapter()); - List fileList = new ArrayList(1); - File file1 = new File("testFileReader1.txt"); - FileUtils.write(file1, "testFileReader1 line1\n"); - FileUtils.write(file1, " nl line12\n", true); - FileUtils.write(file1, "testFileReader1 line2\n", true); - FileUtils.write(file1, "testFileReader1 line3\n", true); - Thread.sleep(500); - FileState state = new FileState(file1); - fileList.add(state); - state.setFields(new Event().addField("testFileReader1", "testFileReader1")); - Map m = new HashMap(); - m.put("pattern", "testFileReader1"); - m.put("negate", "true"); - state.setMultiline(new Multiline(m)); - reader.readFiles(fileList); - reader.readFiles(fileList); - reader.readFiles(fileList); - //FileUtils.forceDelete(file1); - } -} +package info.fetter.logstashforwarder; + +/* + * Copyright 2015 Didier Fetter + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import static junit.framework.TestCase.assertEquals; +import static org.apache.log4j.Level.*; +import info.fetter.logstashforwarder.util.AdapterException; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.RootLogger; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class FileReaderTest { + Logger logger = Logger.getLogger(FileReaderTest.class); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + BasicConfigurator.configure(); + RootLogger.getRootLogger().setLevel(TRACE); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + BasicConfigurator.resetConfiguration(); + } + + @Test + public void testFileReader1() throws IOException, InterruptedException, AdapterException { + FileReader reader = new FileReader(2); + reader.setAdapter(new MockProtocolAdapter()); + List fileList = new ArrayList(1); + File file1 = new File("testFileReader1.txt"); + FileUtils.write(file1, "testFileReader1 line1\n"); + FileUtils.write(file1, " nl line12\n", true); + FileUtils.write(file1, " nl line13\n", true); + FileUtils.write(file1, "testFileReader1 line2\n", true); + FileUtils.write(file1, "testFileReader1 line3\n", true); + Thread.sleep(500); + FileState state = new FileState(file1); + fileList.add(state); + state.setFields(new Event().addField("testFileReader1", "testFileReader1")); + Map m = new HashMap(); + m.put("pattern", " nl"); + m.put("negate", "false"); + state.setMultiline(new Multiline(m)); + assertEquals(2, reader.readFiles(fileList)); + assertEquals(1, reader.readFiles(fileList)); + assertEquals(0, reader.readFiles(fileList)); + //FileUtils.forceDelete(file1); + } + + @Test + public void testFileReader2() throws IOException, InterruptedException, AdapterException { + FileReader reader = new FileReader(2); + reader.setAdapter(new MockProtocolAdapter()); + List fileList = new ArrayList(1); + File file1 = new File("testFileReader1.txt"); + FileUtils.write(file1, "testFileReader1 line1\n"); + FileUtils.write(file1, " nl line12\n", true); + FileUtils.write(file1, " nl line13\n", true); + FileUtils.write(file1, "testFileReader1 line2\n", true); + FileUtils.write(file1, "testFileReader1 line3\n", true); + Thread.sleep(500); + FileState state = new FileState(file1); + fileList.add(state); + state.setFields(new Event().addField("testFileReader1", "testFileReader1")); + Map m = new HashMap(); + m.put("pattern", "testFileReader1"); + m.put("negate", "true"); + state.setMultiline(new Multiline(m)); + assertEquals(2, reader.readFiles(fileList)); + assertEquals(1, reader.readFiles(fileList)); + assertEquals(0, reader.readFiles(fileList)); + //FileUtils.forceDelete(file1); + } +} From 36d8c468cb2d2efe97b64af05c24ae689e878486 Mon Sep 17 00:00:00 2001 From: toliman Date: Tue, 18 May 2021 08:58:23 +0200 Subject: [PATCH 04/11] Added support for Jar with dependencies --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cc2bbe9..ccf2e1a 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,9 @@ 2.5.3 src/assembly/tarball.xml + + jar-with-dependencies + @@ -166,5 +169,4 @@ 1.2 - \ No newline at end of file From 3ec326b8fcc78cd644d9b61561534460fc240f62 Mon Sep 17 00:00:00 2001 From: toliman Date: Sat, 4 Dec 2021 23:58:32 +0100 Subject: [PATCH 05/11] Updated library to support Java 7 and fixed a multiline bug. --- README.md | 6 +- pom.xml | 6 +- .../fetter/logstashforwarder/FileReader.java | 484 +++++++++--------- .../logstashforwarder/FileReaderTest.java | 112 +++- 4 files changed, 357 insertions(+), 251 deletions(-) diff --git a/README.md b/README.md index 4eaf5ae..44df467 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ So logstash-forwarder-java is a solution for those who want a portable, lightwei ## How to install it ? Download one of the following archives : - - [logstash-forwarder-java-0.2.4-bin.zip](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.4/logstash-forwarder-java-0.2.4-bin.zip) - - [logstash-forwarder-java-0.2.4-bin.tar.gz](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.4/logstash-forwarder-java-0.2.4-bin.tar.gz) - - [logstash-forwarder-java-0.2.4-bin.tar.bz2](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.4/logstash-forwarder-java-0.2.4-bin.tar.bz2) + - [logstash-forwarder-java-0.2.6-bin.zip](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.zip) + - [logstash-forwarder-java-0.2.6-bin.tar.gz](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.tar.gz) + - [logstash-forwarder-java-0.2.6-bin.tar.bz2](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.tar.bz2) Or download the maven project and run maven package. Then you can install one of the archives located in the target directory. diff --git a/pom.xml b/pom.xml index ccf2e1a..d7d9ad5 100644 --- a/pom.xml +++ b/pom.xml @@ -3,12 +3,14 @@ 4.0.0 logstash-forwarder-java logstash-forwarder-java - 0.2.5-SNAPSHOT + 0.2.6-SNAPSHOT logstash-forwarder-java Java version of logstash forwarder https://github.com/didfet/logstash-forwarder-java UTF-8 + 1.7 + 1.7 @@ -144,7 +146,7 @@ com.fasterxml.jackson.core jackson-databind - 2.1.5 + 2.13.0 junit diff --git a/src/main/java/info/fetter/logstashforwarder/FileReader.java b/src/main/java/info/fetter/logstashforwarder/FileReader.java index 99f7468..50d5ba1 100644 --- a/src/main/java/info/fetter/logstashforwarder/FileReader.java +++ b/src/main/java/info/fetter/logstashforwarder/FileReader.java @@ -1,242 +1,242 @@ -package info.fetter.logstashforwarder; - -/* - * Copyright 2015 Didier Fetter - * Copyright 2017 Alberto González Palomo https://sentido-labs.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import info.fetter.logstashforwarder.util.AdapterException; -import info.fetter.logstashforwarder.util.RandomAccessFile; - -import java.io.File; -import java.io.IOException; -//import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import org.apache.log4j.Logger; - - -public class FileReader extends Reader { - private static Logger logger = Logger.getLogger(FileReader.class); - private static final byte[] ZIP_MAGIC = new byte[] {(byte) 0x50, (byte) 0x4b, (byte) 0x03, (byte) 0x04}; - private static final byte[] LZW_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x9d}; - private static final byte[] LZH_MAGIC = new byte[] {(byte) 0x1f, (byte) 0xa0}; - private static final byte[] GZ_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x8b, (byte) 0x08}; - private static final byte[][] MAGICS = new byte[][] {ZIP_MAGIC, LZW_MAGIC, LZH_MAGIC, GZ_MAGIC}; - private Map pointerMap; - ByteBuffer bufferedLines = ByteBuffer.allocate(BYTEBUFFER_CAPACITY); - - public FileReader(int spoolSize) { - super(spoolSize); - } - - public int readFiles(Collection fileList) throws AdapterException { - int eventCount = 0; - if(logger.isTraceEnabled()) { - logger.trace("Reading " + fileList.size() + " file(s)"); - } - pointerMap = new HashMap(fileList.size(),1); - for(FileState state : fileList) { - eventCount += readFile(state, spoolSize - eventCount); - } - if(eventCount > 0) { - try { - adapter.sendEvents(eventList); - } catch(AdapterException e) { - eventList.clear(); // Be sure no events will be sent twice after reconnect - throw e; - } - } - for(FileState state : fileList) { - state.setPointer(pointerMap.get(state.getFile())); - } - eventList.clear(); - return eventCount; // Return number of events sent to adapter - } - - private int readFile(FileState state, int spaceLeftInSpool) { - File file = state.getFile(); - long pointer = state.getPointer(); - int numberOfEvents = 0; - try { - if(state.isDeleted() || state.getRandomAccessFile() == null) { // Don't try to read this file - if(logger.isTraceEnabled()) { - logger.trace("File : " + file + " has been deleted"); - } - } else if(state.getRandomAccessFile().isEmpty()) { - if(logger.isTraceEnabled()) { - logger.trace("File : " + file + " is empty"); - } - } else { - int eventListSizeBefore = eventList.size(); - if(logger.isTraceEnabled()) { - logger.trace("File : " + file + " pointer : " + pointer); - logger.trace("Space left in spool : " + spaceLeftInSpool); - } - if(isCompressedFile(state)) { - pointer = file.length(); - } else { - pointer = readLines(state, spaceLeftInSpool); - } - numberOfEvents = eventList.size() - eventListSizeBefore; - } - } catch(IOException e) { - logger.warn("Exception raised while reading file : " + state.getFile(), e); - } - pointerMap.put(file, pointer); - return numberOfEvents; // Return number of events read - } - - private boolean isCompressedFile(FileState state) { - RandomAccessFile reader = state.getRandomAccessFile(); - - try { - for(byte[] magic : MAGICS) { - byte[] fileBytes = new byte[magic.length]; - reader.seek(0); - int read = reader.read(fileBytes); - if (read != magic.length) { - continue; - } - if(Arrays.equals(magic, fileBytes)) { - logger.debug("Compressed file detected : " + state.getFile()); - return true; - } - } - } catch(IOException e) { - logger.warn("Exception raised while reading file : " + state.getFile(), e); - } - return false; - } - - private static byte[] extractBytes(ByteBuffer byteBuffer) - { - byte[] bytes = new byte[byteBuffer.position()]; - byteBuffer.rewind(); - byteBuffer.get(bytes); - byteBuffer.clear(); - return bytes; - } - - private static void copyLineToBuffer(byte[] line, ByteBuffer byteBuffer) { - if(line.length <= byteBuffer.remaining()) { - byteBuffer.put(line); - } else { - byteBuffer.put(line, 0, byteBuffer.remaining()); - } - } - - private long readLines(FileState state, int spaceLeftInSpool) { - RandomAccessFile reader = state.getRandomAccessFile(); - long pos = state.getPointer(); - Multiline multiline = state.getMultiline(); - try { - reader.seek(pos); - byte[] line = readLine(reader); - bufferedLines.clear(); - - if(multiline != null && multiline.isPrevious()) { - spaceLeftInSpool--; - } - while (line != null && spaceLeftInSpool > 0) { - if(logger.isDebugEnabled()) { - logger.debug("-- Read line : " + new String(line)); - logger.debug("-- Space left in spool : " + spaceLeftInSpool); - } - pos = reader.getFilePointer(); - if(multiline == null) { - addEvent(state, pos, line); - spaceLeftInSpool--; - } - else { - if(logger.isDebugEnabled()) { - logger.debug("-- Multiline : " + multiline + " matches " + multiline.isPatternFound(line)); - } - if(multiline.isPatternFound(line)) { - // buffer the line - if(bufferedLines.position() > 0 && bufferedLines.hasRemaining()) { - bufferedLines.put(Multiline.JOINT); - } - copyLineToBuffer(line, bufferedLines); - } - else { - if(multiline.isPrevious()) { - // did not match, so new event started - if (bufferedLines.position() > 0) { - addEvent(state, pos, extractBytes(bufferedLines)); - spaceLeftInSpool--; - } - copyLineToBuffer(line, bufferedLines); - } - else { - // did not match, add the current line - if(bufferedLines.position() > 0) { - if(bufferedLines.hasRemaining()) { - bufferedLines.put(Multiline.JOINT); - } - copyLineToBuffer(line, bufferedLines); - addEvent(state, pos, extractBytes(bufferedLines)); - spaceLeftInSpool--; - } - else { - addEvent(state, pos, line); - spaceLeftInSpool--; - } - } - } - } - line = readLine(reader); - } - if(bufferedLines.position() > 0) { - addEvent(state, pos, extractBytes(bufferedLines)); // send any buffered lines left - } - reader.seek(pos); // Ensure we can re-read if necessary - } catch(IOException e) { - logger.warn("Exception raised while reading file : " + state.getFile(), e); - } - return pos; - } - - private byte[] readLine(RandomAccessFile reader) throws IOException { - byteBuffer.clear(); - int ch; - boolean seenCR = false; - while((ch=reader.read()) != -1) { - switch(ch) { - case '\n': - return extractBytes(byteBuffer); - case '\r': - seenCR = true; - break; - default: - if (seenCR) { - if(byteBuffer.hasRemaining()) { - byteBuffer.put((byte) '\r'); - } - seenCR = false; - } - if(byteBuffer.hasRemaining()) { - byteBuffer.put((byte)ch); - } - } - } - return null; - } - -} +package info.fetter.logstashforwarder; + +/* + * Copyright 2015 Didier Fetter + * Copyright 2017 Alberto González Palomo https://sentido-labs.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import info.fetter.logstashforwarder.util.AdapterException; +import info.fetter.logstashforwarder.util.RandomAccessFile; + +import java.io.File; +import java.io.IOException; +//import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.apache.log4j.Logger; + + +public class FileReader extends Reader { + private static Logger logger = Logger.getLogger(FileReader.class); + private static final byte[] ZIP_MAGIC = new byte[] {(byte) 0x50, (byte) 0x4b, (byte) 0x03, (byte) 0x04}; + private static final byte[] LZW_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x9d}; + private static final byte[] LZH_MAGIC = new byte[] {(byte) 0x1f, (byte) 0xa0}; + private static final byte[] GZ_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x8b, (byte) 0x08}; + private static final byte[][] MAGICS = new byte[][] {ZIP_MAGIC, LZW_MAGIC, LZH_MAGIC, GZ_MAGIC}; + private Map pointerMap; + ByteBuffer bufferedLines = ByteBuffer.allocate(BYTEBUFFER_CAPACITY); + + public FileReader(int spoolSize) { + super(spoolSize); + } + + public int readFiles(Collection fileList) throws AdapterException { + int eventCount = 0; + if(logger.isTraceEnabled()) { + logger.trace("Reading " + fileList.size() + " file(s)"); + } + pointerMap = new HashMap(fileList.size(),1); + for(FileState state : fileList) { + eventCount += readFile(state, spoolSize - eventCount); + } + if(eventCount > 0) { + try { + adapter.sendEvents(eventList); + } catch(AdapterException e) { + eventList.clear(); // Be sure no events will be sent twice after reconnect + throw e; + } + } + for(FileState state : fileList) { + state.setPointer(pointerMap.get(state.getFile())); + } + eventList.clear(); + return eventCount; // Return number of events sent to adapter + } + + private int readFile(FileState state, int spaceLeftInSpool) { + File file = state.getFile(); + long pointer = state.getPointer(); + int numberOfEvents = 0; + try { + if(state.isDeleted() || state.getRandomAccessFile() == null) { // Don't try to read this file + if(logger.isTraceEnabled()) { + logger.trace("File : " + file + " has been deleted"); + } + } else if(state.getRandomAccessFile().isEmpty()) { + if(logger.isTraceEnabled()) { + logger.trace("File : " + file + " is empty"); + } + } else { + int eventListSizeBefore = eventList.size(); + if(logger.isTraceEnabled()) { + logger.trace("File : " + file + " pointer : " + pointer); + logger.trace("Space left in spool : " + spaceLeftInSpool); + } + if(isCompressedFile(state)) { + pointer = file.length(); + } else { + pointer = readLines(state, spaceLeftInSpool); + } + numberOfEvents = eventList.size() - eventListSizeBefore; + } + } catch(IOException e) { + logger.warn("Exception raised while reading file : " + state.getFile(), e); + } + pointerMap.put(file, pointer); + return numberOfEvents; // Return number of events read + } + + private boolean isCompressedFile(FileState state) { + RandomAccessFile reader = state.getRandomAccessFile(); + + try { + for(byte[] magic : MAGICS) { + byte[] fileBytes = new byte[magic.length]; + reader.seek(0); + int read = reader.read(fileBytes); + if (read != magic.length) { + continue; + } + if(Arrays.equals(magic, fileBytes)) { + logger.debug("Compressed file detected : " + state.getFile()); + return true; + } + } + } catch(IOException e) { + logger.warn("Exception raised while reading file : " + state.getFile(), e); + } + return false; + } + + private static byte[] extractBytes(ByteBuffer byteBuffer) + { + byte[] bytes = new byte[byteBuffer.position()]; + byteBuffer.rewind(); + byteBuffer.get(bytes); + byteBuffer.clear(); + return bytes; + } + + private static void copyLineToBuffer(byte[] line, ByteBuffer byteBuffer) { + if(line.length <= byteBuffer.remaining()) { + byteBuffer.put(line); + } else { + byteBuffer.put(line, 0, byteBuffer.remaining()); + } + } + + private long readLines(FileState state, int spaceLeftInSpool) { + RandomAccessFile reader = state.getRandomAccessFile(); + long pos = state.getPointer(); + Multiline multiline = state.getMultiline(); + try { + reader.seek(pos); + byte[] line = readLine(reader); + bufferedLines.clear(); + + while (line != null && spaceLeftInSpool > 0) { + if(logger.isDebugEnabled()) { + logger.debug("-- Read line : " + new String(line)); + logger.debug("-- Space left in spool : " + spaceLeftInSpool); + } + long prevPos = pos; + pos = reader.getFilePointer(); + if(multiline == null) { + addEvent(state, pos, line); + spaceLeftInSpool--; + } + else { + if(logger.isDebugEnabled()) { + logger.debug("-- Multiline : " + multiline + " matches " + multiline.isPatternFound(line)); + } + if(multiline.isPatternFound(line)) { + // buffer the line + if(bufferedLines.position() > 0 && bufferedLines.hasRemaining()) { + bufferedLines.put(Multiline.JOINT); + } + copyLineToBuffer(line, bufferedLines); + } + else { + if(multiline.isPrevious()) { + // did not match, so new event started + if (bufferedLines.position() > 0) { + addEvent(state, prevPos, extractBytes(bufferedLines)); + spaceLeftInSpool--; + } + if (spaceLeftInSpool > 0) + copyLineToBuffer(line, bufferedLines); + else + pos = prevPos; + } + else { + // did not match, add the current line + if(bufferedLines.position() > 0) { + if(bufferedLines.hasRemaining()) { + bufferedLines.put(Multiline.JOINT); + } + copyLineToBuffer(line, bufferedLines); + addEvent(state, pos, extractBytes(bufferedLines)); + } + else { + addEvent(state, pos, line); + } + spaceLeftInSpool--; + } + } + } + line = readLine(reader); + } + if(bufferedLines.position() > 0) { + addEvent(state, pos, extractBytes(bufferedLines)); // send any buffered lines left + } + reader.seek(pos); // Ensure we can re-read if necessary + } catch(IOException e) { + logger.warn("Exception raised while reading file : " + state.getFile(), e); + } + return pos; + } + + private byte[] readLine(RandomAccessFile reader) throws IOException { + byteBuffer.clear(); + int ch; + boolean seenCR = false; + while((ch=reader.read()) != -1) { + switch(ch) { + case '\n': + return extractBytes(byteBuffer); + case '\r': + seenCR = true; + break; + default: + if (seenCR) { + if(byteBuffer.hasRemaining()) { + byteBuffer.put((byte) '\r'); + } + seenCR = false; + } + if(byteBuffer.hasRemaining()) { + byteBuffer.put((byte)ch); + } + } + } + return null; + } + +} diff --git a/src/test/java/info/fetter/logstashforwarder/FileReaderTest.java b/src/test/java/info/fetter/logstashforwarder/FileReaderTest.java index 5b47374..f932ba6 100644 --- a/src/test/java/info/fetter/logstashforwarder/FileReaderTest.java +++ b/src/test/java/info/fetter/logstashforwarder/FileReaderTest.java @@ -72,17 +72,18 @@ public void testFileReader1() throws IOException, InterruptedException, AdapterE assertEquals(2, reader.readFiles(fileList)); assertEquals(1, reader.readFiles(fileList)); assertEquals(0, reader.readFiles(fileList)); - //FileUtils.forceDelete(file1); + FileUtils.forceDelete(file1); } - @Test + @Test public void testFileReader2() throws IOException, InterruptedException, AdapterException { FileReader reader = new FileReader(2); reader.setAdapter(new MockProtocolAdapter()); List fileList = new ArrayList(1); File file1 = new File("testFileReader1.txt"); FileUtils.write(file1, "testFileReader1 line1\n"); - FileUtils.write(file1, " nl line12\n", true); + FileUtils.write(file1, " nl line12\n", true); + FileUtils.write(file1, " nl line13\n", true); FileUtils.write(file1, "testFileReader1 line2\n", true); FileUtils.write(file1, "testFileReader1 line3\n", true); Thread.sleep(500); @@ -96,6 +97,109 @@ public void testFileReader2() throws IOException, InterruptedException, AdapterE assertEquals(2, reader.readFiles(fileList)); assertEquals(1, reader.readFiles(fileList)); assertEquals(0, reader.readFiles(fileList)); - //FileUtils.forceDelete(file1); + FileUtils.forceDelete(file1); + } + + @Test + public void testFileReader3() throws IOException, InterruptedException, AdapterException { + FileReader reader = new FileReader(2); + reader.setAdapter(new MockProtocolAdapter()); + List fileList = new ArrayList(1); + File file1 = new File("testFileReader1.txt"); + FileUtils.write(file1, "1234 line1\n"); + FileUtils.write(file1, " nl line12\n", true); + FileUtils.write(file1, " nl line13\n", true); + FileUtils.write(file1, "2345 line2\n", true); + FileUtils.write(file1, "5524 line3\n", true); + FileUtils.write(file1, "7478 line4\n", true); + FileUtils.write(file1, " nl line42\n", true); + FileUtils.write(file1, "8234 line5\n", true); + FileUtils.write(file1, " nl line52\n", true); + Thread.sleep(500); + FileState state = new FileState(file1); + fileList.add(state); + state.setFields(new Event().addField("testFileReader1", "testFileReader1")); + Map m = new HashMap(); + m.put("pattern", "^[0-9]{4}"); + m.put("negate", "true"); + state.setMultiline(new Multiline(m)); + assertEquals(2, reader.readFiles(fileList)); + assertEquals(2, reader.readFiles(fileList)); + assertEquals(1, reader.readFiles(fileList)); + assertEquals(0, reader.readFiles(fileList)); + FileUtils.forceDelete(file1); + } + + @Test + public void testFileReader4() throws IOException, InterruptedException, AdapterException { + FileReader reader = new FileReader(2); + reader.setAdapter(new MockProtocolAdapter()); + List fileList = new ArrayList(1); + File file1 = new File("testFileReader1.txt"); + FileUtils.write(file1, "1234 line1\n"); + FileUtils.write(file1, " nl line12\n", true); + FileUtils.write(file1, "8234 line5\n", true); + FileUtils.write(file1, " nl line52\n", true); + Thread.sleep(500); + FileState state = new FileState(file1); + fileList.add(state); + state.setFields(new Event().addField("testFileReader1", "testFileReader1")); + Map m = new HashMap(); + m.put("pattern", "^[0-9]{4}"); + m.put("negate", "true"); + state.setMultiline(new Multiline(m)); + assertEquals(2, reader.readFiles(fileList)); + assertEquals(0, reader.readFiles(fileList)); + FileUtils.forceDelete(file1); + } + + @Test + public void testFileReader5() throws IOException, InterruptedException, AdapterException { + FileReader reader = new FileReader(2); + reader.setAdapter(new MockProtocolAdapter()); + List fileList = new ArrayList(1); + File file1 = new File("testFileReader1.txt"); + FileUtils.write(file1, "testFileReader1 line1\n"); + FileUtils.write(file1, "testFileReader1 line2\n", true); + FileUtils.write(file1, "testFileReader1 line3\n", true); + Thread.sleep(500); + FileState state = new FileState(file1); + fileList.add(state); + state.setFields(new Event().addField("testFileReader1", "testFileReader1")); + assertEquals(2, reader.readFiles(fileList)); + assertEquals(1, reader.readFiles(fileList)); + assertEquals(0, reader.readFiles(fileList)); + FileUtils.forceDelete(file1); + } + + @Test + public void testFileReader6() throws IOException, InterruptedException, AdapterException { + FileReader reader = new FileReader(100); + reader.setAdapter(new MockProtocolAdapter()); + List fileList = new ArrayList(1); + File file1 = new File("testFileReader6.txt"); + FileUtils.write(file1, "0000 line1\n"); + for (int i=0; i<700; i+=7) { + FileUtils.write(file1, String.format("%04d line\n", i), true); + FileUtils.write(file1, String.format(" nl line%04d\n", i+1), true); + FileUtils.write(file1, String.format("%04d line\n", i+2), true); + FileUtils.write(file1, String.format("%04d line\n", i+3), true); + FileUtils.write(file1, String.format("%04d line\n", i+4), true); + FileUtils.write(file1, String.format(" nl line%04d\n", i+5), true); + FileUtils.write(file1, String.format(" nl line%04d\n", i+6), true); + } + Thread.sleep(500); + FileState state = new FileState(file1); + fileList.add(state); + state.setFields(new Event().addField("testFileReader6", "testFileReader6")); + Map m = new HashMap(); + m.put("pattern", "^[0-9]{4}"); + m.put("negate", "true"); + state.setMultiline(new Multiline(m)); + for (int i=0; i<4; i++) + assertEquals(100, reader.readFiles(fileList)); + assertEquals(1, reader.readFiles(fileList)); + assertEquals(0, reader.readFiles(fileList)); + FileUtils.forceDelete(file1); } } From 91a67128ae5d52a63910f4c068395fe647f55c3b Mon Sep 17 00:00:00 2001 From: toliman Date: Tue, 7 Dec 2021 22:30:13 +0100 Subject: [PATCH 06/11] Release version --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d7d9ad5..39f9b84 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 logstash-forwarder-java logstash-forwarder-java - 0.2.6-SNAPSHOT + 0.2.6 logstash-forwarder-java Java version of logstash forwarder https://github.com/didfet/logstash-forwarder-java @@ -171,4 +171,4 @@ 1.2 - \ No newline at end of file + From aaacdc4352f36f10eefabd091cf93b7f62a782e5 Mon Sep 17 00:00:00 2001 From: toliman7 Date: Tue, 7 Dec 2021 22:36:04 +0100 Subject: [PATCH 07/11] Updated release path --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 44df467..b5a2bcd 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ So logstash-forwarder-java is a solution for those who want a portable, lightwei ## How to install it ? Download one of the following archives : - - [logstash-forwarder-java-0.2.6-bin.zip](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.zip) - - [logstash-forwarder-java-0.2.6-bin.tar.gz](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.tar.gz) - - [logstash-forwarder-java-0.2.6-bin.tar.bz2](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.tar.bz2) + - [logstash-forwarder-java-0.2.6-bin.zip](https://github.com/toliman7/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.zip) + - [logstash-forwarder-java-0.2.6-bin.tar.gz](https://github.com/toliman7/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.tar.gz) + - [logstash-forwarder-java-0.2.6-bin.tar.bz2](https://github.com/toliman7/logstash-forwarder-java/releases/download/0.2.6/logstash-forwarder-java-0.2.6-bin.tar.bz2) Or download the maven project and run maven package. Then you can install one of the archives located in the target directory. From c77d71565543a9733567f74b09e46ecd85591848 Mon Sep 17 00:00:00 2001 From: toliman Date: Tue, 7 Dec 2021 22:36:30 +0100 Subject: [PATCH 08/11] Prepare next steps --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 39f9b84..ebe4caa 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 logstash-forwarder-java logstash-forwarder-java - 0.2.6 + 0.2.7-SNAPSHOT logstash-forwarder-java Java version of logstash forwarder https://github.com/didfet/logstash-forwarder-java From 91d9c19a23e50323e18307957611f613ca43af46 Mon Sep 17 00:00:00 2001 From: toliman Date: Tue, 15 Feb 2022 10:20:11 +0100 Subject: [PATCH 09/11] Need to stay compatible with Java 1.7 and therefore had to update jackson binding version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ebe4caa..3054ab1 100644 --- a/pom.xml +++ b/pom.xml @@ -146,7 +146,7 @@ com.fasterxml.jackson.core jackson-databind - 2.13.0 + 2.12.0 junit From 6c4495159ef4eba60cee5d0d628e3b105d6a56e9 Mon Sep 17 00:00:00 2001 From: toliman Date: Tue, 15 Feb 2022 10:23:01 +0100 Subject: [PATCH 10/11] Release 0.2.7 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3054ab1..9a72746 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 logstash-forwarder-java logstash-forwarder-java - 0.2.7-SNAPSHOT + 0.2.7 logstash-forwarder-java Java version of logstash forwarder https://github.com/didfet/logstash-forwarder-java From e9daf2720b374d6196c8f644af0e3102799ba7c7 Mon Sep 17 00:00:00 2001 From: toliman Date: Tue, 15 Feb 2022 10:24:46 +0100 Subject: [PATCH 11/11] Prepare next changes --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9a72746..64e597e 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 logstash-forwarder-java logstash-forwarder-java - 0.2.7 + 0.2.8-SNAPSHOT logstash-forwarder-java Java version of logstash forwarder https://github.com/didfet/logstash-forwarder-java