From 052f251ae9265aad21e814a5f532771abfe5684a Mon Sep 17 00:00:00 2001 From: travisbeale Date: Tue, 22 Sep 2020 09:11:15 -0400 Subject: [PATCH] Refine heuristic to filter out tall-ish whitespace elements that can throw off text chunking by considering realistic font sizes --- .../java/technology/tabula/TextStripper.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/technology/tabula/TextStripper.java b/src/main/java/technology/tabula/TextStripper.java index e437146e..329d45a2 100644 --- a/src/main/java/technology/tabula/TextStripper.java +++ b/src/main/java/technology/tabula/TextStripper.java @@ -13,8 +13,11 @@ import java.util.List; public class TextStripper extends PDFTextStripper { + private static final String NBSP = "\u00A0"; private static final float AVG_HEIGHT_MULT_THRESHOLD = 6.0f; + private static final float MAX_BLANK_FONT_SIZE = 40.0f; + private static final float MIN_BLANK_FONT_SIZE = 2.0f; private PDDocument document; public ArrayList textElements; public RectangleSpatialIndex spatialIndex; @@ -69,15 +72,24 @@ protected void writeString(String string, List textPositions) thro this.minCharWidth = (float) Math.min(this.minCharWidth, te.getWidth()); this.minCharHeight = (float) Math.min(this.minCharHeight, te.getHeight()); - + countHeight++; totalHeight += te.getHeight(); float avgHeight = totalHeight / countHeight; - if (avgHeight > 0 - && te.getHeight() >= (avgHeight * AVG_HEIGHT_MULT_THRESHOLD) - && (te.getText() == null || te.getText().trim().equals(""))) { - continue; + //We have an issue where tall blank cells throw off the row height calculation + //Introspect a blank cell a bit here to see if it should be thrown away + if ((te.getText() == null || te.getText().trim().equals(""))) { + //if the cell height is more than AVG_HEIGHT_MULT_THRESHOLDxaverage, throw it away + if (avgHeight > 0 + && te.getHeight() >= (avgHeight * AVG_HEIGHT_MULT_THRESHOLD)) { + continue; + } + + //if the font size is outside of reasonable ranges, throw it away + if (textPosition.getFontSizeInPt() > MAX_BLANK_FONT_SIZE || textPosition.getFontSizeInPt() < MIN_BLANK_FONT_SIZE) { + continue; + } } this.spatialIndex.add(te);