001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellComparatorImpl;
030import org.apache.hadoop.hbase.CompareOperator;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.KeyValue;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.Put;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.RegionInfoBuilder;
038import org.apache.hadoop.hbase.client.Scan;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.regionserver.InternalScanner;
044import org.apache.hadoop.hbase.testclassification.FilterTests;
045import org.apache.hadoop.hbase.testclassification.SmallTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
048import org.junit.jupiter.api.AfterEach;
049import org.junit.jupiter.api.BeforeEach;
050import org.junit.jupiter.api.Tag;
051import org.junit.jupiter.api.Test;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055@Tag(FilterTests.TAG)
056@Tag(SmallTests.TAG)
057public class TestDependentColumnFilter {
058
059  private static final Logger LOG = LoggerFactory.getLogger(TestDependentColumnFilter.class);
060  private static final byte[][] ROWS = { Bytes.toBytes("test1"), Bytes.toBytes("test2") };
061  private static final byte[][] FAMILIES =
062    { Bytes.toBytes("familyOne"), Bytes.toBytes("familyTwo") };
063  private static final long STAMP_BASE = EnvironmentEdgeManager.currentTime();
064  private static final long[] STAMPS = { STAMP_BASE - 100, STAMP_BASE - 200, STAMP_BASE - 300 };
065  private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
066  private static final byte[][] BAD_VALS =
067    { Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3") };
068  private static final byte[] MATCH_VAL = Bytes.toBytes("match");
069  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
070
071  List<KeyValue> testVals;
072  private HRegion region;
073
074  @BeforeEach
075  public void setUp() throws Exception {
076    testVals = makeTestVals();
077
078    TableDescriptor tableDescriptor =
079      TableDescriptorBuilder.newBuilder(TableName.valueOf(this.getClass().getSimpleName()))
080        .setColumnFamily(
081          ColumnFamilyDescriptorBuilder.newBuilder(FAMILIES[0]).setMaxVersions(3).build())
082        .setColumnFamily(
083          ColumnFamilyDescriptorBuilder.newBuilder(FAMILIES[1]).setMaxVersions(3).build())
084        .build();
085    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
086    this.region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
087      TEST_UTIL.getConfiguration(), tableDescriptor);
088    addData();
089  }
090
091  @AfterEach
092  public void tearDown() throws Exception {
093    HBaseTestingUtil.closeRegionAndWAL(this.region);
094  }
095
096  private void addData() throws IOException {
097    Put put = new Put(ROWS[0]);
098    // add in an entry for each stamp, with 2 as a "good" value
099    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
100    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]);
101    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
102    // add in entries for stamps 0 and 2.
103    // without a value check both will be "accepted"
104    // with one 2 will be accepted(since the corresponding ts entry
105    // has a matching value
106    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]);
107    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
108
109    this.region.put(put);
110
111    put = new Put(ROWS[1]);
112    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
113    // there is no corresponding timestamp for this so it should never pass
114    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
115    // if we reverse the qualifiers this one should pass
116    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL);
117    // should pass
118    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]);
119
120    this.region.put(put);
121  }
122
123  private List<KeyValue> makeTestVals() {
124    List<KeyValue> testVals = new ArrayList<>();
125    testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
126    testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
127    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
128    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
129    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));
130
131    return testVals;
132  }
133
134  /**
135   * This shouldn't be confused with TestFilter#verifyScan as expectedKeys is not the per row total,
136   * but the scan total
137   */
138  private void verifyScan(Scan s, long expectedRows, long expectedCells) throws IOException {
139    InternalScanner scanner = this.region.getScanner(s);
140    List<Cell> results = new ArrayList<>();
141    int i = 0;
142    int cells = 0;
143    for (boolean done = true; done; i++) {
144      done = scanner.next(results);
145      Arrays.sort(results.toArray(new Cell[results.size()]), CellComparatorImpl.COMPARATOR);
146      LOG.info("counter=" + i + ", " + results);
147      if (results.isEmpty()) break;
148      cells += results.size();
149      assertTrue(expectedRows > i, "Scanned too many rows! Only expected " + expectedRows
150        + " total but already scanned " + (i + 1));
151      assertTrue(expectedCells >= cells,
152        "Expected " + expectedCells + " cells total but " + "already scanned " + cells);
153      results.clear();
154    }
155    assertEquals(expectedRows, i, "Expected " + expectedRows + " rows but scanned " + i + " rows");
156    assertEquals(expectedCells, cells,
157      "Expected " + expectedCells + " cells but scanned " + cells + " cells");
158  }
159
160  /**
161   * Test scans using a DependentColumnFilter
162   */
163  @Test
164  public void testScans() throws Exception {
165    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
166
167    Scan scan = new Scan();
168    scan.setFilter(filter);
169    scan.readVersions(Integer.MAX_VALUE);
170
171    verifyScan(scan, 2, 8);
172
173    // drop the filtering cells
174    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true);
175    scan = new Scan();
176    scan.setFilter(filter);
177    scan.readVersions(Integer.MAX_VALUE);
178
179    verifyScan(scan, 2, 3);
180
181    // include a comparator operation
182    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false, CompareOperator.EQUAL,
183      new BinaryComparator(MATCH_VAL));
184    scan = new Scan();
185    scan.setFilter(filter);
186    scan.readVersions(Integer.MAX_VALUE);
187
188    /*
189     * expecting to get the following 3 cells row 0 put.add(FAMILIES[0], QUALIFIER, STAMPS[2],
190     * MATCH_VAL); put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); row 1
191     * put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
192     */
193    verifyScan(scan, 2, 3);
194
195    // include a comparator operation and drop comparator
196    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL,
197      new BinaryComparator(MATCH_VAL));
198    scan = new Scan();
199    scan.setFilter(filter);
200    scan.readVersions(Integer.MAX_VALUE);
201
202    /*
203     * expecting to get the following 1 cell row 0 put.add(FAMILIES[1], QUALIFIER, STAMPS[2],
204     * BAD_VALS[2]);
205     */
206    verifyScan(scan, 1, 1);
207
208  }
209
210  /**
211   * Test that the filter correctly drops rows without a corresponding timestamp
212   */
213  @Test
214  public void testFilterDropping() throws Exception {
215    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
216    List<Cell> accepted = new ArrayList<>();
217    for (Cell val : testVals) {
218      if (filter.filterCell(val) == ReturnCode.INCLUDE) {
219        accepted.add(val);
220      }
221    }
222    assertEquals(5, accepted.size(), "check all values accepted from filterCell");
223
224    filter.filterRowCells(accepted);
225    assertEquals(4, accepted.size(),
226      "check filterRow(List<KeyValue>) dropped cell without corresponding column entry");
227
228    // start do it again with dependent column dropping on
229    filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true);
230    accepted.clear();
231    for (KeyValue val : testVals) {
232      if (filter.filterCell(val) == ReturnCode.INCLUDE) {
233        accepted.add(val);
234      }
235    }
236    assertEquals(2, accepted.size(), "check the filtering column cells got dropped");
237
238    filter.filterRowCells(accepted);
239    assertEquals(2, accepted.size(), "check cell retention");
240  }
241
242  /**
243   * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString().
244   */
245  @Test
246  public void testToStringWithNullComparator() {
247    // Test constructor that implicitly sets a null comparator
248    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
249    assertNotNull(filter.toString());
250    assertTrue(filter.toString().contains("null"),
251      "check string contains 'null' as compatator is null");
252
253    // Test constructor with explicit null comparator
254    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, null);
255    assertNotNull(filter.toString());
256    assertTrue(filter.toString().contains("null"),
257      "check string contains 'null' as compatator is null");
258  }
259
260  @Test
261  public void testToStringWithNonNullComparator() {
262    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL,
263      new BinaryComparator(MATCH_VAL));
264    assertNotNull(filter.toString());
265    assertTrue(filter.toString().contains("match"), "check string contains comparator value");
266  }
267
268}